From ecae00fe30e09da757a6836c6d27bb9a3fbcbae5 Mon Sep 17 00:00:00 2001 From: Eugene Aseev Date: Wed, 16 Sep 2026 21:58:23 +0800 Subject: [PATCH 01/19] chainplot: scoped onchain events to a reproducible dataset and dashboard An agent-first toolkit. A project declares which contracts and events it cares about; chainplot indexes exactly that range, exports it to parquet, runs SQL over it, and builds a static dashboard that needs no server. How it holds together: - Plans are digest-bound. `plan` records the project digest and every job boundary; `apply` refuses a plan whose project changed underneath it, so a run either does what was reviewed or nothing at all. - Coverage is proven rather than assumed. Every ingested range is joined back to block hashes from the chain, and the promotion gate refuses to build a release over a range that cannot be shown complete. - Queries run sandboxed. The DuckDB worker reads its snapshots, then closes external access before any project-supplied SQL executes, and admits only statements that survive `json_serialize_sql` as read-only. - Amounts stay exact. A uint256 travels as a decimal string from parquet to the rendered cell; nothing but chart geometry ever touches a double. - Releases are content-addressed and forkable. `publish` writes a release under a digest of its own content; `fork` reconstructs the project from a published release and recomputes it offline. Three examples cover the surface, one per release mode: a year of USDC supply (1.43M mint/burn events), WETH wrap/unwrap flows, and transfer traffic. Co-Authored-By: Claude Opus 5 --- .dockerignore | 11 + .env.example | 26 + .github/workflows/ci.yml | 92 ++ .gitignore | 17 + LICENSE | 21 + NOTICES.md | 21 + README.md | 254 +++ docker/producer.Dockerfile | 29 + docs/acceptance.md | 55 + docs/capabilities.md | 99 ++ docs/compatibility.md | 165 ++ docs/plans/2026-09-12-m1-fixture-cli.md | 1252 ++++++++++++++ docs/plans/2026-09-13-m2-ingest-snapshots.md | 775 +++++++++ ...09-13-m3-models-dashboards-static-build.md | 159 ++ docs/plans/2026-09-13-m4-publish-fork.md | 180 ++ .../2026-09-14-m5-examples-acceptance.md | 87 + docs/security.md | 92 ++ docs/specs/2026-09-12-chainplot-design.md | 1042 ++++++++++++ examples/README.md | 25 + examples/fork/README.md | 41 + examples/protocol-flows/.env.example | 5 + examples/protocol-flows/README.md | 19 + examples/protocol-flows/abis/WETH.json | 20 + examples/protocol-flows/chainplot.yaml | 73 + examples/protocol-flows/compose.yaml | 35 + .../protocol-flows/queries/deposit_count.sql | 1 + .../queries/largest_deposits.sql | 6 + .../queries/withdrawal_count.sql | 1 + examples/transfer-traffic/.env.example | 6 + examples/transfer-traffic/README.md | 39 + examples/transfer-traffic/abis/ERC20.json | 12 + examples/transfer-traffic/chainplot.yaml | 63 + examples/transfer-traffic/compose.yaml | 35 + .../queries/top_transfers.sql | 9 + .../queries/transfer_count.sql | 1 + examples/transfer-traffic/tests/usdc.yaml | 4 + examples/usdc-supply/.env.example | 13 + examples/usdc-supply/README.md | 51 + examples/usdc-supply/abis/ERC20.json | 12 + examples/usdc-supply/chainplot.yaml | 151 ++ examples/usdc-supply/compose.yaml | 45 + examples/usdc-supply/models/daily.sql | 19 + .../usdc-supply/queries/cumulative_net.sql | 5 + examples/usdc-supply/queries/daily_events.sql | 3 + examples/usdc-supply/queries/daily_flow.sql | 3 + .../usdc-supply/queries/largest_mints.sql | 4 + examples/usdc-supply/queries/net_change.sql | 1 + examples/usdc-supply/queries/total_burned.sql | 1 + examples/usdc-supply/queries/total_minted.sql | 1 + package.json | 48 + pnpm-lock.yaml | 1449 +++++++++++++++++ schemas/coverage.schema.json | 58 + schemas/latest.schema.json | 12 + schemas/lock.schema.json | 10 + schemas/manifest.schema.json | 113 ++ schemas/plan.schema.json | 161 ++ schemas/progress.schema.json | 18 + schemas/project.schema.json | 200 +++ schemas/release.schema.json | 105 ++ schemas/result.schema.json | 52 + scripts/m0-probe/.env.example | 3 + scripts/m0-probe/abis/ERC20.json | 12 + scripts/m0-probe/compose.yaml | 27 + scripts/m0-probe/rindexer.yaml | 34 + scripts/write-fixture-parquet.ts | 69 + src/cli/commands/apply.ts | 57 + src/cli/commands/build.ts | 31 + src/cli/commands/capabilities.ts | 48 + src/cli/commands/describe.ts | 31 + src/cli/commands/doctor.ts | 14 + src/cli/commands/fork.ts | 19 + src/cli/commands/init.ts | 76 + src/cli/commands/plan.ts | 68 + src/cli/commands/publish.ts | 41 + src/cli/commands/query.ts | 105 ++ src/cli/commands/refresh.ts | 132 ++ src/cli/commands/runs.ts | 80 + src/cli/commands/schemaShow.ts | 45 + src/cli/commands/serve.ts | 55 + src/cli/commands/templates.ts | 29 + src/cli/commands/test.ts | 6 + src/cli/commands/validate.ts | 33 + src/cli/envelope.ts | 53 + src/cli/main.ts | 32 + src/cli/run.ts | 325 ++++ src/config/env.ts | 27 + src/fork/fetchGuard.ts | 214 +++ src/fork/importRelease.ts | 321 ++++ src/ingest/adapter.ts | 70 + src/ingest/coverage.ts | 84 + src/ingest/coverageStore.ts | 44 + src/ingest/exportWorkerMain.ts | 76 + src/ingest/exporter.ts | 132 ++ src/ingest/rindexer/index.ts | 32 + src/ingest/rindexer/inspectCoverage.ts | 110 ++ src/ingest/rindexer/renderConfig.ts | 48 + src/ingest/rindexer/runBounded.ts | 134 ++ src/plan/apply.ts | 422 +++++ src/plan/digest.ts | 22 + src/plan/errors.ts | 54 + src/plan/generate.ts | 333 ++++ src/project/assertions.ts | 231 +++ src/project/columns.ts | 65 + src/project/limits.ts | 20 + src/project/load.ts | 29 + src/project/modelGraph.ts | 53 + src/project/types.ts | 117 ++ src/project/validate.ts | 152 ++ src/publish/directory.ts | 78 + src/publish/doctor.ts | 126 ++ src/publish/latestPointer.ts | 25 + src/publish/publishRelease.ts | 169 ++ src/publish/s3.ts | 260 +++ src/publish/serve.ts | 68 + src/publish/sourceBundle.ts | 49 + src/publish/target.ts | 39 + src/publish/writeRelease.ts | 421 +++++ src/query/runQuery.ts | 215 +++ src/query/sqlGuard.ts | 181 ++ src/query/workerMain.ts | 226 +++ src/rpc/client.ts | 62 + src/rpc/heads.ts | 62 + src/runtime/journal.ts | 107 ++ src/runtime/locks.ts | 188 +++ src/snapshot/describe.ts | 57 + templates/fixture-transfers/chainplot.yaml | 25 + .../fixture-transfers/queries/raw_amounts.sql | 6 + .../snapshots/amounts.parquet | Bin 0 -> 1183 bytes .../fixture-transfers/tests/amounts.yaml | 4 + templates/ingest-transfers/.env.example | 13 + templates/ingest-transfers/README.md | 45 + templates/ingest-transfers/abis/ERC20.json | 12 + templates/ingest-transfers/chainplot.yaml | 42 + templates/ingest-transfers/compose.yaml | 45 + .../queries/transfer_count.sql | 1 + tests/cli/a10.e2e.test.ts | 80 + tests/cli/a15.e2e.test.ts | 40 + tests/cli/a2.e2e.test.ts | 67 + tests/cli/build.test.ts | 136 ++ tests/cli/capabilities.test.ts | 22 + tests/cli/describe.test.ts | 25 + tests/cli/doctor.test.ts | 54 + tests/cli/exampleCoverage.test.ts | 98 ++ tests/cli/ingestCommands.test.ts | 266 +++ tests/cli/init.test.ts | 30 + tests/cli/initIngest.test.ts | 70 + tests/cli/query.test.ts | 79 + tests/cli/runs.cancel.test.ts | 291 ++++ tests/cli/runs.test.ts | 199 +++ tests/cli/schemaKinds.test.ts | 66 + tests/cli/schemaShow.test.ts | 21 + tests/cli/serve.test.ts | 84 + tests/cli/testCmd.test.ts | 16 + tests/cli/validate.test.ts | 58 + .../projects/follow-plus-depth/chainplot.yaml | 33 + .../follow-plus-depth/queries/raw_amounts.sql | 1 + .../projects/unknown-field/chainplot.yaml | 17 + .../valid-dataset-only/chainplot.yaml | 16 + .../queries/raw_amounts.sql | 1 + tests/fork/fetchGuard.test.ts | 88 + tests/fork/hostileRelease.test.ts | 167 ++ tests/fork/importRelease.test.ts | 225 +++ tests/helpers/fakeRindexer.mjs | 16 + tests/helpers/run.ts | 14 + tests/ingest/coverage.test.ts | 141 ++ tests/ingest/exporter.test.ts | 55 + tests/ingest/inspectCoverage.test.ts | 116 ++ tests/ingest/live/e2e.live.test.ts | 214 +++ tests/ingest/locks.test.ts | 123 ++ tests/ingest/planApply.test.ts | 476 ++++++ tests/ingest/renderConfig.test.ts | 67 + tests/ingest/runBounded.test.ts | 75 + tests/plan/buildIntent.test.ts | 150 ++ tests/project/modelGraph.test.ts | 55 + tests/publish/directory.test.ts | 132 ++ tests/publish/live/s3.live.test.ts | 124 ++ tests/publish/modes.test.ts | 212 +++ tests/publish/publishCommand.test.ts | 162 ++ tests/publish/s3unit.test.ts | 123 ++ tests/query/models.test.ts | 221 +++ tests/query/sqlGuard.test.ts | 137 ++ tests/rpc/client.test.ts | 116 ++ tests/viewer/format.test.ts | 283 ++++ tsconfig.json | 14 + viewer/index.html | 12 + viewer/package.json | 23 + viewer/pnpm-lock.yaml | 1369 ++++++++++++++++ viewer/pnpm-workspace.yaml | 2 + viewer/src/App.tsx | 533 ++++++ viewer/src/data.ts | 68 + viewer/src/echarts.ts | 21 + viewer/src/env.d.ts | 2 + viewer/src/format.ts | 241 +++ viewer/src/main.tsx | 13 + viewer/src/styles.css | 431 +++++ viewer/tsconfig.json | 12 + viewer/vite.config.ts | 11 + vitest.config.ts | 10 + 198 files changed, 22072 insertions(+) create mode 100644 .dockerignore create mode 100644 .env.example create mode 100644 .github/workflows/ci.yml create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 NOTICES.md create mode 100644 README.md create mode 100644 docker/producer.Dockerfile create mode 100644 docs/acceptance.md create mode 100644 docs/capabilities.md create mode 100644 docs/compatibility.md create mode 100644 docs/plans/2026-09-12-m1-fixture-cli.md create mode 100644 docs/plans/2026-09-13-m2-ingest-snapshots.md create mode 100644 docs/plans/2026-09-13-m3-models-dashboards-static-build.md create mode 100644 docs/plans/2026-09-13-m4-publish-fork.md create mode 100644 docs/plans/2026-09-14-m5-examples-acceptance.md create mode 100644 docs/security.md create mode 100644 docs/specs/2026-09-12-chainplot-design.md create mode 100644 examples/README.md create mode 100644 examples/fork/README.md create mode 100644 examples/protocol-flows/.env.example create mode 100644 examples/protocol-flows/README.md create mode 100644 examples/protocol-flows/abis/WETH.json create mode 100644 examples/protocol-flows/chainplot.yaml create mode 100644 examples/protocol-flows/compose.yaml create mode 100644 examples/protocol-flows/queries/deposit_count.sql create mode 100644 examples/protocol-flows/queries/largest_deposits.sql create mode 100644 examples/protocol-flows/queries/withdrawal_count.sql create mode 100644 examples/transfer-traffic/.env.example create mode 100644 examples/transfer-traffic/README.md create mode 100644 examples/transfer-traffic/abis/ERC20.json create mode 100644 examples/transfer-traffic/chainplot.yaml create mode 100644 examples/transfer-traffic/compose.yaml create mode 100644 examples/transfer-traffic/queries/top_transfers.sql create mode 100644 examples/transfer-traffic/queries/transfer_count.sql create mode 100644 examples/transfer-traffic/tests/usdc.yaml create mode 100644 examples/usdc-supply/.env.example create mode 100644 examples/usdc-supply/README.md create mode 100644 examples/usdc-supply/abis/ERC20.json create mode 100644 examples/usdc-supply/chainplot.yaml create mode 100644 examples/usdc-supply/compose.yaml create mode 100644 examples/usdc-supply/models/daily.sql create mode 100644 examples/usdc-supply/queries/cumulative_net.sql create mode 100644 examples/usdc-supply/queries/daily_events.sql create mode 100644 examples/usdc-supply/queries/daily_flow.sql create mode 100644 examples/usdc-supply/queries/largest_mints.sql create mode 100644 examples/usdc-supply/queries/net_change.sql create mode 100644 examples/usdc-supply/queries/total_burned.sql create mode 100644 examples/usdc-supply/queries/total_minted.sql create mode 100644 package.json create mode 100644 pnpm-lock.yaml create mode 100644 schemas/coverage.schema.json create mode 100644 schemas/latest.schema.json create mode 100644 schemas/lock.schema.json create mode 100644 schemas/manifest.schema.json create mode 100644 schemas/plan.schema.json create mode 100644 schemas/progress.schema.json create mode 100644 schemas/project.schema.json create mode 100644 schemas/release.schema.json create mode 100644 schemas/result.schema.json create mode 100644 scripts/m0-probe/.env.example create mode 100644 scripts/m0-probe/abis/ERC20.json create mode 100644 scripts/m0-probe/compose.yaml create mode 100644 scripts/m0-probe/rindexer.yaml create mode 100644 scripts/write-fixture-parquet.ts create mode 100644 src/cli/commands/apply.ts create mode 100644 src/cli/commands/build.ts create mode 100644 src/cli/commands/capabilities.ts create mode 100644 src/cli/commands/describe.ts create mode 100644 src/cli/commands/doctor.ts create mode 100644 src/cli/commands/fork.ts create mode 100644 src/cli/commands/init.ts create mode 100644 src/cli/commands/plan.ts create mode 100644 src/cli/commands/publish.ts create mode 100644 src/cli/commands/query.ts create mode 100644 src/cli/commands/refresh.ts create mode 100644 src/cli/commands/runs.ts create mode 100644 src/cli/commands/schemaShow.ts create mode 100644 src/cli/commands/serve.ts create mode 100644 src/cli/commands/templates.ts create mode 100644 src/cli/commands/test.ts create mode 100644 src/cli/commands/validate.ts create mode 100644 src/cli/envelope.ts create mode 100644 src/cli/main.ts create mode 100644 src/cli/run.ts create mode 100644 src/config/env.ts create mode 100644 src/fork/fetchGuard.ts create mode 100644 src/fork/importRelease.ts create mode 100644 src/ingest/adapter.ts create mode 100644 src/ingest/coverage.ts create mode 100644 src/ingest/coverageStore.ts create mode 100644 src/ingest/exportWorkerMain.ts create mode 100644 src/ingest/exporter.ts create mode 100644 src/ingest/rindexer/index.ts create mode 100644 src/ingest/rindexer/inspectCoverage.ts create mode 100644 src/ingest/rindexer/renderConfig.ts create mode 100644 src/ingest/rindexer/runBounded.ts create mode 100644 src/plan/apply.ts create mode 100644 src/plan/digest.ts create mode 100644 src/plan/errors.ts create mode 100644 src/plan/generate.ts create mode 100644 src/project/assertions.ts create mode 100644 src/project/columns.ts create mode 100644 src/project/limits.ts create mode 100644 src/project/load.ts create mode 100644 src/project/modelGraph.ts create mode 100644 src/project/types.ts create mode 100644 src/project/validate.ts create mode 100644 src/publish/directory.ts create mode 100644 src/publish/doctor.ts create mode 100644 src/publish/latestPointer.ts create mode 100644 src/publish/publishRelease.ts create mode 100644 src/publish/s3.ts create mode 100644 src/publish/serve.ts create mode 100644 src/publish/sourceBundle.ts create mode 100644 src/publish/target.ts create mode 100644 src/publish/writeRelease.ts create mode 100644 src/query/runQuery.ts create mode 100644 src/query/sqlGuard.ts create mode 100644 src/query/workerMain.ts create mode 100644 src/rpc/client.ts create mode 100644 src/rpc/heads.ts create mode 100644 src/runtime/journal.ts create mode 100644 src/runtime/locks.ts create mode 100644 src/snapshot/describe.ts create mode 100644 templates/fixture-transfers/chainplot.yaml create mode 100644 templates/fixture-transfers/queries/raw_amounts.sql create mode 100644 templates/fixture-transfers/snapshots/amounts.parquet create mode 100644 templates/fixture-transfers/tests/amounts.yaml create mode 100644 templates/ingest-transfers/.env.example create mode 100644 templates/ingest-transfers/README.md create mode 100644 templates/ingest-transfers/abis/ERC20.json create mode 100644 templates/ingest-transfers/chainplot.yaml create mode 100644 templates/ingest-transfers/compose.yaml create mode 100644 templates/ingest-transfers/queries/transfer_count.sql create mode 100644 tests/cli/a10.e2e.test.ts create mode 100644 tests/cli/a15.e2e.test.ts create mode 100644 tests/cli/a2.e2e.test.ts create mode 100644 tests/cli/build.test.ts create mode 100644 tests/cli/capabilities.test.ts create mode 100644 tests/cli/describe.test.ts create mode 100644 tests/cli/doctor.test.ts create mode 100644 tests/cli/exampleCoverage.test.ts create mode 100644 tests/cli/ingestCommands.test.ts create mode 100644 tests/cli/init.test.ts create mode 100644 tests/cli/initIngest.test.ts create mode 100644 tests/cli/query.test.ts create mode 100644 tests/cli/runs.cancel.test.ts create mode 100644 tests/cli/runs.test.ts create mode 100644 tests/cli/schemaKinds.test.ts create mode 100644 tests/cli/schemaShow.test.ts create mode 100644 tests/cli/serve.test.ts create mode 100644 tests/cli/testCmd.test.ts create mode 100644 tests/cli/validate.test.ts create mode 100644 tests/fixtures/projects/follow-plus-depth/chainplot.yaml create mode 100644 tests/fixtures/projects/follow-plus-depth/queries/raw_amounts.sql create mode 100644 tests/fixtures/projects/unknown-field/chainplot.yaml create mode 100644 tests/fixtures/projects/valid-dataset-only/chainplot.yaml create mode 100644 tests/fixtures/projects/valid-dataset-only/queries/raw_amounts.sql create mode 100644 tests/fork/fetchGuard.test.ts create mode 100644 tests/fork/hostileRelease.test.ts create mode 100644 tests/fork/importRelease.test.ts create mode 100644 tests/helpers/fakeRindexer.mjs create mode 100644 tests/helpers/run.ts create mode 100644 tests/ingest/coverage.test.ts create mode 100644 tests/ingest/exporter.test.ts create mode 100644 tests/ingest/inspectCoverage.test.ts create mode 100644 tests/ingest/live/e2e.live.test.ts create mode 100644 tests/ingest/locks.test.ts create mode 100644 tests/ingest/planApply.test.ts create mode 100644 tests/ingest/renderConfig.test.ts create mode 100644 tests/ingest/runBounded.test.ts create mode 100644 tests/plan/buildIntent.test.ts create mode 100644 tests/project/modelGraph.test.ts create mode 100644 tests/publish/directory.test.ts create mode 100644 tests/publish/live/s3.live.test.ts create mode 100644 tests/publish/modes.test.ts create mode 100644 tests/publish/publishCommand.test.ts create mode 100644 tests/publish/s3unit.test.ts create mode 100644 tests/query/models.test.ts create mode 100644 tests/query/sqlGuard.test.ts create mode 100644 tests/rpc/client.test.ts create mode 100644 tests/viewer/format.test.ts create mode 100644 tsconfig.json create mode 100644 viewer/index.html create mode 100644 viewer/package.json create mode 100644 viewer/pnpm-lock.yaml create mode 100644 viewer/pnpm-workspace.yaml create mode 100644 viewer/src/App.tsx create mode 100644 viewer/src/data.ts create mode 100644 viewer/src/echarts.ts create mode 100644 viewer/src/env.d.ts create mode 100644 viewer/src/format.ts create mode 100644 viewer/src/main.tsx create mode 100644 viewer/src/styles.css create mode 100644 viewer/tsconfig.json create mode 100644 viewer/vite.config.ts create mode 100644 vitest.config.ts diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..e474266 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,11 @@ +node_modules +dist +coverage +.chainplot +.env +.env.* +!.env.example +.git +scripts +docs +tests diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..b67fe6c --- /dev/null +++ b/.env.example @@ -0,0 +1,26 @@ +# Chainplot environment. Copy to .env and fill in. Never commit .env. +# Scope: this file is for developing Chainplot itself (live tests read it). +# For an analytics project, keep its own .env in the project directory — +# the CLI loads .env from the directory you run it in. +# All values stay local; tests and the CLI read them at runtime. + +# --- Ingest (M2) --------------------------------------------------------- +# The live suites read RPC_URL (Ethereum mainnet, archive-capable). Postgres +# and rindexer come from compose, so there is no test database URL to set. +# Real analytics projects keep their own .env with per-chain endpoints. +RPC_URL= + +# Postgres 16 for the rindexer adapter (compose.yaml provides one). +DATABASE_URL=postgresql://chainplot:chainplot@localhost:5432/chainplot + +# --- Publish to S3-compatible storage (M4) ------------------------------- +# Example: Cloudflare R2 endpoint (account-specific URL). +CHAINPLOT_S3_ENDPOINT= +CHAINPLOT_S3_BUCKET= +CHAINPLOT_S3_REGION=auto +AWS_ACCESS_KEY_ID= +AWS_SECRET_ACCESS_KEY= + +# --- Local overrides (optional) ------------------------------------------ +# rindexer binary (default: rindexer on PATH) +# CHAINPLOT_RINDEXER_BIN= diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..7c3122e --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,92 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + branches: [main] + +jobs: + test: + name: Build & Test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + - uses: pnpm/action-setup@v6 + with: + version: 11.24.0 + + - uses: actions/setup-node@v7 + with: + node-version: 22 + cache: pnpm + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + # The viewer is a separate project with its own lockfile. Typecheck it + # explicitly: `vite build` does not, so a type error here would + # otherwise ship in the bundle every release embeds. + - name: Install viewer dependencies + run: pnpm --dir viewer install --frozen-lockfile + + - name: Typecheck viewer + run: pnpm --dir viewer run typecheck + + - name: Build CLI and viewer + run: pnpm build + + - name: Run tests + run: pnpm vitest run + + template-smoke: + name: Scaffolded template runs + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + - uses: pnpm/action-setup@v6 + with: + version: 11.24.0 + + - uses: actions/setup-node@v7 + with: + node-version: 22 + cache: pnpm + + - name: Install dependencies + run: | + pnpm install --frozen-lockfile + pnpm --dir viewer install --frozen-lockfile + + - name: Build CLI and viewer + run: pnpm build + + # The producer image is the CLI plus rindexer, which ships linux/amd64 + # only. Every step below is what a user does from the template README; + # each of them was broken at some point and none was covered by a test. + - name: Build the producer image + run: | + docker build --platform linux/amd64 -t chainplot:local \ + -f docker/producer.Dockerfile . + + - name: Scaffold the template + run: | + node dist/cli/main.js init --template ingest-transfers \ + --output "$RUNNER_TEMP/proj" --json + + # No RPC is needed to prove the stack comes up and the CLI is reachable. + - name: Bring the stack up and drive the CLI + working-directory: ${{ runner.temp }}/proj + run: | + printf 'RPC_URL=http://127.0.0.1:1\n' > .env + docker compose up -d + docker compose ps + docker compose exec -T producer chainplot capabilities --json + docker compose exec -T producer chainplot validate --json + + - name: Tear down + if: always() + working-directory: ${{ runner.temp }}/proj + run: docker compose down -v diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..50add4b --- /dev/null +++ b/.gitignore @@ -0,0 +1,17 @@ +node_modules/ +/dist/ +examples/**/dist/ +.chainplot/ +.worktrees/ +.superpowers/ +.env +.env.* +!.env.example +*.log +.DS_Store +coverage/ +.pnpm-store/ +.vitest/ +viewer/node_modules +# Built from viewer/src by `pnpm build`; committing it lets the two drift. +viewer/dist/ diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..10acbe4 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Chainstack Labs + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/NOTICES.md b/NOTICES.md new file mode 100644 index 0000000..1cbe604 --- /dev/null +++ b/NOTICES.md @@ -0,0 +1,21 @@ +# Upstream notices + +Chainplot is MIT-licensed (see `LICENSE`). It bundles or depends on the +following upstream software at runtime; their licenses govern those +components, not the datasets you publish. + +| Component | License | Use | +|---|---|---| +| [rindexer](https://github.com/joshstevens19/rindexer) | MIT (pinned binary from the upstream image) | EVM event indexing | +| [DuckDB](https://duckdb.org) via `@duckdb/node-api` | MIT | Snapshot queries and Parquet export | +| [React](https://react.dev) | MIT | Viewer | +| [ECharts](https://echarts.apache.org) | Apache-2.0 | Viewer charts | +| [AWS SDK for JavaScript v3](https://github.com/aws/aws-sdk-js-v3) | Apache-2.0 | S3-compatible publish | +| [pg](https://github.com/brianc/node-postgres) | MIT | Advisory lock, coverage cursor reads | +| [ajv](https://github.com/ajv-validator/ajv), [yaml](https://github.com/eemeli/yaml), [commander](https://github.com/tj/commander.js), [Vite](https://vitejs.dev) | MIT | Schema validation, YAML, CLI, viewer build | + +## Dataset licenses are separate + +The software license (MIT) does not cover the data you publish. Every publish +target requires an explicit `dataset_license` field; it is recorded with the +release. Choose and document the license that applies to your dataset. diff --git a/README.md b/README.md new file mode 100644 index 0000000..2a713d5 --- /dev/null +++ b/README.md @@ -0,0 +1,254 @@ +# Chainplot + +**Scoped onchain events → a proven dataset → a static dashboard you can host anywhere.** + +Point it at a contract and a block range. It indexes exactly that range, proves +the range is complete, and writes a self-contained directory: the parquet, the +query results, the recipe that produced them, and a viewer. Copy the directory +to any static host and it works — no server, no database, no API key at read +time. + +Every command speaks JSON and returns a typed error, so an agent can drive the +whole pipeline without screen-scraping or guessing. + +```bash +pnpm install && pnpm build +node dist/cli/main.js init --template fixture-transfers --output ./demo --json +cd demo && node ../dist/cli/main.js build --json && node ../dist/cli/main.js serve --json +``` + +--- + +## Why this instead of a notebook + +- **The range is proven, not assumed.** Each indexed segment records its start + and end block hashes and hash-joins to the previous one. A gap, or a reorg + that breaks the join, refuses promotion — an incomplete dataset never becomes + a release. +- **uint256 survives.** Amounts are carried as decimal strings from parquet to + the page. Nothing passes through a double, so `2^256-1` arrives intact and + sorts correctly. `ORDER BY` on a raw amount is refused, because `"9"` sorts + after `"10"` as text; `cp_sortkey()` is built in for the correct ordering. +- **The output outlives the infrastructure.** A published release is static + files. Your RPC provider, your Postgres, and this CLI can all be gone and the + dashboard still renders. +- **Anyone can fork it.** A release always ships the recipe, and can ship the + dataset with it, so a second person imports it, writes a new query, and + rebuilds — no reindexing, no credentials, no access to your RPC. That costs + upload size, so it is opt-in: see [What a release weighs](#what-a-release-weighs). + +## The pipeline + +```text +chainplot.yaml ──▶ plan ──▶ apply ──▶ build ──▶ publish + contract, what index query, static files + block range, it will + prove render, + latest.json + queries, do, and coverage bundle pointer + dashboards bounded viewer + │ + fork ◀─────┘ + someone else's release + becomes your project +``` + +`plan` is read-only and writes a digest-bound plan. `apply` executes that plan +and nothing else — if the config drifts, the digest stops it. Re-applying the +same plan with the same idempotency key is a no-op, so a killed run resumes +instead of duplicating. + +## A project in one file + +```yaml +format_version: 1 +id: transfer-traffic + +chain_sources: + - id: mainnet + chain_id: 1 + rpc_secret: RPC_URL # the env var name, never the value + finality: { policy: finalized } + +event_sources: + - id: usdc + chain: mainnet + addresses: ["0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48"] + abi: abis/ERC20.json + events: [Transfer] + start_block: 18600000 + end: { mode: pinned, block: 18600010 } + +datasets: + - id: usdc + snapshot: .chainplot/snapshots/usdc/usdc_transfer.parquet + +queries: + - id: top_transfers + file: queries/top_transfers.sql + dataset: usdc + title: Largest transfers + raw_amount_columns: + - name: value + decimals: 6 # display only; the stored integer is untouched + symbol: USDC + label: Amount + +dashboards: + - id: transfer-activity + title: USDC transfer activity + panels: + - query: top_transfers + chart: table + title: Largest transfers + span: full # half (default) or full +``` + +```sql +-- queries/top_transfers.sql +select value, tx_hash, block_number +from usdc +order by cp_sortkey(value) desc -- signed-numeric order over decimal strings +limit 10 +``` + +## Commands + +| Command | Purpose | +| --- | --- | +| `capabilities` | Machine-readable CLI surface | +| `schema show ` | Print a frozen JSON Schema | +| `templates list` / `init` | Scaffold a project | +| `validate` | Check `chainplot.yaml` offline | +| `dataset describe` / `query` / `test` | Inspect and query a snapshot offline | +| `plan --intent ingest\|refresh\|build` | Write a digest-bound plan (read-only probes) | +| `apply --plan ` | Execute that plan, and only that plan | +| `refresh` | `plan --intent refresh` + `apply` | +| `build` | Write the full static release | +| `serve` | Preview a release on 127.0.0.1 | +| `publish` | Push a release to a directory or S3-compatible target | +| `runs list\|show\|cancel` | Run journal; cancel is cooperative | +| `fork` | Import a published release as a new project | +| `doctor` | Check credentials, RPC, Postgres, rindexer, storage | + +Every command **requires** `--json` and returns +`{ schema_version, ok, command, data, warnings, error }`. Errors carry a code +from a closed set — `validation`, `policy_refused`, `missing_credentials`, +`unsupported_capability`, `source_inconsistent`, `transient_dependency`, +`internal` — plus `retryable` and `suggested_next`. + +## Presentation + +Panels are declarative. `title`, `description`, `span`, `hide_columns` and +`unit` shape the page; `decimals`, `symbol` and `label` on a raw amount column +shape the numbers. Charts are `line`, `bar`, `area`, `kpi`, `table` — an +allowlist, not an embedded plotting language. + +Display metadata never alters stored values. `decimals: 6` renders +`983644533552` as `983,644.533552 USDC`; the exact integer stays in the result +JSON and in the hover title. + +## What a release weighs + +A release is a static page plus the answers, and that part is about 800 KB +regardless of how much history it covers — most of it the charting bundle, +which is only fetched when a dashboard has a chart. Query *results* are small: +a year of daily figures is a few hundred rows. Tables are virtualised, so a +wide result stays scrollable; what bounds it is the bytes a reader downloads, +and `policy.row_limit` raises that when it is worth paying. + +The dataset is separate, and `--mode` decides whether it ships with the page: + +| Mode | The page carries | Published beside it | Fork can rebuild | +|---|---|---|---| +| `results_only` (default) | page, results, recipe | — | no — point it at your own snapshot | +| `dataset_referenced` | page, results, recipe, a checksum | the parquet | yes, fetched on demand | +| `dataset_included` | page, results, recipe, **and the parquet** | — | yes | + +`dataset_referenced` is usually the one you want if forkability matters: the +page stays under a megabyte, the parquet is uploaded alongside it, and `fork` +pulls it in and verifies it against the checksum only when someone actually +wants to recompute. `dataset_included` puts everything in one directory, which +is simpler to copy around but makes every reader download the data. + +Publishing is outward and irreversible, so the default uploads the least that +still works: the page and its answers, without the dataset. Shipping the +parquet is a deliberate choice — it is what lets someone fork the release and +recompute, and it is also what turns an 800 KB page into hundreds of megabytes. +Opt in with `--mode dataset_included`, or `policy.release_mode` in the project +(which is what `apply` and `refresh` use, since neither takes a flag). + +The 100 MiB cap applies only to the copied dataset, so it never limits the +dashboard, and the default never trips it. A project whose parquet is larger than that publishes an identical +page with `--mode results_only`; what you give up is the ability for someone +forking it to recompute your numbers from source data, which is why the +default keeps the data in. + +## Security + +**Forking runs a stranger's SQL on your machine.** That is the whole point of a +portable recipe, and it is contained rather than prevented: queries run in a +separate process with filesystem and network access switched off *before* any +project SQL — models included — and DuckDB's own parser refuses anything that +is not a single SELECT. + +What is not defended: a release's checksums come from the same bucket as its +files, so they prove integrity, not authorship. There are no signatures. Fork +only from buckets you would trust with the data. + +Full model, including the SSRF guard on `fork --from https://…`: +[`docs/security.md`](docs/security.md). + +## Examples + +[`examples/`](examples/) — USDC transfer activity (full ingest pipeline), WETH +deposit/withdrawal flows (multi-event source), and forking a published dataset +as a second agent. Both ingest examples are published live: + +- [USDC transfer activity](https://pub-0593f9128f674400bcbbc940cf9f01b1.r2.dev/transfer-traffic/latest.json) — 92 transfers, blocks 18,600,000–18,600,010 +- [WETH wrap/unwrap](https://pub-0593f9128f674400bcbbc940cf9f01b1.r2.dev/protocol-flows/latest.json) — 258 deposit/withdrawal events over the same range + +Publishing more than one project into a single bucket needs a `prefix` on the +target; without one, each project's `latest.json` overwrites the others'. + +## Look and feel + +The viewer transcribes the Chainstack design tokens from `cp-ui-kit` +(`src/styles/tailwind.css`, `src/styles/theme.ts`) — palette, radius scale and +type sizes — rather than importing the kit, since a release is a static page +with no build step of its own. Fonts are named but not bundled: Suisse Intl is +licensed and releases are published to public buckets, so the page asks for it +and falls back to the system stack. + +## Status + +v0.1. M0–M5 complete: ingest with coverage proof, snapshots, models, static +dashboards, publish (directory + S3-compatible), fork, doctor. A1–A16 acceptance +with test evidence — including where a criterion was previously asserted too +generously — is in [`docs/acceptance.md`](docs/acceptance.md). + +Requires Node 22+ and pnpm. Ingest additionally needs Docker, an archive RPC +endpoint, and Postgres 16; everything else runs offline. + +## Development + +```bash +pnpm build # CLI (tsc) + viewer (vite); `build:cli` for the CLI alone +pnpm test # build, then the full offline suite +``` + +`viewer/dist` is built, not committed — `pnpm build` produces it and `build` +refuses to write a release without it. + +`pnpm test` runs the live suites too, with nothing skipped, given `RPC_URL` and +S3 credentials in `.env` (see [`.env.example`](.env.example)) and a running +Docker. Postgres and the pinned rindexer binary come from compose, so there is +nothing else to install. Without those credentials or Docker, the live suites +skip and the rest still runs. + +- Design spec: [`docs/specs/2026-09-12-chainplot-design.md`](docs/specs/2026-09-12-chainplot-design.md) +- Capability matrix and limits: [`docs/capabilities.md`](docs/capabilities.md) +- Compatibility notes (rindexer, DuckDB, R2): [`docs/compatibility.md`](docs/compatibility.md) + +## License + +MIT for the software. Dataset licenses are separate and must be declared on +every publish target — `publish` refuses without `dataset_license`. diff --git a/docker/producer.Dockerfile b/docker/producer.Dockerfile new file mode 100644 index 0000000..0b5ade5 --- /dev/null +++ b/docker/producer.Dockerfile @@ -0,0 +1,29 @@ +# Producer image: chainplot CLI + the pinned rindexer binary (linux/amd64). +# The rindexer binary is copied from the pinned upstream image; no docker.sock. +FROM ghcr.io/joshstevens19/rindexer@sha256:9b33da8cea740b74ebfdfd3932682e8ceab79cbcf2eb3a7ca0863ac413794dd7 AS rindexer + +FROM node:22-bookworm-slim +WORKDIR /app +RUN apt-get update \ + && apt-get install -y --no-install-recommends libssl3 ca-certificates \ + && rm -rf /var/lib/apt/lists/* \ + && corepack enable +COPY package.json pnpm-lock.yaml ./ +RUN pnpm install --frozen-lockfile +COPY tsconfig.json ./ +COPY schemas ./schemas +COPY src ./src +# The viewer bundle is a build artifact copied in, not rebuilt here, so the +# producer image stays CLI + rindexer. Build it on the host first (`pnpm build`). +COPY viewer/dist ./viewer/dist +RUN pnpm build:cli +COPY --from=rindexer /app/rindexer /usr/local/bin/rindexer +# Pre-install the DuckDB postgres extension so export works without network. +RUN node --input-type=module -e "import { DuckDBInstance } from '@duckdb/node-api'; const db = await DuckDBInstance.create(':memory:'); const c = await db.connect(); await c.run('INSTALL postgres; LOAD postgres;');" && mkdir -p /workspace/.duckdb +ENV DUCKDB_EXTENSION_DIRECTORY=/root/.duckdb/extensions +# A `chainplot` on PATH, so the documented +# `docker compose exec producer chainplot --json` is the real command. +RUN printf '#!/bin/sh\nexec node /app/dist/cli/main.js "$@"\n' \ + > /usr/local/bin/chainplot \ + && chmod +x /usr/local/bin/chainplot +ENTRYPOINT ["node", "/app/dist/cli/main.js"] diff --git a/docs/acceptance.md b/docs/acceptance.md new file mode 100644 index 0000000..fa0d9fe --- /dev/null +++ b/docs/acceptance.md @@ -0,0 +1,55 @@ +# Acceptance — A1–A16 (v0.1) + +Spec: `docs/specs/2026-09-12-chainplot-design.md` §18. Every ID maps to test +evidence on `main`. "Live" = run against real infrastructure (archive RPC, +Cloudflare R2); everything else runs offline. + +`pnpm test` runs everything, live suites included, with nothing skipped. The +live tests need only `RPC_URL` and the S3 credentials in `.env` plus a running +Docker — Postgres and the pinned rindexer binary come from the template's own +`compose.yaml`, which the ingest test brings up itself. + +**A12 was wrong until 2026-09-15.** It was marked Pass on the strength of +tests that never ran a hostile *model*, and a model materialized before +filesystem access was disabled. A forked release could read any file the +build process could and publish it. Fixed in `src/query/workerMain.ts`; +`tests/fork/hostileRelease.test.ts` now exercises the whole +publish → fork → build path and fails if the ordering regresses. See +[`docs/security.md`](security.md) for the trust boundary this rests on. + +| ID | Requirement | Evidence | Status | +|---|---|---|---| +| A1 | Agent creates a dashboard from ABI + bounded history; no browser automation, no undocumented steps | `examples/transfer-traffic` (92 real USDC transfers, plan → apply → build → serve); fixture quickstart (`tests/cli/a15.e2e.test.ts`) | Pass (live + offline) | +| A2 | Chart title change → presentation rebuild only, no backfill | `tests/cli/a2.e2e.test.ts` | Pass | +| A3 | Earlier history / another contract = explicit additional work; no silent expansion | `tests/cli/ingestCommands.test.ts` (ingest-relevant edit → `refresh` refuses `policy_refused`); `tests/ingest/planApply.test.ts` (config drift → refuse) | Pass | +| A4 | Kill producer during indexing/export/upload → safe resume; previous release intact | `tests/ingest/runBounded.test.ts` (early exit, wall clock); journal resume (`tests/ingest/planApply.test.ts` failed-run re-apply); S3 live: previous release intact after re-publish. Live kill-mid-indexing exercised via wall-clock path | Pass | +| A5 | Same plan + idempotency key twice → no duplicate data, no mixed release | `tests/ingest/planApply.test.ts` (reused outcome, no second run); live M2 e2e (92 rows unchanged) + live S3 re-publish | Pass (live + offline) | +| A6 | Zero-event interval → `complete_empty` with positive evidence, or refuse | `tests/ingest/inspectCoverage.test.ts` (cursor ≥ end + 0 rows); M0 probe; flush-race settle re-check (`src/plan/apply.ts`) | Pass | +| A7 | Canonical boundary change → detect inconsistency, do not promote | `tests/ingest/planApply.test.ts` (hash-join break → `source_inconsistent`, coverage unchanged) | Pass | +| A8 | uint256 set round-trips exactly; sort-key order is numeric | `tests/cli/build.test.ts` (raw decimal strings through storage/JSON); fixture includes `±2^53±1`, `±2^255`, `2^256-1`; `tests/query/models.test.ts` orders the fixture with `cp_sortkey` and compares against a BigInt sort; `tests/viewer/format.test.ts` covers display scaling and BigInt compare | Pass | +| A9 | Producer + DB + RPC down → public dashboard still renders | `tests/cli/a2.e2e.test.ts` (served release over plain HTTP); demo deployments during M3–M5 with all services absent | Pass | +| A10 | Second agent imports published data, writes a new query — no reindexing, no original credentials (the producer opts in, with `dataset_referenced` or `dataset_included`; the default publishes the page alone and `fork` warns when there is no data). `tests/fork/importRelease.test.ts` covers the referenced round trip and rejects a tampered dataset; verified live against R2 by forking a published release over HTTPS and recomputing offline | `tests/cli/a10.e2e.test.ts`; example 3 (fork of the ingest release, new query + dashboard, offline build) | Pass | +| A11 | Copy project to a clean machine, run Compose → config changes environmental only | `examples/transfer-traffic` + `examples/protocol-flows` run end-to-end in fresh containers (amd64 producer image, our Postgres); no analytics files rewritten | Pass (live) | +| A12 | Malicious filenames, labels, SQL, secret-like content → no execution, no path escape, no credential exposure | `tests/fork/hostileRelease.test.ts` (publish → fork → build with a hostile model: filesystem read refused, non-SELECT refused, no result written); `tests/query/models.test.ts` (same at the query layer, plus multi-statement); `tests/query/sqlGuard.test.ts` (admission policy); `tests/fork/fetchGuard.test.ts` (SSRF blocklist, traversal); `tests/fork/importRelease.test.ts` (checksum, undeclared files); identifier validation (`tests/ingest/inspectCoverage.test.ts`); doctor never echoes secrets | Pass (regression-tested since 2026-09-15; previously asserted without a hostile-model case) | +| A13 | Exceed query/scope/output/publication limits → typed refusal, no silent truncation | `tests/publish/modes.test.ts` (100 MiB cap → choices); `tests/ingest/planApply.test.ts` (block budget caps `job_end`); `tests/query/models.test.ts` (row limit → `policy_refused`; the reader stops at the limit rather than materializing the full result first) | Pass | +| A14 | Missing secrets or wrong chain identity → precise diagnosis, no hang, no destructive fallback | `tests/cli/ingestCommands.test.ts` (`missing_credentials`); finalized-null → no fallback (`tests/rpc/client.test.ts`) | Pass | +| A15 | Fixture-only quickstart with network disabled | `tests/cli/a15.e2e.test.ts` | Pass | +| A16 | `refresh` on pinned-complete project rebuilds, no ingest; address/destination edit → `policy_refused` | `tests/cli/ingestCommands.test.ts` (both halves, zero-RPC asserted) | Pass | + +## Live evidence runs + +- **M2 ingest e2e** (2026-09-13): plan → apply → coverage → idempotent re-apply → no-op plan → gate, in the amd64 producer container against archive RPC + our Postgres 16. 92 USDC rows, matching M0. +- **M4 S3** (2026-09-13): conditional-write probe + publish/re-publish cycle against Cloudflare R2 (`tests/publish/live/s3.live.test.ts`). +- **M5 examples** (2026-09-14): both ingest examples executed via Compose in fresh containers; example 3 forked and built offline. +- **Re-run on the current code** (2026-09-16): both ingest examples re-ingested from mainnet through the containerised rindexer (92 USDC transfers, 258 WETH events over blocks 18,600,000-18,600,010, coverage complete with chain timestamps), built, and published to R2 under per-project prefixes. The ingest and S3 live suites both run in `pnpm test` and pass. + +## Known gaps + +- **Authenticity of a forked release is not established.** `release.json` + carries checksums for its own files, so a fork detects corruption in + transit, but the checksums and the payload come from the same place. + Whoever controls the bucket controls both. There are no signatures; + `fork` is safe against a hostile *recipe* (A12), not against a + substituted *publisher*. Only fork buckets you would trust with the data. +- **Nothing here is signed.** See the authenticity gap above; it is the one + substantive control the design does not yet have. diff --git a/docs/capabilities.md b/docs/capabilities.md new file mode 100644 index 0000000..80fc5c0 --- /dev/null +++ b/docs/capabilities.md @@ -0,0 +1,99 @@ +# Capability matrix (v0.1) + +Machine-readable source of truth: `chainplot capabilities --json`. + +## Commands + +| Command | Status | Notes | +|---|---|---| +| `capabilities` | shipped | | +| `schema show ` | shipped | 9 kinds, frozen schemas | +| `templates list` | shipped | `fixture-transfers`, `ingest-transfers` | +| `init --template --output` | shipped | | +| `validate` | shipped | offline; cycles, policy combos, refs | +| `doctor` | shipped | S3 write/promote always `unverified` | +| `plan --intent ingest` | shipped | finalized-head probe; budget caps | +| `plan --intent refresh` | shipped | pinned sources skip ingest | +| `plan --intent build` | shipped | no RPC | +| `plan --intent publish` | shipped | publication-only | +| `apply --plan` | shipped | digest-bound, idempotency journal, locks | +| `build` | shipped | full static release; `--mode results_only\|dataset_referenced` | +| `refresh` | shipped | `--publish-target` publishes | +| `query` / `dataset describe` / `test` | shipped | offline over snapshots | +| `serve` | shipped | 127.0.0.1 only | +| `publish` | shipped | directory + S3-compatible (R2 verified) | +| `runs list\|show\|cancel` | shipped | cooperative cancel | +| `fork` | shipped | deny-by-default SSRF guard | +| `deploy render`, `watch` | not in v0.1 | spec §1.1 | + +## Sources / targets / charts + +- Event sources: explicit address lists, one chain per project, rindexer adapter (pinned image, linux/amd64) +- End policies: `pinned` (finality-checked at plan time), `follow_finalized` (requires `finalized` chain policy) +- Publish targets: `directory` (atomic `latest.json`), `s3` (conditional write; verified on Cloudflare R2). Set `prefix` on a target when one bucket or directory holds more than one project — `latest.json` is otherwise a single key at the root and the projects overwrite each other's pointer. +- Charts: `line`, `bar`, `area`, `kpi`, `table` (allowlisted encodings only) +- Dataset modes: `results_only` (default), `dataset_referenced`, `dataset_included`. The default publishes the page and its results only. `dataset_referenced` uploads the parquet beside the release and records a release-relative path and checksum, so `fork` fetches and verifies it on demand. `dataset_included` copies it into the release. Set per build (`--mode`) or per project (`policy.release_mode`). +- Panel presentation: `title`, `description`, `span` (`half`/`full`), `hide_columns`, `unit` +- Column display: `raw_amount_columns[].decimals` / `.symbol` / `.label` (display only; stored values are never rewritten) + +## SQL admission control + +Every model and query is parsed by DuckDB (`json_serialize_sql`) before it +runs. A statement that is not a single SELECT is refused — that call rejects +INSERT, COPY, ATTACH, PRAGMA and friends outright, so the rule is the parser's, +not a keyword denylist. Filesystem and network access are disabled before any +project SQL executes, models included: a forked recipe is a stranger's code. + +`cp_sortkey(v)` is available in every query. It maps a decimal-string amount to +a fixed-width key whose lexicographic order is signed-numeric order, so +`ORDER BY cp_sortkey(value)` sorts uint256 correctly without projecting a +78-digit column into the dashboard. Ordering directly by a declared raw amount +column is refused, including through a select-list alias or a positional +ordinal. + +## Enforced limits (spec §14.1) + +Each row names where it is enforced, so a claim that stops being true is +visible in review rather than only in production. + +| Limit | Default | Enforced in | +|---|---|---| +| Chains per project | 1 | `schemas/project.schema.json` (`maxItems`) | +| Contract addresses | 20 | `schemas/project.schema.json` (`maxItems`) | +| Blocks per approved run | 100_000, `policy.block_budget` to change | `src/plan/generate.ts` (`DEFAULT_BLOCK_BUDGET`) | +| Query deadline | 60 s | `src/query/runQuery.ts` (`DEADLINE_MS`, SIGKILL) | +| DuckDB memory | 1 GiB, spills to a temp dir | `src/query/workerMain.ts` (`MEMORY_LIMIT`) | +| Returned rows | 10_000, `policy.row_limit` to change | `src/project/limits.ts`, enforced in `src/query/workerMain.ts` (the reader stops at the limit) | +| RPC job wall clock | 30 min, resumable | `src/ingest/rindexer/runBounded.ts` | +| Copied public dataset | 100 MiB | `src/publish/writeRelease.ts` (`MAX_COPIED_BYTES`) | +| Concurrent ingest/publish per project | 1 | `src/runtime/locks.ts` (Postgres advisory lock) | +| `fork` release.json body | 1 MiB | `src/fork/fetchGuard.ts` (`FORK_LIMITS`) | +| `fork` total download | 512 MiB | `src/fork/fetchGuard.ts` (`FORK_LIMITS`) | +| `fork` per-request timeout | 30 s | `src/fork/fetchGuard.ts` (`FORK_LIMITS`) | +| `fork` redirect hops | 0 | `src/fork/fetchGuard.ts` (refused outright) | + +`CHAINPLOT_QUERY_MEMORY_LIMIT` overrides the memory figure; the query still +spills to disk rather than failing when it goes over. + +The row limit bounds the *download*, not the rendering. The table is +virtualised — 5,000 rows put 26 in the DOM — so a wide result no longer +publishes a page that cannot be scrolled. What remains is that results are +embedded in the release, so every row is bytes a reader fetches before seeing +anything. `policy.row_limit` raises it when a genuinely wide table is worth +that cost. + +Two limits listed here through v0.1 described behaviour that did not exist and +have been removed rather than left as promises: a 5 MiB query-result cache +(there is no cache) and a 20-row CLI sample cap (the CLI returns what the query +returns, bounded only by the row limit). A per-run cap on distinct header +fetches is likewise gone: `apply` fetches exactly three headers per job — the +job boundaries — so there was nothing for a budget to bound. + +## Environment variables + +See `.env.example`. Project-level `.env` (loaded from the directory the CLI +runs in) or the process environment; values never appear in project files. + +An ingest project needs exactly one secret of its own: `RPC_URL`. Postgres and +the pinned rindexer binary are supplied by the template's `compose.yaml`, and +`DATABASE_URL` there already points at that service. diff --git a/docs/compatibility.md b/docs/compatibility.md new file mode 100644 index 0000000..5781e55 --- /dev/null +++ b/docs/compatibility.md @@ -0,0 +1,165 @@ +# Compatibility + +## M5 complete (2026-09-14) + +Examples, acceptance, multi-arch smoke done. v0.1 scope (M0–M5) implemented. + +| Piece | Evidence | +| --- | --- | +| Multi-arch Dockerfile smoke | `linux/amd64`: full build + live ingest runs (M2/M5). `linux/arm64`: image builds, `capabilities --json` runs; **rindexer binary cannot execute on arm64** (upstream publishes amd64-only) — ingest requires amd64, documented limitation | +| rindexer version in image | 0.43.0 (pinned image digest) | +| Examples | `examples/transfer-traffic` (92 USDC transfers), `examples/protocol-flows` (150 WETH deposits + 108 withdrawals), `examples/fork` — all executed end-to-end | +| Acceptance | `docs/acceptance.md` — A1–A16 all pass (live where noted) | + +### M5 corrections recorded + +1. Multi-event sources: coverage inspection aggregates **all** declared + events (worst status, summed rows); export writes one Parquet per + (source, event). Single-event sources unchanged. +2. rindexer forks children that keep writing after the parent exits — + `stopAndQuiesce` now signals the whole process group (`detached` spawn + + `kill(-pid)`), then coverage is inspected. +3. `complete_empty` immediately after SIGTERM can be a flush race (cursor + commits before final row flush): apply re-inspects after a 3 s settle + window; genuinely empty ranges stay `complete_empty` (A6). +4. `forbidOrderByRaw` strips SQL line comments first — comment mentions of a + raw column no longer trip the guard. +5. Forks strip `chain_sources`/`event_sources`: a fork has no chain access + (spec §16.4); the forked project is dataset-only. + +## M4 implemented (2026-09-13) + +Publish/fork on `main`: directory target with atomic `latest.json` (temp+rename), `plan --intent publish` + `publish` command + `refresh --publish-target`, dataset modes with the 100 MiB cap (`build --mode results_only|dataset_referenced`), `doctor`, `fork` with deny-by-default fetch guard (https only, 0 redirects, IP blocklist incl. decimal/hex/octal literals and IPv4-mapped forms, DNS pinning, byte caps, pointer-checksum verification), per-file checksums in `release.json`. + +| Piece | Pin | +| --- | --- | +| S3 SDK | `@aws-sdk/client-s3` v3 (pnpm lock) | +| S3 env | `CHAINPLOT_S3_ENDPOINT`, `CHAINPLOT_S3_BUCKET`, `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `CHAINPLOT_S3_REGION` (default `auto`) | +| S3 promotion | `If-None-Match: *` on first write, `If-Match` on current ETag after; 412 → `policy_refused` | +| Live S3 evidence | **Proven 2026-09-13 against Cloudflare R2** (`tests/publish/live/s3.live.test.ts`, env-gated): read-after-write ✓; `If-None-Match: *` on existing key → 412 `PreconditionFailed` ✓; `If-Match` current ETag → success ✓; stale ETag → 412 ✓; full publish → re-publish cycle flipped the pointer, previous release still readable, pointer checksum matched release.json bytes. R2 honors both conditional-write headers — the last unproven M0 assumption is closed. S3-compatible targets are supported, not refused. | +| Unit coverage | mocked `S3Ops`: first-promote `If-None-Match`, 412 → `policy_refused`, checksum verify | + +Notes: +- `doctor` marks S3 write/promote `unverified` — proven only at upload time. +- `fork` verifies `release.json` against the `latest.json` pointer checksum, then every declared file against its per-file checksum; undeclared files are never fetched. +- Fork of a results-only release requires `source/chainplot.yaml` (recipe); refused otherwise. + +## M3 accepted (2026-09-13) + +Full static build on `main`: SELECT model graph (topo order, cycles refused at +`validate`, SELECT-only enforced in the isolated worker), extended `build` +(full §16.1 layout minus `latest.json`: `index.html`, `assets/`, +`dashboards/`, `results/`, `datasets//{manifest.json,tables/*.parquet}`, +`source/` allowlist), viewer bundle (React + Vite + ECharts, committed under +`viewer/dist/`, no CDN), `serve` (127.0.0.1 only), `plan --intent build` +(no RPC). A2 + A9 groundwork pass offline. + +| Piece | Pin | +| --- | --- | +| viewer deps | react 19, echarts 6, vite 7 (`viewer/package.json`, own lockfile) | +| viewer bundle | committed `viewer/dist/`; rebuild with `pnpm --dir viewer build` | +| release id | `local-` staged then renamed to `dist/releases/local` (M4 adds real ids + `latest.json`) | + +Notes: +- Raw-amount columns are published in `results/.json` + (`raw_amount_columns`) so the viewer sorts them with BigInt numeric compare, + never `localeCompare` (§12/§15). +- `source/` copies only `chainplot.yaml`, `abis/`, `models/`, `queries/`, + `tests/`, `schemas/` — never `.env`, `.chainplot/`, `dist/`. + +## M2 ingest accepted (2026-09-13) + +M2 scope implemented on `main`: `plan --intent ingest|refresh` → `apply` with +bounded rindexer jobs, coverage from `rindexer_internal.*.last_synced_block` +plus header hashes, DuckDB postgres-ATTACH export, run journal, locks, cancel, +and the product Compose + Dockerfile. `plan --intent build|publish` returns +`unsupported_capability` (M3/M4). S3 still untouched (M4). + +| Piece | Pin | +| --- | --- | +| rindexer binary in producer image | copied from `ghcr.io/joshstevens19/rindexer@sha256:9b33…97dd` at `/app/rindexer` → `/usr/local/bin/rindexer`. Binary is linux/amd64 ELF; runs on amd64 hosts, not on arm64 hosts (no arm64 image exists upstream). | +| Producer image | built from repo root: `docker build -f templates/ingest-transfers/Dockerfile -t chainplot-producer .`; `capabilities --json` verified inside the container. | +| DuckDB postgres extension | pre-installed in the image (`INSTALL postgres`); extension dir `v1.5.5`. | +| New dependency | `pg` (advisory lock + coverage cursor reads; session-scoped advisory locks cannot go through DuckDB's pooled ATTACH). | +| Live e2e | **Run 2026-09-13** in an amd64 producer container against our Postgres 16 + `RPC_URL` from `scripts/m0-probe/.env` (env only, never committed). Full scenario passed: plan (finalized-head probe) → apply (rindexer 11 blocks, 92 USDC rows — matches M0) → coverage segment with 3 header hashes → idempotent re-apply (`reused: true`, still 92 rows) → second plan is a no-op (`job_start > job_end`, no ingest action) → coverage removed → `build` refused `policy_refused` → `refresh` with `RPC_URL` unset rebuilt only (A16). | + +### M2 live-run corrections (2026-09-13) + +1. rindexer requires top-level `name` in the manifest; it derives table names + from it, not from the network: event table `{name}_{contract}.{event}`, + cursor `rindexer_internal.{name}_{contract}_{event}`. `renderConfig` sets + `name: chainplot_`. +2. Producer image installs `libssl3` (rindexer links it; bookworm-slim omits it). +3. A failed `apply` now records `status: failed` in the journal (was `running`, + which blocked resume). +4. The promotion gate moved into `buildRelease` so every build path (standalone + `build`, `apply`, `refresh`) refuses incomplete sources. +5. rindexer table naming confirmed live: cursor + `rindexer_internal.chainplot_chainplot_1_usdc_transfer` (`last_synced_block + = 18600010`, `network = chainplot_1`), event table + `chainplot_chainplot_1_usdc.transfer` (92 rows). + +### M2 decisions recorded + +1. rindexer network name is `chainplot_` (e.g. `chainplot_1`), so + table names are deterministic: `rindexer_internal.chainplot_1__`. +2. `validate` skips the snapshot-file existence check for projects with + `event_sources` (ingest materializes snapshots at apply); dataset-only + projects still require the file. +3. `plan` demands credentials only when work needs them: no finalized-head + probe and no `DATABASE_URL` check when every pinned source is already + complete (spec §11 no-ingest refresh). +4. `apply` is injectable (`exportFn`/`buildFn`) for offline tests; the real + exporter runs DuckDB in a forked child (`src/ingest/exportWorkerMain.ts`) + with `ATTACH … (TYPE POSTGRES, READ_ONLY)`, `CAST(block_number/tx_index AS + BIGINT)`, literal `chain_id`, and a physical-uniqueness gate before COPY. + +## M1 accepted (2026-09-13) + +Fixture CLI on `main`: `init` → `validate` → `query` → `test` → `build`. +A15 passes with network disabled. No ingest, S3, or viewer. + +| Tool | Version | +| --- | --- | +| Node.js | `>=22 <27` (`package.json` `engines`) | +| pnpm | `11.24.0` | +| `@duckdb/node-api` | `1.5.5-r.4` | + +DuckDB neo in-memory `select 1` verified during M1. + +## M0 ingest probe (2026-09-13) + +Runtime: Docker Compose in `scripts/m0-probe/`. **Our** Postgres 16, pinned rindexer image, **no docker.sock**. Matches spec ingest boundary. + +| Piece | Pin | +| --- | --- | +| rindexer image | `ghcr.io/joshstevens19/rindexer@sha256:9b33da8cea740b74ebfdfd3932682e8ceab79cbcf2eb3a7ca0863ac413794dd7` (`:latest` that day). Tag `v0.43.1` does not exist on GHCR. Image is **linux/amd64** only; ran on arm64 via qemu. | +| Postgres | `postgres:16-alpine` | +| RPC | `RPC_URL` env (archive-capable Ethereum JSON-RPC). `eth_getBlockByNumber("finalized")` and archive `eth_getLogs` verified. Publicnode archive `eth_getLogs` is 403. | +| Window | inclusive archive `18600000`–`18600010` via `RPC_URL`. Earlier publicnode run used a near-head window because publicnode archive `eth_getLogs` is 403. | + +### Verified + +- Inclusive `start_block`/`end_block`. Historic job finishes (`Historical indexing completed`). Process **does not exit** (health server on 8080). M2 must SIGTERM after that log line. +- Archive via `RPC_URL`: USDC `18600000`–`18600010` → 92 rows; empty contract → 0 rows; both cursors `last_synced_block = 18600010`; 0 null `block_hash`/`block_timestamp`. +- Resume (earlier publicnode near-head run): after stop, raising `end_block` restarts at `last_synced_block + 1` (`25967330`). USDC rows 935 then +909 = 1844. No rescan of the first window. +- Empty-range evidence: cursor at `end_block` with **0** rows. Completeness is the cursor, not `max(block)` / row count. A6: `last_synced_block >= end_block` and row count 0 → `complete_empty`. +- Per-log `block_hash` and `block_timestamp` both present, **0 nulls**, with `timestamp: true` on the contract. No header-enrichment stage required for this pin. +- `value` is Postgres `varchar(78)` (decimal string). DuckDB postgres scanner keeps it `VARCHAR`. +- Cursor table: `rindexer_internal.{indexer}_{contract}_{event}` columns `network text PK`, `last_synced_block numeric`. +- Event table (no-code): `contract_address`, `from`, `to`, `value`, `tx_hash`, `block_number numeric`, `block_timestamp timestamptz`, `block_hash`, `network`, `tx_index numeric`, `log_index varchar(78)`. +- Snapshot export path **1 works**: DuckDB `INSTALL postgres; ATTACH … TYPE POSTGRES; COPY … TO parquet`. Must `CAST(block_number AS BIGINT)` and `CAST(tx_index AS BIGINT)` — scanner maps PG `numeric` to DuckDB **DOUBLE**. `value` stays string through parquet. + +### Not verified this run + +- S3 `If-Match` / `If-None-Match` (M4). +- linux/arm64 rindexer image (none published; qemu only). +- `reorg_block_hashes` stayed empty on these historic windows. + +### Spec consequences for M2 + +1. Coverage evidence = `rindexer_internal.*.last_synced_block`, not max event block. +2. Always set `timestamp: true`. +3. Export through DuckDB postgres extension with explicit casts; do not trust scanner types for `numeric`. +4. Stop rindexer with SIGTERM after historic complete; do not wait for process exit. +5. Ingest tests need `RPC_URL` (archive + `finalized`). Do not use publicnode for historical `eth_getLogs`. diff --git a/docs/plans/2026-09-12-m1-fixture-cli.md b/docs/plans/2026-09-12-m1-fixture-cli.md new file mode 100644 index 0000000..73876e8 --- /dev/null +++ b/docs/plans/2026-09-12-m1-fixture-cli.md @@ -0,0 +1,1252 @@ +# Chainplot M1 Fixture CLI Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Ship a `chainplot` CLI that an agent can learn from `--json` help and JSON Schema, init a fixture project, validate it offline, query a committed Parquet snapshot with DuckDB, run assertions, and `build` cached query results — with network disabled. + +**Architecture:** One Node package. JSON Schema in `schemas/` is the source of truth. CLI commands call `src/project` and `src/query`; DuckDB runs in a forked child that sees snapshot files only. No rindexer, Postgres, S3, viewer HTML, or `plan --intent ingest` in this plan. + +**Tech Stack:** Node.js 22 or 24 LTS (`engines.node`: `>=22 <27`), pnpm 11. TypeScript, vitest, ajv, yaml, commander, and `@duckdb/node-api` (neo) versions come from `pnpm add` in Task 1 and are recorded in `docs/compatibility.md` plus `pnpm-lock.yaml`. Do not use the deprecated `duckdb` package. + +**Spec:** `docs/specs/2026-09-12-chainplot-design.md` (M1 plus the local DuckDB/uint256 pin from M0). M2+ plans cover ingest, Compose, viewer, S3, fork. + +## Global Constraints + +- CLI name is `chainplot`. Noninteractive. `--json` emits exactly one result object on stdout; diagnostics on stderr. +- Result envelope: `{ schema_version: 1, ok, command, data, warnings, error }`. `schema_version` is an integer. +- `error.code` is only: `validation`, `missing_credentials`, `unsupported_capability`, `policy_refused`, `source_inconsistent`, `transient_dependency`, `internal`. +- Unknown YAML/JSON fields fail validation. `format_version` other than `1` → `unsupported_capability`. +- Dataset-only path: no RPC, no Postgres. A15: fixture quickstart works with network disabled. +- Wide integers in JSON are decimal strings. Use DuckDB `getRowsJson()`. No JavaScript `number` for on-chain integers. +- `ORDER BY` on a raw decimal-string amount column is forbidden. +- DuckDB worker: child process, snapshot files + temp dir only; strip RPC/PG/S3 env. +- M1 `build` emits `release.json`, `manifest.json`, and typed query results. No HTML, no viewer assets. +- `capabilities` advertises **implemented** commands and kinds only. Do not claim rindexer or S3 until those plans land. +- One package. pnpm. MIT already in `LICENSE`. +- Do not add `.markdownlint.yaml`. + +## Later plans (out of this file) + +M2: rindexer adapter, coverage, plan/apply/refresh ingest, Compose, remaining M0 live probes (empty-range evidence, S3 conditional write, header columns). +M3: SELECT models, viewer, `serve`, full `build`. +M4: directory + S3 publish, fork, `latest.json`. +M5: examples, A1–A16, multi-arch smoke. + +--- + +## File structure + +| Path | Responsibility | +|---|---| +| `package.json` | Name `chainplot`, bin, engines, pnpm scripts | +| `pnpm-lock.yaml` | Software lock | +| `tsconfig.json` | `strict`, `NodeNext`, `outDir: dist` | +| `src/cli/main.ts` | Process entry: parse argv, print JSON, `process.exit` | +| `src/cli/run.ts` | `runCli(argv, opts) → CommandResult` (testable, no `process.exit`) | +| `src/cli/envelope.ts` | `okResult`, `failResult`, `ErrorCode` | +| `src/cli/commands/capabilities.ts` | `capabilities` | +| `src/cli/commands/schemaShow.ts` | `schema show ` | +| `src/cli/commands/templates.ts` | `templates list` | +| `src/cli/commands/init.ts` | `init --template --output` | +| `src/cli/commands/validate.ts` | `validate` | +| `src/cli/commands/describe.ts` | `dataset describe` | +| `src/cli/commands/query.ts` | `query --file --snapshot` | +| `src/cli/commands/test.ts` | `test` | +| `src/cli/commands/build.ts` | `build` | +| `src/project/load.ts` | Read `chainplot.yaml`, parse YAML | +| `src/project/validate.ts` | Ajv + graph checks | +| `src/project/types.ts` | Project document types | +| `src/query/workerMain.ts` | Child entry: SQL in, JSON rows out | +| `src/query/runQuery.ts` | Fork worker, enforce limits, parse JSON | +| `src/query/forbidOrderByRaw.ts` | Reject `ORDER BY` on raw-amount columns | +| `schemas/*.schema.json` | One file per kind | +| `templates/fixture-transfers/` | Init scaffold + committed Parquet | +| `tests/cli/*.test.ts` | Command tests | +| `tests/helpers/run.ts` | `runCliJson` helper | +| `docs/compatibility.md` | DuckDB/Node pin evidence from Task 1 | + +--- + +## Shared types (lock these names) + +Every later task uses these. Define them in Task 1 and do not rename. + +```ts +// src/cli/envelope.ts +export const SCHEMA_VERSION = 1 as const; + +export type ErrorCode = + | "validation" + | "missing_credentials" + | "unsupported_capability" + | "policy_refused" + | "source_inconsistent" + | "transient_dependency" + | "internal"; + +export interface CommandError { + code: ErrorCode; + message: string; + resource_id: string | null; + pointer: string | null; + retryable: boolean; + suggested_next: string | null; +} + +export interface CommandResult { + schema_version: typeof SCHEMA_VERSION; + ok: boolean; + command: string; + data: T | null; + warnings: string[]; + error: CommandError | null; +} + +export function okResult(command: string, data: T): CommandResult { + return { + schema_version: SCHEMA_VERSION, + ok: true, + command, + data, + warnings: [], + error: null, + }; +} + +export function failResult( + command: string, + error: CommandError, +): CommandResult { + return { + schema_version: SCHEMA_VERSION, + ok: false, + command, + data: null, + warnings: [], + error, + }; +} +``` + +```ts +// src/cli/run.ts +export interface RunCliOptions { + cwd: string; +} + +export function runCli( + argv: string[], + opts: RunCliOptions, +): Promise; +``` + +`argv` is the args after the binary name, e.g. `["capabilities", "--json"]`. + +```ts +// tests/helpers/run.ts +import { runCli, type CommandResult } from "../../src/cli/run.js"; + +export async function runCliJson( + argv: string[], + cwd: string, +): Promise { + return runCli(argv, { cwd }); +} +``` + +Schema kinds (exact set for M1 `schema show` and `capabilities`): + +`project`, `plan`, `result`, `progress`, `release`, `manifest`, `latest`, `coverage`, `lock` + +M1 implements behavior for `project`, `result`, `release`, `manifest`, `lock`. The others still have schemas so `schema show` works; runtime writers for `plan` / `progress` / `latest` / `coverage` wait for M2/M4. + +--- + +### Task 1: Scaffold, envelope, capabilities + +**Files:** +- Create: `package.json` +- Create: `tsconfig.json` +- Create: `src/cli/envelope.ts` +- Create: `src/cli/run.ts` +- Create: `src/cli/main.ts` +- Create: `src/cli/commands/capabilities.ts` +- Create: `tests/helpers/run.ts` +- Create: `tests/cli/capabilities.test.ts` +- Create: `docs/compatibility.md` + +**Interfaces:** +- Consumes: nothing +- Produces: `runCli`, `okResult`, `failResult`, `CommandResult`, `ErrorCode`, `SCHEMA_VERSION`; `capabilities` data shape below + +Capabilities `data`: + +```ts +export interface CapabilitiesData { + cli_version: string; + schema_kinds: string[]; + commands: string[]; + sources: string[]; + publish_targets: string[]; + chart_types: string[]; + sql_modes: string[]; +} +``` + +M1 values: `schema_kinds` = the nine kinds; `commands` = `["capabilities"]` after this task (later tasks append); `sources` = `[]`; `publish_targets` = `[]`; `chart_types` = `["line", "bar", "kpi", "table"]`; `sql_modes` = `["snapshot"]`. + +- [ ] **Step 1: Write the failing test** + +```ts +// tests/cli/capabilities.test.ts +import { describe, expect, it } from "vitest"; +import { runCliJson } from "../helpers/run.js"; +import { fileURLToPath } from "node:url"; +import path from "node:path"; + +const cwd = path.dirname(fileURLToPath(import.meta.url)); + +describe("capabilities", () => { + it("prints one JSON object with integer schema_version", async () => { + const result = await runCliJson(["capabilities", "--json"], cwd); + expect(result.schema_version).toBe(1); + expect(result.ok).toBe(true); + expect(result.command).toBe("capabilities"); + expect(result.error).toBeNull(); + expect(result.data).toMatchObject({ + schema_kinds: expect.arrayContaining(["project", "progress", "latest"]), + sql_modes: ["snapshot"], + sources: [], + publish_targets: [], + }); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `pnpm exec vitest run tests/cli/capabilities.test.ts` + +Expected: FAIL (no `package.json` / `runCli` yet). If pnpm is missing, install pnpm 11 then retry. + +- [ ] **Step 3: Write minimal implementation** + +`package.json` (exact fields; pin versions with `pnpm add` in this step, do not invent floating `latest`): + +```json +{ + "name": "chainplot", + "version": "0.1.0", + "private": true, + "type": "module", + "engines": { "node": ">=22 <27" }, + "packageManager": "pnpm@11.24.0", + "bin": { "chainplot": "./dist/cli/main.js" }, + "scripts": { + "test": "vitest run", + "build": "tsc -p tsconfig.json" + } +} +``` + +`tsconfig.json`: + +```json +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "strict": true, + "outDir": "dist", + "rootDir": "src", + "declaration": true, + "skipLibCheck": true + }, + "include": ["src"] +} +``` + +Install: `pnpm add commander yaml ajv @duckdb/node-api` and `pnpm add -D typescript vitest @types/node`. Record the resolved versions in `docs/compatibility.md` (Node `process.version`, `pnpm -v`, `@duckdb/node-api` version from lockfile). Prove DuckDB neo loads: + +```ts +import { DuckDBInstance } from "@duckdb/node-api"; +const db = await DuckDBInstance.create(":memory:"); +const conn = await db.connect(); +const reader = await conn.runAndReadAll("select 1 as n"); +``` + +Implement `envelope.ts` as in Shared types. `run.ts` uses commander: require `--json` for M1 (if missing, still print JSON error `validation` on stdout so agents never get text-only stdout). `main.ts`: + +```ts +import { runCli } from "./run.js"; + +const result = await runCli(process.argv.slice(2), { cwd: process.cwd() }); +process.stdout.write(JSON.stringify(result) + "\n"); +process.exit(result.ok ? 0 : 1); +``` + +`capabilities.ts` returns `okResult("capabilities", { ... })`. Register the command in `run.ts`. + +Update `.gitignore` if needed: `node_modules/`, `dist/`, `coverage/`. Keep existing `.chainplot/` and `.env` ignores. + +- [ ] **Step 4: Run test to verify it passes** + +Run: `pnpm test` + +Expected: PASS. `docs/compatibility.md` lists Node, pnpm, `@duckdb/node-api`. + +- [ ] **Step 5: Commit** + +```bash +git add package.json pnpm-lock.yaml tsconfig.json src tests docs/compatibility.md .gitignore +git commit -m "feat: add chainplot CLI capabilities --json" +``` + +--- + +### Task 2: JSON Schema files and `schema show` + +**Files:** +- Create: `schemas/project.schema.json` +- Create: `schemas/plan.schema.json` +- Create: `schemas/result.schema.json` +- Create: `schemas/progress.schema.json` +- Create: `schemas/release.schema.json` +- Create: `schemas/manifest.schema.json` +- Create: `schemas/latest.schema.json` +- Create: `schemas/coverage.schema.json` +- Create: `schemas/lock.schema.json` +- Create: `src/cli/commands/schemaShow.ts` +- Create: `tests/cli/schemaShow.test.ts` +- Modify: `src/cli/run.ts` (register command; add `schema show` to capabilities `commands`) + +**Interfaces:** +- Consumes: `runCli`, `okResult`, `failResult` +- Produces: `SCHEMA_KINDS` constant; `schemaShow(kind: string) → CommandResult` + +```ts +export const SCHEMA_KINDS = [ + "project", + "plan", + "result", + "progress", + "release", + "manifest", + "latest", + "coverage", + "lock", +] as const; +export type SchemaKind = (typeof SCHEMA_KINDS)[number]; +``` + +Unknown kind → `failResult` with `code: "validation"`, `pointer: "/kind"`. + +- [ ] **Step 1: Write the failing test** + +```ts +import { describe, expect, it } from "vitest"; +import { runCliJson } from "../helpers/run.js"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const cwd = path.dirname(fileURLToPath(import.meta.url)); + +describe("schema show", () => { + it("returns the project schema with additionalProperties false", async () => { + const result = await runCliJson(["schema", "show", "project", "--json"], cwd); + expect(result.ok).toBe(true); + const schema = result.data as { additionalProperties: boolean }; + expect(schema.additionalProperties).toBe(false); + }); + + it("rejects unknown kind", async () => { + const result = await runCliJson(["schema", "show", "nope", "--json"], cwd); + expect(result.ok).toBe(false); + expect(result.error?.code).toBe("validation"); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `pnpm exec vitest run tests/cli/schemaShow.test.ts` + +Expected: FAIL (`schema show` not registered). + +- [ ] **Step 3: Write minimal implementation** + +Each schema file is JSON Schema draft 2020-12. Required for `project.schema.json`: + +```json +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://chainplot.dev/schema/project", + "type": "object", + "additionalProperties": false, + "required": ["format_version", "id"], + "properties": { + "format_version": { "type": "integer", "const": 1 }, + "id": { "type": "string", "minLength": 1 }, + "datasets": { "type": "array", "items": { "$ref": "#/$defs/dataset" } }, + "queries": { "type": "array", "items": { "$ref": "#/$defs/query" } }, + "dashboards": { "type": "array", "items": { "$ref": "#/$defs/dashboard" } }, + "models": { "type": "array", "items": { "$ref": "#/$defs/model" } }, + "chain_sources": { "type": "array", "maxItems": 1, "items": { "$ref": "#/$defs/chain_source" } }, + "event_sources": { "type": "array", "maxItems": 20, "items": { "$ref": "#/$defs/event_source" } }, + "publish_targets": { "type": "array", "items": { "$ref": "#/$defs/publish_target" } }, + "policy": { "type": "object", "additionalProperties": false, "properties": { + "block_budget": { "type": "integer", "minimum": 1 } + }} + }, + "$defs": { + "dataset": { + "type": "object", + "additionalProperties": false, + "required": ["id", "snapshot"], + "properties": { + "id": { "type": "string" }, + "snapshot": { "type": "string" }, + "schema": { "type": "string" } + } + }, + "query": { + "type": "object", + "additionalProperties": false, + "required": ["id", "file", "dataset"], + "properties": { + "id": { "type": "string" }, + "file": { "type": "string" }, + "dataset": { "type": "string" }, + "raw_amount_columns": { + "type": "array", + "items": { "type": "string" } + } + } + }, + "dashboard": { + "type": "object", + "additionalProperties": false, + "required": ["id", "title", "panels"], + "properties": { + "id": { "type": "string" }, + "title": { "type": "string" }, + "description": { "type": "string" }, + "panels": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["query", "chart"], + "properties": { + "query": { "type": "string" }, + "chart": { "enum": ["line", "bar", "kpi", "table"] } + } + } + } + } + }, + "model": { + "type": "object", + "additionalProperties": false, + "required": ["id", "file", "depends_on"], + "properties": { + "id": { "type": "string" }, + "file": { "type": "string" }, + "depends_on": { "type": "array", "items": { "type": "string" } } + } + }, + "chain_source": { + "type": "object", + "additionalProperties": false, + "required": ["id", "chain_id", "rpc_secret", "finality"], + "properties": { + "id": { "type": "string" }, + "chain_id": { "type": "integer" }, + "rpc_secret": { "type": "string" }, + "finality": { + "oneOf": [ + { "type": "object", "additionalProperties": false, "required": ["policy"], + "properties": { "policy": { "const": "finalized" } } }, + { "type": "object", "additionalProperties": false, "required": ["policy", "depth"], + "properties": { "policy": { "const": "confirmation_depth" }, "depth": { "type": "integer", "minimum": 1 } } } + ] + } + } + }, + "event_source": { + "type": "object", + "additionalProperties": false, + "required": ["id", "chain", "addresses", "abi", "events", "start_block", "end"], + "properties": { + "id": { "type": "string" }, + "chain": { "type": "string" }, + "addresses": { "type": "array", "minItems": 1, "maxItems": 20, "items": { "type": "string" } }, + "abi": { "type": "string" }, + "events": { "type": "array", "items": { "type": "string" } }, + "start_block": { "type": "integer", "minimum": 0 }, + "end": { + "oneOf": [ + { "type": "object", "additionalProperties": false, "required": ["mode", "block"], + "properties": { "mode": { "const": "pinned" }, "block": { "type": "integer" } } }, + { "type": "object", "additionalProperties": false, "required": ["mode"], + "properties": { "mode": { "const": "follow_finalized" } } } + ] + } + } + }, + "publish_target": { + "type": "object", + "additionalProperties": false, + "required": ["id", "type"], + "properties": { + "id": { "type": "string" }, + "type": { "enum": ["directory", "s3"] }, + "path": { "type": "string" }, + "bucket": { "type": "string" } + } + } + } +} +``` + +`result.schema.json` matches `CommandResult`. `progress.schema.json`: `{ schema_version, type: "progress", run_id, stage, message }` plus optional numeric fields. Stub the rest as objects with `schema_version` integer, `additionalProperties` true only where the spec has not yet frozen fields (`plan`, `coverage`); `latest` requires `prefix` and `release_json_checksum` strings; `lock` requires `format_version` integer; `manifest` requires `snapshot_id` string and `mode` enum `results_only` | `dataset_included` | `dataset_referenced`; `release` requires `schema_version`, `project_id`, `mode`. + +`schemaShow` reads `schemas/.schema.json` from the package root (resolve via `import.meta.url` relative to `src/`, then `../../schemas`). + +- [ ] **Step 4: Run test to verify it passes** + +Run: `pnpm exec vitest run tests/cli/schemaShow.test.ts` + +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add schemas src/cli tests/cli/schemaShow.test.ts +git commit -m "feat: add JSON Schema kinds and schema show" +``` + +--- + +### Task 3: Load and validate `chainplot.yaml` + +**Files:** +- Create: `src/project/types.ts` +- Create: `src/project/load.ts` +- Create: `src/project/validate.ts` +- Create: `src/cli/commands/validate.ts` +- Create: `tests/cli/validate.test.ts` +- Create: `tests/fixtures/projects/valid-dataset-only/chainplot.yaml` +- Create: `tests/fixtures/projects/unknown-field/chainplot.yaml` +- Create: `tests/fixtures/projects/follow-plus-depth/chainplot.yaml` +- Modify: `src/cli/run.ts` + +**Interfaces:** +- Consumes: project JSON Schema from Task 2 +- Produces: + +```ts +export function loadProject(projectDir: string): unknown; +export function validateProject(doc: unknown, projectDir: string): { + ok: true; + project: ProjectDocument; +} | { + ok: false; + error: CommandError; +}; +``` + +`ProjectDocument` fields match the schema `$defs` names: `format_version`, `id`, `datasets`, `queries`, `dashboards`, optional `models`, `chain_sources`, `event_sources`, `publish_targets`, `policy`. + +- [ ] **Step 1: Write the failing test** + +```ts +import { describe, expect, it } from "vitest"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { runCliJson } from "../helpers/run.js"; + +const fixtures = path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + "../fixtures/projects", +); + +describe("validate", () => { + it("accepts a dataset-only project", async () => { + const result = await runCliJson(["validate", "--json"], path.join(fixtures, "valid-dataset-only")); + expect(result.ok).toBe(true); + expect(result.command).toBe("validate"); + }); + + it("rejects unknown fields", async () => { + const result = await runCliJson(["validate", "--json"], path.join(fixtures, "unknown-field")); + expect(result.ok).toBe(false); + expect(result.error?.code).toBe("validation"); + }); + + it("rejects follow_finalized plus confirmation_depth", async () => { + const result = await runCliJson(["validate", "--json"], path.join(fixtures, "follow-plus-depth")); + expect(result.ok).toBe(false); + expect(result.error?.code).toBe("unsupported_capability"); + }); +}); +``` + +`valid-dataset-only/chainplot.yaml`: + +```yaml +format_version: 1 +id: fixture-transfers +datasets: + - id: amounts + snapshot: snapshots/amounts.parquet +queries: + - id: raw_amounts + file: queries/raw_amounts.sql + dataset: amounts + raw_amount_columns: [amount] +dashboards: + - id: overview + title: Amounts + panels: + - query: raw_amounts + chart: table +``` + +`unknown-field/chainplot.yaml`: same plus `extra: true`. + +`follow-plus-depth/chainplot.yaml`: one `chain_sources` entry with `finality.policy: confirmation_depth` and `depth: 12`, one `event_sources` entry with `end.mode: follow_finalized`, plus dummy datasets/queries so the rest of the schema passes. + +- [ ] **Step 2: Run test to verify it fails** + +Run: `pnpm exec vitest run tests/cli/validate.test.ts` + +Expected: FAIL (`validate` not registered). + +- [ ] **Step 3: Write minimal implementation** + +`load.ts`: read `path.join(projectDir, "chainplot.yaml")` with `fs.readFileSync` + `yaml.parse`. Missing file → throw a `CommandError` with `validation`. + +`validate.ts`: compile `project.schema.json` with Ajv `{ allErrors: true, strict: true }`. Map Ajv `additionalProperties` errors to `pointer` JSON Pointer. After schema pass: if any `event_sources[].end.mode === "follow_finalized"` and the referenced chain `finality.policy === "confirmation_depth"`, return `unsupported_capability`. If `format_version !== 1`, `unsupported_capability`. Resolve query `file` and dataset `snapshot` paths relative to `projectDir`; missing files → `validation` with `resource_id` set to the query/dataset id. + +`validate` command: `loadProject(opts.cwd)` then `validateProject`. + +- [ ] **Step 4: Run test to verify it passes** + +Run: `pnpm exec vitest run tests/cli/validate.test.ts` + +Expected: PASS. (Missing parquet is OK if validate only checks the YAML path exists — create empty placeholder files `snapshots/.keep` in the valid fixture, or skip snapshot existence until Task 5. **This task: YAML + schema + policy combo only.** Do not require parquet yet. Document that in `validate.ts`: snapshot-file existence is Task 5.) + +- [ ] **Step 5: Commit** + +```bash +git add src/project src/cli/commands/validate.ts tests/cli/validate.test.ts tests/fixtures +git commit -m "feat: validate chainplot.yaml against JSON Schema" +``` + +--- + +### Task 4: `templates list` and `init` + +**Files:** +- Create: `templates/fixture-transfers/chainplot.yaml` +- Create: `templates/fixture-transfers/queries/raw_amounts.sql` +- Create: `templates/fixture-transfers/dashboards/overview.yaml` (optional; panels may live in `chainplot.yaml` only — keep panels in `chainplot.yaml` to avoid two sources of truth) +- Create: `templates/fixture-transfers/tests/amounts.yaml` +- Create: `src/cli/commands/templates.ts` +- Create: `src/cli/commands/init.ts` +- Create: `tests/cli/init.test.ts` +- Modify: `src/cli/run.ts` + +**Interfaces:** +- Consumes: `loadProject` / `validateProject` from Task 3 +- Produces: `listTemplates() → { id, required_inputs, limitations }[]`; `initTemplate(id, outputDir)` + +Template id: `fixture-transfers`. + +Limitations string: `dataset-only fixture; no RPC; no ingest`. + +- [ ] **Step 1: Write the failing test** + +```ts +import { describe, expect, it } from "vitest"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { runCliJson } from "../helpers/run.js"; + +describe("init", () => { + it("lists fixture-transfers", async () => { + const result = await runCliJson(["templates", "list", "--json"], process.cwd()); + expect(result.ok).toBe(true); + const ids = (result.data as { templates: { id: string }[] }).templates.map((t) => t.id); + expect(ids).toContain("fixture-transfers"); + }); + + it("creates a project and fails on collision", async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "chainplot-init-")); + const first = await runCliJson( + ["init", "--template", "fixture-transfers", "--output", dir, "--json"], + process.cwd(), + ); + expect(first.ok).toBe(true); + expect(fs.existsSync(path.join(dir, "chainplot.yaml"))).toBe(true); + const second = await runCliJson( + ["init", "--template", "fixture-transfers", "--output", dir, "--json"], + process.cwd(), + ); + expect(second.ok).toBe(false); + expect(second.error?.code).toBe("validation"); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `pnpm exec vitest run tests/cli/init.test.ts` + +Expected: FAIL. + +- [ ] **Step 3: Write minimal implementation** + +Copy files from `templates/fixture-transfers/` into `--output` with `fs.cpSync(..., { recursive: true, errorOnExist: true, force: false })`. If the output directory exists and is non-empty **or** any destination file exists, fail `validation` (`message` names the colliding path). After copy, the parquet may still be missing until Task 5 — include `snapshots/` in the template only after Task 5 generates it. For this task, copy yaml + sql + tests. + +`chainplot.yaml` in the template matches Task 3 valid fixture. + +`queries/raw_amounts.sql`: + +```sql +SELECT amount +FROM amounts +ORDER BY amount_sort +``` + +(`amount_sort` column lands in Task 5 parquet. For this task the SQL file only needs to exist.) + +Unknown `--template` → `validation`. + +- [ ] **Step 4: Run test to verify it passes** + +Run: `pnpm exec vitest run tests/cli/init.test.ts` + +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add templates src/cli/commands/templates.ts src/cli/commands/init.ts tests/cli/init.test.ts src/cli/run.ts +git commit -m "feat: add templates list and init" +``` + +--- + +### Task 5: Fixture Parquet, `dataset describe`, snapshot existence + +**Files:** +- Create: `src/snapshot/describe.ts` +- Create: `src/cli/commands/describe.ts` +- Create: `scripts/write-fixture-parquet.ts` +- Create: `templates/fixture-transfers/snapshots/amounts.parquet` (generated, committed) +- Create: `tests/cli/describe.test.ts` +- Modify: `src/project/validate.ts` (require snapshot files to exist) +- Modify: `tests/cli/validate.test.ts` (valid fixture must include parquet or point at the template) + +**Interfaces:** +- Consumes: `validateProject` +- Produces: + +```ts +export interface DatasetDescribeData { + id: string; + mode: "dataset_included"; + snapshot: string; + columns: { name: string; logical_type: string }[]; + coverage: null; +} + +export function describeDataset( + projectDir: string, + datasetId: string, +): Promise; +``` + +M1 fixture mode is always `dataset_included` (files on disk). `coverage` is `null` (no ingest). + +Parquet columns (exact): + +| name | physical | logical | +|---|---|---| +| amount | VARCHAR | decimal-string int256 | +| amount_sort | VARCHAR | zero-padded sign-aware sort key, width 78 + 1 sign | + +Rows (exact amount strings): `0`, `1`, `-1`, `9007199254740993` (2^53+1), `-9007199254740993`, `57896044618658097711785492504343953926634992332820282019728792003956564819967` (2^255-1), `-57896044618658097711785492504343953926634992332820282019728792003956564819968` (-2^255), `115792089237316195423570985008687907853269984665640564039457584007913129639935` (2^256-1). + +Sort key: `'0'` + 78-digit zero-padded absolute value for non-negative; `'1'` + 78-digit zero-padded `(10^78 - abs)` for negative (standard decimal two's-complement style so `ORDER BY amount_sort` is numeric). Document the formula in `scripts/write-fixture-parquet.ts` comments. + +- [ ] **Step 1: Write the failing test** + +```ts +import { describe, expect, it } from "vitest"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { runCliJson } from "../helpers/run.js"; + +const template = path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + "../../templates/fixture-transfers", +); + +describe("dataset describe", () => { + it("describes the fixture snapshot", async () => { + const result = await runCliJson( + ["dataset", "describe", "amounts", "--json"], + template, + ); + expect(result.ok).toBe(true); + expect(result.data).toMatchObject({ + id: "amounts", + mode: "dataset_included", + }); + const cols = (result.data as { columns: { name: string }[] }).columns.map((c) => c.name); + expect(cols).toEqual(expect.arrayContaining(["amount", "amount_sort"])); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `pnpm exec vitest run tests/cli/describe.test.ts` + +Expected: FAIL (no parquet / command). + +- [ ] **Step 3: Write minimal implementation** + +`scripts/write-fixture-parquet.ts` uses `@duckdb/node-api`: `CREATE TABLE amounts (amount VARCHAR, amount_sort VARCHAR)`, insert the eight rows, `COPY amounts TO 'templates/fixture-transfers/snapshots/amounts.parquet' (FORMAT PARQUET)`. Run: `pnpm exec tsx scripts/write-fixture-parquet.ts` (add `tsx` as a devDependency if `pnpm exec tsx` is missing). + +`describe.ts`: open parquet with DuckDB in-process **only in this command's parent** is not allowed — use the worker from Task 6. For Task 5, read parquet metadata via a one-shot worker helper `describeParquet(path)` that will move into `src/query/runQuery.ts` in Task 6. To keep this task shippable without the worker yet, spawn the same child module path Task 6 will use: if `workerMain.ts` does not exist, implement a minimal `src/query/inspectParquet.ts` that creates an in-process connection, runs `DESCRIBE SELECT * FROM read_parquet(?)`, returns column names, then closes. Task 6 replaces in-process inspect with the worker; **do not leave in-process query execution after Task 6.** + +Unknown dataset id → `validation`, `resource_id` = the id. + +- [ ] **Step 4: Run test to verify it passes** + +Run: `pnpm exec vitest run tests/cli/describe.test.ts tests/cli/validate.test.ts` + +Expected: PASS. Commit the generated parquet (binary is required; it is the fixture). + +- [ ] **Step 5: Commit** + +```bash +git add templates/fixture-transfers/snapshots src/snapshot src/cli/commands/describe.ts scripts tests/cli/describe.test.ts src/project/validate.ts +git commit -m "feat: add fixture parquet and dataset describe" +``` + +--- + +### Task 6: DuckDB child worker, `query`, A8 strings and sort key + +**Files:** +- Create: `src/query/workerMain.ts` +- Create: `src/query/runQuery.ts` +- Create: `src/query/forbidOrderByRaw.ts` +- Create: `src/cli/commands/query.ts` +- Create: `tests/cli/query.test.ts` +- Modify: `src/snapshot/describe.ts` (call worker, remove in-process DuckDB if introduced in Task 5) +- Modify: `src/cli/run.ts` + +**Interfaces:** +- Consumes: dataset snapshot path from `ProjectDocument` +- Produces: + +```ts +export interface QueryRequest { + sql: string; + tables: Record; + analysisTimestamp: string; + rawAmountColumns: string[]; + rowLimit: number; +} + +export interface QuerySuccess { + columns: { name: string; logical_type: string }[]; + rows: unknown[][]; + snapshot: string; +} + +export function runQuery(req: QueryRequest): Promise; +``` + +Worker protocol (stdin/stdout JSON lines, one request, one response, then exit): + +Parent writes: `JSON.stringify({ sql, tables, analysisTimestamp }) + "\n"`. +Child prints: `JSON.stringify({ ok: true, columns, rows })` or `{ ok: false, message }` using `getRowsJson()`. +`tables` maps SQL table name → parquet path. Child `CREATE VIEW AS SELECT * FROM read_parquet(path)` for each. No `INSTALL` / `LOAD` of httpfs or postgres. Child starts with `env` stripped to `PATH`, `HOME`, `LANG` only. + +Limits: `rowLimit` default `10000`. More rows → `policy_refused`. Deadline 60s: parent `timeout` kills the child, `transient_dependency`. + +`forbidOrderByRaw(sql, rawAmountColumns)`: if a raw amount column appears as an `ORDER BY` expression (token match on column name after `ORDER BY`), throw `validation`. Do not parse a full SQL grammar; split on `ORDER BY` (case-insensitive) and reject if any raw column name appears in that tail. + +- [ ] **Step 1: Write the failing test** + +```ts +import { describe, expect, it } from "vitest"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { runCliJson } from "../helpers/run.js"; + +const template = path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + "../../templates/fixture-transfers", +); + +describe("query", () => { + it("returns 2^256-1 as a decimal string, not a number", async () => { + const sql = path.join(os.tmpdir(), "q-a8.sql"); + fs.writeFileSync(sql, "SELECT amount FROM amounts ORDER BY amount_sort"); + const result = await runCliJson( + ["query", "--file", sql, "--snapshot", "amounts", "--json"], + template, + ); + expect(result.ok).toBe(true); + const rows = (result.data as { rows: string[][] }).rows; + const amounts = rows.map((r) => r[0]); + expect(typeof amounts[0]).toBe("string"); + expect(amounts).toContain( + "115792089237316195423570985008687907853269984665640564039457584007913129639935", + ); + expect(amounts[0]).toBe( + "-57896044618658097711785492504343953926634992332820282019728792003956564819968", + ); + }); + + it("rejects ORDER BY on a raw amount column", async () => { + const sql = path.join(os.tmpdir(), "q-bad.sql"); + fs.writeFileSync(sql, "SELECT amount FROM amounts ORDER BY amount"); + const result = await runCliJson( + ["query", "--file", sql, "--snapshot", "amounts", "--json"], + template, + ); + expect(result.ok).toBe(false); + expect(result.error?.code).toBe("validation"); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `pnpm exec vitest run tests/cli/query.test.ts` + +Expected: FAIL. + +- [ ] **Step 3: Write minimal implementation** + +`workerMain.ts`: read one JSON line from stdin, open in-memory DuckDB, create views, `runAndReadAll`, `getRowsJson()`, write one JSON object, exit. Catch errors into `{ ok: false, message }`. + +`runQuery.ts`: `fork(workerPath, { env: { PATH, HOME, LANG } })`. `workerPath` is `new URL("./workerMain.js", import.meta.url)` after compile; in vitest use `tsx` by setting `execArgv: ["--import", "tsx"]` if running TS, or point vitest `vite-node` at the TS file via `fork(fileURLToPath(new URL("./workerMain.ts", import.meta.url)), { execArgv: ["--import", "tsx"] })`. Pick one and keep it for all tests. + +`query` command: `--file` required, `--snapshot` is the dataset id. Load project from `cwd`. Resolve parquet. `analysisTimestamp` = snapshot file mtime as UTC ISO (M1 has no ingest analysis time). + +- [ ] **Step 4: Run test to verify it passes** + +Run: `pnpm exec vitest run tests/cli/query.test.ts` + +Expected: PASS. Confirm the huge integer is a string in the parsed JSON (`typeof === "string"`). + +- [ ] **Step 5: Commit** + +```bash +git add src/query src/cli/commands/query.ts tests/cli/query.test.ts src/snapshot +git commit -m "feat: query snapshots in a DuckDB child process" +``` + +--- + +### Task 7: `test` assertions + +**Files:** +- Create: `src/project/assertions.ts` +- Create: `src/cli/commands/test.ts` +- Create: `tests/cli/testCmd.test.ts` +- Modify: `templates/fixture-transfers/tests/amounts.yaml` +- Modify: `src/cli/run.ts` + +**Interfaces:** +- Consumes: `runQuery`, `validateProject` +- Produces: + +```ts +export interface AssertionFile { + dataset: string; + query?: string; + expect: { + row_count?: number; + columns?: string[]; + }; +} + +export function runAssertions(projectDir: string): Promise; +``` + +Assertion YAML (`tests/*.yaml`): `additionalProperties: false`. M1 supports `row_count` and `columns` only. + +Fixture `tests/amounts.yaml`: + +```yaml +dataset: amounts +expect: + row_count: 8 + columns: [amount, amount_sort] +``` + +Missing local data → `validation` (do not fetch). `test` command network: none. + +- [ ] **Step 1: Write the failing test** + +```ts +import { describe, expect, it } from "vitest"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { runCliJson } from "../helpers/run.js"; + +const template = path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + "../../templates/fixture-transfers", +); + +describe("test", () => { + it("passes fixture assertions", async () => { + const result = await runCliJson(["test", "--json"], template); + expect(result.ok).toBe(true); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `pnpm exec vitest run tests/cli/testCmd.test.ts` + +Expected: FAIL. + +- [ ] **Step 3: Write minimal implementation** + +Glob `tests/*.yaml` under `projectDir`. For each file, load YAML, require `dataset`, `SELECT * FROM ` via `runQuery` with `rawAmountColumns: []` and `ORDER BY` omitted (use `SELECT * FROM t` without ORDER BY). Compare `rows.length` to `row_count` and column names. First failure: `ok: false`, `code: validation`, `resource_id` = dataset id, `pointer` = the assertion file path as a string. + +- [ ] **Step 4: Run test to verify it passes** + +Run: `pnpm exec vitest run tests/cli/testCmd.test.ts` + +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add src/project/assertions.ts src/cli/commands/test.ts tests/cli/testCmd.test.ts templates/fixture-transfers/tests +git commit -m "feat: add chainplot test assertions" +``` + +--- + +### Task 8: M1 `build` (results + manifests, no HTML) + +**Files:** +- Create: `src/cli/commands/build.ts` +- Create: `src/publish/writeRelease.ts` +- Create: `tests/cli/build.test.ts` +- Modify: `src/cli/run.ts` + +**Interfaces:** +- Consumes: `validateProject`, `runQuery` +- Produces: + +```ts +export function buildRelease(projectDir: string): Promise<{ + distDir: string; + files: string[]; +}>; +``` + +Output layout (M1): + +```text +dist/releases/local/ + release.json + results/.json + datasets//manifest.json +``` + +`release.json`: `{ schema_version: 1, project_id, mode: "dataset_included", queries: [] }`. +`manifest.json`: `{ snapshot_id: , mode: "dataset_included", files: ["tables/amounts.parquet"] }` — M1 does **not** copy parquet into dist (results-only would be wrong; included means the project already has the snapshot locally). Set `mode` to `dataset_included` and `files` relative to the project snapshot path recorded as `source_path`. Do not write `index.html`. + +Each `results/.json`: `{ schema_version: 1, query_id, columns, rows, snapshot }` from `runQuery` on the query SQL file. Use `getRowsJson` strings. + +If any query fails, `build` fails; do not write a partial `release.json` (write to a temp dir then rename). + +- [ ] **Step 1: Write the failing test** + +```ts +import { describe, expect, it } from "vitest"; +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { runCliJson } from "../helpers/run.js"; + +const template = path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + "../../templates/fixture-transfers", +); + +describe("build", () => { + it("writes release.json and query results without HTML", async () => { + const result = await runCliJson(["build", "--json"], template); + expect(result.ok).toBe(true); + const dist = path.join(template, "dist/releases/local"); + expect(fs.existsSync(path.join(dist, "release.json"))).toBe(true); + expect(fs.existsSync(path.join(dist, "index.html"))).toBe(false); + const raw = JSON.parse( + fs.readFileSync(path.join(dist, "results/raw_amounts.json"), "utf8"), + ); + expect(typeof raw.rows[0][0]).toBe("string"); + }); +}); +``` + +Add `dist/` under `templates/` to `.gitignore` if tests write there, **or** write to a copied temp project in the test. Prefer copy the template to `mkdtemp` so the git tree stays clean. + +- [ ] **Step 2: Run test to verify it fails** + +Run: `pnpm exec vitest run tests/cli/build.test.ts` + +Expected: FAIL. + +- [ ] **Step 3: Write minimal implementation** + +`build.ts`: validate, for each query `runQuery`, write staging dir `dist/releases/local/.tmp-*`, then `fs.renameSync` onto `dist/releases/local`. Command data: `{ distDir, files }`. + +- [ ] **Step 4: Run test to verify it passes** + +Run: `pnpm exec vitest run tests/cli/build.test.ts` + +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add src/cli/commands/build.ts src/publish tests/cli/build.test.ts .gitignore +git commit -m "feat: build cached query results for fixture projects" +``` + +--- + +### Task 9: A15 network-disabled e2e and README command list + +**Files:** +- Create: `tests/cli/a15.e2e.test.ts` +- Modify: `README.md` +- Modify: `src/cli/commands/capabilities.ts` (commands list = all M1 commands) + +**Interfaces:** +- Consumes: all M1 commands +- Produces: documented agent path; A15 green + +- [ ] **Step 1: Write the failing test** + +```ts +import { describe, expect, it } from "vitest"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { spawnSync } from "node:child_process"; + +describe("A15", () => { + it("init, validate, test, build with network disabled", () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "chainplot-a15-")); + const env = { + ...process.env, + NO_NETWORK: "1", + http_proxy: "http://127.0.0.1:1", + https_proxy: "http://127.0.0.1:1", + HTTP_PROXY: "http://127.0.0.1:1", + HTTPS_PROXY: "http://127.0.0.1:1", + }; + const bin = path.resolve("src/cli/main.ts"); + const run = (args: string[], cwd: string) => + spawnSync("pnpm", ["exec", "tsx", bin, ...args, "--json"], { + cwd, + env, + encoding: "utf8", + }); + const init = run( + ["init", "--template", "fixture-transfers", "--output", dir], + process.cwd(), + ); + expect(init.status).toBe(0); + for (const cmd of [["validate"], ["test"], ["build"]]) { + const r = run(cmd, dir); + expect(r.status, r.stderr).toBe(0); + const json = JSON.parse(r.stdout); + expect(json.ok).toBe(true); + expect(json.schema_version).toBe(1); + } + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `pnpm exec vitest run tests/cli/a15.e2e.test.ts` + +Expected: FAIL until spawn + tsx path is wired; fix the bin invocation to match Task 6 (`pnpm exec tsx src/cli/main.ts`). + +- [ ] **Step 3: Write minimal implementation** + +No new features. Fix spawn if `runCli` vs process entry disagrees. Update `capabilities.commands` to: + +`["capabilities", "schema show", "templates list", "init", "validate", "dataset describe", "query", "test", "build"]` + +README: replace "design only" with the M1 command list and: + +```text +pnpm install +pnpm exec tsx src/cli/main.ts init --template fixture-transfers --output ./demo --json +cd demo +pnpm exec tsx ../src/cli/main.ts validate --json +pnpm exec tsx ../src/cli/main.ts test --json +pnpm exec tsx ../src/cli/main.ts build --json +``` + +Use repo-relative commands. Do not mention machine-specific paths. + +- [ ] **Step 4: Run test to verify it passes** + +Run: `pnpm test` + +Expected: all tests PASS, including A15. + +- [ ] **Step 5: Commit** + +```bash +git add tests/cli/a15.e2e.test.ts README.md src/cli/commands/capabilities.ts +git commit -m "test: A15 fixture quickstart with network disabled" +``` + +--- + +## Self-review + +**Spec coverage (M1 only):** +- Agent `--json` envelope, integer `schema_version`, closed error codes: Task 1 +- JSON Schema kinds + `schema show`: Task 2 +- Unknown fields, format_version, `follow_finalized`+`confirmation_depth`: Task 3 +- `init` / `templates list`: Task 4 +- Dataset describe, fixture snapshot: Task 5 +- DuckDB child, uint256 strings, ORDER BY raw forbidden, A8 values: Task 6 +- `test` no-fetch assertions: Task 7 +- M1 `build` without HTML: Task 8 +- A15: Task 9 +- Not in this plan (M2+): rindexer, coverage completeness, plan/apply ingest, refresh ingest, doctor, publish, fork, serve, S3, Compose, viewer, A1–A4, A6, A7, A9–A14, A16 + +**Placeholders:** none. Versions pinned via pnpm lock in Task 1; DuckDB neo pin recorded in `docs/compatibility.md`. + +**Type consistency:** `runCli` / `CommandResult` / `ErrorCode` / `ProjectDocument` / `runQuery` / `QueryRequest` names are stable across tasks. diff --git a/docs/plans/2026-09-13-m2-ingest-snapshots.md b/docs/plans/2026-09-13-m2-ingest-snapshots.md new file mode 100644 index 0000000..e17671f --- /dev/null +++ b/docs/plans/2026-09-13-m2-ingest-snapshots.md @@ -0,0 +1,775 @@ +# Chainplot M2 Ingest and Snapshots Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Ship ingest: `plan --intent ingest|refresh` → `apply` runs a bounded rindexer job against our Postgres, proves coverage from `rindexer_internal.*.last_synced_block` plus header hashes, exports a Parquet snapshot through DuckDB, and refuses to promote incomplete data. Plus run journal, locks, cancel, and the product Compose + Dockerfile. + +**Architecture:** One package. `src/ingest/` owns the `IngestAdapter` interface and the rindexer implementation (generated gitignored config, subprocess, coverage inspection). `src/rpc/` is a minimal JSON-RPC client for finalized head and coverage-boundary headers. `src/plan/` generates digest-bound plans and executes them. Coverage segments live in `.chainplot/coverage.json`; run state in `.chainplot/runs//`. The DuckDB **query worker** stays isolated (no PG/RPC env, unchanged from M1); the **exporter** is a separate DuckDB child that may `ATTACH` Postgres. + +**Tech Stack:** Unchanged from M1 (`Node >=22 <27`, pnpm 11, TypeScript, vitest, ajv, yaml, commander, `@duckdb/node-api` 1.5.5-r.4). New dependency: `pg` (Postgres client) for the advisory lock and coverage-cursor reads — session-scoped advisory locks cannot go through DuckDB's pooled postgres ATTACH. rindexer runs as a subprocess binary (`CHAINPLOT_RINDEXER_BIN`, default `rindexer` on PATH); the Compose Dockerfile copies the pinned rindexer binary out of the pinned image. No AWS SDK yet (M4). + +**Spec:** `docs/specs/2026-09-12-chainplot-design.md` is authority. M0 evidence: `docs/compatibility.md` (cursor table, resume = `last_synced_block + 1`, SIGTERM after `Historical indexing completed`, DuckDB postgres ATTACH with `CAST(block_number/tx_index AS BIGINT)`, `timestamp: true` gives native `block_hash`/`block_timestamp`, empty-range evidence = cursor at `end_block` with 0 rows). + +## Global Constraints + +- CLI envelope, error codes, `--json`/`--jsonl` rules: unchanged from M1. New commands register in `capabilities` only when implemented. +- **Never** commit or document an RPC URL. Live tests read `RPC_URL` / `CHAINPLOT_TEST_DATABASE_URL` from the environment and `it.skip` when absent. `.env` stays gitignored. +- Jobs only append. No overlap-tail re-ingest. `job_start = last_proven_complete_block + 1`, `job_end = min(job_target_end, job_start + block_budget - 1)`. +- `follow_finalized` requires chain finality `finalized`. `follow_finalized` + `confirmation_depth` → `unsupported_capability` (already refused at `validate`; `plan` re-checks). +- A `pinned` `end_block` must satisfy finality at plan time (`≤ finalized` or `≤ head − depth`), else `policy_refused`. +- Completeness evidence is the rindexer cursor, never `max(block_number)` or row count. Zero rows + cursor ≥ `end_block` → `complete_empty`. +- Coverage segments record `start_block_hash`, `end_block_hash`, `start_block_parent_hash` from `eth_getBlockByNumber` header fetches (3 per segment: `job_start-1`, `job_start`, `job_end`). Adjacent segments hash-join iff number-adjacent **and** parent link matches; otherwise `source_inconsistent`, not complete. +- Generated rindexer config: `project_type: no-code`, Postgres storage enabled, GraphQL disabled, explicit addresses only, `include_events` limited to declared signatures, `timestamp: true`, always explicit `start_block`/`end_block`, RPC via `${RPC_URL}` env interpolation. Config file is gitignored. +- Stop rindexer with SIGTERM after the `Historical indexing completed` log line; do not wait for process exit. Wall clock 30 min → SIGTERM + `transient_dependency`, resumable. +- Export SQL: `CAST(block_number AS BIGINT)`, `CAST(tx_index AS BIGINT)`; `value` and other uint256 stay VARCHAR decimal strings; add `chain_id` as a literal column. Physical uniqueness `(chain_id, block_number, tx_hash, log_index)`; same key with different `block_hash` → `source_inconsistent`. +- Block budget applies to the first ingest too. Default budget 100_000 (`policy.block_budget` overrides). A budget-split pinned range is finished by repeated `plan --intent ingest` + `apply`, never by `refresh`. +- `build` never talks to RPC. Promotion gate: a snapshot whose sources are not complete over `[start_block, required_end]` must not produce a release; `policy_refused` naming the source and missing range. Hash-join break → `source_inconsistent`. +- One active ingest writer per project: local lock file (`.chainplot/locks/ingest.lock`, `O_EXCL`) **plus** Postgres advisory lock (`pg_try_advisory_lock`). Concurrent `apply` → `policy_refused`. +- No docker.sock anywhere. Compose: `producer` (CLI + rindexer binary) + `postgres:16-alpine`. +- Do not restore any RPC URL or provider hostname into the repo, docs, commits, or test fixtures. +- Do not add `.markdownlint.yaml`. + +## Later plans (out of this file) + +M3: SELECT model graph re-export, viewer, `serve`, full static `build`, `plan --intent build` behavior. +M4: directory + S3 publish, `latest.json` conditional promotion, `fork`, `plan --intent publish`, S3 conditional-write probe. +M5: examples, A1–A16, multi-arch smoke. + +In M2, `plan --intent build|publish` returns `unsupported_capability` (typed, not silent). + +--- + +## File structure + +| Path | Responsibility | +|---|---| +| `schemas/plan.schema.json` | Frozen plan kind (replaces M1 stub) | +| `schemas/coverage.schema.json` | Frozen coverage kind (replaces M1 stub) | +| `schemas/progress.schema.json` | Frozen progress-event envelope (replaces M1 stub) | +| `src/rpc/client.ts` | JSON-RPC over HTTP, closed error mapping | +| `src/rpc/heads.ts` | `getFinalizedHead`, `getHeader` | +| `src/ingest/adapter.ts` | `IngestAdapter` interface + shared job/report types | +| `src/ingest/coverage.ts` | Segment store, hash-join, `lastProvenCompleteBlock`, completeness | +| `src/ingest/rindexer/renderConfig.ts` | `chainplot.yaml` → rindexer YAML text | +| `src/ingest/rindexer/runBounded.ts` | Spawn, watch for historic-complete, wall clock, signals | +| `src/ingest/rindexer/inspectCoverage.ts` | Cursor + row count via `pg`, classify 4 statuses | +| `src/ingest/exporter.ts` | DuckDB ATTACH → Parquet + uniqueness + manifest fields | +| `src/plan/digest.ts` | Canonical JSON + sha256 helpers | +| `src/plan/generate.ts` | Plan generation for `ingest`/`refresh` intents | +| `src/plan/apply.ts` | Verify, lock, execute actions, record coverage | +| `src/plan/refresh.ts` | Authorization classification for `refresh` | +| `src/runtime/journal.ts` | `.chainplot/runs//` plan copy, status, checkpoints | +| `src/runtime/locks.ts` | Local lock file + pg advisory lock | +| `src/cli/commands/plan.ts` | `plan --intent` | +| `src/cli/commands/apply.ts` | `apply --plan` | +| `src/cli/commands/refresh.ts` | `refresh [--publish-target]` | +| `src/cli/commands/runs.ts` | `runs list|show|cancel` | +| `templates/ingest-transfers/` | Ingest scaffold: `chainplot.yaml`, ABI, `.env.example`, `compose.yaml`, `Dockerfile` | +| `tests/rpc/*.test.ts` | Mock-server RPC tests | +| `tests/ingest/*.test.ts` | Coverage, renderConfig, runBounded (fake binary), plan/apply | +| `tests/ingest/live/*.test.ts` | Env-gated live tests (RPC + Postgres) | + +--- + +## Shared types (lock these names) + +```ts +// src/ingest/adapter.ts +export type CoverageStatus = + | "not_indexed" + | "incomplete" + | "complete_empty" + | "complete_with_rows"; + +export interface BoundedJob { + sourceId: string; + contractName: string; // lowercase PG identifier; drives table names + networkName: string; // `chainplot_` + chainId: number; + addresses: string[]; + abiPath: string; // absolute, inside project dir + events: string[]; + jobStart: number; // inclusive + jobEnd: number; // inclusive + rpcUrl: string; + databaseUrl: string; + workDir: string; // .chainplot/ingest// (gitignored) +} + +export interface RunHandle { + job: BoundedJob; + pid: number; + completedLogSeen: boolean; +} + +export interface CoverageReport { + status: CoverageStatus; + lastSyncedBlock: number | null; // from rindexer_internal cursor; null = no row + rowCount: number; +} + +export interface IngestAdapter { + renderConfig(job: BoundedJob): string; + runBounded(job: BoundedJob, opts: RunOptions): Promise; + stopAndQuiesce(handle: RunHandle): Promise; + inspectCoverage(job: BoundedJob): Promise; +} +``` + +```ts +// src/ingest/coverage.ts +export interface CoverageSegment { + start_block: number; + end_block: number; + start_block_hash: string; + end_block_hash: string; + start_block_parent_hash: string; + status: "complete_empty" | "complete_with_rows"; + row_count?: number; +} + +export interface SourceCoverage { + source_id: string; + segments: CoverageSegment[]; +} + +export interface CoverageFile { + schema_version: 1; + chain_id: number; + sources: SourceCoverage[]; +} + +export function lastProvenCompleteBlock( + segments: CoverageSegment[], + projectStartBlock: number, +): number; +export function hashJoinOk(prev: CoverageSegment, next: CoverageSegment): boolean; +export function requiredEnd( + end: EventEnd, + segments: CoverageSegment[], +): number | null; // pinned → declared block; follow_finalized → max segment end or null +export function isComplete( + segments: CoverageSegment[], + projectStartBlock: number, + end: EventEnd, +): { complete: boolean; reason: string | null }; +``` + +```ts +// src/plan/digest.ts +export function canonicalJson(value: unknown): string; // sorted keys, no whitespace +export function sha256Hex(text: string): string; +``` + +```ts +// src/plan/generate.ts +export interface PlanSource { + source_id: string; + end_mode: "pinned" | "follow_finalized"; + job_start: number; + job_end: number; // resolved inclusive bound for THIS job + job_target_end: number; // declared end_block or resolved_safe_end + required_end: number | null; + blocks_remaining: number; // job_target_end - last_proven_complete_block, ≥ 0 +} + +export interface PlanDocument { + schema_version: 1; + plan_id: string; // sha256 of canonical plan without plan_id + intent: "ingest" | "refresh"; + project_id: string; + project_digest: string; // sha256 of raw chainplot.yaml bytes + created_at: string; // UTC ISO + chain: { chain_id: number; finality: Finality }; + sources: PlanSource[]; + actions: Array< + | { type: "ingest"; source_id: string } + | { type: "export"; source_id: string } + | { type: "build_results" } + >; + limits: { block_budget: number; blocks_in_range: number }; + deletes_data: boolean; + makes_data_public: boolean; + state_assumptions: { + sources: Record; + }; + publish_target: string | null; +} +``` + +Plan file lives at `.chainplot/plans/.json`. `plan` prints the path in `data.plan_path`; `apply --plan` accepts a path or a plan id. + +--- + +### Task 1: Freeze `plan`, `coverage`, `progress` schemas + +**Files:** +- Modify: `schemas/plan.schema.json` +- Modify: `schemas/coverage.schema.json` +- Modify: `schemas/progress.schema.json` +- Create: `tests/cli/schemaKinds.test.ts` + +**Interfaces:** +- Consumes: `schema show` from M1 +- Produces: draft 2020-12 schemas with `additionalProperties: false`; `coverage`/`plan` shapes above. + +`progress.schema.json`: required `schema_version` (const 1), `type` (const `"progress"`), `run_id`, `stage` (string); optional `message`, `rows`, `output_bytes`, `retries`, `ts` — all typed, no free-form fields. + +- [ ] **Step 1: Write the failing test** + +```ts +// tests/cli/schemaKinds.test.ts +import { describe, expect, it } from "vitest"; +import { runCliJson } from "../helpers/run.js"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const cwd = path.dirname(fileURLToPath(import.meta.url)); + +describe("frozen M2 schema kinds", () => { + for (const kind of ["plan", "coverage", "progress"]) { + it(`${kind} is closed (additionalProperties false)`, async () => { + const result = await runCliJson(["schema", "show", kind, "--json"], cwd); + expect(result.ok).toBe(true); + const schema = result.data as { additionalProperties: boolean }; + expect(schema.additionalProperties).toBe(false); + }); + } + + it("coverage requires segment boundary hashes", async () => { + const result = await runCliJson(["schema", "show", "coverage", "--json"], cwd); + const schema = result.data as { $defs: Record }; + expect(schema.$defs.segment.required).toEqual( + expect.arrayContaining([ + "start_block_hash", + "end_block_hash", + "start_block_parent_hash", + "status", + ]), + ); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `pnpm exec vitest run tests/cli/schemaKinds.test.ts` + +Expected: FAIL (stubs have `additionalProperties: true`). + +- [ ] **Step 3: Write the schemas** per Shared types. `plan.schema.json` freezes the `PlanDocument` shape (without `plan_id` in `required`, since the id is computed over the rest — keep `plan_id` as an optional string property so `apply` can validate a finished plan). + +- [ ] **Step 4: Run test to verify it passes** + +Run: `pnpm test` + +Expected: PASS (16 existing + new). + +- [ ] **Step 5: Commit** + +```bash +git add schemas tests/cli/schemaKinds.test.ts +git commit -m "feat: freeze plan, coverage, progress schemas" +``` + +--- + +### Task 2: RPC client — finalized head and headers + +**Files:** +- Create: `src/rpc/client.ts` +- Create: `src/rpc/heads.ts` +- Create: `tests/rpc/client.test.ts` + +**Interfaces:** + +```ts +// src/rpc/client.ts +export class RpcError extends Error { + constructor( + public readonly retryable: boolean, + message: string, + ) { + super(message); + } +} +export interface RpcClient { + call(method: string, params: unknown[]): Promise; +} +export function createRpcClient(url: string, fetchImpl?: typeof fetch): RpcClient; +``` + +Network/HTTP failure or JSON-RPC error object → `RpcError(retryable: true)` (maps to `transient_dependency` at call sites). Malformed response → `RpcError(false)` → `internal`. + +```ts +// src/rpc/heads.ts +export interface BlockHeader { + number: number; + hash: string; // lowercase hex + parentHash: string; // lowercase hex +} +export async function getFinalizedHead(client: RpcClient): Promise; +// eth_getBlockByNumber ["finalized", false]; null result → RpcError(false) "node cannot supply finalized block" — no fallback +export async function getHeader(client: RpcClient, blockNumber: number): Promise; +// eth_getBlockByNumber [hexQuantity, false]; null → RpcError(false) "block vanished" (source_inconsistent at call sites) +``` + +- [ ] **Step 1: Write the failing test** — spin `node:http` createServer on port 0 returning canned JSON-RPC responses; assert happy path, JSON-RPC error → retryable, connection refused → retryable, `finalized` → `null` → non-retryable, hex quantity parsing (`"0x5f5e0ff"` → 100000000... use small numbers). +- [ ] **Step 2: Run to verify FAIL** (`pnpm exec vitest run tests/rpc/client.test.ts`). +- [ ] **Step 3: Implement** with global `fetch`, 30 s per-request timeout (`AbortSignal.timeout`), no redirects followed (`redirect: "error"`). +- [ ] **Step 4: Run to verify PASS** (`pnpm test`). +- [ ] **Step 5: Commit** + +```bash +git add src/rpc tests/rpc/client.test.ts +git commit -m "feat: add JSON-RPC client for finalized head and headers" +``` + +--- + +### Task 3: Coverage segments — hash-join and proven-complete derivation + +**Files:** +- Create: `src/ingest/coverage.ts` +- Create: `tests/ingest/coverage.test.ts` + +Pure functions only; no I/O in this task. Table-driven cases (spec §9.2, §11): + +1. No segments → `lastProvenCompleteBlock = start_block - 1`. +2. Single segment `[start, end]` with `start_block = projectStart` → proven = end. +3. Segment starting after `projectStart` (gap at the front) → proven = `start_block - 1`. +4. Two number-adjacent segments with matching parent link → proven = second end. +5. Two number-adjacent segments, parent link mismatch → proven = first end only (the join is `source_inconsistent`; `isComplete` returns `complete: false, reason: "hash_join_broken"`). +6. Gap between segments → proven = first end. +7. Pinned, proven < declared end → `isComplete` false, reason `"truncated"` (a pinned range killed at 60% stays incomplete). +8. Pinned, proven ≥ declared end → complete. +9. `follow_finalized` with zero segments → `requiredEnd` null, `isComplete` false, reason `"not_indexed"`. +10. `follow_finalized` with contiguous segments → complete; `requiredEnd` = max segment end (lag vs `resolved_safe_end` is freshness, not incompleteness). + +- [ ] **Step 1: Write the failing test** (all 10 cases as `it.each` rows with explicit segment fixtures). +- [ ] **Step 2: Run to verify FAIL.** +- [ ] **Step 3: Implement** the four exported functions. `hashJoinOk(a, b)`: `b.start_block === a.end_block + 1 && b.start_block_parent_hash === a.end_block_hash`. +- [ ] **Step 4: Run to verify PASS.** +- [ ] **Step 5: Commit** + +```bash +git add src/ingest/coverage.ts tests/ingest/coverage.test.ts +git commit -m "feat: coverage segments with hash-join completeness" +``` + +--- + +### Task 4: rindexer `renderConfig` + +**Files:** +- Create: `src/ingest/rindexer/renderConfig.ts` +- Create: `tests/ingest/renderConfig.test.ts` + +**Interfaces:** `renderConfig(job: BoundedJob): string` — pure, returns YAML text. + +Golden assertions on the output for a two-address USDC-style job: + +- `project_type: no-code` +- `networks:` exactly one entry, `name: chainplot_1`, `chain_id: 1`, `rpc: ${RPC_URL}` +- `storage.postgres.enabled: true`; `graphql.enabled: false`; no `streams`, `chatbots`, `csv`, or docker-socket keys anywhere in the text +- one contract block per source: addresses quoted lowercase, explicit `start_block`/`end_block` = `jobStart`/`jobEnd`, `abi:` path, `include_events` exactly the declared signatures, `timestamp: true` +- deterministic: same input → byte-identical output (needed for plan digests) + +- [ ] **Step 1: Write the failing test.** +- [ ] **Step 2: Run to verify FAIL.** +- [ ] **Step 3: Implement** using the `yaml` package (`stringify` of a plain object; addresses as quoted strings). +- [ ] **Step 4: Run to verify PASS.** +- [ ] **Step 5: Commit** + +```bash +git add src/ingest/rindexer/renderConfig.ts tests/ingest/renderConfig.test.ts +git commit -m "feat: render bounded rindexer config" +``` + +--- + +### Task 5: `runBounded` + `stopAndQuiesce` with a fake rindexer binary + +**Files:** +- Create: `src/ingest/rindexer/runBounded.ts` +- Create: `tests/ingest/runBounded.test.ts` +- Create: `tests/helpers/fakeRindexer.mjs` + +**Behavior:** + +1. Write `renderConfig(job)` to `/rindexer.yaml` before spawn. +2. Spawn `${rindexerBin} start -p ${workDir} indexer` with env: `RPC_URL=job.rpcUrl`, `DATABASE_URL=job.databaseUrl`; strip other inherited secrets except `PATH`, `HOME`, `LANG`, `TMPDIR`. +3. `runBounded` resolves a `RunHandle` once the child is spawned. +4. Watch stdout for `/Historical indexing completed/` → set `completedLogSeen`, then the caller invokes `stopAndQuiesce`: SIGTERM, wait ≤ 10 s for exit, escalate SIGKILL. +5. Wall clock (`opts.wallClockMs`, default 30 min): on expiry SIGTERM the child and reject with `RpcError(true, "rpc job wall clock exceeded")` — job stays resumable, no coverage recorded. +6. Parent `SIGINT`/`SIGTERM` handlers (registered per apply-run, removed after) forward the signal to the child and exit non-zero; journal keeps recoverable state. +7. Child exit before the completed line → reject `RpcError(true, "rindexer exited before historic completion")`. + +**Fake binary** (`tests/helpers/fakeRindexer.mjs`): parses `start -p indexer`; modes selected by env `FAKE_RINDEXER_MODE`: `complete` (print the log line, then sleep forever), `hang` (sleep forever, never print), `exit-early` (print nothing, exit 3). Used to test SIGTERM-after-line, wall clock, and early exit. + +- [ ] **Step 1: Write the failing tests** (three modes; assert `completedLogSeen`, exit within timeout, rejection codes). +- [ ] **Step 2: Run to verify FAIL.** +- [ ] **Step 3: Implement.** +- [ ] **Step 4: Run to verify PASS.** +- [ ] **Step 5: Commit** + +```bash +git add src/ingest/rindexer/runBounded.ts tests/ingest/runBounded.test.ts tests/helpers/fakeRindexer.mjs +git commit -m "feat: bounded rindexer run with SIGTERM quiesce" +``` + +--- + +### Task 6: `inspectCoverage` — cursor evidence via `pg` + +**Files:** +- Add dependency: `pnpm add pg` and `pnpm add -D @types/pg` +- Create: `src/ingest/rindexer/inspectCoverage.ts` +- Create: `tests/ingest/inspectCoverage.test.ts` (unit: classification + SQL shape) +- Create: `tests/ingest/live/inspectCoverage.live.test.ts` (env-gated) + +**Evidence (M0-verified):** cursor table `rindexer_internal.{networkName}_{contractName}_{event}` (lowercased), columns `network text PK`, `last_synced_block numeric`. Event table `{networkName}_{contractName}_{event}` for row count. + +**Behavior:** + +1. `lastSyncedBlock`: `SELECT last_synced_block FROM rindexer_internal. WHERE network = $1` — read as a **string** (pg returns `numeric` as string), parse with `BigInt` → `Number` only after range check; missing row → `null`. +2. `rowCount`: `SELECT count(*)::bigint AS n FROM ` — identifiers are constructed only from validated lowercase `[a-z0-9_]` names (reject anything else at `BoundedJob` construction: `validation`). +3. Classification (pure function `classifyCoverage(lastSyncedBlock, rowCount, jobEnd)`): + - `lastSyncedBlock === null` → `not_indexed` + - `lastSyncedBlock < jobEnd` → `incomplete` + - `lastSyncedBlock >= jobEnd && rowCount === 0` → `complete_empty` (A6) + - else → `complete_with_rows` +4. Connection failure → `RpcError(true, ...)` → `transient_dependency`. +5. Close the pg pool; never leave connections open. + +- [ ] **Step 1: Write the failing unit test** for `classifyCoverage` (4 statuses) and identifier validation (rejects `Drop;--`). +- [ ] **Step 2: Run to verify FAIL.** +- [ ] **Step 3: Implement.** +- [ ] **Step 4: Run to verify PASS** (unit only; live test skips without env). +- [ ] **Step 5: Commit** + +```bash +git add package.json pnpm-lock.yaml src/ingest/rindexer/inspectCoverage.ts tests/ingest +git commit -m "feat: coverage inspection from rindexer cursor" +``` + +--- + +### Task 7: Snapshot exporter — DuckDB ATTACH → Parquet + +**Files:** +- Create: `src/ingest/exporter.ts` +- Create: `tests/ingest/exporter.test.ts` (SQL generation, pure) +- Create: `tests/ingest/live/exporter.live.test.ts` (env-gated: needs Postgres with an rindexer-shaped table) + +**Interfaces:** + +```ts +export interface ExportResult { + parquetPath: string; + rowCount: number; +} +export async function exportEventTable( + job: BoundedJob, + outDir: string, + duckdbPath: string, // disposable; .chainplot/duckdb/export.duckdb +): Promise; +``` + +**SQL (M0 path 1, one coherent read):** + +```sql +ATTACH '' AS pg_db (TYPE POSTGRES, READ_ONLY); +COPY ( + SELECT + AS chain_id, + contract_address, + "from" AS from_address, + "to" AS to_address, + value, + tx_hash, + CAST(block_number AS BIGINT) AS block_number, + block_timestamp, + block_hash, + network, + CAST(tx_index AS BIGINT) AS tx_index, + log_index + FROM pg_db.__ +) TO '/.parquet' (FORMAT PARQUET); +``` + +- Column list is derived from the ABI event inputs + the fixed envelope columns (M0-verified set); unknown ABI types fail `unsupported_capability` at `validate`/plan time, not here. +- Reserved/keyword input names (`from`, `to`) get explicit aliases; nested/tuple ABI types are refused (`unsupported_capability`) in M2 — only elementary types pass. +- Uniqueness gate in the same DuckDB session before COPY succeeds: + +```sql +SELECT count(*) AS total, + count(DISTINCT (contract_address, block_number, tx_hash, log_index)) AS distinct_keys +``` + +`total !== distinct_keys` → inspect whether conflicting rows differ in `block_hash`: yes → `source_inconsistent` (reorg duplicate); identical duplicates → deduplicate on the unique key (duplicate delivery must not produce duplicate snapshot rows, spec §9.2). + +- `databaseUrl` never appears in the Parquet or manifest; connection string is passed via ATTACH only. +- DuckDB postgres extension: `INSTALL postgres; LOAD postgres;` — first host run needs network; the Dockerfile pre-installs it. Export child gets `DATABASE_URL` env; it is **not** the M1 query worker and never touches snapshot query paths. + +- [ ] **Step 1: Write the failing test** for SQL text generation (casts present, `chain_id` literal, aliases, identifier validation) — pure function `buildExportSql(job, table)`. +- [ ] **Step 2: Run to verify FAIL.** +- [ ] **Step 3: Implement** with `@duckdb/node-api` in a child process (`fork`-style, same pattern as `src/query/workerMain.ts` but a separate entry `src/ingest/exportWorkerMain.ts` that receives the database URL via env — the query worker env-stripping rule applies to the *query* worker, not this one). +- [ ] **Step 4: Run to verify PASS** (unit; live gated). +- [ ] **Step 5: Commit** + +```bash +git add src/ingest/exporter.ts src/ingest/exportWorkerMain.ts tests/ingest +git commit -m "feat: export event tables to parquet via DuckDB postgres attach" +``` + +--- + +### Task 8: Plan generation — `ingest` and `refresh` intents + +**Files:** +- Create: `src/plan/digest.ts` +- Create: `src/plan/generate.ts` +- Create: `tests/ingest/generatePlan.test.ts` + +**Rules (spec §8, §9.2, §14.1):** + +- Load project; exactly one `chain_source` (`validation` otherwise). Resolve `rpc_secret` → env var name; missing env → `missing_credentials`. +- `pinned`: probe finalized head (finality `finalized`) or head − depth (`confirmation_depth`); `end_block` beyond that → `policy_refused`, `pointer: "/event_sources//end"`. +- `follow_finalized`: chain finality must be `finalized` (else `unsupported_capability`); `resolved_safe_end` = finalized head; `< start_block` → `policy_refused`. +- Per source: `job_start = lastProvenCompleteBlock + 1`; `job_end = min(job_target_end, job_start + budget - 1)`; `job_start > job_end` → **no `ingest` action** for that source (no-op ingest, coverage unchanged). Budget split: `blocks_remaining > 0` after capping → plan stays intent `ingest`, `blocks_remaining` reported. +- `refresh` on a project where every source is `pinned` and complete → actions are `export` + `build_results` only, no RPC ingest (finalized-head probe still allowed only if some source is `follow_finalized`; all-pinned refresh does **no** RPC, spec §11). +- `state_assumptions` snapshot the current `lastProvenCompleteBlock` per source from `.chainplot/coverage.json` (missing file → all sources at `start_block - 1`). +- `plan_id = sha256(canonicalJson(plan without plan_id))`. +- Estimates that cannot be computed are `unknown`; never invented. Plan shows resolved interval and blocks remaining. + +**Tests** (mock RPC server from Task 2 helpers; no live RPC): + +1. Pinned end ≤ finalized → plan OK, `job_end = end_block`, actions include `ingest`. +2. Pinned `end_block` > finalized → `policy_refused`. +3. `confirmation_depth` chain, `end_block > head - depth` → `policy_refused`. +4. `follow_finalized` + `confirmation_depth` → `unsupported_capability` at plan time. +5. `follow_finalized` + `finalized`: `job_target_end` = finalized head; head < start → `policy_refused`. +6. Budget 10, range 35, no coverage → `job_end = start + 9`, `blocks_remaining = 25`. +7. Existing coverage proven to block X → `job_start = proven + 1` (resume, no rescan). +8. All sources pinned + already complete → plan has no `ingest` action and made **zero** RPC calls (assert mock server hit count). +9. Plan id stable: same inputs → same `plan_id`; any field change → different id. + +- [ ] **Step 1: Write the failing test.** +- [ ] **Step 2: Run to verify FAIL.** +- [ ] **Step 3: Implement** `generate.ts` + `digest.ts`. +- [ ] **Step 4: Run to verify PASS.** +- [ ] **Step 5: Commit** + +```bash +git add src/plan tests/ingest/generatePlan.test.ts +git commit -m "feat: digest-bound ingest and refresh plans" +``` + +--- + +### Task 9: Journal, locks, `apply` + +**Files:** +- Create: `src/runtime/journal.ts` +- Create: `src/runtime/locks.ts` +- Create: `src/plan/apply.ts` +- Create: `tests/ingest/apply.test.ts` + +**Journal** (`.chainplot/runs//`): `plan.json` (copy), `status.json` (`running | succeeded | failed | canceled`), `checkpoints.jsonl`, `child.pid`. Idempotency key defaults to `plan_id`; `--idempotency-key` overrides. Second `apply` with same key + **different** plan digest → `policy_refused`. Same key + same digest with `succeeded` status → return the recorded outcome without re-executing (A5). Same key + `running` → `policy_refused` (lock held). + +**Locks:** `.chainplot/locks/ingest.lock` via `open(..., "wx")` with pid; released by `unlink` in `finally`. Advisory lock: `SELECT pg_try_advisory_lock(hashtext($1))` with `hashtext('')`; false → `policy_refused`. PG lock is skipped (with a warning) when no `DATABASE_URL` is configured — publish-only flows never take it. + +**`apply` sequence:** + +1. Load plan file; validate against frozen `plan` schema. +2. Recompute `project_digest` from current `chainplot.yaml` → mismatch → `policy_refused` ("configuration drifted"). +3. Recompute per-source `last_proven_complete_block` from `.chainplot/coverage.json` → mismatch with `state_assumptions` → `policy_refused` (state drifted; re-plan). +4. Acquire local lock + advisory lock. +5. Idempotency check against journal (above). +6. Execute `actions` in order: + - `ingest`: `renderConfig` → `runBounded` → on `Historical indexing completed` → `stopAndQuiesce` → `inspectCoverage`. Status `complete_empty|complete_with_rows` → fetch 3 headers (`jobStart-1`, `jobStart`, `jobEnd`), append segment via `hashJoinOk` against the tail (broken join → `source_inconsistent`, segment **not** appended, run fails, prior coverage intact). Status `incomplete|not_indexed` → run fails `transient_dependency`, no segment. + - `export`: `exportEventTable` per source with new coverage. + - `build_results`: reuse M1 build pipeline over the exported snapshot; **promotion gate**: `isComplete` per source must be true over `[start_block, required_end]`, else `policy_refused` naming source + missing range. +7. Write updated `.chainplot/coverage.json`; journal status `succeeded` with release path. +8. Release locks in `finally`. + +**Tests** (offline, fake adapter + fake binary where needed): + +- Digest drift: edit `chainplot.yaml` after `plan` → `apply` fails `policy_refused`, nothing executed. +- State drift: coverage file changed after `plan` → `policy_refused`. +- Same key + same digest, already succeeded → no second execution (counter in fake adapter). +- Same key + different digest → `policy_refused`. +- Concurrent apply → `policy_refused` from local lock. +- Hash-join break injected via fake header fetch → `source_inconsistent`, coverage file unchanged. +- Incomplete pinned (fake adapter reports cursor < end) → run fails, no segment, no release. + +- [ ] **Step 1: Write failing tests.** +- [ ] **Step 2: Run to verify FAIL.** +- [ ] **Step 3: Implement** `journal.ts`, `locks.ts`, `apply.ts`. +- [ ] **Step 4: Run to verify PASS.** +- [ ] **Step 5: Commit** + +```bash +git add src/runtime src/plan/apply.ts tests/ingest/apply.test.ts +git commit -m "feat: apply plans with journal, locks, coverage recording" +``` + +--- + +### Task 9: CLI `plan`, `apply`, `refresh` + +**Files:** +- Create: `src/cli/commands/plan.ts`, `src/cli/commands/apply.ts`, `src/cli/commands/refresh.ts` +- Modify: `src/cli/run.ts` (register; capabilities `commands` += `plan`, `apply`, `refresh`) +- Create: `tests/cli/plan.test.ts`, `tests/cli/refresh.test.ts` + +**Command surface:** + +- `plan --intent ingest|refresh|build|publish [--json|--jsonl]` — `build|publish` → `unsupported_capability` (M3/M4). Read-only probes only; writes only the plan file. +- `apply --plan [--idempotency-key ] [--json|--jsonl]` +- `refresh [--publish-target ] [--json|--jsonl]` — sugar: classify authorization (Task below), generate `plan --intent refresh`, apply. + +**`refresh` authorization (spec §9.2):** + +- Find last successfully applied plan in the journal. Compare its stored project copy against the current one: + - Only `models`/`queries`/`dashboards` content changed → allowed; actions become `export` + `build_results` (no ingest for pinned sources). + - `addresses`, `start_block`, `end`, public columns, `publish_targets`, or chain identity changed → `policy_refused` (out-of-band `plan` + `apply` required). +- No last applied plan → derive from the project file alone: first `follow_finalized` run from `start_block` allowed; nothing else widens. +- All sources `pinned` → refresh does zero RPC (A16). Any `follow_finalized` source → exactly one finalized-head probe. +- `--publish-target` must name a target already in `chainplot.yaml`, else `policy_refused`. + +**Tests:** + +- Pinned complete project + dashboard title edit → `refresh` succeeds, actions contain no `ingest`. +- Address edit → `refresh` fails `policy_refused`. +- `--publish-target nope` → `policy_refused`. +- All-pinned refresh makes no HTTP calls (mock server request counter = 0). +- `plan --intent build` → `unsupported_capability`. + +- [ ] **Step 1: Write failing tests.** +- [ ] **Step 2: Run to verify FAIL.** +- [ ] **Step 3: Implement** commands + `src/plan/refresh.ts`. +- [ ] **Step 4: Run to verify PASS.** +- [ ] **Step 5: Commit** + +```bash +git add src/cli src/plan tests/cli +git commit -m "feat: plan, apply, refresh commands" +``` + +--- + +### Task 10: Progress events and `runs list|show|cancel` + +**Files:** +- Create: `src/runtime/journal.ts` (if not fully landed in Task 8, finish here) +- Create: `src/cli/commands/runs.ts` +- Create: `tests/cli/runs.test.ts` + +**Behavior:** + +- `--jsonl` on `apply`/`refresh`: one `progress` event per stage transition (`plan_verified`, `ingest_started`, `ingest_completed`, `coverage_recorded`, `export_completed`, `release_written`), then the final result object. Events validate against `progress.schema.json`. No invented completion percentages. +- `runs list` → journal entries: key, plan id, intent, status, started/updated timestamps. +- `runs show ` → plan copy + status + checkpoints. +- `runs cancel ` → write `cancel_requested`; a running `apply` checks the flag between stages and after each ingest log line, forwards SIGTERM to the child, marks `canceled`, keeps coverage written so far (recoverable state). Cancel of a succeeded run → `validation`. + +- [ ] **Step 1: Write failing tests** (list/show roundtrip; cancel flag observed by a fake long stage). +- [ ] **Step 2: Run to verify FAIL.** +- [ ] **Step 3: Implement.** +- [ ] **Step 4: Run to verify PASS.** +- [ ] **Step 5: Commit** + +```bash +git add src/cli/commands/runs.ts src/runtime tests/cli/runs.test.ts +git commit -m "feat: run journal commands and cooperative cancel" +``` + +--- + +### Task 11: Ingest template, Compose, Dockerfile + +**Files:** +- Create: `templates/ingest-transfers/chainplot.yaml` +- Create: `templates/ingest-transfers/abis/ERC20.json` +- Create: `templates/ingest-transfers/.env.example` (`RPC_URL=`, `DATABASE_URL=` — comments only, no real endpoints) +- Create: `templates/ingest-transfers/compose.yaml` +- Create: `templates/ingest-transfers/Dockerfile` +- Create: `templates/ingest-transfers/.dockerignore` +- Modify: `src/cli/commands/templates.ts`, `src/cli/commands/init.ts` (register `ingest-transfers`; `init` copies `compose.yaml` + `Dockerfile` for ingest templates) +- Modify: `src/cli/commands/capabilities.ts` (`templates` list) +- Create: `tests/cli/initIngest.test.ts` + +**`chainplot.yaml` (template):** one chain source (`finality: finalized`, `rpc_secret: RPC_URL`), one event source (USDC address, `start_block: 18600000`, `end: {mode: pinned, block: 18600010}` — small, cheap, matches M0 evidence), `policy.block_budget: 100000`, one directory publish target. Queries/dashboards mirror the fixture template so `build` works after apply. + +**`compose.yaml`:** `postgres` (`postgres:16-alpine`, healthcheck, named volume) + `producer` (`build: .`, `platform: linux/amd64`, `env_file: .env`, project dir mounted at `/workspace`, `command: sleep infinity` — the agent execs CLI commands inside; no docker.sock; no ports exposed beyond loopback needs). + +**`Dockerfile`:** + +```dockerfile +FROM ghcr.io/joshstevens19/rindexer@sha256:9b33da8cea740b74ebfdfd3932682e8ceab79cbcf2eb3a7ca0863ac413794dd7 AS rindexer +FROM node:22-bookworm-slim +# ... install pnpm, copy package.json + lock, pnpm install --frozen-lockfile +# ... copy source, pnpm build +# COPY --from=rindexer /usr/local/bin/rindexer +# RUN node --input-type=module -e "install duckdb postgres extension into /app/.duckdb" && ENV DUCKDB_EXTENSION_DIR +``` + +- [ ] **Step 1: Locate the rindexer binary path inside the pinned image** — `docker create` + `docker export` (or `docker inspect`) the pinned digest; record the path and the fact in `docs/compatibility.md`. This is a verification step, not a guess. +- [ ] **Step 2: Write failing test** — `init --template ingest-transfers` produces `compose.yaml`, `Dockerfile`, `.env.example`, `chainplot.yaml`; `validate` passes on the scaffold offline (no RPC needed for `validate`). +- [ ] **Step 3: Implement template + init wiring.** +- [ ] **Step 4: Run to verify PASS** (`pnpm test`). +- [ ] **Step 5: Build the image and smoke it**: `docker build -t chainplot-producer templates/ingest-transfers && docker run --rm chainplot-producer capabilities --json` parses. Record Node/rindexer paths in `docs/compatibility.md`. +- [ ] **Step 6: Commit** + +```bash +git add templates src/cli tests/cli/initIngest.test.ts docs/compatibility.md +git commit -m "feat: ingest template with Compose and producer Dockerfile" +``` + +--- + +### Task 12: Live-gated end-to-end ingest + docs + +**Files:** +- Create: `tests/ingest/live/e2e.live.test.ts` +- Modify: `docs/compatibility.md` +- Modify: `README.md` (status line + commands table) + +**Gate:** `CHAINPLOT_TEST_DATABASE_URL` (our Postgres 16) **and** `RPC_URL` (archive-capable) both set; otherwise `describe.skip`. No URLs in the repo — the test reads env only. + +**Scenario (M0 replay through the product):** + +1. `init --template ingest-transfers` into a temp dir; `.env` from env vars. +2. `plan --intent ingest --json` → plan with `job_start: 18600000`, `job_end: 18600010`. +3. `apply --plan --json` → rindexer indexes 11 blocks; coverage segment `complete_with_rows` (USDC) recorded with 3 header hashes; snapshot parquet exists; release written. +4. Re-`apply` same plan + key → no duplicate rows (row count unchanged), status returned from journal (A5). +5. `plan --intent ingest` again → `job_start > job_end` → no-op plan, coverage unchanged. +6. Truncation probe: hand-edit coverage to drop the segment → `build` fails `policy_refused` (incomplete cannot be promoted — the M2 gate). +7. `refresh` on the pinned complete project → succeeds with zero RPC requests (A16 offline half). + +Record in `docs/compatibility.md`: rindexer binary path in image, image build date, live test evidence (row counts), any deviation from this plan with a one-line reason. + +- [ ] **Step 1: Write the live test (skipped by default).** +- [ ] **Step 2: Run with env set; fix product code until green.** +- [ ] **Step 3: Run `pnpm test` without env — all skip, suite green.** +- [ ] **Step 4: Update docs.** +- [ ] **Step 5: Commit** + +```bash +git add tests/ingest/live docs/compatibility.md README.md +git commit -m "test: live-gated ingest end-to-end and M2 docs" +``` + +--- + +## Verification checklist (M2 gate) + +- [ ] `pnpm test` green with no env (live tests skip). +- [ ] `pnpm test` green with `RPC_URL` + `CHAINPLOT_TEST_DATABASE_URL` set (live path). +- [ ] Incomplete pinned range cannot produce a release (`policy_refused`). +- [ ] Hash-join break → `source_inconsistent`, coverage not corrupted. +- [ ] Two concurrent applies → one `policy_refused`. +- [ ] No tracked file contains an RPC URL or provider hostname; `.env` gitignored. +- [ ] `capabilities` lists exactly the implemented commands. +- [ ] No docker.sock in any compose/Dockerfile/test. diff --git a/docs/plans/2026-09-13-m3-models-dashboards-static-build.md b/docs/plans/2026-09-13-m3-models-dashboards-static-build.md new file mode 100644 index 0000000..327e635 --- /dev/null +++ b/docs/plans/2026-09-13-m3-models-dashboards-static-build.md @@ -0,0 +1,159 @@ +# Chainplot M3 Models, Dashboards, Static Build Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Ship the full local build: SELECT model graph materialized in dependency order, `plan --intent build`, a static viewer (React + Vite + ECharts) that renders cached results with freshness/coverage/finality chips, provenance + sanitized `source/` in the release, and `serve` on loopback. Gate: the built dashboard works with producer, database, RPC, and CDN unavailable. + +**Architecture:** Models are SELECT-only SQL files materialized as DuckDB tables inside the same isolated worker that runs queries (topo order, cycles refused at `validate`). `build` extends the M1 results-only release to the §16.1 release layout minus `latest.json` (M4): `index.html`, `assets/` (prebuilt viewer bundle), `release.json`, `dashboards/`, `results/`, `datasets//manifest.json` + `tables/*.parquet`, `source/`. The viewer bundle is built once from `viewer/` (React + Vite + ECharts) and committed; `build` copies it — no node toolchain at user runtime, no CDN. + +**Tech Stack:** Unchanged for the CLI. Viewer adds `react`, `react-dom`, `echarts`, `vite`, `@vitejs/plugin-react` in `viewer/package.json` (own lockfile, `viewer/node_modules` gitignored). + +**Spec:** `docs/specs/2026-09-12-chainplot-design.md` §8 (change classes), §12 (wide integers, sort keys), §13 (models/queries), §14 (isolation/limits), §15 (viewer), §16.1 (release layout), §9.1 (`serve` loopback-only). + +## Global Constraints + +- All M1/M2 constraints hold: envelope, closed error codes, `--json`, offline `validate`, uint256 decimal strings, no `ORDER BY` on raw amount columns, DuckDB worker isolation (snapshot files + temp dir only, stripped env, external access disabled). +- Models are `SELECT` only. The worker rejects any model SQL whose first keyword is not `SELECT` (no DDL/DML/COPY/ATTACH/PRAGMA). Cycles fail `validate` with `validation`. +- `build` never talks to RPC. Promotion gate (M2) still applies to ingest projects. +- `plan --intent build` makes no network probes and authorizes only `build_results` (+ `export` never). `plan --intent publish` stays `unsupported_capability` (M4). +- Viewer: no third-party CDN, no user JavaScript formatters, no arbitrary HTML. Sort of a raw-amount column uses numeric compare of the decimal string (or a published sort key), never `localeCompare`. A control must not imply it can fetch missing history or rerun SQL. +- `serve` binds `127.0.0.1` only; refuses `0.0.0.0`; serves a directory read-only. +- Release layout (M3 subset of §16.1, no `latest.json` yet): + `index.html`, `assets/`, `release.json`, `dashboards/.json`, `results/.json`, `datasets//manifest.json`, `datasets//tables/*.parquet`, `source/`. +- `source/` is an explicit allowlist: `chainplot.yaml`, `abis/`, `models/`, `queries/`, `tests/`, `schemas/` (project-local). Never `.env`, `.chainplot/`, `dist/`, run logs, absolute machine paths. +- Do not add `.markdownlint.yaml`. Do not commit viewer `node_modules`. + +## Later plans (out of this file) + +M4: publish targets (directory + S3), `latest.json` conditional promotion, fork with SSRF rules, dataset modes per manifest, S3 conditional-write probe. +M5: examples, A1–A16 full pass, multi-arch smoke. + +--- + +## File structure + +| Path | Responsibility | +|---|---| +| `src/project/modelGraph.ts` | Dep resolution, cycle detection, topo order | +| `src/query/runQuery.ts` | Extend worker request: `models: {id, sql}[]` materialized before the query | +| `src/query/workerMain.ts` | Materialize models in order (SELECT-only), then run query | +| `src/publish/writeRelease.ts` | Full release: models, dashboards, static assets, provenance, `source/` | +| `src/publish/sourceBundle.ts` | Allowlist copy for `source/` | +| `src/cli/commands/serve.ts` | Loopback static server | +| `src/cli/commands/plan.ts` | Accept `--intent build` | +| `src/plan/generate.ts` | `build` intent: no probes, `build_results` action only | +| `viewer/` | React + Vite + ECharts viewer source; committed bundle in `viewer/dist/` | +| `schemas/release.schema.json` | Extend: dashboards, generated_at, snapshots, coverage | +| `schemas/manifest.schema.json` | Extend: coverage, finality label, freshness | + +--- + +## Shared types (lock these names) + +```ts +// src/project/modelGraph.ts +export interface ModelNode { + id: string; + file: string; + depends_on: string[]; +} + +// returns model ids in dependency order (deps first) +export function topoSortModels(models: ModelNode[]): string[]; // throws CommandError validation on cycle/unknown dep +``` + +```ts +// worker request extension (src/query/runQuery.ts QueryRequest) +export interface QueryRequest { + sql: string; + tables: Record; + analysisTimestamp: string; + rawAmountColumns: string[]; + rowLimit: number; + models?: { id: string; sql: string }[]; // materialized in array order before sql +} +``` + +```ts +// release.json (extended) +export interface ReleaseDocument { + schema_version: 1; + project_id: string; + mode: "results_only" | "dataset_included" | "dataset_referenced"; + queries: string[]; + dashboards: string[]; + generated_at: string; // build time (UTC) + snapshots: { dataset_id: string; snapshot_id: string }[]; + coverage: { source_id: string; start_block: number; end_block: number; status: string }[]; + finality: { policy: string } | { policy: "confirmation_depth"; depth: number } | null; +} +``` + +--- + +### Task 1: Model graph — deps, cycles, topo order + +**Files:** Create `src/project/modelGraph.ts`, `tests/project/modelGraph.test.ts`; modify `src/project/validate.ts` (validate model deps exist + acyclic; model `expected columns` field optional string array in project schema + types). + +Cases: linear chain; diamond; unknown dep → `validation`; self-dep → cycle; two-node cycle; order correct (deps before dependents); models depending on datasets are allowed (dataset ids are roots). + +- [ ] Failing test → implement → pass → commit `feat: model graph validation and topo order` + +### Task 2: Model materialization in the DuckDB worker + +**Files:** Modify `src/query/runQuery.ts` (accept `models`), `src/query/workerMain.ts` (materialize in order: `CREATE TABLE AS ()`; first-keyword-SELECT enforcement), `tests/query/models.test.ts`. + +Cases: model consumed by query (`WITH`-free reuse); model on top of model in order; non-SELECT model SQL → worker fails `validation`; model failure fails the query with the model id in the message; existing no-model requests unchanged; raw-amount `ORDER BY` guard still applies to query SQL. + +- [ ] Failing test → implement → pass → commit `feat: materialize SELECT models in the DuckDB worker` + +### Task 3: `plan --intent build` and apply build-only + +**Files:** Modify `src/plan/generate.ts` (intent `build`: require no `event_sources` ingest — sources may exist but no ingest/export actions; actions = `[build_results]`; zero RPC probes; state assumptions from coverage), `src/cli/commands/plan.ts` (accept `build`; `publish` stays unsupported), `tests/plan/buildIntent.test.ts`. + +Cases: build plan on fixture project → ok, no RPC calls, actions exactly `[build_results]`; apply executes build only; `plan --intent publish` → `unsupported_capability`; build plan on ingest project with incomplete coverage → apply refused by the promotion gate (M2 behavior, regression guard). + +- [ ] Failing test → implement → pass → commit `feat: plan --intent build` + +### Task 4: Viewer bundle (React + Vite + ECharts) + +**Files:** Create `viewer/package.json`, `viewer/vite.config.ts`, `viewer/tsconfig.json`, `viewer/src/main.tsx`, `viewer/src/App.tsx` (dashboards, panels, chips, table sort, KPI, line/bar via ECharts), `viewer/index.html`; commit built `viewer/dist/`; `.gitignore` += `viewer/node_modules`. + +Behavior: fetch `release.json` → for each dashboard fetch `dashboards/.json` → per panel fetch `results/.json`; chips: freshness (`generated_at`), coverage (from release `coverage`), finality label (confirmation-based labeled), dataset mode. Table sort on raw-amount columns uses numeric compare of decimal strings (BigInt compare, sign-aware); KPI shows exact decimal string. Works under a base path (relative fetches only). No CDN, no eval, no user HTML. + +- [ ] Scaffold + implement viewer → `pnpm build` in `viewer/` → commit `feat: viewer bundle (React + Vite + ECharts)` + +### Task 5: Full build — static release layout, provenance, source/ + +**Files:** Modify `src/publish/writeRelease.ts` (models re-export, dashboards data, copy viewer bundle → `index.html` + `assets/`, provenance in `release.json`, coverage/finality/freshness into manifest), create `src/publish/sourceBundle.ts` (allowlist copy), extend `schemas/release.schema.json` + `schemas/manifest.schema.json`, `tests/build/fullBuild.test.ts`. + +Cases: fixture project build produces the full layout (index.html, assets/, release.json with dashboards+coverage, dashboards/.json, results/, datasets//{manifest.json,tables/*.parquet}, source/ with allowlisted files only); `source/` never contains `.env` or `.chainplot`; dashboard title change → rebuild reflects it, no ingest (A2); release.json validates against the frozen schema; manifest carries coverage + finality label + freshness. + +- [ ] Failing test → implement → pass → commit `feat: full static build with provenance and source bundle` + +### Task 6: `serve` — loopback static preview + +**Files:** Create `src/cli/commands/serve.ts`, register in `run.ts` + capabilities, `tests/cli/serve.test.ts`. + +Behavior: `serve [--dir ] [--port ]`; binds `127.0.0.1` only (refuse/ignore other hosts — no `--host` flag at all); serves index.html + static files with content types; path traversal rejected (`..` segments resolve inside root); `--json` prints `{url}` and keeps serving until SIGINT (test uses an ephemeral port + real HTTP fetch). + +- [ ] Failing test → implement → pass → commit `feat: serve loopback static preview` + +### Task 7: Acceptance checks + docs + +**Files:** Modify `README.md`, `docs/compatibility.md`; `tests/cli/a2.e2e.test.ts`. + +A2: init fixture → build → edit dashboard title → build again → release reflects new title, no network, no ingest actions. A15 regression: fixture quickstart still green with network disabled. A9 groundwork: served release renders via plain HTTP fetch of index.html + release.json (no other services). + +- [ ] Tests → docs → commit `test: A2 presentation rebuild and served-release checks` + +--- + +## Verification checklist (M3 gate) + +- [ ] `pnpm test` green, no env, no network. +- [ ] Fixture build output renders via `serve` with only local HTTP (A9 groundwork). +- [ ] Cycle in models → `validate` fails before any build. +- [ ] `source/` allowlist: no `.env`, no `.chainplot`, no absolute paths. +- [ ] Viewer: no CDN references in committed bundle (`grep -r "https://" viewer/dist` → only license comments/none). +- [ ] `capabilities` lists `serve`; `plan --intent build` works; `--intent publish` still `unsupported_capability`. diff --git a/docs/plans/2026-09-13-m4-publish-fork.md b/docs/plans/2026-09-13-m4-publish-fork.md new file mode 100644 index 0000000..5a3489d --- /dev/null +++ b/docs/plans/2026-09-13-m4-publish-fork.md @@ -0,0 +1,180 @@ +# Chainplot M4 Publish and Fork Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Ship publish (directory + one S3-compatible adapter), `latest.json` conditional promotion, dataset modes with size caps, `doctor`, and `fork` with deny-by-default SSRF rules. Gate: a failed upload leaves the previous complete release usable; another agent forks using only published data. + +**Architecture:** `src/publish/` gains a `PublishTarget` interface with two implementations (`directory`, `s3`). Publication protocol per spec §16.2: staging release already exists from `build` → validate → upload immutable files → verify → promote `latest.json` **last** (single small pointer: release prefix + checksum of that release's `release.json`). Directory promotion is atomic rename; S3 promotion is a conditional PUT (`If-Match` on current pointer, `If-None-Match: *` for first publish). `fork` imports a release from a local dir or https URL into a new project with pinned snapshots; fetch rules are deny-by-default (§16.4). + +**Tech Stack:** Adds `@aws-sdk/client-s3` (v3) — pinned via `pnpm add`. Everything else unchanged. + +**Spec:** `docs/specs/2026-09-12-chainplot-design.md` §9.1 (`doctor`, `publish`, `fork`), §14.1 (caps), §16 (release/publish/fork), §16.3 (S3 adapter), §16.4 (fork fetch rules). + +## Global Constraints + +- All M1–M3 constraints hold. +- **No endpoint URLs or credentials in the repo.** S3 endpoint/keys/bucket come from env (`CHAINPLOT_S3_ENDPOINT`, `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `CHAINPLOT_S3_BUCKET`, optional `CHAINPLOT_S3_REGION`); tests read env and `it.skip` when absent. `.env` stays gitignored. +- Publication order is fixed: immutable files → verify → promote `latest.json` last. Interrupted upload leaves the previous complete release. +- `latest.json` is never stored inside `releases//`. Promotion is a single object write. Never copy a release into a `latest/` directory. +- S3 write/promote permission is proven only at upload time; `doctor` marks S3 `unverified` (HeadBucket at most). +- If an S3-compatible target cannot do conditional writes + read-after-write, refuse the target (`unsupported_capability`), never last-write-wins. +- Dataset size cap: copied public dataset 100 MiB compressed. Exceeding it → typed refusal with choices (reference mode, reduce data, results-only). Never silent truncation. +- `fork` fetch rules (§16.4, deny by default): local dir or https only; redirects forbidden (0 hops); resolve DNS and validate every address against the blocklist (loopback, link-local, ULA, RFC1918, CGNAT, unspecified, IPv4-mapped forms; reject decimal/octal/hex IP literals decoding to blocked addresses); pin the resolved address for the connection; block path traversal and undeclared files; validate `latest.json` → `release.json` → manifests against frozen schemas and §14.1 caps **before** downloading bodies; stream bodies with hard byte caps (`release.json` ≤ 1 MiB, total ≤ 512 MiB, per-request timeout 30 s); verify checksums as bytes arrive; escape hatch for private nets is an explicit documented flag, default off. +- `fork` never executes imported recipes, installs extensions, or runs hooks. Unreviewed imported SQL is never auto-run. +- Software license (MIT) is not dataset license: publishing requires an explicit `dataset_license` field; `source/` descriptions are in the allowlist review. +- v0.1 retention is manual; no `prune`. +- Do not add `.markdownlint.yaml`. + +## Later plans (out of this file) + +M5: three examples, A1–A16 full pass, multi-arch Compose smoke, capability matrix, upstream notices + dataset license statements. + +--- + +## File structure + +| Path | Responsibility | +|---|---| +| `schemas/publish-target.schema.json`? | No — publish targets live in `project.schema.json` (`publish_target` def gains fields) | +| `src/publish/target.ts` | `PublishTarget` interface + target resolution from project doc | +| `src/publish/directory.ts` | Directory target: copy immutable files, atomic `latest.json` | +| `src/publish/s3.ts` | S3-compatible target: upload, verify, conditional promote | +| `src/publish/latestPointer.ts` | `latest.json` shape + checksum computation | +| `src/publish/publishRelease.ts` | Protocol orchestration (validate → upload → verify → promote) | +| `src/publish/doctor.ts` | `doctor` command logic | +| `src/fork/fetchGuard.ts` | DNS/IP blocklist, pinning, no-redirect fetch with byte caps | +| `src/fork/importRelease.ts` | Validate pointers/manifests, stream + verify, write new project | +| `src/cli/commands/publish.ts` | `publish` command | +| `src/cli/commands/fork.ts` | `fork` command | +| `src/cli/commands/doctor.ts` | `doctor` command | +| `tests/publish/*.test.ts` | Directory target, pointer, protocol, modes/caps | +| `tests/publish/live/*.test.ts` | Env-gated S3 (R2) tests | +| `tests/fork/*.test.ts` | fetchGuard (mock DNS/HTTP), importRelease | +| `tests/cli/doctor.test.ts` | doctor offline paths | + +--- + +## Shared types (lock these names) + +```ts +// src/publish/target.ts +export interface PublishTargetDoc { + id: string; + type: "directory" | "s3"; + path?: string; // directory target + bucket?: string; // s3 target (bucket name; endpoint/keys via env) + dataset_license?: string; // required at publish time + public_base_url?: string; +} + +export interface LatestPointer { + schema_version: 1; + release_prefix: string; // e.g. "releases/local-1731..." + release_json_checksum: string; // sha256 hex of that release's release.json bytes +} + +export interface PublishResult { + target_id: string; + release_prefix: string; + latest_url: string | null; // public_base_url + latest.json when configured + files_uploaded: number; + promoted: boolean; +} + +export interface PublishTarget { + uploadFiles(releaseDir: string, prefix: string, files: string[]): Promise; + verifyFiles(prefix: string, files: string[], checksums: Record): Promise; + readLatest(): Promise; + promoteLatest(pointer: LatestPointer): Promise; // conditional write +} +``` + +```ts +// src/fork/fetchGuard.ts +export interface FetchGuardOptions { + allowPrivateNetworks?: boolean; // default false + maxBytes: number; + timeoutMs: number; // per request, default 30_000 +} +export function assertAllowedUrl(url: string, opts: FetchGuardOptions): Promise; +export function guardedFetch(url: string, opts: FetchGuardOptions): Promise; // no redirects, byte-capped stream +``` + +--- + +### Task 1: `latest.json` pointer + directory target + +**Files:** Create `src/publish/latestPointer.ts`, `src/publish/directory.ts`, `src/publish/target.ts`; extend `project.schema.json` `publish_target` def (`dataset_license`, `public_base_url`); `tests/publish/directory.test.ts`. + +Cases: pointer checksum = sha256 of release.json bytes; directory publish copies files under `releases//`, writes `latest.json` at root via temp+rename (assert no partial state on simulated crash: temp name left behind is ignored); second publish of a new release flips the pointer atomically; previous release directory untouched; `latest.json` never inside `releases//`. + +- [ ] Failing test → implement → pass → commit `feat: latest pointer and directory publish target` + +### Task 2: `plan --intent publish`, `publish` command, refresh wiring + +**Files:** Modify `src/plan/generate.ts` (publish intent: actions `[{type:"publish"}]`, requires an existing built release + target in project; no RPC), `src/plan/apply.ts` (execute publish action via target), `src/cli/commands/publish.ts` (sugar: plan + apply), `src/cli/commands/refresh.ts` (`--publish-target` now publishes), `src/cli/run.ts`, capabilities; `tests/publish/publishCommand.test.ts`. + +Cases: publish without prior build → `validation` ("run build first"); publish with `dataset_license` missing → `policy_refused` naming the field; `--publish-target` not in project → `policy_refused`; publish plan makes no RPC; refresh with valid target publishes after rebuild (A16 complete path); publish plan is digest-bound (config drift refused). + +- [ ] Failing test → implement → pass → commit `feat: publish command and publish plans` + +### Task 3: Dataset modes + size caps + +**Files:** Modify `src/publish/writeRelease.ts` (mode selection: `dataset_included` default; `results_only` when total parquet size > 100 MiB unless project opts into reference mode; `dataset_referenced` writes a pointer manifest instead of tables), `tests/publish/modes.test.ts`. + +Cases: small fixture → `dataset_included` with tables; oversized (inject a >100 MiB parquet via sparse file) → build refuses with typed refusal listing choices; `results_only` mode omits `datasets//tables/` and manifest says so; `dataset_referenced` manifest carries external snapshot pointer + checksum, no tables. + +- [ ] Failing test → implement → pass → commit `feat: dataset modes and size caps` + +### Task 4: `doctor` + +**Files:** Create `src/publish/doctor.ts`, `src/cli/commands/doctor.ts`, register; `tests/cli/doctor.test.ts`. + +Checks (each reported as `{name, status: ok|fail|unverified|skipped, detail}`): project file parses; secrets present (RPC_URL env ref, DATABASE_URL) without values echoed; chain id + `eth_chainId` + finalized head probe (skipped offline); rindexer binary on PATH or `CHAINPLOT_RINDEXER_BIN` (version probe, skipped when absent); writable storage (touch `.chainplot/doctor.tmp`); S3: HeadBucket only, write/promote marked `unverified`. `doctor` must not start indexing. Offline run: RPC + rindexer + S3 skipped, exit ok. + +- [ ] Failing test → implement → pass → commit `feat: doctor command` + +### Task 5: S3 conditional-write probe (R2) — env-gated + +**Files:** Create `tests/publish/live/s3probe.live.test.ts`; record evidence in `docs/compatibility.md`. + +Needs env: `CHAINPLOT_S3_ENDPOINT` (the R2 URL), `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `CHAINPLOT_S3_BUCKET=chainplot-test`, optional `CHAINPLOT_S3_REGION=auto`. Skips when absent. + +Probe: PutObject `probe/initial` → GetObject read-after-write → conditional PutObject with `If-None-Match: *` (must succeed on new key, must fail 412 on existing key) → conditional PutObject with `If-Match` of the current ETag (must succeed; stale ETag must fail 412) → DeleteObject cleanup. Record which conditions R2 honors; if any required condition is unsupported, record `unsupported_capability` verdict for S3 targets in `docs/compatibility.md` (directory publish remains the v0.1 path). + +- [ ] Write probe → run with creds from the human → record evidence → commit `test: S3 conditional-write probe against R2` + +### Task 6: S3 publisher + +**Files:** `pnpm add @aws-sdk/client-s3`; create `src/publish/s3.ts`; `tests/publish/live/s3publish.live.test.ts` (env-gated, uses the probe bucket). + +Behavior: `uploadFiles` PutObjects under `/`; `verifyFiles` HeadObject + checksum compare (sha256 of body for small files, ETag+size for large); `readLatest` GetObject `latest.json` (404 → null); `promoteLatest` conditional PUT (`If-None-Match: *` when no pointer exists, `If-Match: ` otherwise; 412 → `policy_refused` "concurrent promotion detected"). Offline unit tests with a mocked S3Client interface; live test does a real publish + re-publish cycle against R2 and asserts the pointer flipped and 412 on stale ETag. + +- [ ] Unit tests (mocked client) → implement → live test with creds → commit `feat: S3-compatible publish target` + +### Task 7: `fork` — fetch guard + import + +**Files:** Create `src/fork/fetchGuard.ts`, `src/fork/importRelease.ts`, `src/cli/commands/fork.ts`, register; `tests/fork/fetchGuard.test.ts`, `tests/fork/importRelease.test.ts`. + +fetchGuard cases: http URL refused; redirect refused (mock server 302 → error); loopback/`127.0.0.0/8`/`::1`/RFC1918/CGNAT/link-local/ULA/unspecified refused; decimal IP literal (`2130706433`) resolving to 127.0.0.1 refused; hex/octal literals refused; IPv4-mapped IPv6 refused; `allowPrivateNetworks: true` escape hatch permits them; byte cap aborts mid-stream; timeout aborts. + +importRelease cases: local directory import of a built fixture release → new project with pinned snapshot (`datasets//tables/*.parquet` copied), copied `chainplot.yaml` (ids preserved), no `.env`, no recipe execution (assert no hooks run — none exist); `release.json` > 1 MiB refused before body; undeclared file in release dir refused; path-traversal filename in manifest refused; checksum mismatch refused; forked project passes `validate` offline and `build` works over the pinned snapshot (A10 core). + +- [ ] Failing tests → implement → pass → commit `feat: fork with deny-by-default fetch guard` + +### Task 8: Acceptance + docs + +**Files:** `tests/cli/a10.e2e.test.ts` (second agent: fork a published directory release, write a new query over the pinned snapshot, build a new dashboard — no RPC/Postgres/credentials); `README.md`, `docs/compatibility.md`. + +- [ ] A10 test → docs → commit `test: A10 fork acceptance and M4 docs` + +--- + +## Verification checklist (M4 gate) + +- [ ] `pnpm test` green offline (S3/fork live tests skip). +- [ ] Failed S3 upload leaves previous release + pointer intact (live test). +- [ ] 412 on stale ETag promotion → `policy_refused` (live test). +- [ ] `fork` of the R2-published release works with only the public URL (needs public access or a second signed read — record which; if the bucket is private, fork test uses the local-directory path and URL fork is documented as needing public read). +- [ ] No endpoint URLs or keys in tracked files (no tracked file mentions the S3 endpoint hostname or keys). +- [ ] `capabilities` lists `publish`, `fork`, `doctor`. diff --git a/docs/plans/2026-09-14-m5-examples-acceptance.md b/docs/plans/2026-09-14-m5-examples-acceptance.md new file mode 100644 index 0000000..ac95aaf --- /dev/null +++ b/docs/plans/2026-09-14-m5-examples-acceptance.md @@ -0,0 +1,87 @@ +# Chainplot M5 Examples and Acceptance Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Ship the three v0.1 examples, run A1–A16, smoke the Compose Dockerfile on Linux x86-64 and ARM64, and finish docs (capability matrix, upstream notices, dataset license statements). Gate: fix interface and correctness problems before new source types or a human editor. + +**Architecture:** Examples live in `examples/` as ordinary Chainplot projects (committed `chainplot.yaml` + queries; snapshots are produced by running the pipeline, not committed — each example README documents the run command). Acceptance A1–A16 map to existing automated tests plus a small number of new e2e tests; the matrix in `docs/acceptance.md` records evidence per ID. + +**Spec:** `docs/specs/2026-09-12-chainplot-design.md` §18 (definition of done), §19 (M5), §14.1, §17 (Compose operation). + +## Global Constraints + +- All M1–M4 constraints hold. No RPC URLs, endpoints, or keys in tracked files. +- Live examples run against the operator's `.env` (RPC_URL, DATABASE_URL) and a local Postgres; they are documented runbooks, not CI tests. +- rindexer binary is linux/amd64-only: multi-arch smoke = Dockerfile builds on both architectures; rindexer execution is amd64-only (recorded limitation, follow-on: upstream arm64 image). + +--- + +### Task 1: M2 live RPC e2e through the product + +**Files:** `tests/ingest/live/e2e.live.test.ts` (already written, env-gated on `CHAINPLOT_TEST_DATABASE_URL` + `RPC_URL`). + +Run with a disposable Docker Postgres (`docker run -d --name chainplot-test-pg -p 127.0.0.1:5433:5432 postgres:16-alpine`), `CHAINPLOT_TEST_DATABASE_URL=postgresql://chainplot:chainplot@localhost:5433/chainplot`, `RPC_URL` from `.env`. Scenario: plan → apply (92 USDC rows) → idempotent re-apply → no-op plan → truncation gate. Record evidence in `docs/compatibility.md`. + +- [ ] Run green → commit `test: M2 live RPC e2e green` + +### Task 2: Example 1 — transfer activity (full ingest pipeline) + +**Files:** Create `examples/transfer-activity/` (chainplot.yaml, abis/, queries/, dashboards/, tests/, README.md) + `examples/README.md` index. Run end-to-end: init-style scaffold → plan → apply (real rindexer, USDC 18600000–18600010) → build → serve. Evidence in `docs/acceptance.md` (A1). + +- [ ] Example runs green → commit `feat: transfer activity example` + +### Task 3: Example 2 — protocol deposits/withdrawals (WETH) + +**Files:** Create `examples/weth-activity/` — WETH9 (`0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2`), events `Deposit(address,uint256)` + `Withdrawal(address,uint256)`, small pinned range, two queries (deposits count/sum, withdrawals count/sum), dashboard with KPI + table. Same live run path as Task 2. + +- [ ] Example runs green → commit `feat: deposit/withdrawal example (WETH)` + +### Task 4: Example 3 — fork of a published dataset + +**Files:** `examples/fork/README.md` documenting the flow; the A10 e2e already covers the mechanics. Publish example 1 to the R2 target (or local dir), fork it as a second agent, add a new query/dashboard, build offline. + +- [ ] Documented + run → commit `docs: fork example` + +### Task 5: A1–A16 acceptance matrix + +**Files:** Create `docs/acceptance.md` mapping every A-id to its evidence (test file + status). New tests where gaps remain: + +- A1 (agent creates dashboard from ABI + bounded fixture): covered by fixture quickstart + examples. +- A3 (earlier history / another contract = explicit additional work): test that widening `start_block` requires a new ingest plan and `refresh` refuses — extend `tests/cli/ingestCommands.test.ts`. +- A4 (kill producer mid-indexing): covered by runBounded wall-clock/early-exit tests + journal resume; add an integration note. +- A12 (malicious filenames/SQL): fork path traversal test exists; add a model-SQL rejection test (non-SELECT model refused — exists in models.test.ts). +- A13 (limits): size-cap test exists; add block-budget split test (exists in planApply). +- A14 (missing secrets): missing_credentials tests exist. +- A6/A7/A8/A5/A16/A15/A9/A2/A10/A11: existing tests (record pointers). + +- [ ] Matrix complete, gaps closed → commit `test: A1–A16 acceptance matrix` + +### Task 6: Capability matrix + license docs + +**Files:** `docs/capabilities.md` (commands × status, sources, targets, chart types, limits table from §14.1); `NOTICES.md` (upstream: rindexer MIT, DuckDB MIT, ECharts Apache-2.0, React MIT, AWS SDK Apache-2.0); README dataset-license note (already in publish targets). + +- [ ] Docs → commit `docs: capability matrix and upstream notices` + +### Task 7: Multi-arch Compose smoke + +**Files:** `docs/compatibility.md` evidence. + +`docker build --platform linux/amd64` (already proven in M2 live run) + `docker build --platform linux/arm64` (build-only smoke; rindexer binary cannot execute on arm64 — documented limitation, no arm64 upstream image). Record both. + +- [ ] Both builds green → commit `test: multi-arch Dockerfile smoke` + +### Task 8: Final verification + +`pnpm test` green; live S3 + live RPC suites green; `docs/acceptance.md` complete; README final status. + +- [ ] Commit `docs: M5 complete` + +--- + +## Verification checklist (M5 gate = v0.1 done) + +- [ ] A1–A16 pass on fixtures (A11 via Compose), live smoke opt-in and bounded. +- [ ] Three examples run from their READMEs. +- [ ] Capability matrix matches `capabilities` output. +- [ ] Multi-arch evidence recorded. +- [ ] No secrets or endpoint URLs in tracked files. diff --git a/docs/security.md b/docs/security.md new file mode 100644 index 0000000..409a15c --- /dev/null +++ b/docs/security.md @@ -0,0 +1,92 @@ +# Security model + +What Chainplot defends, what it does not, and where the boundary sits. + +## The one thing to understand + +**A `fork` imports someone else's SQL, and the next `build` runs it.** + +`fork` copies `source/chainplot.yaml`, `source/queries/` and `source/models/` +out of a published release into a new project. Those files are the recipe; +running `build` executes them against DuckDB on your machine. Forking an +untrusted release is therefore closer to running a downloaded script than to +downloading a CSV, and the containment below is what makes it safe rather +than the file format. + +## Containment + +Query execution happens in a forked child process (`src/query/workerMain.ts`), +never in the CLI process: + +| Control | Where | +|---|---| +| Separate process, env stripped to `PATH`/`HOME`/`LANG` | `src/query/runQuery.ts` | +| In-memory DuckDB; no database file on disk | `workerMain.ts` | +| Extension autoinstall and autoload disabled | `workerMain.ts` | +| `enable_external_access=false` **before any project SQL runs** | `workerMain.ts` | +| Single-SELECT admission control, via DuckDB's parser | `src/query/sqlGuard.ts` | +| 60 s deadline, SIGKILL on expiry | `runQuery.ts` | +| Row limit enforced by stopping the reader, not by truncating after | `workerMain.ts` | + +Ordering matters and is the part that was wrong before 2026-09-15. Snapshots +are read first, because `read_parquet` needs filesystem access. External +access is then disabled, and only after that are models materialized and the +query run. Models are project-supplied SQL like any other, so they must land +on the closed side of that door; DuckDB does not allow external access to be +re-enabled within a session. + +### Admission control + +Every model and query is parsed before it executes: + +```sql +SELECT json_serialize_sql('') +``` + +That call fails on anything that is not a SELECT — `Only SELECT statements can +be serialized to json!` — which covers INSERT, UPDATE, COPY, ATTACH, PRAGMA, +SET, INSTALL and LOAD without maintaining a keyword denylist. It also reports +the statement count, so `SELECT 1; DROP TABLE t` is refused as two statements +rather than passing a check aimed at the first. The statement is parsed, not +run, by this call. + +## What is *not* defended + +- **Authenticity of a published release.** `release.json` lists a SHA-256 for + every file, and `fork` verifies each one. That detects corruption in + transit; it does not establish provenance, because the checksums and the + files come from the same bucket. There are no signatures. Whoever can write + to the bucket can serve a consistent, hostile release. **Fork only from + buckets you would trust with the data itself.** +- **Denial of service by a hostile recipe.** Bounded, not eliminated: a forked + query gets 60 s, a row limit, and a 1 GiB memory cap that spills to a temp + directory rather than failing. A release can still make your build slow, and + can still fill that temp directory. +- **Secrets you place inside the recipe directories.** The source bundle is an + allowlist — `chainplot.yaml`, `abis/`, `models/`, `queries/`, `tests/`, + `schemas/` — and nothing else is copied into a release. Everything inside + those directories *is* published. `.env` is never among them. +- **The snapshot's own contents.** Chainplot publishes what you indexed. + Deciding whether onchain data is publishable is yours. + +## Fetching (`fork --from https://…`) + +`src/fork/fetchGuard.ts` is deny-by-default: + +- HTTPS only. +- DNS is resolved once and the connection is pinned to that address, so a + rebind between check and connect cannot redirect it. +- Loopback, link-local, ULA, and RFC1918 ranges are refused, in IPv4, IPv6, + and IPv4-mapped IPv6 form. Decimal, hex, and octal IP literals are + normalized before the check. +- Redirects are refused outright rather than followed. +- 1 MiB cap on `release.json`, 512 MiB on the release, 30 s per request. A + `dataset_referenced` release adds one fetch per dataset, counted against the + same 512 MiB total and verified against the checksum the manifest records. +- `--allow-private-networks` is the documented, explicit escape hatch for + testing against a local server. + +## Reporting + +Chainplot is an internal tool. Raise anything you find in the repo's issue +tracker, or directly with the maintainers if it is exploitable. diff --git a/docs/specs/2026-09-12-chainplot-design.md b/docs/specs/2026-09-12-chainplot-design.md new file mode 100644 index 0000000..18375ff --- /dev/null +++ b/docs/specs/2026-09-12-chainplot-design.md @@ -0,0 +1,1042 @@ +# Chainplot v0.1 design + +**Date:** 2026-09-12 +**Status:** Implementation baseline. No code, versions, or benchmark results are implied. +**Working name:** Chainplot. CLI: `chainplot`. Name, package, and domain clearance are a launch check, not a v0.1 gate. + +Chainplot is an open-source, agent-first toolkit: scoped EVM events become a reproducible Parquet snapshot, SQL models, and a static dashboard. An agent authors files, validates, indexes a bounded range, publishes, and refreshes. Humans read the dashboard and own credentials. No graphical editor, model runtime, or hosted control plane. + +> Define the data. Build the insight. Publish anywhere. + +--- + +## 1. Goal + +An agent with filesystem and shell access, given this project's documentation, a contract ABI, an explicitly bounded source scope, and authorized credentials, creates and publishes a correct dashboard without browser automation or undocumented manual steps. A second agent forks the published dataset and builds a different dashboard without RPC, Postgres, or the original credentials. + +Refresh of a published dashboard is a command the agent, a human, or external cron runs. Chainplot does not sleep in a loop in v0.1. + +### 1.1 Non-goals (v0.1) + +No Kubernetes manifests, systemd units, `watch` daemon, MCP server, human editor, embedded chatbot, second indexer implementation, plugin loader, public SQL service, multi-tenant hosting, cloud provisioning, traces, chain-wide discovery, factory-address expansion, non-EVM sources, DuckDB-Wasm in the browser, IPFS, or a global dashboard directory. + +Unsupported requests return a typed capability error. Do not substitute a plausible metric. + +--- + +## 2. Users + +| Role | Does | +|---|---| +| Agent (primary operator) | Author files, run CLI, apply plans, publish, fork, diagnose | +| Human reviewer | Read dashboards, review definitions, own infra and secrets | +| Public reader | Open a static URL. No account, wallet, RPC, or producer | + +--- + +## 3. Invariants + +1. **Agent-first.** Files and CLI are the authoring interface. A web editor is not required for usefulness. +2. **Index-once on the read path.** Viewing a dashboard or querying a published snapshot must not generate historical RPC. Refresh is a separate write path. +3. **No mandatory hosted component.** Losing the project website must not stop an existing deployment or published dashboard. +4. **Snapshot is the public data contract.** Models, queries, the viewer, and forks never name indexer tables. +5. **Open artifacts.** SQL, JSON/YAML, Parquet, ABIs, and ordinary static web assets are the portability boundary. +6. **Correctness before convenience.** Missing data, incomplete coverage, stale results, approximation, and unsupported capabilities are explicit. +7. **Small footprint.** Reuse rindexer and DuckDB. Do not build a general orchestrator, indexer, or warehouse. +8. **No required model runtime.** The calling agent supplies intelligence. Chainplot supplies deterministic tools, contracts, and evidence. + +--- + +## 4. Scope + +### 4.1 In v0.1 + +- Declarative project, JSON Schema, CLI with `--json` / `--jsonl` +- One ingest adapter implementation: rindexer (no-code, Postgres, bounded jobs) +- Thin `IngestAdapter` interface so a later similar indexer is a new module, not a core rewrite +- Postgres only while ingesting; not required for dataset-only / fork projects +- Consistent Parquet snapshot; DuckDB only on snapshots +- Static dashboard (line, bar, KPI, table) +- Publish to a local directory and to one S3-compatible adapter +- `refresh` command that resumes indexing, rebuilds, and optionally publishes +- Docker Compose as the tested ingest runtime. The ingest template ships + `compose.yaml` plus a Dockerfile that packages the CLI and the pinned + rindexer binary. M2 must pass on one architecture; M5 must smoke Linux + x86-64 and ARM64. macOS via Compose is acceptable. Native Windows is not + claimed. A published multi-arch registry image is not a v0.1 product. + +### 4.2 Deliberately out + +See §1.1. Also out: automatic proxy-upgrade interpretation, curated identity/price feeds, arbitrary historical state reconstruction, mixing latest files from different releases. + +--- + +## 5. Architecture + +Two run modes, one CLI: + +| Mode | Needs | Does not need | +|---|---|---| +| Ingest project | RPC, Postgres, pinned rindexer binary | — | +| Dataset-only / fork | Snapshot files + CLI | RPC, Postgres, rindexer | + +```mermaid +flowchart TD + agent["Agent / cron / human"] + cli["chainplot CLI"] + adapter["IngestAdapter rindexer subprocess"] + rpc["JSON-RPC"] + pg["Postgres adapter-private"] + snap["Parquet snapshot + schema"] + duck["DuckDB worker"] + dist["Static release"] + s3["S3-compatible optional"] + + agent --> cli + cli --> adapter + adapter --> rpc + adapter --> pg + cli --> snap + pg --> snap + cli --> duck + snap --> duck + duck --> dist + cli --> dist + dist --> s3 +``` + +Compose for local ingest: `producer` (CLI + pinned rindexer) + `postgres`. Dataset-only skips Compose. + +Logical ownership: + +| Component | Owns | Must not own | +|---|---|---| +| Project core | Validation, plans, identity, schemas, policy | Model-provider calls, UI-only config | +| Ingest adapter | Generated upstream config, bounded run, coverage inspection | Dashboard presentation, a second indexer | +| Postgres | Indexed events, upstream checkpoints, advisory lock | Public query endpoint | +| Snapshot / query | Typed export, SQL models, bounded analytics | A second live database of record | +| Builder / publisher | Static artifacts, manifests, `latest` promotion | Ingestion or cloud provisioning | +| Viewer | Cached results, provenance, downloads | RPC, remote SQL, credentials | + +--- + +## 6. Repository layout + +One package until a split is forced. Directories are modules, not services. + +```text +chainplot/ + docs/specs/ # this design; later docs/plans/ + schemas/ # JSON Schema for YAML/JSON documents + src/cli/ + src/project/ + src/ingest/ # IngestAdapter + rindexer impl + src/snapshot/ + src/query/ + src/publish/ + src/runtime/ # subprocess, locks, run journal + viewer/ # React + Vite + ECharts + templates/ # init scaffolds, including Compose + tests/ + LICENSE # MIT +``` + +Create `src/` only when implementation starts. This spec does not require those files to exist today. + +Tooling defaults (pinned in Milestone 0, not floating `latest`): + +- TypeScript on a supported Node.js LTS +- pnpm +- `@duckdb/node-api` (DuckDB Node neo). Do not use the deprecated `duckdb` package +- AWS SDK v3 for the S3 adapter +- Viewer: React, Vite, ECharts +- rindexer: pinned release binary, `project_type: no-code` + +--- + +## 7. Project files + +```text +my-analytics/ + chainplot.yaml + chainplot.lock.json + abis/ + schemas/ + models/ + queries/ + dashboards/ + tests/ + .env.example + compose.yaml # from template; ingest projects only + .chainplot/ # gitignored work, plans, runs + dist/ # gitignored built releases +``` + +An agent edits files directly. There are no field-level mutation commands. + +`chainplot.yaml` is the source of truth. JSON Schema is authoritative for every YAML/JSON document. Kinds: +`project`, `plan`, `result`, `progress`, `release`, `manifest`, +`latest`, `coverage`, `lock`. + +`chainplot schema show ` accepts exactly that set. `capabilities` +advertises it. The same schemas generate types, examples, and docs. +Unknown fields fail validation. All `schema_version` / `format_version` +fields are integers, not strings. + +`progress` is the JSONL progress-event envelope, not an event-source +resource. `release` is `release.json`. `manifest` is +`datasets//manifest.json`. `latest` is the `latest.json` pointer. + +Two lock files, different jobs: + +- `pnpm-lock.yaml` — Chainplot software dependencies (Node packages). +- `chainplot.lock.json` — the analytics project: tool versions, ABI digests, + imported snapshot ids, transformation digests. Rebuilding from this lock + must not silently fetch a publisher's latest data. Updating it is an + explicit action. + +`chainplot.yaml` `format_version` is an integer. Unknown or newer versions +fail `validate` with `unsupported_capability`. v0.1 does not ship a +migrator. + +Secrets enter through environment or file references named in `.env.example`. Never command-line literals, never committed values, never public project files. + +--- + +## 8. Resources + +Every resource has a **stable ID** distinct from its title. Renaming a dashboard does not recreate an event table. + +| Resource | Required | +|---|---| +| Project | Format version, stable id, policy and publication defaults | +| Chain source | Numeric chain id, RPC secret reference, finality policy | +| Event source | Addresses, ABI path and digest, event signatures, inclusive start block, end policy (see below) | +| Dataset schema | Id/version, columns, physical and logical types, nullability, grain, key, units, coverage, provenance | +| SQL model | Id, SQL file, explicit dependencies, expected output schema, assertions | +| Query | Id, SQL file, dataset/model refs, expected columns/types, resource limits, metric definition | +| Dashboard | Id, title, description, ordered panels, query refs, allowlisted chart encodings | +| Publish target | `directory` or `s3`, credential references, public-data selection, optional public base URL | + +v0.1 event sources: **explicit address lists** on **one chain**. No factory +discovery, no chain-wide event filters, no multi-network project. + +End policy on each event source is one of: + +- `pinned` — `end_block` is a number. `refresh` does not ingest further + (it may still rebuild queries/presentation). +- `follow_finalized` — each refresh resolves `resolved_safe_end` (see + below), capped by the run's block budget, and sets that as `end_block` + for the rindexer job. + +`resolved_safe_end` is the finalized block. `follow_finalized` therefore +requires chain finality policy `finalized` (§9.2). `confirmation_depth` +applies only to `pinned` sources, whose declared `end_block` must +already satisfy ≤ head − confirmation_depth at plan time. + +`start_block` and `end_block` are inclusive integers. A run covers +`end - start + 1` blocks. Both ends are always written into the generated +rindexer config. There is no "live index from now" mode. + +At plan time a `pinned` `end_block` must already satisfy the chain +source's finality policy (≤ finalized, or ≤ head − confirmation_depth). +Otherwise `plan` fails with `policy_refused`. Pinning into the unfinalized +zone is not a labeled special case in v0.1; it is refused. + +Source event schemas are derived from the ABI and verified against indexed columns. Agents may rename columns and define derived schemas. They cannot redefine an ABI type without an explicit checked transformation. + +Change classification in plans: + +| Class | Example | Command / intent | +|---|---|---| +| Presentation-only | Dashboard title | `plan --intent build` then `apply` (or `build`) | +| Query rebuild | SQL text | `plan --intent build` then `apply` (or `build`) | +| Snapshot rebuild | Model graph | `plan --intent build` then `apply` (or `build`) | +| Additional backfill | Earlier start, new address | `plan --intent ingest` then `apply` | +| Continue pinned ingest | Proven end below declared `end_block` (budget split or crash) | Repeated `plan --intent ingest` then `apply`. Not `refresh`. | +| Follow-head ingest | `follow_finalized` advance | `plan --intent refresh` then `apply` (or `refresh`) | +| Publish existing release | Upload `dist/` | `plan --intent publish` then `apply` (or `publish`) | +| Incompatible | Schema/grain/chain identity | Refuse; new version or separately authorized destructive plan | + +`build` never talks to RPC. It produces a staging release under `dist/` +from an existing complete snapshot (re-export models if the graph +changed, re-run queries, write static assets). A2 maps to `build`. + +Never automatically drop existing data to fit an incompatible change. + +--- + +## 9. CLI + +All commands are noninteractive. `--json` emits exactly one result object on stdout; diagnostics go to stderr. Long operations also accept `--jsonl`: versioned progress events, then one final result. Never mix spinner animation or upstream plaintext into structured stdout. + +Result envelope: + +```json +{ + "schema_version": 1, + "ok": true, + "command": "validate", + "data": {}, + "warnings": [], + "error": null +} +``` + +On failure, `ok` is false and `error` is: + +```json +{ + "code": "policy_refused", + "message": "human readable", + "resource_id": "src.transfers", + "pointer": "/event_sources/0/end", + "retryable": false, + "suggested_next": "plan --intent ingest" +} +``` + +`error.code` is a closed enum. v0.1 values: + +`validation`, `missing_credentials`, `unsupported_capability`, +`policy_refused`, `source_inconsistent`, `transient_dependency`, +`internal`. + +Do not invent ad-hoc codes per command. Details go in `message`, +`resource_id`, and `pointer`. + +### 9.1 Commands + +| Command | Network | Job | +|---|---|---| +| `capabilities` | no | Versions, supported sources/targets/charts, limits, schema versions | +| `schema show ` | no | Full local JSON Schema for one of: `project`, `plan`, `result`, `progress`, `release`, `manifest`, `latest`, `coverage`, `lock` | +| `templates list` | no | Template ids, required inputs, limitations | +| `init --template --output` | no | Scaffold; fail if files would collide | +| `validate` | no | Syntax, schema, refs, ABI, model graph, and static policy combos (including `follow_finalized` + `confirmation_depth` → `unsupported_capability`) | +| `doctor` | yes | Creds present, chain id, RPC methods, Postgres, rindexer binary, writable storage, S3 if configured. S3 check is a non-mutating API call (e.g. HeadBucket), never a write. Output must mark S3 **write/promote** capability as `unverified`. Write permission is proven only at upload time. | +| `plan --intent ingest\|refresh\|build\|publish` | read-only probes | Write a plan file; mutate nothing | +| `apply --plan` | as planned | Execute that plan only | +| `build` | no RPC; HTTPS snapshot fetch only in `dataset_referenced` mode (§16.4, §14.1 caps) | Sugar over `plan --intent build` plus `apply`. Staging release in `dist/` from an existing snapshot. Cache already present → no network. | +| `refresh [--publish-target]` | as planned | Sugar over `plan --intent refresh` plus `apply`. Optional publish of the new release. | +| `query --file --snapshot` | no RPC; HTTPS snapshot fetch only in `dataset_referenced` mode (§16.4, §14.1 caps) | DuckDB on a materialized snapshot. Cache already present → no network. | +| `dataset describe ` | no RPC; HTTPS snapshot fetch only in `dataset_referenced` mode (§16.4, §14.1 caps) | Schema, coverage, freshness, dataset mode; optional sample. Cache already present → no network. | +| `test` | no | Declared assertions against data already on disk. Missing local data fails with `validation`; `test` does not fetch. | +| `publish` | as targeted | Publication-only plan for an already-built release | +| `fork` | optional HTTPS fetch | Import local dir or release URL; do not execute recipes | +| `serve` | loopback only (`127.0.0.1`) | Preview trusted static output. Refuse `0.0.0.0`. | +| `runs list\|show\|cancel` | no | Journal. Cancel is cooperative: forward signal, keep recoverable state | + +No `deploy render`. No `watch`. `init` copies `compose.yaml` for ingest templates. + +`validate` must work with RPC, Postgres, and credentials absent. `doctor` must not start unbounded indexing. + +### 9.2 Plan and apply + +A plan is digest-bound authorization. It includes: + +- Project, source, and dependency digests +- Current-state assumptions +- Resolved chain and block interval +- Ordered actions, affected resources, whether any action deletes data or makes data public +- Enforced limits; estimates that cannot be computed are `unknown`, never invented +- Public file/column selection and destination when publishing + +`apply` refuses if configuration, policy, source identity, or state +assumptions drifted. Routine movement of chain head does not invalidate a +**finalized** pinned range (the pin was already ≤ finalized at plan time). +A changed canonical boundary or changed source definition does. + +A plan does not authorize extra spend, a wider backfill, installing +software, provisioning cloud resources, or publishing to another +destination. No flag widens a plan's scope or destination. Destructive and +public-data actions must appear in the plan. + +`refresh` may execute only actions already authorized by the project file +and the last successfully applied plan: same addresses, same public column +selection, same publish destination, block count ≤ budget. The generated +plan is still written to the run journal. `--publish-target` must name a +target already in `chainplot.yaml`. Anything else fails with +`policy_refused` and needs an out-of-band `plan` + `apply`. + +Same plan digest + idempotency key: resume or return the existing outcome. +Do not duplicate ingestion or mix files from two releases. Do not claim +globally exactly-once execution. + +Idempotency key defaults to the plan digest. Override with +`--idempotency-key`. Journal path: +`.chainplot/runs//` containing the plan copy, status, +child pid, and checkpoints. A second `apply` with the same key and a +**different** plan digest fails with `policy_refused`. + +`last_proven_complete_block` is per event source, derived from coverage +rows (not a separate checkpoint): the end of the contiguous complete +segment chain that starts at that source's project `start_block`. +Sources on the same project may differ. With **no** coverage rows, +`last_proven_complete_block` is `start_block − 1`, so the first run +begins at the project `start_block`. + +Every ingest job (refresh or `plan --intent ingest`) uses inclusive +bounds: + +- `job_start = last_proven_complete_block + 1` +- `job_end = min(job_target_end, job_start + block_budget - 1)` + where `job_target_end` is the declared `end_block` for `pinned` + and `resolved_safe_end` for `follow_finalized` + +If `job_start > job_end`, ingest is a no-op and coverage is unchanged. + +v0.1 does **not** re-ingest an overlap tail. Jobs only append. Interior +reorg handling for a live unfinalized window is follow-on. + +Therefore `follow_finalized` requires chain finality policy `finalized`. +`follow_finalized` + `confirmation_depth` is `unsupported_capability` at +`plan` time. `confirmation_depth` remains valid for **pinned** sources: +the pinned `end_block` must already satisfy the depth at plan time +(§8). Those snapshots are labeled confirmation-based. + +`pinned` sources skip ingest on **refresh**. A pinned source with +`last_proven_complete_block < end_block` is finished by repeated +out-of-band `plan --intent ingest` + `apply` (budget split or resume +after a consumed plan). + +Coverage is stored per contiguous segment. Each coverage segment records `start_block`, `end_block`, +`start_block_hash`, `end_block_hash`, and `start_block_parent_hash` +(from the same header fetches as §12; no extra RPC). Adjacent segments +A then B are **hash-joined** iff `B.start_block = A.end_block + 1` and +`B.start_block_parent_hash = A.end_block_hash`. Number-adjacent without +that parent link is `source_inconsistent`, not complete. + +A snapshot is **complete** for a source only when contiguous hash-joined +complete segments cover `[start_block, required_end]`: + +- `pinned`: `required_end` is the declared `end_block` +- `follow_finalized`: `required_end` is the maximum `end_block` over + that source's recorded coverage segments. A job that wrote no coverage + segment contributes nothing. This is not `resolved_safe_end` (that is + `job_target_end`). With no coverage segments, `required_end` is + undefined and the source is `not_indexed`, never `complete`. A release + containing it must not be promoted. Under append-only hash-joined + segments this equals `last_proven_complete_block` whenever coverage is + contiguous. + +If `resolved_safe_end < start_block` for a `follow_finalized` source, +`plan` fails with `policy_refused` (start is in the future relative to +finalized head). + +A gap, hash mismatch, or proven end below that `required_end` is +`incomplete` and must not be promoted. A pinned source killed at 60% of +its range is incomplete even though it has a single gap-free segment. + +`resolved_safe_end > required_end` is a normal lagging-but-complete +state. Report it as freshness lag in `manifest.json`, `dataset describe`, +and the viewer chip — not as `incomplete`. Catch-up is later refresh +jobs, each ≤ block budget. + +`refresh` vs a dirty working tree: + +- Model SQL, query SQL, dashboard presentation: rebuild (`build` class). + Cron may publish those; they do not widen ingest or public-data + selection. +- Addresses, start/end, public columns, destination, or chain identity: + `policy_refused`. Needs an out-of-band plan. + +If there is **no** last successfully applied plan (fresh clone, first +machine, fork that only ran `build`), `refresh` derives authorization +from the project file alone and may ingest only if that does not widen +beyond the file (first `follow_finalized` run from `start_block` is +allowed; changing destination is not). + +Dedup on resume or re-apply of a partial segment: physical uniqueness +from §12. Duplicate delivery of the **same canonical log** must not +produce duplicate snapshot rows. A row that matches the unique key with +a **different** `block_hash` is a reorg: `source_inconsistent`. + +New addresses, earlier start, new public columns, or a new destination +require an out-of-band ingest or publish plan. + +Progress events report completed coverage, current stage, rows, output bytes, retries, heartbeat, last successful release. Do not invent a completion percentage when the total is unknown. Samples default to 20 rows. + +--- + +## 10. Ingest adapter + +### 10.1 Interface + +One TypeScript interface, one v0.1 implementation (`rindexer`). No plugin loader, no user-facing `indexer:` driver field, no second stub. + +```ts +interface IngestAdapter { + renderConfig(input: RenderConfigInput): GeneratedConfig; + runBounded(job: BoundedJob): Promise; + stopAndQuiesce(handle: RunHandle): Promise; + inspectCoverage(job: BoundedJob): Promise; +} +``` + +Call `inspectCoverage` only after `stopAndQuiesce`. The report must +distinguish `not_indexed`, `incomplete`, `complete_empty`, and +`complete_with_rows`. Maximum observed event block is not completeness. + +A later indexer that can run a bounded EVM-event job, stop writing, and prove coverage including empty ranges is a new module behind this interface. Indexers that only head-follow, only expose GraphQL, or cannot prove empty-range coverage do not fit; those datasets enter via snapshot import. + +### 10.2 rindexer implementation + +Generate gitignored upstream YAML from `chainplot.yaml`. Do not copy rindexer's full config surface into Chainplot. + +Required generated settings (Milestone 0 must verify against the pinned version): + +- `project_type: no-code` +- Postgres storage enabled; GraphQL, streams, chatbots, CSV, and Docker-socket DB provisioning disabled +- **Always set `start_block` and `end_block`.** Upstream without `start_block` indexes from "now" and does not track last-synced block across restarts. Upstream without `end_block` live-indexes. Chainplot never uses those modes in v0.1 +- Explicit contract addresses only (not `factory`, not chain-wide `filter`) +- `include_events` limited to the declared signatures +- RPC URL from the secret reference, not from committed files + +Parent process: forward termination signals, reap children, impose the RPC-job wall-clock limit, persist run state. + +One active ingest writer per project. Postgres advisory lock plus a +local lock file. Concurrent `apply` on the same project fails with +`policy_refused`. Publish without Postgres uses the local lock plus the +S3 conditional write on `latest.json` (§16.2). + +Dashboard SQL never depends on rindexer's table-naming (contract `name` drives upstream table names). The snapshot exporter maps to logical tables defined in `schemas/`. + +--- + +## 11. Coverage and finality + +A published **complete** snapshot requires evidence of complete ingestion +for every selected source over `[start_block, required_end]` as defined +in §9.2, including ranges with zero matching events. A truncated pinned +range is incomplete. + +M0 must name the concrete upstream evidence (which rindexer table/column, +or which Chainplot-written coverage row) that proves last-synced block +per contract. "Max event block" is not that evidence. If the pinned +rindexer cannot prove empty-range completeness, `plan` refuses that +configuration with `unsupported_capability` before ingest. v0.1 never +labels an interval complete without positive evidence. + +**Finality policy** is explicit on the chain source: + +- `finalized` — use the RPC finalized block. If the node cannot supply it, fail. Do not fall back. +- `confirmation_depth` — integer depth, labeled confirmation-based in every manifest and viewer chip. Not equivalent to protocol finality. + +Never silently index unconfirmed head. + +Record start/end block numbers, `start_block_hash`, `end_block_hash`, +`start_block_parent_hash`, source ids, ABI digest, chain id, finality +policy, generation time, and transformation digests. + +Before a refresh that **ingests** (advances proven coverage): the +previous published end-boundary hash must still be canonical. Mismatch +blocks promotion. Keep the last complete release. Repair is a +documented rebuild, not a silent skip. + +A refresh that performs no ingest performs no canonicality check. It +performs no RPC when every source is `pinned`, or in dataset-only / +fork / presentation-rebuild projects. A `follow_finalized` source still costs one finalized-head probe per +refresh to resolve `resolved_safe_end`, **in addition to** any +canonicality and coverage-boundary header fetches when that refresh +ingests. If the head probe fails, refresh fails with +`transient_dependency` and coverage is unchanged. `job_start > job_end` +is decided **after** that probe. + +Milestone 0 must test: interrupted batches, duplicate delivery, +zero-event ranges, restart resume from `last_proven_complete_block + 1`, +inclusive `end_block` arithmetic, a pinned range stopped below +`end_block` (must stay `incomplete`), a controlled canonical-boundary +change, a two-segment cross-reorg that breaks +`start_block_parent_hash` join (do not treat as complete), a zero-event +segment that still records both boundary hashes and fails promotion on +a boundary-hash change, and that `follow_finalized` + +`confirmation_depth` is refused at plan time. If the pinned rindexer +cannot meet a required safety for a configuration, reject that +configuration. Do not write a second indexer to paper over it. + +--- + +## 12. Snapshot and types + +Default snapshot happens after the bounded upstream run has **stopped writing**. Read selected tables and coverage metadata from one coherent database snapshot. Do not stitch independent reads and call them one point in time. + +Milestone 0 chooses **one** export path and records it: + +1. DuckDB Postgres extension, if type mapping and transaction behavior pass tests, or +2. One explicit Postgres read transaction to stage tables, then DuckDB + +Do not maintain both forever. + +Each event table includes: `chain_id`, `contract_address`, `block_number`, +`block_hash`, `tx_hash`, `log_index`, `block_timestamp`. Do not infer +timestamps from average block time. + +Milestone 0 must check whether the pinned rindexer persists per-log +`block_hash` and `block_timestamp`. If yes, use them and skip enrichment. +If no, Chainplot runs an enrichment stage **after** `stopAndQuiesce` and +**before** snapshot export. + +In **either** case, header fetches for coverage always include +`segment start_block` and `segment end_block`, even when those blocks +have zero matching logs (A6). Plus distinct `block_number` values that +did produce logs, if timestamps/hashes are not already on the log +rows. Those fetches count against the §14.1 distinct-header limit. A +missing header makes the segment `incomplete` (`transient_dependency` +if the RPC failed; `source_inconsistent` if the block vanished). +Boundary hashes used in §11 come from these header fetches, not from +“some log happened to land on the boundary.” + +Physical uniqueness: `(chain_id, block_number, tx_hash, log_index)`. +That key is one canonical log. The same key with a different +`block_hash` is a reorg, not a second row: `source_inconsistent`. Do +not keep both. Logical grain is declared on the dataset schema. + +Canonical formatting: lowercase hex for addresses and hashes, UTC timestamps, declared nullability. Overloaded event signatures and reserved column names are handled explicitly. Nested ABI types we cannot map fail validation. + +**Wide integers (`uint256`, `int256`, and any integer wider than JS +safe integer):** + +- Portable JSON, CLI, and Parquet raw amounts: validated decimal + **string**. Signed values use a leading `-` for negatives. No JavaScript + `number`. +- Do not auto-cast every uint256/int256 to DuckDB `DECIMAL` (max 38 + digits; 256-bit needs 78). +- Raw display (KPI raw, table raw column, A8) must show the exact + decimal string. +- `ORDER BY` on a raw-amount column is forbidden. Models that need + order over amounts must emit a dedicated sort key (zero-padded + fixed-width decimal string, sign-aware) and `ORDER BY` that key. +- Analytical columns that need numeric ops declare: token decimals, + precision class (`exact` or `approximate`), and target DuckDB type. + A checked cast that overflows an `exact` column fails the query; the + cached result is an error object, and the viewer shows that error — not + a rounded number, not an empty chart pretending success. + `approximate` is allowed only when the metric schema says so; the + viewer chip must say approximated. +- Use DuckDB neo `getRowsJson()` / JSON converters for CLI JSON. + +Acceptance A8: `0`, `1`, `-1`, `2^53+1`, `-2^53-1`, `2^255-1`, +`-2^255`, `2^256-1` round-trip as raw strings. A9-adjacent ordering +fixture: sort those values via the sort key, not lexicographic strings. + +--- + +## 13. SQL models and queries + +Materialize scoped source tables from the snapshot, apply declared `SELECT` models in dependency order, execute queries against that snapshot. + +- No arbitrary DDL, no shell lifecycle hooks, no incremental-model framework in v0.1 +- Cycles fail `validate` +- Queries and results identify snapshot id, schema version, query digest, output types, units, execution stats +- Published chart/table output requires explicit stable `ORDER BY`. + Do not `ORDER BY` a raw decimal-string amount column (see §12). +- Time-dependent SQL uses the snapshot's analysis timestamp, not `now()` +- Parameters are typed and bound, never concatenated + +Assertions cover keys, nullability, types/ranges, expected columns, and template-specific invariants. A dataset can be well-formed and semantically wrong; metric descriptions and exclusions make interpretation reviewable, not automatically true. + +--- + +## 14. Isolation, trust, limits + +Project owners and their agents may write SQL. Public viewers cannot submit SQL to the producer. Imported manifests, ABI text, data values, and dashboard copy are untrusted data, not instructions. + +`fork` copies files and pins snapshots. It does not execute imported recipes, install extensions, or run hooks. The schema format cannot embed shell commands or JavaScript callbacks. + +DuckDB runs in a child process with snapshot files and a temp dir only. RPC, Postgres, and publication credentials are stripped from its environment. Disable extension install and external access after setup. Apply query time, memory, thread, and output limits. + +This is a trusted-owner tool, not a multi-tenant sandbox. Unreviewed imported SQL is never auto-run. If a deployment cannot isolate the worker, refuse untrusted-execution requests rather than claim parser safety. Do not mount a Docker socket to wrap each query. + +### 14.1 Enforced v0.1 limits + +These are safety limits, not performance promises. Query, output, and +publication-size limits are typed refusals with choices (narrow scope, +publish aggregates, raise the policy). Never silent truncation of a +published metric. Never synthetic live data outside visibly marked +fixtures. + +The **block budget** is the exception: it **caps** `job_end` per §9.2. +`plan` must show the resolved interval and blocks remaining. A +200_000-block pinned range becomes two sequential authorized ingest +plans, not `policy_refused` on the first. + +| Limit | Default | +|---|---| +| Chains per project | 1 | +| Contract addresses | 20 | +| Blocks in the resolved inclusive `[start, end]` of one approved run | 100_000 | +| Distinct block-header fetches per approved run (coverage boundaries always; enrichment when needed) | 100_000 | +| Query deadline | 60 s | +| DuckDB memory target | 1 GiB | +| Returned rows | 10_000 | +| RPC job wall clock | 30 min, resumable | +| Copied public dataset | 100 MiB compressed | +| Cached query results | 5 MiB total | +| CLI sample rows | 20 | +| Concurrent ingest/publish per project | 1 | +| `fork` `release.json` body | 1 MiB | +| `fork` total download | 512 MiB, streaming abort, independent of declared size | +| `fork` per-request timeout | 30 s | +| `fork` redirect hops | 0 (redirects forbidden) | + +The block budget applies to the **first ingest** as well as refresh. + +A block-range limit is not an RPC billing cap. Do not invent dollar costs. + +--- + +## 15. Viewer + +Ship a responsive viewer with line charts, bar charts, KPI values, and tables. Allowlisted chart encodings only. No user JavaScript formatters, no arbitrary HTML. + +Display: freshness, coverage, finality policy (including +confirmation-based labeling), units, query source, and dataset mode +(`results_only` | `dataset_included` | `dataset_referenced`). The viewer at the target root loads `latest.json` (via the bootstrap in +§16.1) to resolve which release to render. A pinned URL opens +`releases//index.html` and skips the pointer. + +v0.1 interactions operate only on published result rows: sort, toggle +series, displayed range, table search. Sort of a raw-amount column must +use the published sort key or a numeric compare of the decimal string, +never JavaScript/`localeCompare` string order. A control must not imply +it can fetch missing history or rerun SQL when that capability is absent. + +Readers need no account, wallet, RPC, producer, or database. Works on ordinary HTTP(S) under a configurable base path. `file://` is not promised. No third-party CDN. + +--- + +## 16. Release, publish, fork + +### 16.1 Release layout + +```text +/ + index.html # bootstrap; fetches latest.json + assets/ # release-independent viewer loader + latest.json # pointer: prefix + checksum + releases// + index.html + assets/ + release.json + dashboards/ + results/ + datasets// + manifest.json + tables/*.parquet + source/ +``` + +`latest.json` is never stored inside `releases//`. + +Root `index.html` + root `assets/` are a version-stable bootstrap: fetch +`latest.json`, which carries the release prefix and the checksum of that +release's `release.json`. Verify `release.json` against that checksum; +`release.json` carries per-file checksums used before loading assets. +Promotion stays a single `latest.json` write. Rewrite the bootstrap only +when the viewer loader itself changes, not when data refreshes. Opening +`releases//index.html` is the pinned URL and skips the pointer. + +`source/` is an explicit allowlist of sanitized recipe files. No secrets, no `.env`, no run logs, no absolute machine paths. + +A release may copy selected public data (standalone) or reference an external pinned dataset. Exceeding the size limit fails unless the agent chooses reference mode, reduces data, or publishes results-only. + +Manifests distinguish the three modes. The viewer must not claim arbitrary SQL remixing when only cached results were published. Forking results-only yields a recipe, not the missing source data. + +### 16.2 Publication protocol + +1. `build` writes a staging release (or `apply` of a build/refresh plan does) +2. Validate schemas, references, query success, allowlist, checksums, size. + S3 write/promote permission is proven only here, not by `doctor`. +3. Upload immutable release files +4. Verify +5. Promote `latest` last + +Prefer immutable release addresses for citations. `latest` is **one +small pointer object** `latest.json` containing the immutable release +prefix and the checksum of that prefix's `release.json`. The viewer resolves it at load. Promotion is a +single object write after all immutable files are uploaded and verified. +Never copy a whole release into a `latest/` directory. + +Filesystem target: atomic replace of `latest.json` on the same +filesystem (write to a temp name, `rename`). S3-compatible target: +conditional write of `latest.json` (`If-Match` / `If-None-Match` on the +current pointer). Milestone 0 must verify **conditional-write and +read-after-write** on AWS S3 and at least one S3-compatible endpoint. If +the target cannot do that, refuse the target. + +Dataset-only publish has no Postgres. Cross-host exclusion for S3 is the +conditional write on `latest.json` plus a local lock file (same-host +only). Two hosts without a working conditional write are unsupported; +refuse rather than last-write-wins. + +Interrupted upload leaves the previous complete release. Checksums +detect corruption; they are not proof of analytical truth or authorship. + +v0.1 retention is **manual**. No `prune` command. Automated GC is +follow-on. Operators who delete objects under a prefix still referenced +by `latest.json` or by a cached pointer get a broken URL; that is +outside the product until prune exists. + +Software license (MIT) is not dataset license. Publishing requires an explicit dataset-license field. Schema descriptions can leak; they are in the allowlist review. + +Secret-shaped scanning is best-effort, not a proof that data is safe to publish. + +### 16.3 S3 adapter + +One adapter using AWS SDK v3 against an S3 API. Bucket already provisioned. Credentials via env/file reference. Chainplot does not create buckets or IAM. + +### 16.4 Fork + +Start from `release.json`, not scraped HTML. + +Order: validate `latest.json` against kind `latest` (if present), then +`release.json` against kind `release`, then each dataset `manifest.json` +against kind `manifest`. Check format version, allowlisted paths, and +declared sizes against §14.1 **before** downloading bodies; +stream bodies with a hard byte cap; verify checksums as bytes arrive; +abort on cap, timeout, or mismatch. Declared size is not trusted as an +upper bound. + +Fetch rules (deny by default): + +- Local directory, or `https` URL +- Redirects forbidden +- Resolve DNS; validate **every** resolved address against a blocklist: + loopback (IPv4 `127.0.0.0/8`, IPv6 `::1`), link-local (`169.254.0.0/16`, + `fe80::/10`), ULA `fc00::/7`, RFC1918, CGNAT `100.64.0.0/10`, + unspecified (`0.0.0.0`, `::`), IPv4-mapped IPv6 forms of any of the + above. Reject decimal/octal/hex IP literals that decode to a blocked + address. +- Pin the resolved address for the connection (or re-validate on + connect) to defeat DNS rebinding +- Block path traversal and undeclared files +- Cross-origin references require explicit allowed origins +- Escape hatch for private networks is an explicit, documented flag; + default off + +Fork creates a new project with pinned snapshots and copied +query/dashboard definitions. No chain access is required for new SQL +over included data. Expanding history or changing contracts is a new +ingest plan. + +Dataset mode is the one vocabulary. `dataset describe` and fork output +use the same strings as the viewer: + +| Mode | What the fork obtained | +|---|---| +| `results_only` | Recipe and cached results. No snapshot. New SQL over source tables is impossible until data is obtained another way. | +| `dataset_referenced` | Pointer to an external pinned snapshot. The CLI, not DuckDB, fetches it into `.chainplot/cache/snapshots//` under §16.4 rules and §14.1 fork caps, verifies the pinned checksum, then hands **local files** to the isolated worker. Failed fetch: `transient_dependency`. `query` / `dataset describe` / `build` materialize the cache first. `test` still requires data already on disk and does not fetch. | +| `dataset_included` | Copied snapshot. New SQL over included data, no RPC. | +| (not a mode) | Running the recipe to keep freshness is a **new ingest project**, not implied by fork. | + +--- + +## 17. Compose and operation + +v0.1 tested ingest runtime: Docker Compose with producer + Postgres, explicit volumes, secret references, no Docker socket mounted into the producer. + +`init` ingest template includes `compose.yaml`. The agent (or human) applies it with Docker. Chainplot does not SSH, does not speak Kubernetes, and does not install operators. + +Dataset-only path: CLI + imported snapshot, network disabled, A15. + +Persistence: + +- Postgres: indexed data and upstream progress +- Project + lockfile: version-controlled inputs +- `.chainplot/`: run journal and caches +- `dist/` or the publish target: public output +- Native DuckDB files: disposable + +Switching RPC URL must not invalidate otherwise compatible chain data. Changing chain id fails validation. + +Refresh is a command. Document a cron one-liner that runs `chainplot refresh --publish-target ` from the project directory. Do not ship `watch`. + +--- + +## 18. Definition of done + +v0.1 is done when A1–A16 pass on fixtures (and Compose for A11). Live-chain smoke is opt-in and bounded, not the only evidence. Expected results are independently specified, not generated solely by the pipeline under test. + +| ID | Test | Required outcome | +|---|---|---| +| A1 | Agent creates a dashboard from ABI, contract scope, bounded fixture history | No browser automation or undocumented manual step; correct dataset and charts | +| A2 | Agent changes a chart title and runs `build` | Presentation rebuild only; no backfill | +| A3 | Agent requests earlier history or another contract | Explicit additional work; no silent expansion | +| A4 | Kill producer during indexing, export, and upload | Safe resume; previous complete public release remains usable | +| A5 | Apply the same plan and idempotency key twice | No duplicate logical data or mixed release | +| A6 | Source with no matching events for the whole interval | Either coverage is `complete_empty`, or `plan` refuses the configuration with `unsupported_capability` before ingest. Never report complete without positive evidence. | +| A7 | Change the canonical chain boundary in a fixture | Detect inconsistency; do not promote | +| A8 | Export `0`, `1`, `-1`, `2^53+1`, `-2^53-1`, `2^255-1`, `-2^255`, `2^256-1` as raw amounts | Exact round-trip through storage, dataset, JSON, and displayed raw values. Sort-key order over that set is numeric, not lexicographic. | +| A9 | Stop producer and database; block RPC | Public dashboard still renders | +| A10 | Second agent imports published data and writes a new query | New dashboard without reindexing or original credentials | +| A11 | Copy the project onto a clean machine and run Compose | Config changes are environmental, not rewritten analytics | +| A12 | Malicious filenames, labels, metadata, SQL, secret-like content | No automatic recipe execution, path escape, credential exposure, or unsupported safety claim | +| A13 | Exceed query, scope, output, or publication limits | Typed refusal; no silent truncation or partial success marked complete | +| A14 | Missing secrets or wrong chain identity | Precise diagnosis, no hanging prompt, no destructive fallback | +| A15 | Fixture-only quickstart with network disabled | Works from packaged assets and local data | +| A16 | After a local SQL/dashboard edit, `refresh` on a project whose sources are `pinned` and already complete, with an existing publish target | Rebuilds and may publish; no ingest. After an address or destination edit, `refresh` returns `policy_refused`. | + +Measure later, do not promise now: agent completion rate, interventions, time to first valid dashboard, extra RPC on refresh, publication size, recovery success. + +--- + +## 19. Implementation sequence + +Each milestone is independently reviewable. Viewer work can start against fixture snapshots once schema contracts from M1 exist. Do not parallelize contradictory snapshot or identity designs. + +### M0 — Prove the dependency path (throwaway probe) + +Pin Node LTS, rindexer, DuckDB, `@duckdb/node-api`, AWS SDK. Choose the +snapshot export path. Verify: + +- rindexer inclusive `start_block`/`end_block`, resume from + `last_proven_complete_block + 1`, empty-range evidence source (exact + table/column) +- whether rindexer persists per-log `block_hash` / `block_timestamp` +- `follow_finalized` requires `finalized`; `confirmation_depth` is + pinned-only in v0.1 +- uint256/int256 through Postgres as strings +- DuckDB max exact decimal width vs overflow display +- S3 conditional-write (`If-Match` / `If-None-Match`) and + read-after-write, not just overwrite/list + +**Gate:** Failed assumption narrows this spec with a recorded amendment. Do not build the rest of the CLI around a lie. + +### M1 — Agent contract and fixture-only product + +JSON Schema (including `progress` events), CLI +discovery/init/validate/describe/query/test/`build`, structured errors. +One fixture project: one table, one chart, no RPC. + +M1 `build` is narrower than the full §8 definition: it emits typed +query results plus `release.json` / `manifest.json` for the fixture +snapshot. It does **not** re-export a model graph or write viewer static +assets. M3 extends `build` to those. The M1 fixture chart is a cached +result document, not a rendered HTML dashboard. + +**Gate:** An agent can learn the interface from installed help and schemas without reading `src/`. + +### M2 — Ingest and snapshots + +rindexer adapter, generated config, coverage, plan/apply/refresh, locks, +exporter, run journal, cancel, Compose + Dockerfile, hard scope limits. +Pass on one CPU architecture. + +**Gate:** Incomplete data cannot be promoted as a complete snapshot. + +### M3 — Models, dashboards, static build + +SELECT model graph, typed results, assertions, viewer, provenance, +sanitized source bundle, `serve`. Extend `build` to re-export models and +write static assets into `dist/`. + +**Gate:** Public dashboard works with producer, database, RPC, and CDN unavailable. + +### M4 — Publish and fork + +Release manifests, immutable prefixes, `latest.json` conditional +promotion, directory + S3, publication plans, dataset modes, fork/import +with SSRF rules, checksums. + +**Gate:** Failed upload leaves previous release; another agent forks using only published data. + +### M5 — Examples and acceptance + +Three examples: transfer activity; protocol deposit/withdrawal; fork of a +published dataset. Docs, capability matrix, MIT + upstream notices, +separate dataset license statements. Run A1–A16. Smoke the Compose +Dockerfile on Linux x86-64 and ARM64. + +**Gate:** Fix interface and correctness problems before new source types or a human editor. + +--- + +## 20. Key decisions + +| Decision | Rationale | +|---|---| +| v0.1 = local loop + one S3 publish | Sharing beyond the laptop without K8s/systemd/watch | +| Refresh is a command, not a daemon | Updates the published dash; scheduler stays with cron/CI | +| End policy `pinned` vs `follow_finalized` | Refresh must not invent an end block for a pinned range | +| Block budget applies to first ingest | No silent "first run is unlimited" exception | +| Compose + Dockerfile, not a published image | Matches the ops cut; A11 still has a reproducible runtime | +| Two lock files | Software pin (`pnpm-lock.yaml`) vs project pin (`chainplot.lock.json`) | +| TypeScript throughout | Viewer is TS; JSON Schema → types; DuckDB/rindexer already own native work. “Static binary” is illusory (libduckdb + rindexer still ship) | +| Thin `IngestAdapter`, rindexer-only impl | Snapshot is the swap boundary. A cousin indexer is a later module. A plugin system is unused abstraction | +| Snapshot-centric; DuckDB never on live Postgres | Index-once read path; consistent export | +| Always pass rindexer `start_block` and `end_block` | Upstream live-index modes do not resume safely | +| Explicit addresses, one chain | Factory/filter/multi-network are discovery, not v0.1 | +| Wide integers as decimal strings; sort keys; overflow is an error result | JS `number` and DuckDB `DECIMAL(38)` are lossy; lexicographic `ORDER BY` is wrong | +| Plan digest is authorization; `refresh` only replays already-authorized class | No flag widens scope; cron cannot expand public data | +| `build` is the no-RPC path to `dist/` | Only a `dataset_referenced` cache miss touches the network | +| Refresh ingest starts at `last_proven_complete_block + 1` | Re-scanning from project start blows the budget and duplicates rows | +| `latest.json` single-object pointer + S3 conditional write | Multi-object `latest/` copy cannot be atomic | +| Pinned `end_block` must already be finalized | Otherwise “complete” snapshots sit on reorg-able head | +| Fork deny-by-default destinations, no redirects, streaming byte cap | `release.json` is attacker-controlled | +| `dataset_referenced` fetch is CLI cache, not DuckDB HTTP | Isolation stays; referenced mode still works | +| Completeness is `[start, required_end]`, not “any gap-free prefix” | Truncated pinned ingest must not promote | +| `follow_finalized` completeness `required_end` is ingested coverage max, not `job_target_end` | Budget-capped catch-up is lag, not incomplete | +| Root bootstrap `index.html` resolves `latest.json` | Public URL has an entry point; promotion stays one object | +| Unique key is `(chain_id, block_number, tx_hash, log_index)`; hash conflict is reorg | Do not keep both rows | +| `follow_finalized` requires protocol `finalized`; jobs only append | Overlap-replace is a second write path into live event tables. Live unfinalized window is follow-on. | +| M1 `build` emits results JSON; M3 adds static assets | Milestone gates stay evaluable | +| v0.1 retention is manual; no `prune` | Destructive public delete needs a later plan class | +| Compose only as tested runtime | Matches v0.1 ops cut | +| One package, modules by responsibility | Split when a boundary is forced, not upfront | +| pnpm, ECharts, `@duckdb/node-api` | Defaults; versions locked in M0 | +| MIT for software; dataset license is a publish field | Mixing them hides reuse terms | +| Docs live in `docs/specs/` and later `docs/plans/` | No skill-branded folder | + +--- + +## 21. Open questions + +Resolved in this spec unless Milestone 0 contradicts them. Remaining: + +1. **Exact pins** — Node/`@duckdb/node-api` in M1 lockfile. rindexer image + digest in `docs/compatibility.md` (GHCR `v0.43.1` tag missing; `:latest` + digest pinned 2026-09-13). +2. **Launch name/domain/npm** — not a v0.1 implementation gate. +3. **Empty-range evidence** — **resolved:** + `rindexer_internal.{indexer}_{contract}_{event}.last_synced_block`. + Zero rows + cursor ≥ `end_block` is `complete_empty`. +4. **Header columns** — **resolved:** `block_hash` and `block_timestamp` + on the event table when `timestamp: true`. No enrichment stage. +5. **S3 conditional write** — still open; directory publish until M4 + evidence. +6. **DuckDB postgres scanner** — PG `numeric` arrives as DOUBLE; CAST + `block_number`/`tx_index` to BIGINT on export. `value` stays VARCHAR. + +No product-scope question remains open for v0.1. Implementation plans follow this document after human review of the spec file. + +--- + +## 22. Follow-on (after the core loop works) + +- `follow_finalized` under `confirmation_depth` (overlap-tail replace) +- `chainplot watch` and/or generated systemd/CronJob snippets +- Kubernetes manifests +- DuckDB-Wasm browser exploration +- A second ingest adapter when a real user needs a cousin indexer +- Thin MCP adapter over the same core functions +- Optional visual authoring +- Signing/verification of releases +- `prune` with an explicit destructive plan (cannot remove a release + named by `latest.json`) + +--- + +## 23. References + +Inspected 2026-09-12. Recheck during Milestone 0. These substantiate upstream capabilities, not an unbuilt integration. + +- rindexer: +- rindexer install: +- rindexer contracts YAML (`start_block` / `end_block` / factory / filter): +- DuckDB Node neo (`@duckdb/node-api`, JSON converters): +- DuckDB Postgres extension: +- DuckDB numeric types (`DECIMAL` width): +- Securing DuckDB: + +Adjacent name, not this project: ChainPlots.jl (neural-network visualization). diff --git a/examples/README.md b/examples/README.md new file mode 100644 index 0000000..eb9d926 --- /dev/null +++ b/examples/README.md @@ -0,0 +1,25 @@ +# Chainplot examples + +| Example | What it shows | Needs network? | +|---|---|---| +| [`transfer-traffic`](./transfer-traffic) | Full ingest pipeline: USDC `Transfer` events → Parquet snapshot → dashboard. 92 real transfers over blocks 18,600,000–18,600,010. | Yes (archive RPC) for ingest; offline afterwards | +| [`protocol-flows`](./protocol-flows) | Multi-event source: WETH `Deposit` + `Withdrawal` (150 + 108 events over the same range), two datasets, KPI dashboard. | Yes (archive RPC) for ingest; offline afterwards | +| [`fork`](./fork/README.md) | Second agent forks a published release and builds a new dashboard with its own query — no RPC, no Postgres, no credentials. | No | + +Each example directory is a complete Chainplot project: `chainplot.yaml`, +`abis/`, `queries/`, `tests/`, `compose.yaml` (Postgres + producer with the +pinned rindexer binary), and `.env.example`. + +## Quickstart (transfer-traffic) + +```bash +cd examples/transfer-traffic +cp .env.example .env # fill in RPC_URL +docker compose up -d +docker compose exec producer node /app/dist/cli/main.js plan --intent ingest --json +docker compose exec producer node /app/dist/cli/main.js apply --plan --json +docker compose exec producer node /app/dist/cli/main.js build --json +docker compose exec producer node /app/dist/cli/main.js test --json +``` + +Then `serve` the release and open the printed URL. diff --git a/examples/fork/README.md b/examples/fork/README.md new file mode 100644 index 0000000..4f3718b --- /dev/null +++ b/examples/fork/README.md @@ -0,0 +1,41 @@ +# Example 3 — fork a published dataset + +A second agent takes a published release and builds a different dashboard — +no RPC, no Postgres, no original credentials (acceptance A10). + +## Prerequisites + +Example 1 (`examples/transfer-traffic`) built and published. Publishing works +to a directory target or an S3-compatible target (see its `chainplot.yaml`). + +> **Note on URL forks:** `fork --from https://…` requires the bucket to allow +> public (unauthenticated) reads. The example R2 bucket is private, so this +> walkthrough forks from the local publish output. The fetch rules are +> identical (checksums, caps, no redirects). + +## Fork + +```bash +chainplot fork --from ./dist/releases/local --output ../forked-dashboard --json +# or, against a publish root with latest.json: +chainplot fork --from ./published --output ../forked-dashboard --json +# or, against a public https release root: +chainplot fork --from https://your-bucket.example.com/prefix --output ../forked-dashboard --json +``` + +The fork copies the pinned snapshot (`datasets//tables/*.parquet`), the +recipe (`chainplot.yaml`, queries, tests), and **strips the chain/event +sources** — a fork has no chain access. Expanding history or changing +contracts is a new ingest project, not implied by the fork. + +## Second agent adds a query + +```bash +cd ../forked-dashboard +mkdir -p queries +echo "select count(*) as transfer_count from usdc" > queries/second_agent_count.sql +# add the query + a dashboard panel to chainplot.yaml, then: +chainplot build --json +``` + +`validate` and `build` run fully offline over the pinned snapshot. diff --git a/examples/protocol-flows/.env.example b/examples/protocol-flows/.env.example new file mode 100644 index 0000000..e5f6d59 --- /dev/null +++ b/examples/protocol-flows/.env.example @@ -0,0 +1,5 @@ +# Archive-capable Ethereum JSON-RPC (mainnet). Required for plan/apply. +RPC_URL= + +# Postgres for the rindexer adapter (compose.yaml provides this service). +DATABASE_URL=postgresql://chainplot:chainplot@postgres:5432/chainplot diff --git a/examples/protocol-flows/README.md b/examples/protocol-flows/README.md new file mode 100644 index 0000000..3f10511 --- /dev/null +++ b/examples/protocol-flows/README.md @@ -0,0 +1,19 @@ +# Example 2 — WETH deposit/withdrawal flows + +Two events from one contract (`WETH9`, `0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2`): +`Deposit(address,uint256)` and `Withdrawal(address,uint256)` — the wrap/unwrap +activity of wrapped Ether. + +**Range:** blocks 18,600,000–18,600,010 (pinned, finalized). + +## Run it + +Same runtime as example 1 (see its README): `docker compose up -d`, then +`plan --intent ingest` → `apply` → `test` → `build` → `serve` inside the +producer container. Requires `.env` with an archive-capable `RPC_URL`. + +## What it shows + +- Multiple declared events on one contract (two tables: `weth_deposit`, + `weth_withdrawal`). +- Two datasets, one dashboard with a KPI per flow and a comparison table. diff --git a/examples/protocol-flows/abis/WETH.json b/examples/protocol-flows/abis/WETH.json new file mode 100644 index 0000000..40308fd --- /dev/null +++ b/examples/protocol-flows/abis/WETH.json @@ -0,0 +1,20 @@ +[ + { + "anonymous": false, + "inputs": [ + { "indexed": true, "name": "dst", "type": "address" }, + { "indexed": false, "name": "wad", "type": "uint256" } + ], + "name": "Deposit", + "type": "event" + }, + { + "anonymous": false, + "inputs": [ + { "indexed": true, "name": "src", "type": "address" }, + { "indexed": false, "name": "wad", "type": "uint256" } + ], + "name": "Withdrawal", + "type": "event" + } +] diff --git a/examples/protocol-flows/chainplot.yaml b/examples/protocol-flows/chainplot.yaml new file mode 100644 index 0000000..6fbc353 --- /dev/null +++ b/examples/protocol-flows/chainplot.yaml @@ -0,0 +1,73 @@ +format_version: 1 +id: protocol-flows +policy: + block_budget: 100000 + # The page stays small and the parquet is published beside it, so a fork + # can fetch and verify it on demand. Compare transfer-traffic, which puts the + # data inside the release, and usdc-supply, which publishes no data at all. + release_mode: dataset_referenced +chain_sources: + - id: mainnet + chain_id: 1 + rpc_secret: RPC_URL + finality: + policy: finalized +event_sources: + - id: weth + chain: mainnet + addresses: + - "0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2" + abi: abis/WETH.json + events: + - Deposit + - Withdrawal + start_block: 18600000 + end: + mode: pinned + block: 18600010 +datasets: + - id: weth_deposit + snapshot: .chainplot/snapshots/weth/weth_deposit.parquet + - id: weth_withdrawal + snapshot: .chainplot/snapshots/weth/weth_withdrawal.parquet +queries: + - id: deposit_count + file: queries/deposit_count.sql + dataset: weth_deposit + - id: withdrawal_count + file: queries/withdrawal_count.sql + dataset: weth_withdrawal + - id: largest_deposits + file: queries/largest_deposits.sql + dataset: weth_deposit + title: Largest deposits + raw_amount_columns: + - name: wad + decimals: 18 + symbol: WETH + label: Amount +dashboards: + - id: flows + title: WETH wrap/unwrap + description: Deposits into and withdrawals from WETH9. + panels: + - query: deposit_count + chart: kpi + title: Deposits + unit: wraps + - query: withdrawal_count + chart: kpi + title: Withdrawals + unit: unwraps + - query: largest_deposits + chart: table + title: Largest deposits + description: Amounts shown at 18 decimals; ordered with cp_sortkey. + span: full +publish_targets: + - id: r2 + type: s3 + bucket: chainplot-test + prefix: protocol-flows + public_base_url: https://pub-0593f9128f674400bcbbc940cf9f01b1.r2.dev + dataset_license: CC-BY-4.0 diff --git a/examples/protocol-flows/compose.yaml b/examples/protocol-flows/compose.yaml new file mode 100644 index 0000000..d9893ea --- /dev/null +++ b/examples/protocol-flows/compose.yaml @@ -0,0 +1,35 @@ +# Example 2 runtime: identical shape to example 1. +services: + postgres: + image: postgres:16-alpine + environment: + POSTGRES_USER: chainplot + POSTGRES_PASSWORD: chainplot + POSTGRES_DB: chainplot + volumes: + - postgres-data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U chainplot -d chainplot"] + interval: 2s + timeout: 5s + retries: 30 + + producer: + build: + context: ../.. + dockerfile: docker/producer.Dockerfile + platform: linux/amd64 + depends_on: + postgres: + condition: service_healthy + env_file: .env + environment: + DATABASE_URL: postgresql://chainplot:chainplot@postgres:5432/chainplot + volumes: + - ./:/workspace + working_dir: /workspace + entrypoint: ["sleep"] + command: ["infinity"] + +volumes: + postgres-data: diff --git a/examples/protocol-flows/queries/deposit_count.sql b/examples/protocol-flows/queries/deposit_count.sql new file mode 100644 index 0000000..84b86dd --- /dev/null +++ b/examples/protocol-flows/queries/deposit_count.sql @@ -0,0 +1 @@ +select count(*) as deposit_count from weth_deposit diff --git a/examples/protocol-flows/queries/largest_deposits.sql b/examples/protocol-flows/queries/largest_deposits.sql new file mode 100644 index 0000000..e79c001 --- /dev/null +++ b/examples/protocol-flows/queries/largest_deposits.sql @@ -0,0 +1,6 @@ +select + wad, + tx_hash +from weth_deposit +order by cp_sortkey(wad) desc +limit 5 diff --git a/examples/protocol-flows/queries/withdrawal_count.sql b/examples/protocol-flows/queries/withdrawal_count.sql new file mode 100644 index 0000000..43642cd --- /dev/null +++ b/examples/protocol-flows/queries/withdrawal_count.sql @@ -0,0 +1 @@ +select count(*) as withdrawal_count from weth_withdrawal diff --git a/examples/transfer-traffic/.env.example b/examples/transfer-traffic/.env.example new file mode 100644 index 0000000..0c3981a --- /dev/null +++ b/examples/transfer-traffic/.env.example @@ -0,0 +1,6 @@ +# Archive-capable Ethereum JSON-RPC (mainnet). Required for plan/apply. +# Copy to .env (gitignored). Never commit real endpoints. +RPC_URL= + +# Postgres for the rindexer adapter (compose.yaml provides this service). +DATABASE_URL=postgresql://chainplot:chainplot@postgres:5432/chainplot diff --git a/examples/transfer-traffic/README.md b/examples/transfer-traffic/README.md new file mode 100644 index 0000000..a6274a2 --- /dev/null +++ b/examples/transfer-traffic/README.md @@ -0,0 +1,39 @@ +# Example 1 — USDC transfer activity + +Scoped onchain events → dataset → static dashboard, end to end. + +**Chain:** Ethereum mainnet · **Source:** USDC (`0xa0b86991…`) `Transfer` events +**Range:** blocks 18,600,000–18,600,010 (pinned, finalized) · **Expected:** 92 transfers + +## Run it + +Requires: Docker, and an archive-capable Ethereum RPC in `.env` (see `.env.example`). + +```bash +# 1. Start Postgres + producer (CLI + pinned rindexer, no docker.sock). +# This example builds the producer image from the repo; a scaffolded +# project uses `image: ${CHAINPLOT_IMAGE:-chainplot:local}` instead. +docker compose up -d --build + +# 2. In the producer container: plan → apply → build +docker compose exec producer chainplot plan --intent ingest --json +docker compose exec producer chainplot apply --plan --json +docker compose exec producer chainplot build --json + +# 3. Preview the dashboard (from the host, or serve inside the container) +docker compose exec producer chainplot serve --port 4173 --json +``` + +Coverage evidence comes from `rindexer_internal.*.last_synced_block` plus +boundary header hashes; the release is refused unless the whole pinned range +is proven complete. + +## Files + +| File | Purpose | +|---|---| +| `chainplot.yaml` | Project definition (source scope, queries, dashboard, publish target) | +| `abis/ERC20.json` | ABI for the `Transfer` event | +| `queries/transfer_count.sql` | KPI: number of transfers | +| `queries/top_transfers.sql` | Table: largest transfers, ordered with `cp_sortkey` | +| `tests/` | Assertions over the snapshot | diff --git a/examples/transfer-traffic/abis/ERC20.json b/examples/transfer-traffic/abis/ERC20.json new file mode 100644 index 0000000..cef2eb7 --- /dev/null +++ b/examples/transfer-traffic/abis/ERC20.json @@ -0,0 +1,12 @@ +[ + { + "anonymous": false, + "inputs": [ + { "indexed": true, "name": "from", "type": "address" }, + { "indexed": true, "name": "to", "type": "address" }, + { "indexed": false, "name": "value", "type": "uint256" } + ], + "name": "Transfer", + "type": "event" + } +] diff --git a/examples/transfer-traffic/chainplot.yaml b/examples/transfer-traffic/chainplot.yaml new file mode 100644 index 0000000..c40358d --- /dev/null +++ b/examples/transfer-traffic/chainplot.yaml @@ -0,0 +1,63 @@ +format_version: 1 +id: transfer-traffic +policy: + block_budget: 100000 + # Ship the parquet: this example exists to be forked and recomputed. + # The default is results_only, which publishes the page without the data. + release_mode: dataset_included +chain_sources: + - id: mainnet + chain_id: 1 + rpc_secret: RPC_URL + finality: + policy: finalized +event_sources: + - id: usdc + chain: mainnet + addresses: + - "0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48" + abi: abis/ERC20.json + events: + - Transfer + start_block: 18600000 + end: + mode: pinned + block: 18600010 +datasets: + - id: usdc + snapshot: .chainplot/snapshots/usdc/usdc_transfer.parquet +queries: + - id: transfer_count + file: queries/transfer_count.sql + dataset: usdc + title: Transfers observed + - id: top_transfers + file: queries/top_transfers.sql + dataset: usdc + title: Largest transfers + raw_amount_columns: + - name: value + decimals: 6 + symbol: USDC + label: Amount +dashboards: + - id: transfer-activity + title: USDC transfer activity + description: Every USDC transfer in blocks 18,600,000-18,600,010. + panels: + - query: transfer_count + chart: kpi + title: Transfers + unit: transfers + - query: top_transfers + chart: table + title: Largest transfers + description: Ranked by value; amounts shown at 6 decimals. + span: full +publish_targets: + - id: r2 + type: s3 + bucket: chainplot-test + prefix: transfer-traffic + public_base_url: https://pub-0593f9128f674400bcbbc940cf9f01b1.r2.dev + dataset_license: CC-BY-4.0 diff --git a/examples/transfer-traffic/compose.yaml b/examples/transfer-traffic/compose.yaml new file mode 100644 index 0000000..70858b6 --- /dev/null +++ b/examples/transfer-traffic/compose.yaml @@ -0,0 +1,35 @@ +# Example 1 runtime: our Postgres + producer (CLI + pinned rindexer binary). +services: + postgres: + image: postgres:16-alpine + environment: + POSTGRES_USER: chainplot + POSTGRES_PASSWORD: chainplot + POSTGRES_DB: chainplot + volumes: + - postgres-data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U chainplot -d chainplot"] + interval: 2s + timeout: 5s + retries: 30 + + producer: + build: + context: ../.. + dockerfile: docker/producer.Dockerfile + platform: linux/amd64 + depends_on: + postgres: + condition: service_healthy + env_file: .env + environment: + DATABASE_URL: postgresql://chainplot:chainplot@postgres:5432/chainplot + volumes: + - ./:/workspace + working_dir: /workspace + entrypoint: ["sleep"] + command: ["infinity"] + +volumes: + postgres-data: diff --git a/examples/transfer-traffic/queries/top_transfers.sql b/examples/transfer-traffic/queries/top_transfers.sql new file mode 100644 index 0000000..36e9d67 --- /dev/null +++ b/examples/transfer-traffic/queries/top_transfers.sql @@ -0,0 +1,9 @@ +-- No projected sort column: cp_sortkey() is applied in ORDER BY, so the +-- 78-digit key never reaches the dashboard. +select + value, + tx_hash, + block_number +from usdc +order by cp_sortkey(value) desc +limit 10 diff --git a/examples/transfer-traffic/queries/transfer_count.sql b/examples/transfer-traffic/queries/transfer_count.sql new file mode 100644 index 0000000..aad8c53 --- /dev/null +++ b/examples/transfer-traffic/queries/transfer_count.sql @@ -0,0 +1 @@ +select count(*) as transfer_count from usdc diff --git a/examples/transfer-traffic/tests/usdc.yaml b/examples/transfer-traffic/tests/usdc.yaml new file mode 100644 index 0000000..7bd3b6a --- /dev/null +++ b/examples/transfer-traffic/tests/usdc.yaml @@ -0,0 +1,4 @@ +dataset: usdc +expect: + row_count: 92 + columns: [rindexer_id, contract_address, from, to, value, tx_hash, block_number, block_timestamp, block_hash, network, tx_index, log_index, chain_id] diff --git a/examples/usdc-supply/.env.example b/examples/usdc-supply/.env.example new file mode 100644 index 0000000..d9d4714 --- /dev/null +++ b/examples/usdc-supply/.env.example @@ -0,0 +1,13 @@ +# Archive-capable Ethereum JSON-RPC (mainnet). Required for plan/apply. +# publicnode archive eth_getLogs is 403; use an archive endpoint. +# Copy to .env (gitignored). Never commit real endpoints. +RPC_URL= + +# Postgres for the rindexer adapter. compose.yaml provides this service; +# nothing to install or point at yourself. +DATABASE_URL=postgresql://chainplot:chainplot@postgres:5432/chainplot + +# Producer image tag. Build it once from a chainplot checkout: +# docker build --platform linux/amd64 -t chainplot:local \ +# -f docker/producer.Dockerfile . +# CHAINPLOT_IMAGE=chainplot:local diff --git a/examples/usdc-supply/README.md b/examples/usdc-supply/README.md new file mode 100644 index 0000000..0686c16 --- /dev/null +++ b/examples/usdc-supply/README.md @@ -0,0 +1,51 @@ +# Example 3 — USDC supply changes over a year + +The widest example: a year of Ethereum mainnet, and the only one that uses +indexed filters, a model, and time-series charts. + +**Chain:** Ethereum mainnet · **Source:** USDC (`0xa0b86991…`) `Transfer` +**Range:** blocks 23,400,000–25,987,000 (~1 year, pinned, finalized) +**Events:** ~6.8M (4.8M mints, 2.0M burns) + +## Why it is shaped this way + +- **Indexed filters do the work.** A mint is a `Transfer` from the zero + address and a burn is one to it, so each source filters on a different + indexed position. USDC emits ~82 transfers per block; filtering cuts that by + 99%, which is what makes a year tractable at all. Two sources, because + `eth_getLogs` cannot express "from = 0 OR to = 0" in one call. +- **`block_budget: 400000`.** A year is ~2.6M blocks, far past the 100k + default. 400k keeps each job inside the 30 minute wall clock, so every run + closes and records coverage rather than being killed mid-job. Expect ~7 runs + of `plan` → `apply`; each one resumes where the last finished. +- **`release_mode: results_only`.** The parquet is ~163 MB; the page it feeds + is under a megabyte. This is the default, stated here because the contrast + is the point. +- **A model spanning both datasets.** `models/daily.sql` unions mints and + burns into one row per UTC day. Every dataset is in scope in every query, so + a model may read across them. + +## Run it + +Build the producer image once from a chainplot checkout, then: + +```bash +cp .env.example .env # fill in RPC_URL +docker compose up -d + +# Repeat until a plan reports no ingest work left; each run advances coverage. +docker compose exec producer chainplot plan --intent ingest --json +docker compose exec producer chainplot apply --plan --json +``` + +The run that closes the range also exports and builds. Intermediate runs +ingest only — they cannot build, because the promotion gate refuses anything +short of complete coverage. + +## Files + +| File | Purpose | +|---|---| +| `chainplot.yaml` | Two filtered sources, a model, seven queries, five panel types | +| `models/daily.sql` | One row per day, unioned across mints and burns | +| `queries/` | KPIs, a daily bar series, a cumulative area series, a line series, a top-N table | diff --git a/examples/usdc-supply/abis/ERC20.json b/examples/usdc-supply/abis/ERC20.json new file mode 100644 index 0000000..cef2eb7 --- /dev/null +++ b/examples/usdc-supply/abis/ERC20.json @@ -0,0 +1,12 @@ +[ + { + "anonymous": false, + "inputs": [ + { "indexed": true, "name": "from", "type": "address" }, + { "indexed": true, "name": "to", "type": "address" }, + { "indexed": false, "name": "value", "type": "uint256" } + ], + "name": "Transfer", + "type": "event" + } +] diff --git a/examples/usdc-supply/chainplot.yaml b/examples/usdc-supply/chainplot.yaml new file mode 100644 index 0000000..76278fa --- /dev/null +++ b/examples/usdc-supply/chainplot.yaml @@ -0,0 +1,151 @@ +format_version: 1 +id: usdc-supply +policy: + # A year is ~2.6M blocks, well past the 100k default. 400k keeps each job + # inside the 30 min wall clock, so every run closes and records coverage. + block_budget: 400000 + # Default anyway; stated because this project's parquet is ~163 MB and the + # page it feeds is under a megabyte. + release_mode: results_only +chain_sources: + - id: mainnet + chain_id: 1 + rpc_secret: RPC_URL + finality: + policy: finalized +event_sources: + - id: mint + chain: mainnet + addresses: ["0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48"] + abi: abis/ERC20.json + events: [Transfer] + indexed_filters: + - event_name: Transfer + indexed_1: ["0x0000000000000000000000000000000000000000"] + start_block: 23400000 + end: + mode: pinned + block: 25987000 + - id: burn + chain: mainnet + addresses: ["0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48"] + abi: abis/ERC20.json + events: [Transfer] + indexed_filters: + - event_name: Transfer + indexed_2: ["0x0000000000000000000000000000000000000000"] + start_block: 23400000 + end: + mode: pinned + block: 25987000 +datasets: + - id: mint + snapshot: .chainplot/snapshots/mint/mint_transfer.parquet + - id: burn + snapshot: .chainplot/snapshots/burn/burn_transfer.parquet +queries: + - id: net_change + file: queries/net_change.sql + dataset: mint + title: Net supply change + raw_amount_columns: + - name: net_change + decimals: 6 + symbol: USDC + - id: total_minted + file: queries/total_minted.sql + dataset: mint + title: Total minted + raw_amount_columns: + - name: minted + decimals: 6 + symbol: USDC + - id: total_burned + file: queries/total_burned.sql + dataset: burn + title: Total burned + raw_amount_columns: + - name: burned + decimals: 6 + symbol: USDC + - id: daily_flow + file: queries/daily_flow.sql + dataset: mint + title: Daily mint and burn + raw_amount_columns: + - name: minted + decimals: 6 + symbol: USDC + label: Minted + - name: burned + decimals: 6 + symbol: USDC + label: Burned + - id: cumulative_net + file: queries/cumulative_net.sql + dataset: mint + title: Cumulative net issuance + raw_amount_columns: + - name: cumulative_net + decimals: 6 + symbol: USDC + label: Cumulative net + - id: daily_events + file: queries/daily_events.sql + dataset: mint + title: Daily mint and burn events + - id: largest_mints + file: queries/largest_mints.sql + dataset: mint + title: Largest single mints + raw_amount_columns: + - name: value + decimals: 6 + symbol: USDC + label: Amount +models: + - id: daily + file: models/daily.sql + depends_on: [] +dashboards: + - id: supply + title: USDC supply changes + description: >- + Every USDC mint and burn on Ethereum mainnet over roughly the last year, + indexed directly from Transfer events to and from the zero address. + panels: + - query: net_change + chart: kpi + title: Net issuance + description: Minted minus burned across the window. + - query: total_minted + chart: kpi + title: Minted + - query: total_burned + chart: kpi + title: Burned + - query: cumulative_net + chart: area + title: Cumulative net issuance + description: Running total of mints minus burns. + span: full + - query: daily_flow + chart: bar + title: Daily mint and burn + span: full + - query: daily_events + chart: line + title: Daily event counts + description: How many mint and burn events settled each day. + span: full + - query: largest_mints + chart: table + title: Largest single mints + span: full +publish_targets: + - id: r2 + type: s3 + bucket: chainplot-test + prefix: usdc-supply + public_base_url: https://pub-0593f9128f674400bcbbc940cf9f01b1.r2.dev + dataset_license: CC-BY-4.0 diff --git a/examples/usdc-supply/compose.yaml b/examples/usdc-supply/compose.yaml new file mode 100644 index 0000000..621bbbe --- /dev/null +++ b/examples/usdc-supply/compose.yaml @@ -0,0 +1,45 @@ +# Chainplot ingest runtime: our Postgres + producer with the CLI and the +# pinned rindexer binary. No docker.sock anywhere. +services: + postgres: + image: postgres:16-alpine + environment: + POSTGRES_USER: chainplot + POSTGRES_PASSWORD: chainplot + POSTGRES_DB: chainplot + volumes: + - postgres-data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U chainplot -d chainplot"] + interval: 2s + timeout: 5s + retries: 30 + + producer: + # The producer image is the chainplot CLI plus the pinned rindexer binary. + # It is built from the chainplot repo, not from this project — a scaffolded + # project has no CLI sources to build from. Once, from a chainplot checkout: + # + # docker build --platform linux/amd64 \ + # -t chainplot:local -f docker/producer.Dockerfile . + # + # Point CHAINPLOT_IMAGE at your own tag to use a different build. + image: ${CHAINPLOT_IMAGE:-chainplot:local} + platform: linux/amd64 + depends_on: + postgres: + condition: service_healthy + env_file: .env + environment: + DATABASE_URL: postgresql://chainplot:chainplot@postgres:5432/chainplot + volumes: + - ./:/workspace + working_dir: /workspace + # The image entrypoint is the CLI itself, so a bare `command:` would be + # read as CLI arguments. Hold the container open and drive it with + # `docker compose exec producer chainplot --json`. + entrypoint: ["sleep"] + command: ["infinity"] + +volumes: + postgres-data: diff --git a/examples/usdc-supply/models/daily.sql b/examples/usdc-supply/models/daily.sql new file mode 100644 index 0000000..32ebfaf --- /dev/null +++ b/examples/usdc-supply/models/daily.sql @@ -0,0 +1,19 @@ +-- One row per UTC day, combining both datasets. Amounts stay decimal strings: +-- USDC fits HUGEINT comfortably, but the result is cast straight back to text +-- so nothing downstream sees a narrowed numeric type. +SELECT + d, + sum(minted)::VARCHAR AS minted, + sum(burned)::VARCHAR AS burned, + sum(mint_events) AS mint_events, + sum(burn_events) AS burn_events +FROM ( + SELECT date_trunc('day', block_timestamp) AS d, + value::HUGEINT AS minted, 0::HUGEINT AS burned, 1 AS mint_events, 0 AS burn_events + FROM mint + UNION ALL + SELECT date_trunc('day', block_timestamp), + 0::HUGEINT, value::HUGEINT, 0, 1 + FROM burn +) +GROUP BY d diff --git a/examples/usdc-supply/queries/cumulative_net.sql b/examples/usdc-supply/queries/cumulative_net.sql new file mode 100644 index 0000000..eb50862 --- /dev/null +++ b/examples/usdc-supply/queries/cumulative_net.sql @@ -0,0 +1,5 @@ +SELECT + strftime(d, '%Y-%m-%d') AS day, + (sum(minted::HUGEINT - burned::HUGEINT) OVER (ORDER BY d))::VARCHAR AS cumulative_net +FROM daily +ORDER BY d diff --git a/examples/usdc-supply/queries/daily_events.sql b/examples/usdc-supply/queries/daily_events.sql new file mode 100644 index 0000000..7345be0 --- /dev/null +++ b/examples/usdc-supply/queries/daily_events.sql @@ -0,0 +1,3 @@ +SELECT strftime(d, '%Y-%m-%d') AS day, mint_events, burn_events +FROM daily +ORDER BY d diff --git a/examples/usdc-supply/queries/daily_flow.sql b/examples/usdc-supply/queries/daily_flow.sql new file mode 100644 index 0000000..6f205c6 --- /dev/null +++ b/examples/usdc-supply/queries/daily_flow.sql @@ -0,0 +1,3 @@ +SELECT strftime(d, '%Y-%m-%d') AS day, minted, burned +FROM daily +ORDER BY d diff --git a/examples/usdc-supply/queries/largest_mints.sql b/examples/usdc-supply/queries/largest_mints.sql new file mode 100644 index 0000000..639305c --- /dev/null +++ b/examples/usdc-supply/queries/largest_mints.sql @@ -0,0 +1,4 @@ +SELECT value, "to" AS recipient, block_number, strftime(block_timestamp, '%Y-%m-%d') AS day +FROM mint +ORDER BY cp_sortkey(value) DESC +LIMIT 15 diff --git a/examples/usdc-supply/queries/net_change.sql b/examples/usdc-supply/queries/net_change.sql new file mode 100644 index 0000000..5d7424f --- /dev/null +++ b/examples/usdc-supply/queries/net_change.sql @@ -0,0 +1 @@ +SELECT (sum(minted::HUGEINT) - sum(burned::HUGEINT))::VARCHAR AS net_change FROM daily diff --git a/examples/usdc-supply/queries/total_burned.sql b/examples/usdc-supply/queries/total_burned.sql new file mode 100644 index 0000000..45bb190 --- /dev/null +++ b/examples/usdc-supply/queries/total_burned.sql @@ -0,0 +1 @@ +SELECT sum(burned::HUGEINT)::VARCHAR AS burned FROM daily diff --git a/examples/usdc-supply/queries/total_minted.sql b/examples/usdc-supply/queries/total_minted.sql new file mode 100644 index 0000000..25058b6 --- /dev/null +++ b/examples/usdc-supply/queries/total_minted.sql @@ -0,0 +1 @@ +SELECT sum(minted::HUGEINT)::VARCHAR AS minted FROM daily diff --git a/package.json b/package.json new file mode 100644 index 0000000..8593895 --- /dev/null +++ b/package.json @@ -0,0 +1,48 @@ +{ + "name": "chainplot", + "version": "0.1.0", + "description": "Agent-first toolkit: scoped onchain events → reproducible dataset → static dashboard", + "license": "MIT", + "repository": { + "type": "git", + "url": "https://github.com/chainstacklabs/chainplot" + }, + "private": true, + "type": "module", + "engines": { + "node": ">=22 <27" + }, + "packageManager": "pnpm@11.24.0", + "bin": { + "chainplot": "./dist/cli/main.js" + }, + "files": [ + "dist", + "schemas", + "templates", + "viewer/dist", + "LICENSE", + "NOTICES.md", + "README.md" + ], + "scripts": { + "build": "tsc -p tsconfig.json && pnpm run build:viewer", + "build:cli": "tsc -p tsconfig.json", + "build:viewer": "pnpm --dir viewer install --frozen-lockfile && pnpm --dir viewer run build", + "test": "pnpm build && vitest run" + }, + "dependencies": { + "@aws-sdk/client-s3": "^3.1131.0", + "@duckdb/node-api": "1.5.5-r.4", + "ajv": "^8.20.0", + "commander": "^15.0.0", + "pg": "^8.23.0", + "yaml": "^2.9.0" + }, + "devDependencies": { + "@types/node": "^22.20.2", + "@types/pg": "^8.23.1", + "typescript": "^7.0.2", + "vitest": "^5.0.0" + } +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml new file mode 100644 index 0000000..53e2fa9 --- /dev/null +++ b/pnpm-lock.yaml @@ -0,0 +1,1449 @@ +lockfileVersion: '9.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +importers: + + .: + dependencies: + '@aws-sdk/client-s3': + specifier: ^3.1131.0 + version: 3.1131.0 + '@duckdb/node-api': + specifier: 1.5.5-r.4 + version: 1.5.5-r.4 + ajv: + specifier: ^8.20.0 + version: 8.20.0 + commander: + specifier: ^15.0.0 + version: 15.0.0 + pg: + specifier: ^8.23.0 + version: 8.23.0 + yaml: + specifier: ^2.9.0 + version: 2.9.0 + devDependencies: + '@types/node': + specifier: ^22.20.2 + version: 22.20.2 + '@types/pg': + specifier: ^8.23.1 + version: 8.23.1 + typescript: + specifier: ^7.0.2 + version: 7.0.2 + vitest: + specifier: ^5.0.0 + version: 5.0.0(@types/node@22.20.2)(vite@8.3.0(@types/node@22.20.2)(yaml@2.9.0)) + +packages: + + '@aws-sdk/checksums@3.1001.0': + resolution: {integrity: sha512-6uTniZc87q+B5eXouGTl+7Tmc482rEeCcvxpsvREP8EfF0gvloRZ41UOA9sbSJlyy8TbqIBXb3kKfKarEArUQA==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/client-s3@3.1131.0': + resolution: {integrity: sha512-jMw3q5sYNvWJRngkNwdS3H2WPBcdqhdrOez+7fBUB26fpH/2+TaJcDdlzdjgLTMhFTq2eOyXBSy1T/4A6CL/Lw==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/core@3.978.0': + resolution: {integrity: sha512-2yX9LUmxPklVjSGTb8dfnWRJSiFQ3TeH2nn7G1mdKHTfnabzF0+gfrS8rYfLWmZrQ8A3mEcxMJjRc51dL5KWaA==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-env@3.972.71': + resolution: {integrity: sha512-JN+JHruYZw3GUZB8YGAlDk4wTDPOEAEEdEzj5nS0xodWR4smzHsN7PnK2j6IeOsDIj2aqua5DSbhXl9Gtf90FQ==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-http@3.972.73': + resolution: {integrity: sha512-uyYYnJOnlis8uQzaYGPd7N1JoioCoNpXgnkXYixsWJXHXgXyYi8WXJSDfofxJeWfQIGWLe2Nwyq60Uc7MZdVOg==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-ini@3.973.16': + resolution: {integrity: sha512-i++ly+0Uxa+u3ebSSyr0S/3CFhFJDxCXT3+Zj+mW2bXenEx5bKGCdTIKFu39SgXBNhWDjex/8cXUx9MUTMCrTw==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-login@3.972.78': + resolution: {integrity: sha512-eUtswnXu0+Ii9ieRK+0L7aPFV3Z/dnW2VntJzjBP9xs8s+8p5nBNuymIXtXwZ+5r5+XJP3e32nMkuZ/r0HozEA==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-node@3.972.83': + resolution: {integrity: sha512-jdso7ejzfRnatxMUZK4S/U6KbaDPCvfIV4XL+IQAPFDBt5rj5Fq595euqlK8Le4lNCMFR9oUpt+1l0aMgaayOQ==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-process@3.972.71': + resolution: {integrity: sha512-lYmXJa4gvq4xN1lrT5NiP5vIYYKcGWAdj8y+8o6dlcateB5eF3Dn8DtmjjHKfMBrTPAMr2pebIiX/UOj8c1/UA==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-sso@3.973.15': + resolution: {integrity: sha512-6Jhcf4v0pSFdjk1EW2kvzuEBKD+UZ2uNcHUIglKKLndD20YhvkL2kdmDOV5/j4mYuWWwe/a1FQ1aomU86/Cg5Q==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-web-identity@3.972.77': + resolution: {integrity: sha512-uylIQSUWpfLuH2LovxEEfwzJGM/SabLOfLMg6YXu/E8jJEKUdpdILCVCQCdFvHyu/7dLJOHPMfrSwduxO56NkQ==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/middleware-sdk-s3@3.972.76': + resolution: {integrity: sha512-NfnTkVUTBKTBuBgqaapFK9r3YdkKt1b2oRvgLzZq91bwNKh6ZS0S7sEcheguttREaL4iyfs/xQnqD7Z7AsWSsA==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/nested-clients@3.997.45': + resolution: {integrity: sha512-mooq9Q+jLa18VoM7HouczmslZU60iiB0aKc/Ztnq/luIL1ud0z4DnYprLR/ZO1gp331S9tJctM1HZr7u6YKBXQ==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/signature-v4-multi-region@3.996.46': + resolution: {integrity: sha512-L+2xZTye/2T96f3lwCws0Zw6GG2JHZW9e8FpVgGBeeExSKyeoZ6CWRpBml/7DNiK/O26jrgPM9F+Ay8VkgzUWQ==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/token-providers@3.1129.0': + resolution: {integrity: sha512-Sbl3rpzQdsG4ZK2zh0JWUYyZPKKorJlVOddA2T0DVbKJFrsW8J6wgnslxxUH04+WaBMr4A1HzJZvZX0xUvkniA==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/types@3.974.5': + resolution: {integrity: sha512-LkwLL2BLbC6wNNm4JaH9mbEqBMdOZCct6VAYqhdN4U1xrWM+fUJQEfbHwQgDypapOWTRtlk25akb5afM0P8CIQ==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/xml-builder@3.972.40': + resolution: {integrity: sha512-wlFmCIGUlwF4zx/kncw+bmxTQh1HeSJq4mYV/V5cZUSJadDP3kXvGW8Rn21cimj/7y9ju+47oYWXi97vF7czaA==} + engines: {node: '>=20.0.0'} + + '@aws/lambda-invoke-store@0.3.0': + resolution: {integrity: sha512-sl4Bm6yiMNYrZKkqqDFWN0UfnWhlS8ivKxrYl+6t0gCLrqr8y3B2IqZZbFRkfaVVp7C/baApyh71P+LeE1A2sQ==} + engines: {node: '>=18.0.0'} + + '@duckdb/node-api@1.5.5-r.4': + resolution: {integrity: sha512-8v0CZNo7aM6GQCNUHERGTrZWIfss8xKzZPF3ACNhPdMMrt7UjkNI5nRQ9FTFO7Yx8ghxYxVpdotmBBMQ5vXUOA==} + + '@duckdb/node-bindings-darwin-arm64@1.5.5-r.4': + resolution: {integrity: sha512-4OdO3pkoJzAZDnZ2iyehi67XL4+WqHPCGYWbyAsYyhGJS+WscGrAnFhMhHudMM2XF8pIEM2tuOKMmwWnTNN2wQ==} + cpu: [arm64] + os: [darwin] + + '@duckdb/node-bindings-darwin-x64@1.5.5-r.4': + resolution: {integrity: sha512-WHg3E+TupdujG31LGujMMcmtPIdW6rqRHeihlKgJR8EMetHycoYkwYxRud0niitJvS+uV0du/6pdemzs3HG9GQ==} + cpu: [x64] + os: [darwin] + + '@duckdb/node-bindings-linux-arm64-musl@1.5.5-r.4': + resolution: {integrity: sha512-Tswnf+/XWpOcFJNUUu1fkFIX/su4tMcnNz0ZV0Nru3/CkJKIMwBh+VL4SM9YV4zPK90GC4Lm8gzafjc1qDmfyQ==} + cpu: [arm64] + os: [linux] + libc: [musl] + + '@duckdb/node-bindings-linux-arm64@1.5.5-r.4': + resolution: {integrity: sha512-VIeHMpYAKpGiWZ4QYsOhODAHDEcXcn089aKty0MFsMEKaEfRJWixKwnwXy/ba2/wwFmnPiSFhKNqygj2BrQbqw==} + cpu: [arm64] + os: [linux] + libc: [glibc] + + '@duckdb/node-bindings-linux-x64-musl@1.5.5-r.4': + resolution: {integrity: sha512-h0ixrgGHtHh+C/Fu1eAL9hX4iCf8yuyJN6Y24Gh8axXXP8UuSh0rQRAQUQTYarQ8pm2qSG/67ZYjyYYydD+J/w==} + cpu: [x64] + os: [linux] + libc: [musl] + + '@duckdb/node-bindings-linux-x64@1.5.5-r.4': + resolution: {integrity: sha512-EY+CL/4h8MQZd9MxTBq+98m3U9osmvHBwhE9b5fMWQGt2I6p1j8fvX0SXiwy9KBfQIbjVUOf5XvGdXncI54KSg==} + cpu: [x64] + os: [linux] + libc: [glibc] + + '@duckdb/node-bindings-win32-arm64@1.5.5-r.4': + resolution: {integrity: sha512-uorAnySIMwWkRDuUhM1yTDxWIislnX8mndhFIw6r4VKhVW61HEOOMX5oAgkSQFII4RNWpC5PCcUAEhSATgNvVQ==} + cpu: [arm64] + os: [win32] + + '@duckdb/node-bindings-win32-x64@1.5.5-r.4': + resolution: {integrity: sha512-X9XGcWQ10P3mvUIaMXXk2bi94Cow7b/ziTPMKxJ0U8U3wQPxsLzEUP7D7C/KrBWINl5am2k+5SkDVyA/THUgPg==} + cpu: [x64] + os: [win32] + + '@duckdb/node-bindings@1.5.5-r.4': + resolution: {integrity: sha512-n+4hEfjp4vny3BuWn5p1Gh5CzHaPRxoI8TzTytxL1GMlIKKrXcg/o6sSAjrsROUXGAM4WQCiWPLKnPsjJJSggg==} + + '@jridgewell/resolve-uri@3.1.2': + resolution: {integrity: sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==} + engines: {node: '>=6.0.0'} + + '@jridgewell/sourcemap-codec@1.6.0': + resolution: {integrity: sha512-T7jf+5zgsZHwNJ4lvQ7/aezbyk0nNX+zJVWpmHA7VYsEx7a7qr5Rg5IbtJFqkgze5Y2sruq1RUY8Q837Od7iFw==} + + '@jridgewell/trace-mapping@0.3.31': + resolution: {integrity: sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==} + + '@oxc-project/types@0.149.0': + resolution: {integrity: sha512-Efcc+iF0j3Bf67YjEqIqWXbX5XddXoK/Mw4K1/JuXwRCZ8N16VR7iT23nlCc9XrveFVh/E5Rqs2StT0V8v9LdA==} + + '@rolldown/binding-android-arm-eabi@1.2.8': + resolution: {integrity: sha512-tN5aztYkKCte4i5SIrrz5yK/HMjEuCqCSCJa418jOV8tZ1cBY3YF2otxB1ktPxzsLA1BeTqwapK0bfjxNvHJVw==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm] + os: [android] + + '@rolldown/binding-android-arm64@1.2.8': + resolution: {integrity: sha512-dIYTWl9XprMUiQFoc55KUyk/oS8SKYH3zFl0LTR7RT0Xj4hgSVyuJcroH8JUu8RcpF8fTB6E0aOwCkZoYPcDSQ==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [android] + + '@rolldown/binding-darwin-arm64@1.2.8': + resolution: {integrity: sha512-PCSDQGXD2IyTEFrcgPyBM8jJuGmrbCMuoIOXdbEGVemruKACXoLQJrb+A45Z0L5t1RQkdfJprAYPkikbh7dzdA==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [darwin] + + '@rolldown/binding-darwin-x64@1.2.8': + resolution: {integrity: sha512-Uk7lRsGhPFHVX/sAUC6D5H9Ol30dFHd6iquokll2th3LpdJ3F5CzQB+7DHn0Ri2mG+U7k2zXiPHDrwZenXhwSA==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [x64] + os: [darwin] + + '@rolldown/binding-freebsd-x64@1.2.8': + resolution: {integrity: sha512-DjszaTEVogPqA5bYzsEeqDCQxbcp2fexQwKcRspYji2yzR68fCf+e4fx6kBSRDwX5/brZaHw/hWS9+A/+/w9sQ==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [x64] + os: [freebsd] + + '@rolldown/binding-linux-arm-gnueabihf@1.2.8': + resolution: {integrity: sha512-zmwa7FTmdzB6aaEEuuls18H6Ap5JmJPSoPTuXixeJZV6tG40SyLkApQtz1g8ptZtiEKqj9OM0oNLPh1AgvE31Q==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm] + os: [linux] + + '@rolldown/binding-linux-arm64-gnu@1.2.8': + resolution: {integrity: sha512-KdYQDPHwJVnbFwdTGMgxsI9SqblBlz6STGM+w1We/d5B8OWWidYH0MwkU/uA1wM5fIpO2MkOVxXrNzzuZhw9ew==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [linux] + libc: [glibc] + + '@rolldown/binding-linux-arm64-musl@1.2.8': + resolution: {integrity: sha512-jFJTifHnNPY+yzOoNZQfSIysrVyXzEQPhPnOUjmD1bcQGHH6s7c8cViKWar8YplQImE5N9JRqMCLrM2CdxOrZA==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [linux] + libc: [musl] + + '@rolldown/binding-linux-ppc64-gnu@1.2.8': + resolution: {integrity: sha512-FhiOziBDWPBjbcmRzfLyIJnaP7AVMFXT7YCXPjXxj7wKU3vx24RjrCNN/zjvVa+N2vVoHJwCoUBvsrN/DG3zIA==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [ppc64] + os: [linux] + libc: [glibc] + + '@rolldown/binding-linux-s390x-gnu@1.2.8': + resolution: {integrity: sha512-WnHfADMzOV2Y55wlx1hzzQnar/wDt/VdvWSD99r18Mz9ylNieIGOkRx3UV21h7m/eJvjySYJkO26VvGNFkwsIQ==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [s390x] + os: [linux] + libc: [glibc] + + '@rolldown/binding-linux-x64-gnu@1.2.8': + resolution: {integrity: sha512-H9tRr5ibfXFVLxbPOseVewewFpl28zcEdjRDt2FTUZU7odxP0gEv1ki4/kGmcGOh78oRwZuuQllGLZ9zTJp84g==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [x64] + os: [linux] + libc: [glibc] + + '@rolldown/binding-linux-x64-musl@1.2.8': + resolution: {integrity: sha512-UefiqfM3D6IVNlZ8tSGs9+Ejjud2T+oxO0IHADU45Y+lyEjD2dVFyZHbkfX0LUb5Zugo/oIv1eCO/KVYhgYJYA==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [x64] + os: [linux] + libc: [musl] + + '@rolldown/binding-openharmony-arm64@1.2.8': + resolution: {integrity: sha512-637Ke4kWSy6rp9cxQ9gMOXlxPgIw/c1beASV4M//3+9I4uwBVOOl74G+e3zyU3u19U7RkRl/HuewixZ/Z6+Rjg==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [openharmony] + + '@rolldown/binding-win32-arm64-msvc@1.2.8': + resolution: {integrity: sha512-xWBkPOF1Q9k/Gv1nQXnVdLxKu74jXppuOM4Z3mnypVUJJJwLsMl7hNJGRAUJoG8A5MgOI1ACKM+wBFxSJzKy4A==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [win32] + + '@rolldown/binding-win32-x64-msvc@1.2.8': + resolution: {integrity: sha512-uz2ZvfgXbxqNwijjjbxrnvALwpyODDcgc1T1N8N3rf/DXKQmaFwmB4LX4yyjggpwN2obdQLb2rgirX5ffCWYng==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [x64] + os: [win32] + + '@rolldown/pluginutils@1.0.1': + resolution: {integrity: sha512-2j9bGt5Jh8hj+vPtgzPtl72j0yRxHAyumoo6TNfAjsLB04UtpSvPbPcDcBMxz7n+9CYB0c1GxQFxYRg2jimqGw==} + + '@smithy/core@3.34.1': + resolution: {integrity: sha512-dLcOUxz8YCv1RZUMKq6GbyUf95pLbrqh34bPvpCZ1+CByFF31BEAFewZjsGCnVsZTKdThNENfGyAgk2TJqVwSw==} + engines: {node: '>=18.0.0'} + + '@smithy/credential-provider-imds@4.5.2': + resolution: {integrity: sha512-A9uSdn72ozbRUSit0eib0TW7nXuNPlaeM0zcGkJ+nE6tFcSDbnmtwoxbTCFBukVQcszDAyvsd7+rTduPTXpygg==} + engines: {node: '>=18.0.0'} + + '@smithy/fetch-http-handler@5.8.0': + resolution: {integrity: sha512-ycSJu3tFAQ4v04CBB0agqFMVsSQ1iG3yw+SpgxRqKfaURpQD4CZ8Wn0zPMmSnOuTpTh65Vz+EA0rMrw089wvkA==} + engines: {node: '>=18.0.0'} + + '@smithy/node-http-handler@4.12.1': + resolution: {integrity: sha512-ThMkboGeONWXAelq9FvGsuJC4rOi+qyC4/zhUF58xYpxUg5sQKx2VXZYJmtNjr4dSuBJ1HeJXETQILCz3wOHvw==} + engines: {node: '>=18.0.0'} + + '@smithy/signature-v4@5.7.3': + resolution: {integrity: sha512-7ImGm+FkHRLcBaRttIAMZ6bzJZWb2cJGoYjq46F2UjycujWzrL9GEN9h4w7eQyXJYnltrUhxbbieBAIRrdqpow==} + engines: {node: '>=18.0.0'} + + '@smithy/types@4.18.0': + resolution: {integrity: sha512-CgB6HHWer/vrKps24ulRIbpcpb7K4xAU7SkZ7YHzBPlwHsvsrCJFEXK421s+cJzX+ZrqtA/TuU5w1HzI7k9N8A==} + engines: {node: '>=18.0.0'} + + '@types/chai@5.2.3': + resolution: {integrity: sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==} + + '@types/deep-eql@4.0.2': + resolution: {integrity: sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw==} + + '@types/estree@1.0.9': + resolution: {integrity: sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==} + + '@types/node@22.20.2': + resolution: {integrity: sha512-xlvWf4Vs9n1PEVYwP1n4vvG07M6y8WgvJ2t0vbrWTmijsIHp1cS+uJ2kMIRdY3nHZK0nCYKrPeD171+SzF4/zw==} + + '@types/pg@8.23.1': + resolution: {integrity: sha512-fKVHpikPdg4GKks3JuLEhvwSyvwzF23hnabPy6DD8ljVbC7+6J5dQzdv4arV6jqq57djnMgs1HKBxX4P8aBI3A==} + + '@typescript/typescript-aix-ppc64@7.0.2': + resolution: {integrity: sha512-MTKKkWB7p/0E9xi1d1tHtZ5PiLkGEMIq88pK2CubZjOsLtYTLqhgIgi6zepFa+9GHZ6h05NMCkQxGKiPXMxXtQ==} + engines: {node: '>=16.20.0'} + cpu: [ppc64] + os: [aix] + + '@typescript/typescript-darwin-arm64@7.0.2': + resolution: {integrity: sha512-gowzar9MwS/aRWp6f3a4KUqzRjAZjOsmGNCM6LcTgXum+dBfgsBVMN+AgvOCCbguXyick6LJhpBszxMebJ8syA==} + engines: {node: '>=16.20.0'} + cpu: [arm64] + os: [darwin] + + '@typescript/typescript-darwin-x64@7.0.2': + resolution: {integrity: sha512-SZ9xZInqApNlNGc9s0W1VSsktYSOe9cFqNOIqmN1Gs8SmkjKZYFt017G4VwPxASInODuAdbTW7sXiFUf893RgA==} + engines: {node: '>=16.20.0'} + cpu: [x64] + os: [darwin] + + '@typescript/typescript-freebsd-arm64@7.0.2': + resolution: {integrity: sha512-W5NH4y/J0plIIS5b2xvTEkU7JFxyqdMAOgf+Ilhl0vHQXKO5dZoxd+C/jEtq56c4F3wk71RB4BMRQ2XdI+bwYQ==} + engines: {node: '>=16.20.0'} + cpu: [arm64] + os: [freebsd] + + '@typescript/typescript-freebsd-x64@7.0.2': + resolution: {integrity: sha512-UMGDx5sTpzNw3WiPebH7l90IWfJggEd+egHt/q6p7/Cm3zqoV7VxkGXt+3DxPIw8CcmvAB0j3sVVfbhX+M4Tpw==} + engines: {node: '>=16.20.0'} + cpu: [x64] + os: [freebsd] + + '@typescript/typescript-linux-arm64@7.0.2': + resolution: {integrity: sha512-Qh4eU4/y3yDjnfjjyPYihMj5/ODIlmt+Bzu17OI+fiSRDW57QmU5SiN63exPRNJPKUzcc1INa1NXdrJ+MqHjUQ==} + engines: {node: '>=16.20.0'} + cpu: [arm64] + os: [linux] + + '@typescript/typescript-linux-arm@7.0.2': + resolution: {integrity: sha512-gffT3xPz9sR7j/YJExkyPntrI0P2EP9XbOyWzth2/Gs0RstK+90RBcO0ncXoXy/beYll1SXw846Nf2zdnEz0QQ==} + engines: {node: '>=16.20.0'} + cpu: [arm] + os: [linux] + + '@typescript/typescript-linux-loong64@7.0.2': + resolution: {integrity: sha512-uEHck9i8hoAzXPiYRib1O7miOnz23SxIeVl6F4LXox+qov1K35jHcEW6VHKvZI+pyvl7fZEP4MCU5LYvIq1GuQ==} + engines: {node: '>=16.20.0'} + cpu: [loong64] + os: [linux] + + '@typescript/typescript-linux-mips64el@7.0.2': + resolution: {integrity: sha512-R4KvAMnE43W5Qeqb0Ly56O3mWMWIAgsMyz36DCaycd5nbg/9kzm0liw3JocfRqyJY0KPmzFjbswozXyW0DnIYA==} + engines: {node: '>=16.20.0'} + cpu: [mips64el] + os: [linux] + + '@typescript/typescript-linux-ppc64@7.0.2': + resolution: {integrity: sha512-DORx5b3sd/4S7eayxm4FQv+A7CrkUIGRaHiwI8oiHTAI1fAPWhF4J0vAlkC8biAlHSVVwxMQ3tjZ2/DVbnQiiA==} + engines: {node: '>=16.20.0'} + cpu: [ppc64] + os: [linux] + + '@typescript/typescript-linux-riscv64@7.0.2': + resolution: {integrity: sha512-wf0jqEDOjrPRnKwYRyyJDRo11KMbvMFrU+q4zqKyChODBzvlkbhNQfKvLxQCcwTpdDaXSHZTVuh0JoCrKCUMHQ==} + engines: {node: '>=16.20.0'} + cpu: [riscv64] + os: [linux] + + '@typescript/typescript-linux-s390x@7.0.2': + resolution: {integrity: sha512-IkwJc3L7yhytWd/ewjyxNDfOmswCm9GWMJT/ue/dU4aZNbwZeYAetq42VyLmsmSjvoX7z74X6ZaYCtzAr0EuGw==} + engines: {node: '>=16.20.0'} + cpu: [s390x] + os: [linux] + + '@typescript/typescript-linux-x64@7.0.2': + resolution: {integrity: sha512-EYdf2cNg7rgCWJnxCdJ+F3V39O8ihb37eHAu1LK8oAFizgTQbPOK7zHHXbPt8rX24COqODXeI3sIf0fCXG7H/A==} + engines: {node: '>=16.20.0'} + cpu: [x64] + os: [linux] + + '@typescript/typescript-netbsd-arm64@7.0.2': + resolution: {integrity: sha512-+polYF4MF04aPpO5FTkHran9yUQDSXqy5GiSDKpsll5jy3l3+g9QLhpf39T+ePtefhXLOGrLl0QIjkQP6VnelA==} + engines: {node: '>=16.20.0'} + cpu: [arm64] + os: [netbsd] + + '@typescript/typescript-netbsd-x64@7.0.2': + resolution: {integrity: sha512-8YIT0EHM/3dq10ZOVF/A7pc/YSMtbcecct4rWtexrnSCHOPcpC2KTLXfTCR6vDpnSiY12heNb1GiN/wu+T/FyA==} + engines: {node: '>=16.20.0'} + cpu: [x64] + os: [netbsd] + + '@typescript/typescript-openbsd-arm64@7.0.2': + resolution: {integrity: sha512-APT8+ClYnuYm1u9+kgGXoMj2VzWzcymwh2gNSQVySHfkRDGOTVkoWLjCmOQSaO+PoqQ57B0flRp9SA+7GnnkzQ==} + engines: {node: '>=16.20.0'} + cpu: [arm64] + os: [openbsd] + + '@typescript/typescript-openbsd-x64@7.0.2': + resolution: {integrity: sha512-yX7s+Q0Dln0Dt9tEzZsAjXXR/+ytBM7AlglaqyeMPxQszJ1JhlJdZ6jLA+IzldHtflX81em7lDao1xXu+aRRkg==} + engines: {node: '>=16.20.0'} + cpu: [x64] + os: [openbsd] + + '@typescript/typescript-sunos-x64@7.0.2': + resolution: {integrity: sha512-dLJDGaLZ1D4HPQn62u1n8mBDkJREwMsAkCdkwd4Ieqw+x3TUyTsqY0YiBCtE6H6OzzgGk3iuZ3vFWRS+E8/d1g==} + engines: {node: '>=16.20.0'} + cpu: [x64] + os: [sunos] + + '@typescript/typescript-win32-arm64@7.0.2': + resolution: {integrity: sha512-Gyl1Vy6OsWesLzmq+EP0Fb7b4Nid5232AvcA2SFcdYreldpNtYFFofPjnt62y9hQy7VTaZp65ICJjuAQRaVcIQ==} + engines: {node: '>=16.20.0'} + cpu: [arm64] + os: [win32] + + '@typescript/typescript-win32-x64@7.0.2': + resolution: {integrity: sha512-0BQ3HkAHHlKLSp1qRvf3SUhGpGsDuhB/jgFw75guyqbxJqEaS0Cw/VFO8i2nHglJUzQCRtMMR/IBAKE3ETMC4g==} + engines: {node: '>=16.20.0'} + cpu: [x64] + os: [win32] + + '@vitest/mocker@5.0.0': + resolution: {integrity: sha512-66PGTMIiVJP3t4a5yxU9qPtf7MdTBs8jmToMvy+HVflB3Yy13WJZTtPePdvU+wjRV02SKK5doLbSA6o9pwOmiA==} + peerDependencies: + msw: ^2.4.9 + vite: ^6.0.0 || ^7.0.0 || ^8.0.0 + peerDependenciesMeta: + msw: + optional: true + vite: + optional: true + + '@vitest/spy@5.0.0': + resolution: {integrity: sha512-uy+luWBAPw9XfthoHi5AkfHUnuPYEESjl0p/r+meoBnU8bxg5GDQ3Ey8MjcJ6sqahkL4PFyrvfMJJBw7LbU06g==} + + ajv@8.20.0: + resolution: {integrity: sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==} + + assertion-error@2.0.1: + resolution: {integrity: sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==} + engines: {node: '>=12'} + + bowser@2.14.1: + resolution: {integrity: sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg==} + + chai@6.2.2: + resolution: {integrity: sha512-NUPRluOfOiTKBKvWPtSD4PhFvWCqOi0BGStNWs57X9js7XGTprSmFoz5F0tWhR4WPjNeR9jXqdC7/UpSJTnlRg==} + engines: {node: '>=18'} + + commander@15.0.0: + resolution: {integrity: sha512-z67u4ZhzCL/Tydu1lJARtEZYWbWaN7oYLHbsuzocr6y4N6WZAagG3RQ4FW61V1/0+jImpj293XfrcYnd1qxtPg==} + engines: {node: '>=22.12.0'} + + detect-libc@2.1.2: + resolution: {integrity: sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==} + engines: {node: '>=8'} + + es-module-lexer@2.3.2: + resolution: {integrity: sha512-poHGpORABojJJucnV9KbOavETW8lBVnphkW77ER5/BQ5Fz7oXSoCNek7IH3vR5nRjdsEz926ibFYX8KtLQmdyw==} + + estree-walker@3.0.3: + resolution: {integrity: sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==} + + expect-type@1.4.0: + resolution: {integrity: sha512-KfYbmpRm0VbLjEvVa9yGwCi9GI34xvi7A/HXYWQO65CSD2u3MczUJSuwXKFIxlGsgBQizV9q5J9NHj4VG0n+pA==} + engines: {node: '>=12.0.0'} + + fast-deep-equal@3.1.3: + resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} + + fast-uri@3.1.7: + resolution: {integrity: sha512-dOvZVzjdZdz7phd9v6jCbwxrBW3fK6n8Rc0CtdmM4bumzMnxywBYhuph6J819RRw/ku+rLbelwfMunktuzVVHg==} + + fdir@6.5.0: + resolution: {integrity: sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==} + engines: {node: '>=12.0.0'} + peerDependencies: + picomatch: ^3 || ^4 + peerDependenciesMeta: + picomatch: + optional: true + + fsevents@2.3.3: + resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} + engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} + os: [darwin] + + json-schema-traverse@1.0.0: + resolution: {integrity: sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==} + + lightningcss-android-arm64@1.33.0: + resolution: {integrity: sha512-gEpRTalKdosp4Bb8qWtc2iOgE5SeIHlpS1up9bFq2wAyYhl1UdTObYiHe98zEM9SQvSoqQZ1IQD0JNpg3Ml5pg==} + engines: {node: '>= 12.0.0'} + cpu: [arm64] + os: [android] + + lightningcss-darwin-arm64@1.33.0: + resolution: {integrity: sha512-Sciaz8eenNTKn9b3t7+xr0ipTp9YxKQY4npwQ3mrRuL0BAVHBLyZxofhaKBAVtzmtRZ/zTyo0/to4B1uWG/Djg==} + engines: {node: '>= 12.0.0'} + cpu: [arm64] + os: [darwin] + + lightningcss-darwin-x64@1.33.0: + resolution: {integrity: sha512-Z5UPAxzrjlWNNyGy6i65cJzzvgJ5D3T6wMvs+gWpY9d7qRhANrxqAp6LhxIgZhWEw18RfJTGcRxjuLIBr+m8XQ==} + engines: {node: '>= 12.0.0'} + cpu: [x64] + os: [darwin] + + lightningcss-freebsd-x64@1.33.0: + resolution: {integrity: sha512-QQM/Ti/hQajJwCY+RiWuCZ9sdtI/XQk7nDK5vC8kkdwixezOlDgvDx7+RT+QjK6FcFT4MpsuoBnHIo/O3StRRg==} + engines: {node: '>= 12.0.0'} + cpu: [x64] + os: [freebsd] + + lightningcss-linux-arm-gnueabihf@1.33.0: + resolution: {integrity: sha512-N7FVBe6iS24MlM6R/4RBTxGhQheZGs7tiQ9U32UtF75NzP5Q7xWPRqLBCKxlRQRk3rY1jCIPLzx7WzOhuUIRLQ==} + engines: {node: '>= 12.0.0'} + cpu: [arm] + os: [linux] + + lightningcss-linux-arm64-gnu@1.33.0: + resolution: {integrity: sha512-j2v/itmy4HlNxlc6voKXYgBqNi0Ng2LShg4z7GufpEgs05P+2suBVyi9I6YHq5uoVFx9ETin3eCEhLVyXGQnKg==} + engines: {node: '>= 12.0.0'} + cpu: [arm64] + os: [linux] + libc: [glibc] + + lightningcss-linux-arm64-musl@1.33.0: + resolution: {integrity: sha512-yiO5ROMuYQgXbC60yjZU5CYSFZGKXL0HFATXt9mHJn1+zW55oCtMI9NfcVhYLMFDL7gV7oBPon/EmMMGg2OvtQ==} + engines: {node: '>= 12.0.0'} + cpu: [arm64] + os: [linux] + libc: [musl] + + lightningcss-linux-x64-gnu@1.33.0: + resolution: {integrity: sha512-ar+Ju7LmcN0Jo4FpL4hpFybwNG9/3A/Br5KW2n2jyODg3MEZXaDYADdemoNS+BDNfMgKvylJLj4S5tyRActuAg==} + engines: {node: '>= 12.0.0'} + cpu: [x64] + os: [linux] + libc: [glibc] + + lightningcss-linux-x64-musl@1.33.0: + resolution: {integrity: sha512-RYiYbkokw0trfKqqzfF55lginwEPrD3OJDfTuJzFs1MK6iFnDenaz1fqLLtX4ITG3OktJQXOeTaw1awrBAlZPw==} + engines: {node: '>= 12.0.0'} + cpu: [x64] + os: [linux] + libc: [musl] + + lightningcss-win32-arm64-msvc@1.33.0: + resolution: {integrity: sha512-1K+MPfLSFVpphzpdbfkhlWk6wBrTObBzS2T6db10PNOZgR9GoVsAWzwNyuhUYYbTp23j+4RrncfujZ4uAzXvwA==} + engines: {node: '>= 12.0.0'} + cpu: [arm64] + os: [win32] + + lightningcss-win32-x64-msvc@1.33.0: + resolution: {integrity: sha512-OlEICDx/Xl0FqSp4bry8zFnCvGpig3Gl4gCquvYwHuqJKEC1+n9NgDniFvqHGmMv1ZkqDJrDqKKSykTDX+ehuA==} + engines: {node: '>= 12.0.0'} + cpu: [x64] + os: [win32] + + lightningcss@1.33.0: + resolution: {integrity: sha512-WkUDrojuJs0xkgGf2udWxa3yGBRxPtxUkB79i6aCZLRgc7PM8fZe9TosfPDcvEpQZbuFASnHYmRLBLUbmLOIIA==} + engines: {node: '>= 12.0.0'} + + magic-string@1.3.1: + resolution: {integrity: sha512-rm91zr2Ou+XueDTohjQQjdQEcYM6zVi8KVUCG8Ec3vHwUEKrhSdCNyfuIywkA6hcCAteIn0ZOtAHA6eGpiX+Pg==} + + nanoid@3.3.19: + resolution: {integrity: sha512-Y2tUNy4ouw6tq5oDSKeQYGOyhkUBhNOcGV/02KC+6kd9eDGqdZd++mjMiIDilrBYvjEnCYvVtsuHCuP+okSfug==} + engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} + hasBin: true + + obug@2.2.1: + resolution: {integrity: sha512-XrsrhT5sybtKI6wakr2SPOlGZWWYbUXZ7a0jT8/QOeAPau+1X/bSegNe5YR75oJmEZQbKningirmGOEJCIk61Q==} + engines: {node: '>=12.20.0'} + + pg-cloudflare@1.4.0: + resolution: {integrity: sha512-Vo7z/6rrQYxpNRylp4Tlob2elzbh+N/MOQbxFVWCxS7oEx6jF53GTJFxK2WWpKuBRkmiin4Mt+xofFDjx09R0A==} + + pg-connection-string@2.14.0: + resolution: {integrity: sha512-XwWDGcLRGCXAR8F/AM5bG7Q+A3Wm2s6QeEjlOKZLlH3UYcguiqCWKyWXVag5TLTIjR7oOJUY8kcADaZgWPyLeg==} + + pg-int8@1.0.1: + resolution: {integrity: sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==} + engines: {node: '>=4.0.0'} + + pg-pool@3.14.0: + resolution: {integrity: sha512-gKtPkFdQPU3DksooVLi9LsjZxrsBUZIpa+7aVx+LV5pNh0KzP4Zleud2po+ConrxbuXGBJ6Hfer6hdgpIBpBaw==} + peerDependencies: + pg: '>=8.0' + + pg-protocol@1.16.0: + resolution: {integrity: sha512-sILXutLVjCLjcDuOmvhX5e2Z4cS5qG/6Bu3VkpFwdf/633ElGLpEh9bgmuI5I4sqKqkifQiGyiCcx1HdtrK7tg==} + + pg-types@2.2.0: + resolution: {integrity: sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==} + engines: {node: '>=4'} + + pg@8.23.0: + resolution: {integrity: sha512-Ip2EQCngowJLGOfCwkFhPXU7/ljlhn6Rxlmy4XYfL2Y+vyRM59+8uR2xqRWKdYmbXmxCFOAmKxBuSUCdF34qLg==} + engines: {node: '>= 16.0.0'} + peerDependencies: + pg-native: '>=3.0.1' + peerDependenciesMeta: + pg-native: + optional: true + + pgpass@1.0.5: + resolution: {integrity: sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==} + + picocolors@1.1.1: + resolution: {integrity: sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==} + + picomatch@4.0.7: + resolution: {integrity: sha512-qcJu88Q2IWqJsDD529JKMdwGm/dvInW4HvQnRwiH9JtihJvzGOscDtHE3x1pBKeUOTysQ8kVmLnJ2kJu7yhcGA==} + engines: {node: '>=12'} + + postcss@8.5.28: + resolution: {integrity: sha512-RRuzqDtt5Y9h3quz5hWhK+TPnsmVs6WwSU6LkJMeY4HstUEDuYTG8UJSdawMRzmzAtV+KEoG8N3Qg2qLy5vM/A==} + engines: {node: ^10 || ^12 || >=14} + + postgres-array@2.0.0: + resolution: {integrity: sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==} + engines: {node: '>=4'} + + postgres-bytea@1.0.1: + resolution: {integrity: sha512-5+5HqXnsZPE65IJZSMkZtURARZelel2oXUEO8rH83VS/hxH5vv1uHquPg5wZs8yMAfdv971IU+kcPUczi7NVBQ==} + engines: {node: '>=0.10.0'} + + postgres-date@1.0.7: + resolution: {integrity: sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==} + engines: {node: '>=0.10.0'} + + postgres-interval@1.2.0: + resolution: {integrity: sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==} + engines: {node: '>=0.10.0'} + + require-from-string@2.0.2: + resolution: {integrity: sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==} + engines: {node: '>=0.10.0'} + + rolldown@1.2.8: + resolution: {integrity: sha512-Z67nTmhZe7anqnM/EjI392w5i/ANUinjip7QYsOyN37oayduxt3ksdX0hf5OOamkAd53BiIHfbfSzfUmzKFQqQ==} + engines: {node: ^20.19.0 || >=22.12.0} + hasBin: true + + siginfo@2.0.0: + resolution: {integrity: sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==} + + source-map-js@1.2.1: + resolution: {integrity: sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==} + engines: {node: '>=0.10.0'} + + split2@4.2.0: + resolution: {integrity: sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==} + engines: {node: '>= 10.x'} + + stackback@0.0.2: + resolution: {integrity: sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==} + + std-env@4.2.0: + resolution: {integrity: sha512-oCUKSupKTHX53EyjDtuZQ64pjLJ6yYCtpmEw0goYxtjG9KpbRe8KAsl2tBUGU9DyMcJ0RwJ8GqJAFzMXcXW1Rw==} + + tinybench@6.1.4: + resolution: {integrity: sha512-9APumHG7r4yOk4X4WlkmE71aZcv1gvin1czO3OQ1U9iJcFA5Ja/ygyb0vPOVHTthFozUYs8CLoLUlM8grb2lTQ==} + engines: {node: '>=20.0.0'} + + tinyexec@1.3.0: + resolution: {integrity: sha512-QKAl9m8gWWGHV8jZcPeym6j+XULi6tOf1mT83WYJ4Lk2ytW/uwAWkrP0uFsdoYMdueVJ0qs26wZ+23xeB4ibNQ==} + engines: {node: '>=18'} + + tinyglobby@0.2.17: + resolution: {integrity: sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==} + engines: {node: '>=12.0.0'} + + tslib@2.8.1: + resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==} + + typescript@7.0.2: + resolution: {integrity: sha512-8FYau96o3NKOhbjKi/qNvG/W5jhzxkbdm5sj9AbZ/5T5sWqn3hJgLfGx27sRKZWTvyzCP8dLRBTf5tBTSRVUNA==} + engines: {node: '>=16.20.0'} + hasBin: true + + undici-types@6.21.0: + resolution: {integrity: sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==} + + vite@8.3.0: + resolution: {integrity: sha512-lhZBVvEHefgE+HQZC9O7EBJgCU/nVzFNl7vkS4RE0APtWLP02/8QVIkQtzBxPquh7lq5/78NHipTj7ODQ6XuyQ==} + engines: {node: ^20.19.0 || >=22.12.0} + hasBin: true + peerDependencies: + '@types/node': ^20.19.0 || >=22.12.0 + '@vitejs/devtools': ^0.7.1 + esbuild: ^0.27.0 || ^0.28.0 + jiti: '>=1.21.0' + less: ^4.0.0 + sass: ^1.70.0 + sass-embedded: ^1.70.0 + stylus: '>=0.54.8' + sugarss: ^5.0.0 + terser: ^5.16.0 + tsx: ^4.8.1 + yaml: ^2.4.2 + peerDependenciesMeta: + '@types/node': + optional: true + '@vitejs/devtools': + optional: true + esbuild: + optional: true + jiti: + optional: true + less: + optional: true + sass: + optional: true + sass-embedded: + optional: true + stylus: + optional: true + sugarss: + optional: true + terser: + optional: true + tsx: + optional: true + yaml: + optional: true + + vitest@5.0.0: + resolution: {integrity: sha512-gpsMNoRhMjMktVxPtstOH4/PJuPyovVaMDr4oDilXaGH1EcqM2OE96SoHT2VIQ6fTGtTjqmHDrEu2X9RQiXf8Q==} + engines: {node: ^22.12.0 || ^24.0.0 || >=26.0.0} + hasBin: true + peerDependencies: + '@edge-runtime/vm': '*' + '@opentelemetry/api': ^1.9.0 + '@types/node': ^22.0.0 || >=24.0.0 + '@vitest/browser-playwright': 5.0.0 + '@vitest/browser-preview': 5.0.0 + '@vitest/browser-webdriverio': ^5.0.0-beta.5 || >=5.0.0 + '@vitest/coverage-istanbul': 5.0.0 + '@vitest/coverage-v8': 5.0.0 + '@vitest/ui': 5.0.0 + happy-dom: '*' + jsdom: '*' + vite: ^6.4.0 || ^7.0.0 || ^8.0.0 + peerDependenciesMeta: + '@edge-runtime/vm': + optional: true + '@opentelemetry/api': + optional: true + '@types/node': + optional: true + '@vitest/browser-playwright': + optional: true + '@vitest/browser-preview': + optional: true + '@vitest/browser-webdriverio': + optional: true + '@vitest/coverage-istanbul': + optional: true + '@vitest/coverage-v8': + optional: true + '@vitest/ui': + optional: true + happy-dom: + optional: true + jsdom: + optional: true + + why-is-node-running@2.3.0: + resolution: {integrity: sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w==} + engines: {node: '>=8'} + hasBin: true + + xtend@4.0.2: + resolution: {integrity: sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==} + engines: {node: '>=0.4'} + + yaml@2.9.0: + resolution: {integrity: sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA==} + engines: {node: '>= 14.6'} + hasBin: true + +snapshots: + + '@aws-sdk/checksums@3.1001.0': + dependencies: + '@aws-sdk/core': 3.978.0 + '@aws-sdk/types': 3.974.5 + '@smithy/core': 3.34.1 + '@smithy/types': 4.18.0 + tslib: 2.8.1 + + '@aws-sdk/client-s3@3.1131.0': + dependencies: + '@aws-sdk/checksums': 3.1001.0 + '@aws-sdk/core': 3.978.0 + '@aws-sdk/credential-provider-node': 3.972.83 + '@aws-sdk/middleware-sdk-s3': 3.972.76 + '@aws-sdk/signature-v4-multi-region': 3.996.46 + '@aws-sdk/types': 3.974.5 + '@smithy/core': 3.34.1 + '@smithy/fetch-http-handler': 5.8.0 + '@smithy/node-http-handler': 4.12.1 + '@smithy/types': 4.18.0 + tslib: 2.8.1 + + '@aws-sdk/core@3.978.0': + dependencies: + '@aws-sdk/types': 3.974.5 + '@aws-sdk/xml-builder': 3.972.40 + '@aws/lambda-invoke-store': 0.3.0 + '@smithy/core': 3.34.1 + '@smithy/signature-v4': 5.7.3 + '@smithy/types': 4.18.0 + bowser: 2.14.1 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-env@3.972.71': + dependencies: + '@aws-sdk/core': 3.978.0 + '@aws-sdk/types': 3.974.5 + '@smithy/core': 3.34.1 + '@smithy/types': 4.18.0 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-http@3.972.73': + dependencies: + '@aws-sdk/core': 3.978.0 + '@aws-sdk/types': 3.974.5 + '@smithy/core': 3.34.1 + '@smithy/fetch-http-handler': 5.8.0 + '@smithy/node-http-handler': 4.12.1 + '@smithy/types': 4.18.0 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-ini@3.973.16': + dependencies: + '@aws-sdk/core': 3.978.0 + '@aws-sdk/credential-provider-env': 3.972.71 + '@aws-sdk/credential-provider-http': 3.972.73 + '@aws-sdk/credential-provider-login': 3.972.78 + '@aws-sdk/credential-provider-process': 3.972.71 + '@aws-sdk/credential-provider-sso': 3.973.15 + '@aws-sdk/credential-provider-web-identity': 3.972.77 + '@aws-sdk/nested-clients': 3.997.45 + '@aws-sdk/types': 3.974.5 + '@smithy/core': 3.34.1 + '@smithy/credential-provider-imds': 4.5.2 + '@smithy/types': 4.18.0 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-login@3.972.78': + dependencies: + '@aws-sdk/core': 3.978.0 + '@aws-sdk/nested-clients': 3.997.45 + '@aws-sdk/types': 3.974.5 + '@smithy/core': 3.34.1 + '@smithy/types': 4.18.0 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-node@3.972.83': + dependencies: + '@aws-sdk/credential-provider-env': 3.972.71 + '@aws-sdk/credential-provider-http': 3.972.73 + '@aws-sdk/credential-provider-ini': 3.973.16 + '@aws-sdk/credential-provider-process': 3.972.71 + '@aws-sdk/credential-provider-sso': 3.973.15 + '@aws-sdk/credential-provider-web-identity': 3.972.77 + '@aws-sdk/types': 3.974.5 + '@smithy/core': 3.34.1 + '@smithy/credential-provider-imds': 4.5.2 + '@smithy/types': 4.18.0 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-process@3.972.71': + dependencies: + '@aws-sdk/core': 3.978.0 + '@aws-sdk/types': 3.974.5 + '@smithy/core': 3.34.1 + '@smithy/types': 4.18.0 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-sso@3.973.15': + dependencies: + '@aws-sdk/core': 3.978.0 + '@aws-sdk/nested-clients': 3.997.45 + '@aws-sdk/token-providers': 3.1129.0 + '@aws-sdk/types': 3.974.5 + '@smithy/core': 3.34.1 + '@smithy/types': 4.18.0 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-web-identity@3.972.77': + dependencies: + '@aws-sdk/core': 3.978.0 + '@aws-sdk/nested-clients': 3.997.45 + '@aws-sdk/types': 3.974.5 + '@smithy/core': 3.34.1 + '@smithy/types': 4.18.0 + tslib: 2.8.1 + + '@aws-sdk/middleware-sdk-s3@3.972.76': + dependencies: + '@aws-sdk/core': 3.978.0 + '@aws-sdk/signature-v4-multi-region': 3.996.46 + '@aws-sdk/types': 3.974.5 + '@smithy/core': 3.34.1 + '@smithy/types': 4.18.0 + tslib: 2.8.1 + + '@aws-sdk/nested-clients@3.997.45': + dependencies: + '@aws-sdk/core': 3.978.0 + '@aws-sdk/signature-v4-multi-region': 3.996.46 + '@aws-sdk/types': 3.974.5 + '@smithy/core': 3.34.1 + '@smithy/fetch-http-handler': 5.8.0 + '@smithy/node-http-handler': 4.12.1 + '@smithy/types': 4.18.0 + tslib: 2.8.1 + + '@aws-sdk/signature-v4-multi-region@3.996.46': + dependencies: + '@aws-sdk/types': 3.974.5 + '@smithy/signature-v4': 5.7.3 + '@smithy/types': 4.18.0 + tslib: 2.8.1 + + '@aws-sdk/token-providers@3.1129.0': + dependencies: + '@aws-sdk/core': 3.978.0 + '@aws-sdk/nested-clients': 3.997.45 + '@aws-sdk/types': 3.974.5 + '@smithy/core': 3.34.1 + '@smithy/types': 4.18.0 + tslib: 2.8.1 + + '@aws-sdk/types@3.974.5': + dependencies: + '@smithy/types': 4.18.0 + tslib: 2.8.1 + + '@aws-sdk/xml-builder@3.972.40': + dependencies: + '@smithy/types': 4.18.0 + tslib: 2.8.1 + + '@aws/lambda-invoke-store@0.3.0': {} + + '@duckdb/node-api@1.5.5-r.4': + dependencies: + '@duckdb/node-bindings': 1.5.5-r.4 + + '@duckdb/node-bindings-darwin-arm64@1.5.5-r.4': + optional: true + + '@duckdb/node-bindings-darwin-x64@1.5.5-r.4': + optional: true + + '@duckdb/node-bindings-linux-arm64-musl@1.5.5-r.4': + optional: true + + '@duckdb/node-bindings-linux-arm64@1.5.5-r.4': + optional: true + + '@duckdb/node-bindings-linux-x64-musl@1.5.5-r.4': + optional: true + + '@duckdb/node-bindings-linux-x64@1.5.5-r.4': + optional: true + + '@duckdb/node-bindings-win32-arm64@1.5.5-r.4': + optional: true + + '@duckdb/node-bindings-win32-x64@1.5.5-r.4': + optional: true + + '@duckdb/node-bindings@1.5.5-r.4': + dependencies: + detect-libc: 2.1.2 + optionalDependencies: + '@duckdb/node-bindings-darwin-arm64': 1.5.5-r.4 + '@duckdb/node-bindings-darwin-x64': 1.5.5-r.4 + '@duckdb/node-bindings-linux-arm64': 1.5.5-r.4 + '@duckdb/node-bindings-linux-arm64-musl': 1.5.5-r.4 + '@duckdb/node-bindings-linux-x64': 1.5.5-r.4 + '@duckdb/node-bindings-linux-x64-musl': 1.5.5-r.4 + '@duckdb/node-bindings-win32-arm64': 1.5.5-r.4 + '@duckdb/node-bindings-win32-x64': 1.5.5-r.4 + + '@jridgewell/resolve-uri@3.1.2': {} + + '@jridgewell/sourcemap-codec@1.6.0': {} + + '@jridgewell/trace-mapping@0.3.31': + dependencies: + '@jridgewell/resolve-uri': 3.1.2 + '@jridgewell/sourcemap-codec': 1.6.0 + + '@oxc-project/types@0.149.0': {} + + '@rolldown/binding-android-arm-eabi@1.2.8': + optional: true + + '@rolldown/binding-android-arm64@1.2.8': + optional: true + + '@rolldown/binding-darwin-arm64@1.2.8': + optional: true + + '@rolldown/binding-darwin-x64@1.2.8': + optional: true + + '@rolldown/binding-freebsd-x64@1.2.8': + optional: true + + '@rolldown/binding-linux-arm-gnueabihf@1.2.8': + optional: true + + '@rolldown/binding-linux-arm64-gnu@1.2.8': + optional: true + + '@rolldown/binding-linux-arm64-musl@1.2.8': + optional: true + + '@rolldown/binding-linux-ppc64-gnu@1.2.8': + optional: true + + '@rolldown/binding-linux-s390x-gnu@1.2.8': + optional: true + + '@rolldown/binding-linux-x64-gnu@1.2.8': + optional: true + + '@rolldown/binding-linux-x64-musl@1.2.8': + optional: true + + '@rolldown/binding-openharmony-arm64@1.2.8': + optional: true + + '@rolldown/binding-win32-arm64-msvc@1.2.8': + optional: true + + '@rolldown/binding-win32-x64-msvc@1.2.8': + optional: true + + '@rolldown/pluginutils@1.0.1': {} + + '@smithy/core@3.34.1': + dependencies: + '@smithy/types': 4.18.0 + tslib: 2.8.1 + + '@smithy/credential-provider-imds@4.5.2': + dependencies: + '@smithy/core': 3.34.1 + '@smithy/types': 4.18.0 + tslib: 2.8.1 + + '@smithy/fetch-http-handler@5.8.0': + dependencies: + '@smithy/core': 3.34.1 + '@smithy/types': 4.18.0 + tslib: 2.8.1 + + '@smithy/node-http-handler@4.12.1': + dependencies: + '@smithy/core': 3.34.1 + '@smithy/types': 4.18.0 + tslib: 2.8.1 + + '@smithy/signature-v4@5.7.3': + dependencies: + '@smithy/core': 3.34.1 + '@smithy/types': 4.18.0 + tslib: 2.8.1 + + '@smithy/types@4.18.0': + dependencies: + tslib: 2.8.1 + + '@types/chai@5.2.3': + dependencies: + '@types/deep-eql': 4.0.2 + assertion-error: 2.0.1 + + '@types/deep-eql@4.0.2': {} + + '@types/estree@1.0.9': {} + + '@types/node@22.20.2': + dependencies: + undici-types: 6.21.0 + + '@types/pg@8.23.1': + dependencies: + '@types/node': 22.20.2 + pg-protocol: 1.16.0 + pg-types: 2.2.0 + + '@typescript/typescript-aix-ppc64@7.0.2': + optional: true + + '@typescript/typescript-darwin-arm64@7.0.2': + optional: true + + '@typescript/typescript-darwin-x64@7.0.2': + optional: true + + '@typescript/typescript-freebsd-arm64@7.0.2': + optional: true + + '@typescript/typescript-freebsd-x64@7.0.2': + optional: true + + '@typescript/typescript-linux-arm64@7.0.2': + optional: true + + '@typescript/typescript-linux-arm@7.0.2': + optional: true + + '@typescript/typescript-linux-loong64@7.0.2': + optional: true + + '@typescript/typescript-linux-mips64el@7.0.2': + optional: true + + '@typescript/typescript-linux-ppc64@7.0.2': + optional: true + + '@typescript/typescript-linux-riscv64@7.0.2': + optional: true + + '@typescript/typescript-linux-s390x@7.0.2': + optional: true + + '@typescript/typescript-linux-x64@7.0.2': + optional: true + + '@typescript/typescript-netbsd-arm64@7.0.2': + optional: true + + '@typescript/typescript-netbsd-x64@7.0.2': + optional: true + + '@typescript/typescript-openbsd-arm64@7.0.2': + optional: true + + '@typescript/typescript-openbsd-x64@7.0.2': + optional: true + + '@typescript/typescript-sunos-x64@7.0.2': + optional: true + + '@typescript/typescript-win32-arm64@7.0.2': + optional: true + + '@typescript/typescript-win32-x64@7.0.2': + optional: true + + '@vitest/mocker@5.0.0(vite@8.3.0(@types/node@22.20.2)(yaml@2.9.0))': + dependencies: + '@jridgewell/trace-mapping': 0.3.31 + '@vitest/spy': 5.0.0 + estree-walker: 3.0.3 + magic-string: 1.3.1 + optionalDependencies: + vite: 8.3.0(@types/node@22.20.2)(yaml@2.9.0) + + '@vitest/spy@5.0.0': {} + + ajv@8.20.0: + dependencies: + fast-deep-equal: 3.1.3 + fast-uri: 3.1.7 + json-schema-traverse: 1.0.0 + require-from-string: 2.0.2 + + assertion-error@2.0.1: {} + + bowser@2.14.1: {} + + chai@6.2.2: {} + + commander@15.0.0: {} + + detect-libc@2.1.2: {} + + es-module-lexer@2.3.2: {} + + estree-walker@3.0.3: + dependencies: + '@types/estree': 1.0.9 + + expect-type@1.4.0: {} + + fast-deep-equal@3.1.3: {} + + fast-uri@3.1.7: {} + + fdir@6.5.0(picomatch@4.0.7): + optionalDependencies: + picomatch: 4.0.7 + + fsevents@2.3.3: + optional: true + + json-schema-traverse@1.0.0: {} + + lightningcss-android-arm64@1.33.0: + optional: true + + lightningcss-darwin-arm64@1.33.0: + optional: true + + lightningcss-darwin-x64@1.33.0: + optional: true + + lightningcss-freebsd-x64@1.33.0: + optional: true + + lightningcss-linux-arm-gnueabihf@1.33.0: + optional: true + + lightningcss-linux-arm64-gnu@1.33.0: + optional: true + + lightningcss-linux-arm64-musl@1.33.0: + optional: true + + lightningcss-linux-x64-gnu@1.33.0: + optional: true + + lightningcss-linux-x64-musl@1.33.0: + optional: true + + lightningcss-win32-arm64-msvc@1.33.0: + optional: true + + lightningcss-win32-x64-msvc@1.33.0: + optional: true + + lightningcss@1.33.0: + dependencies: + detect-libc: 2.1.2 + optionalDependencies: + lightningcss-android-arm64: 1.33.0 + lightningcss-darwin-arm64: 1.33.0 + lightningcss-darwin-x64: 1.33.0 + lightningcss-freebsd-x64: 1.33.0 + lightningcss-linux-arm-gnueabihf: 1.33.0 + lightningcss-linux-arm64-gnu: 1.33.0 + lightningcss-linux-arm64-musl: 1.33.0 + lightningcss-linux-x64-gnu: 1.33.0 + lightningcss-linux-x64-musl: 1.33.0 + lightningcss-win32-arm64-msvc: 1.33.0 + lightningcss-win32-x64-msvc: 1.33.0 + + magic-string@1.3.1: + dependencies: + '@jridgewell/sourcemap-codec': 1.6.0 + + nanoid@3.3.19: {} + + obug@2.2.1: {} + + pg-cloudflare@1.4.0: + optional: true + + pg-connection-string@2.14.0: {} + + pg-int8@1.0.1: {} + + pg-pool@3.14.0(pg@8.23.0): + dependencies: + pg: 8.23.0 + + pg-protocol@1.16.0: {} + + pg-types@2.2.0: + dependencies: + pg-int8: 1.0.1 + postgres-array: 2.0.0 + postgres-bytea: 1.0.1 + postgres-date: 1.0.7 + postgres-interval: 1.2.0 + + pg@8.23.0: + dependencies: + pg-connection-string: 2.14.0 + pg-pool: 3.14.0(pg@8.23.0) + pg-protocol: 1.16.0 + pg-types: 2.2.0 + pgpass: 1.0.5 + optionalDependencies: + pg-cloudflare: 1.4.0 + + pgpass@1.0.5: + dependencies: + split2: 4.2.0 + + picocolors@1.1.1: {} + + picomatch@4.0.7: {} + + postcss@8.5.28: + dependencies: + nanoid: 3.3.19 + picocolors: 1.1.1 + source-map-js: 1.2.1 + + postgres-array@2.0.0: {} + + postgres-bytea@1.0.1: {} + + postgres-date@1.0.7: {} + + postgres-interval@1.2.0: + dependencies: + xtend: 4.0.2 + + require-from-string@2.0.2: {} + + rolldown@1.2.8: + dependencies: + '@oxc-project/types': 0.149.0 + '@rolldown/pluginutils': 1.0.1 + optionalDependencies: + '@rolldown/binding-android-arm-eabi': 1.2.8 + '@rolldown/binding-android-arm64': 1.2.8 + '@rolldown/binding-darwin-arm64': 1.2.8 + '@rolldown/binding-darwin-x64': 1.2.8 + '@rolldown/binding-freebsd-x64': 1.2.8 + '@rolldown/binding-linux-arm-gnueabihf': 1.2.8 + '@rolldown/binding-linux-arm64-gnu': 1.2.8 + '@rolldown/binding-linux-arm64-musl': 1.2.8 + '@rolldown/binding-linux-ppc64-gnu': 1.2.8 + '@rolldown/binding-linux-s390x-gnu': 1.2.8 + '@rolldown/binding-linux-x64-gnu': 1.2.8 + '@rolldown/binding-linux-x64-musl': 1.2.8 + '@rolldown/binding-openharmony-arm64': 1.2.8 + '@rolldown/binding-win32-arm64-msvc': 1.2.8 + '@rolldown/binding-win32-x64-msvc': 1.2.8 + + siginfo@2.0.0: {} + + source-map-js@1.2.1: {} + + split2@4.2.0: {} + + stackback@0.0.2: {} + + std-env@4.2.0: {} + + tinybench@6.1.4: {} + + tinyexec@1.3.0: {} + + tinyglobby@0.2.17: + dependencies: + fdir: 6.5.0(picomatch@4.0.7) + picomatch: 4.0.7 + + tslib@2.8.1: {} + + typescript@7.0.2: + optionalDependencies: + '@typescript/typescript-aix-ppc64': 7.0.2 + '@typescript/typescript-darwin-arm64': 7.0.2 + '@typescript/typescript-darwin-x64': 7.0.2 + '@typescript/typescript-freebsd-arm64': 7.0.2 + '@typescript/typescript-freebsd-x64': 7.0.2 + '@typescript/typescript-linux-arm': 7.0.2 + '@typescript/typescript-linux-arm64': 7.0.2 + '@typescript/typescript-linux-loong64': 7.0.2 + '@typescript/typescript-linux-mips64el': 7.0.2 + '@typescript/typescript-linux-ppc64': 7.0.2 + '@typescript/typescript-linux-riscv64': 7.0.2 + '@typescript/typescript-linux-s390x': 7.0.2 + '@typescript/typescript-linux-x64': 7.0.2 + '@typescript/typescript-netbsd-arm64': 7.0.2 + '@typescript/typescript-netbsd-x64': 7.0.2 + '@typescript/typescript-openbsd-arm64': 7.0.2 + '@typescript/typescript-openbsd-x64': 7.0.2 + '@typescript/typescript-sunos-x64': 7.0.2 + '@typescript/typescript-win32-arm64': 7.0.2 + '@typescript/typescript-win32-x64': 7.0.2 + + undici-types@6.21.0: {} + + vite@8.3.0(@types/node@22.20.2)(yaml@2.9.0): + dependencies: + lightningcss: 1.33.0 + picomatch: 4.0.7 + postcss: 8.5.28 + rolldown: 1.2.8 + tinyglobby: 0.2.17 + optionalDependencies: + '@types/node': 22.20.2 + fsevents: 2.3.3 + yaml: 2.9.0 + + vitest@5.0.0(@types/node@22.20.2)(vite@8.3.0(@types/node@22.20.2)(yaml@2.9.0)): + dependencies: + '@types/chai': 5.2.3 + '@vitest/mocker': 5.0.0(vite@8.3.0(@types/node@22.20.2)(yaml@2.9.0)) + chai: 6.2.2 + es-module-lexer: 2.3.2 + expect-type: 1.4.0 + magic-string: 1.3.1 + obug: 2.2.1 + picomatch: 4.0.7 + std-env: 4.2.0 + tinybench: 6.1.4 + tinyexec: 1.3.0 + tinyglobby: 0.2.17 + vite: 8.3.0(@types/node@22.20.2)(yaml@2.9.0) + why-is-node-running: 2.3.0 + optionalDependencies: + '@types/node': 22.20.2 + transitivePeerDependencies: + - msw + + why-is-node-running@2.3.0: + dependencies: + siginfo: 2.0.0 + stackback: 0.0.2 + + xtend@4.0.2: {} + + yaml@2.9.0: {} diff --git a/schemas/coverage.schema.json b/schemas/coverage.schema.json new file mode 100644 index 0000000..39aa236 --- /dev/null +++ b/schemas/coverage.schema.json @@ -0,0 +1,58 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://chainplot.dev/schema/coverage", + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "chain_id", "sources"], + "properties": { + "schema_version": { "type": "integer", "const": 1 }, + "chain_id": { "type": "integer" }, + "sources": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["source_id", "segments"], + "properties": { + "source_id": { "type": "string" }, + "segments": { + "type": "array", + "items": { "$ref": "#/$defs/segment" } + } + } + } + } + }, + "$defs": { + "segment": { + "type": "object", + "additionalProperties": false, + "required": [ + "start_block", + "end_block", + "start_block_hash", + "end_block_hash", + "start_block_parent_hash", + "status" + ], + "properties": { + "start_block": { "type": "integer", "minimum": 0 }, + "end_block": { "type": "integer", "minimum": 0 }, + "start_block_hash": { "type": "string", "pattern": "^0x[0-9a-f]{64}$" }, + "end_block_hash": { "type": "string", "pattern": "^0x[0-9a-f]{64}$" }, + "start_block_parent_hash": { + "type": "string", + "pattern": "^0x[0-9a-f]{64}$" + }, + "status": { "enum": ["complete_empty", "complete_with_rows"] }, + "row_count": { "type": "integer", "minimum": 0 }, + "end_block_timestamp": { + "description": "Unix seconds of end_block, from the chain. Freshness is a property of the data, not of the file on disk.", + "type": "integer", + "minimum": 0 + }, + "indexed_at": { "type": "string", "format": "date-time" } + } + } + } +} diff --git a/schemas/latest.schema.json b/schemas/latest.schema.json new file mode 100644 index 0000000..5711239 --- /dev/null +++ b/schemas/latest.schema.json @@ -0,0 +1,12 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://chainplot.dev/schema/latest", + "type": "object", + "additionalProperties": false, + "required": ["prefix", "release_json_checksum"], + "properties": { + "schema_version": { "type": "integer" }, + "prefix": { "type": "string" }, + "release_json_checksum": { "type": "string" } + } +} diff --git a/schemas/lock.schema.json b/schemas/lock.schema.json new file mode 100644 index 0000000..68f9380 --- /dev/null +++ b/schemas/lock.schema.json @@ -0,0 +1,10 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://chainplot.dev/schema/lock", + "type": "object", + "additionalProperties": false, + "required": ["format_version"], + "properties": { + "format_version": { "type": "integer" } + } +} diff --git a/schemas/manifest.schema.json b/schemas/manifest.schema.json new file mode 100644 index 0000000..d8d6eba --- /dev/null +++ b/schemas/manifest.schema.json @@ -0,0 +1,113 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://chainplot.dev/schema/manifest", + "type": "object", + "additionalProperties": false, + "required": [ + "snapshot_id", + "mode", + "files", + "source_path" + ], + "properties": { + "schema_version": { + "type": "integer" + }, + "snapshot_id": { + "type": "string" + }, + "mode": { + "enum": [ + "results_only", + "dataset_included", + "dataset_referenced" + ] + }, + "files": { + "type": "array", + "items": { + "type": "string" + } + }, + "source_path": { + "type": "string" + }, + "coverage": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "source_id", + "start_block", + "end_block", + "status" + ], + "properties": { + "source_id": { + "type": "string" + }, + "start_block": { + "type": "integer" + }, + "end_block": { + "type": "integer" + }, + "status": { + "type": "string" + } + } + } + }, + "finality": { + "oneOf": [ + { + "type": "null" + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "policy" + ], + "properties": { + "policy": { + "enum": [ + "finalized", + "confirmation_depth" + ] + }, + "depth": { + "type": "integer", + "minimum": 1 + } + } + } + ] + }, + "freshness": { + "type": "string" + }, + "external": { + "type": "object", + "additionalProperties": false, + "required": [ + "path", + "checksum" + ], + "properties": { + "path": { + "type": "string" + }, + "checksum": { + "type": "string" + }, + "bytes": { + "type": "integer", + "minimum": 0 + } + }, + "description": "A dataset published alongside the release rather than inside it. `path` is relative to the release root, so a consumer resolves it against the base URL it fetched the release from." + } + } +} diff --git a/schemas/plan.schema.json b/schemas/plan.schema.json new file mode 100644 index 0000000..d380eba --- /dev/null +++ b/schemas/plan.schema.json @@ -0,0 +1,161 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://chainplot.dev/schema/plan", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "intent", + "project_id", + "project_digest", + "created_at", + "chain", + "sources", + "actions", + "limits", + "deletes_data", + "makes_data_public", + "state_assumptions" + ], + "properties": { + "schema_version": { "type": "integer", "const": 1 }, + "plan_id": { "type": "string", "minLength": 1 }, + "intent": { "enum": ["ingest", "refresh", "build", "publish"] }, + "project_id": { "type": "string", "minLength": 1 }, + "project_digest": { "type": "string", "minLength": 1 }, + "created_at": { "type": "string" }, + "chain": { + "oneOf": [ + { "type": "null" }, + { "$ref": "#/$defs/chain" } + ] + }, + "sources": { "type": "array", "items": { "$ref": "#/$defs/plan_source" } }, + "actions": { "type": "array", "items": { "$ref": "#/$defs/action" } }, + "limits": { "$ref": "#/$defs/limits" }, + "deletes_data": { "type": "boolean" }, + "makes_data_public": { "type": "boolean" }, + "state_assumptions": { "$ref": "#/$defs/state_assumptions" }, + "publish_target": { "type": ["string", "null"] }, + "release_digest": { + "description": "content_digest of the built release a publish plan would upload. Binds the plan to the bytes, so rebuilt content is not mistaken for an already-applied run.", + "type": ["string", "null"] + } + }, + "$defs": { + "chain": { + "type": "object", + "additionalProperties": false, + "required": ["chain_id", "finality"], + "properties": { + "chain_id": { "type": "integer" }, + "finality": { + "oneOf": [ + { + "type": "object", + "additionalProperties": false, + "required": ["policy"], + "properties": { "policy": { "const": "finalized" } } + }, + { + "type": "object", + "additionalProperties": false, + "required": ["policy", "depth"], + "properties": { + "policy": { "const": "confirmation_depth" }, + "depth": { "type": "integer", "minimum": 1 } + } + } + ] + } + } + }, + "plan_source": { + "type": "object", + "additionalProperties": false, + "required": [ + "source_id", + "end_mode", + "job_start", + "job_end", + "job_target_end" + ], + "properties": { + "source_id": { "type": "string" }, + "end_mode": { "enum": ["pinned", "follow_finalized"] }, + "job_start": { "type": "integer", "minimum": 0 }, + "job_end": { "type": "integer", "minimum": -1 }, + "job_target_end": { "type": "integer", "minimum": 0 }, + "required_end": { "type": ["integer", "null"] }, + "blocks_remaining": { "type": "integer", "minimum": 0 } + } + }, + "action": { + "oneOf": [ + { + "type": "object", + "additionalProperties": false, + "required": ["type", "source_id"], + "properties": { + "type": { "const": "ingest" }, + "source_id": { "type": "string" } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": ["type", "source_id"], + "properties": { + "type": { "const": "export" }, + "source_id": { "type": "string" } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": ["type"], + "properties": { "type": { "const": "build_results" } } + }, + { + "type": "object", + "additionalProperties": false, + "required": ["type", "target_id"], + "properties": { + "type": { "const": "publish" }, + "target_id": { "type": "string" } + } + } + ] + }, + "limits": { + "type": "object", + "additionalProperties": false, + "required": ["block_budget"], + "properties": { + "block_budget": { "type": "integer", "minimum": 1 }, + "blocks_in_range": { "type": "integer", "minimum": 0 } + } + }, + "state_assumptions": { + "type": "object", + "additionalProperties": false, + "required": ["sources"], + "properties": { + "sources": { + "type": "object", + "additionalProperties": false, + "patternProperties": { + "^[a-z0-9][a-z0-9_-]*$": { + "type": "object", + "additionalProperties": false, + "required": ["last_proven_complete_block"], + "properties": { + "last_proven_complete_block": { "type": "integer", "minimum": -1 } + } + } + } + } + } + } + } +} diff --git a/schemas/progress.schema.json b/schemas/progress.schema.json new file mode 100644 index 0000000..d0662ed --- /dev/null +++ b/schemas/progress.schema.json @@ -0,0 +1,18 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://chainplot.dev/schema/progress", + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "type", "run_id", "stage"], + "properties": { + "schema_version": { "type": "integer", "const": 1 }, + "type": { "const": "progress" }, + "run_id": { "type": "string", "minLength": 1 }, + "stage": { "type": "string", "minLength": 1 }, + "message": { "type": "string" }, + "rows": { "type": "integer", "minimum": 0 }, + "output_bytes": { "type": "integer", "minimum": 0 }, + "retries": { "type": "integer", "minimum": 0 }, + "ts": { "type": "string" } + } +} diff --git a/schemas/project.schema.json b/schemas/project.schema.json new file mode 100644 index 0000000..c4c8d10 --- /dev/null +++ b/schemas/project.schema.json @@ -0,0 +1,200 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://chainplot.dev/schema/project", + "type": "object", + "additionalProperties": false, + "required": ["format_version", "id"], + "properties": { + "format_version": { "type": "integer", "const": 1 }, + "id": { "type": "string", "minLength": 1 }, + "datasets": { "type": "array", "items": { "$ref": "#/$defs/dataset" } }, + "queries": { "type": "array", "items": { "$ref": "#/$defs/query" } }, + "dashboards": { "type": "array", "items": { "$ref": "#/$defs/dashboard" } }, + "models": { "type": "array", "items": { "$ref": "#/$defs/model" } }, + "chain_sources": { "type": "array", "maxItems": 1, "items": { "$ref": "#/$defs/chain_source" } }, + "event_sources": { "type": "array", "maxItems": 20, "items": { "$ref": "#/$defs/event_source" } }, + "publish_targets": { "type": "array", "items": { "$ref": "#/$defs/publish_target" } }, + "policy": { "type": "object", "additionalProperties": false, "properties": { + "block_budget": { "type": "integer", "minimum": 1 }, + "row_limit": { + "description": "Rows a single query may return. Bounds the viewer, which renders every row: results ride inside the release and the table is not virtualised. Going over is refused, never truncated.", + "type": "integer", + "minimum": 1, + "maximum": 1000000, + "default": 10000 + }, + "release_mode": { + "description": "Mode `build` uses when none is given on the command line. The default publishes the page and its results without the dataset; declare dataset_included to ship the parquet so a fork can recompute from it.", + "enum": ["dataset_included", "results_only", "dataset_referenced"], + "default": "results_only" + } + }} + }, + "$defs": { + "dataset": { + "type": "object", + "additionalProperties": false, + "required": ["id", "snapshot"], + "properties": { + "id": { "type": "string" }, + "snapshot": { "type": "string" }, + "schema": { "type": "string" } + } + }, + "query": { + "type": "object", + "additionalProperties": false, + "required": ["id", "file", "dataset"], + "properties": { + "id": { "type": "string" }, + "file": { "type": "string" }, + "dataset": { "type": "string" }, + "title": { "type": "string", "maxLength": 120 }, + "raw_amount_columns": { + "description": "Columns holding integer token amounts as decimal strings. A bare name declares the column raw; the object form also drives display scaling.", + "type": "array", + "items": { "$ref": "#/$defs/raw_amount_column" } + } + } + }, + "raw_amount_column": { + "oneOf": [ + { "type": "string", "minLength": 1 }, + { + "type": "object", + "additionalProperties": false, + "required": ["name"], + "properties": { + "name": { "type": "string", "minLength": 1 }, + "decimals": { + "description": "Token decimals. Display divides by 10^decimals; the stored value is never rewritten.", + "type": "integer", + "minimum": 0, + "maximum": 77 + }, + "symbol": { "type": "string", "maxLength": 16 }, + "label": { "type": "string", "maxLength": 64 } + } + } + ] + }, + "dashboard": { + "type": "object", + "additionalProperties": false, + "required": ["id", "title", "panels"], + "properties": { + "id": { "type": "string" }, + "title": { "type": "string" }, + "description": { "type": "string" }, + "panels": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["query", "chart"], + "properties": { + "query": { "type": "string" }, + "chart": { "enum": ["line", "bar", "area", "kpi", "table"] }, + "title": { "type": "string", "maxLength": 120 }, + "description": { "type": "string", "maxLength": 240 }, + "span": { + "description": "Grid width. Tables and wide charts usually want 'full'.", + "enum": ["half", "full"], + "default": "half" + }, + "hide_columns": { + "description": "Columns the query computes but the panel does not show, such as an explicit sort key.", + "type": "array", + "items": { "type": "string" } + }, + "unit": { "type": "string", "maxLength": 16 } + } + } + } + } + }, + "model": { + "type": "object", + "additionalProperties": false, + "required": ["id", "file", "depends_on"], + "properties": { + "id": { "type": "string" }, + "file": { "type": "string" }, + "depends_on": { "type": "array", "items": { "type": "string" } }, + "columns": { "type": "array", "items": { "type": "string" } } + } + }, + "chain_source": { + "type": "object", + "additionalProperties": false, + "required": ["id", "chain_id", "rpc_secret", "finality"], + "properties": { + "id": { "type": "string" }, + "chain_id": { "type": "integer" }, + "rpc_secret": { "type": "string" }, + "finality": { + "oneOf": [ + { "type": "object", "additionalProperties": false, "required": ["policy"], + "properties": { "policy": { "const": "finalized" } } }, + { "type": "object", "additionalProperties": false, "required": ["policy", "depth"], + "properties": { "policy": { "const": "confirmation_depth" }, "depth": { "type": "integer", "minimum": 1 } } } + ] + } + } + }, + "event_source": { + "type": "object", + "additionalProperties": false, + "required": ["id", "chain", "addresses", "abi", "events", "start_block", "end"], + "properties": { + "id": { "type": "string" }, + "chain": { "type": "string" }, + "addresses": { "type": "array", "minItems": 1, "maxItems": 20, "items": { "type": "string" } }, + "abi": { "type": "string" }, + "events": { "type": "array", "items": { "type": "string" } }, + "start_block": { "type": "integer", "minimum": 0 }, + "end": { + "oneOf": [ + { "type": "object", "additionalProperties": false, "required": ["mode", "block"], + "properties": { "mode": { "const": "pinned" }, "block": { "type": "integer" } } }, + { "type": "object", "additionalProperties": false, "required": ["mode"], + "properties": { "mode": { "const": "follow_finalized" } } } + ] + }, + "indexed_filters": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["event_name"], + "properties": { + "event_name": { "type": "string" }, + "indexed_1": { "type": "array", "items": { "type": "string" } }, + "indexed_2": { "type": "array", "items": { "type": "string" } }, + "indexed_3": { "type": "array", "items": { "type": "string" } } + } + } + } + } + }, + "publish_target": { + "type": "object", + "additionalProperties": false, + "required": ["id", "type"], + "properties": { + "id": { "type": "string" }, + "type": { "enum": ["directory", "s3"] }, + "path": { "type": "string" }, + "bucket": { "type": "string" }, + "prefix": { + "description": "Key prefix inside the target. Required when one bucket or directory holds more than one project: without it each project's latest.json overwrites the others.", + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*(/[A-Za-z0-9._-]+)*$", + "maxLength": 128 + }, + "dataset_license": { "type": "string" }, + "public_base_url": { "type": "string" } + } + } + } +} diff --git a/schemas/release.schema.json b/schemas/release.schema.json new file mode 100644 index 0000000..64b7e0f --- /dev/null +++ b/schemas/release.schema.json @@ -0,0 +1,105 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://chainplot.dev/schema/release", + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "project_id", "mode", "queries"], + "properties": { + "schema_version": { "type": "integer" }, + "project_id": { "type": "string" }, + "mode": { + "enum": ["results_only", "dataset_included", "dataset_referenced"] + }, + "queries": { + "type": "array", + "items": { "type": "string" } + }, + "dashboards": { + "type": "array", + "items": { "type": "string" } + }, + "content_digest": { + "description": "sha256 over the sorted file checksums plus project id and mode. Stable across rebuilds of identical inputs, unlike generated_at.", + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "generated_at": { "type": "string" }, + "freshness": { + "description": "How current the data is, and on whose authority. 'chain' answers from the last block proven complete; 'snapshot_mtime' admits it only knows the file's timestamp.", + "type": "object", + "additionalProperties": false, + "required": ["kind", "snapshot_mtime"], + "properties": { + "kind": { "enum": ["chain", "snapshot_mtime"] }, + "data_through": { + "oneOf": [ + { "type": "null" }, + { + "type": "object", + "additionalProperties": false, + "required": ["block"], + "properties": { + "block": { "type": "integer", "minimum": 0 }, + "timestamp": { "type": ["string", "null"] } + } + } + ] + }, + "indexed_at": { "type": ["string", "null"] }, + "snapshot_mtime": { "type": "string" } + } + }, + "snapshots": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["dataset_id", "snapshot_id"], + "properties": { + "dataset_id": { "type": "string" }, + "snapshot_id": { "type": "string" } + } + } + }, + "coverage": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["source_id", "start_block", "end_block", "status"], + "properties": { + "source_id": { "type": "string" }, + "start_block": { "type": "integer" }, + "end_block": { "type": "integer" }, + "status": { "type": "string" } + } + } + }, + "finality": { + "oneOf": [ + { "type": "null" }, + { + "type": "object", + "additionalProperties": false, + "required": ["policy"], + "properties": { + "policy": { "enum": ["finalized", "confirmation_depth"] }, + "depth": { "type": "integer", "minimum": 1 } + } + } + ] + }, + "files": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["path", "checksum"], + "properties": { + "path": { "type": "string" }, + "checksum": { "type": "string" } + } + } + } + } +} diff --git a/schemas/result.schema.json b/schemas/result.schema.json new file mode 100644 index 0000000..c15c88e --- /dev/null +++ b/schemas/result.schema.json @@ -0,0 +1,52 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://chainplot.dev/schema/result", + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "ok", "command", "data", "warnings", "error"], + "properties": { + "schema_version": { "type": "integer", "const": 1 }, + "ok": { "type": "boolean" }, + "command": { "type": "string" }, + "data": {}, + "warnings": { + "type": "array", + "items": { "type": "string" } + }, + "error": { + "oneOf": [ + { "type": "null" }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "code", + "message", + "resource_id", + "pointer", + "retryable", + "suggested_next" + ], + "properties": { + "code": { + "enum": [ + "validation", + "missing_credentials", + "unsupported_capability", + "policy_refused", + "source_inconsistent", + "transient_dependency", + "internal" + ] + }, + "message": { "type": "string" }, + "resource_id": { "type": ["string", "null"] }, + "pointer": { "type": ["string", "null"] }, + "retryable": { "type": "boolean" }, + "suggested_next": { "type": ["string", "null"] } + } + } + ] + } + } +} diff --git a/scripts/m0-probe/.env.example b/scripts/m0-probe/.env.example new file mode 100644 index 0000000..7f8a4c8 --- /dev/null +++ b/scripts/m0-probe/.env.example @@ -0,0 +1,3 @@ +# Archive-capable Ethereum JSON-RPC (mainnet). Required. +# Copy to .env (gitignored). Do not commit real endpoints. +RPC_URL= diff --git a/scripts/m0-probe/abis/ERC20.json b/scripts/m0-probe/abis/ERC20.json new file mode 100644 index 0000000..cef2eb7 --- /dev/null +++ b/scripts/m0-probe/abis/ERC20.json @@ -0,0 +1,12 @@ +[ + { + "anonymous": false, + "inputs": [ + { "indexed": true, "name": "from", "type": "address" }, + { "indexed": true, "name": "to", "type": "address" }, + { "indexed": false, "name": "value", "type": "uint256" } + ], + "name": "Transfer", + "type": "event" + } +] diff --git a/scripts/m0-probe/compose.yaml b/scripts/m0-probe/compose.yaml new file mode 100644 index 0000000..aceeca7 --- /dev/null +++ b/scripts/m0-probe/compose.yaml @@ -0,0 +1,27 @@ +# M0 probe only. Our Postgres, pinned rindexer image, no docker.sock. +# Spec ingest runtime: Compose, Chainplot-owned DB, no rindexer docker-socket provisioning. +services: + postgres: + image: postgres:16-alpine + environment: + POSTGRES_USER: chainplot + POSTGRES_PASSWORD: chainplot + POSTGRES_DB: chainplot + healthcheck: + test: ["CMD-SHELL", "pg_isready -U chainplot -d chainplot"] + interval: 2s + timeout: 5s + retries: 30 + + rindexer: + platform: linux/amd64 + image: ghcr.io/joshstevens19/rindexer@sha256:9b33da8cea740b74ebfdfd3932682e8ceab79cbcf2eb3a7ca0863ac413794dd7 + depends_on: + postgres: + condition: service_healthy + environment: + DATABASE_URL: postgresql://chainplot:chainplot@postgres:5432/chainplot + RPC_URL: ${RPC_URL:?set RPC_URL to an archive-capable Ethereum JSON-RPC} + volumes: + - ./:/app/project_path + command: ["start", "-p", "/app/project_path", "indexer"] diff --git a/scripts/m0-probe/rindexer.yaml b/scripts/m0-probe/rindexer.yaml new file mode 100644 index 0000000..474834f --- /dev/null +++ b/scripts/m0-probe/rindexer.yaml @@ -0,0 +1,34 @@ +# Archive window. publicnode eth_getLogs on old ranges returns 403; set RPC_URL. +name: ChainplotM0 +description: Throwaway M0 probe +project_type: no-code +networks: + - name: ethereum + chain_id: 1 + rpc: ${RPC_URL} +storage: + postgres: + enabled: true +graphql: + enabled: false +contracts: + - name: Usdc + details: + - network: ethereum + address: "0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48" + start_block: 18600000 + end_block: 18600010 + abi: ./abis/ERC20.json + include_events: + - Transfer + timestamp: true + - name: Empty + details: + - network: ethereum + address: "0x000000000000000000000000000000000000dEaD" + start_block: 18600000 + end_block: 18600010 + abi: ./abis/ERC20.json + include_events: + - Transfer + timestamp: true diff --git a/scripts/write-fixture-parquet.ts b/scripts/write-fixture-parquet.ts new file mode 100644 index 0000000..01a0ee8 --- /dev/null +++ b/scripts/write-fixture-parquet.ts @@ -0,0 +1,69 @@ +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { DuckDBInstance } from "@duckdb/node-api"; + +// A8 fixture amounts as decimal strings (int256 range, including 2^256-1). +const AMOUNTS = [ + "0", + "1", + "-1", + "9007199254740993", // 2^53+1 + "-9007199254740993", + "57896044618658097711785492504343953926634992332820282019728792003956564819967", // 2^255-1 + "-57896044618658097711785492504343953926634992332820282019728792003956564819968", // -2^255 + "115792089237316195423570985008687907853269984665640564039457584007913129639935", // 2^256-1 +] as const; + +// amount_sort must equal cp_sortkey(amount) exactly — that macro, defined in +// src/query/workerMain.ts, is the one definition of this key. A precomputed +// column and an in-query call have to agree, or a snapshot and a query sort +// the same data differently. +// +// negative → '0' + nines-complement of the zero-padded magnitude +// non-negative → '1' + zero-padded magnitude +// +// The leading sign digit puts every negative first; complementing the +// magnitude reverses its order, so -10 sorts before -9. +const SORT_WIDTH = 78; +const SORT_RADIX = 10n ** BigInt(SORT_WIDTH); + +function amountSort(amount: string): string { + const value = BigInt(amount); + if (value >= 0n) { + return "1" + value.toString().padStart(SORT_WIDTH, "0"); + } + const complement = SORT_RADIX - 1n + value; + return "0" + complement.toString().padStart(SORT_WIDTH, "0"); +} + +const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const outPath = path.join( + repoRoot, + "templates/fixture-transfers/snapshots/amounts.parquet", +); + +fs.mkdirSync(path.dirname(outPath), { recursive: true }); + +const instance = await DuckDBInstance.create(":memory:"); +const conn = await instance.connect(); +try { + await conn.run("CREATE TABLE amounts (amount VARCHAR, amount_sort VARCHAR)"); + const insert = await conn.prepare( + "INSERT INTO amounts (amount, amount_sort) VALUES (?, ?)", + ); + for (const amount of AMOUNTS) { + insert.bindVarchar(1, amount); + insert.bindVarchar(2, amountSort(amount)); + await insert.run(); + } + insert.destroySync(); + + const copy = await conn.prepare("COPY amounts TO ? (FORMAT PARQUET)"); + copy.bindVarchar(1, outPath); + await copy.run(); + copy.destroySync(); +} finally { + conn.closeSync(); + instance.closeSync(); +} diff --git a/src/cli/commands/apply.ts b/src/cli/commands/apply.ts new file mode 100644 index 0000000..fe99c6f --- /dev/null +++ b/src/cli/commands/apply.ts @@ -0,0 +1,57 @@ +import { okResult, failResult, type CommandResult } from "../envelope.js"; +import { applyPlan } from "../../plan/apply.js"; +import { rindexerAdapter } from "../../ingest/rindexer/index.js"; +import { createRpcClient } from "../../rpc/client.js"; +import { loadProject } from "../../project/load.js"; +import { validateProject } from "../../project/validate.js"; +import { isCommandError } from "./build.js"; + +export async function applyCommand( + cwd: string, + planRef: string, + idempotencyKey?: string, + jsonl = false, +): Promise { + try { + const outcome = await applyPlan({ + cwd, + planRef, + idempotencyKey, + adapter: rindexerAdapter(), + rindexerBin: process.env.CHAINPLOT_RINDEXER_BIN ?? "rindexer", + rpcClient: createRpcClient(rpcUrlFor(cwd)), + onProgress: jsonl ? writeProgressLine : undefined, + }); + return okResult("apply", outcome); + } catch (err) { + if (isCommandError(err)) return failResult("apply", err); + throw err; + } +} + +function writeProgressLine(event: { + run_id: string; + stage: string; + message?: string; + rows?: number; +}): void { + process.stdout.write( + JSON.stringify({ + schema_version: 1, + type: "progress", + run_id: event.run_id, + stage: event.stage, + ...(event.message !== undefined ? { message: event.message } : {}), + ...(event.rows !== undefined ? { rows: event.rows } : {}), + ts: new Date().toISOString(), + }) + "\n", + ); +} + +function rpcUrlFor(cwd: string): string { + const doc = loadProject(cwd); + const validated = validateProject(doc, cwd); + if (!validated.ok) return ""; + const secret = validated.project.chain_sources?.[0]?.rpc_secret; + return (secret && process.env[secret]) || ""; +} diff --git a/src/cli/commands/build.ts b/src/cli/commands/build.ts new file mode 100644 index 0000000..93581f3 --- /dev/null +++ b/src/cli/commands/build.ts @@ -0,0 +1,31 @@ +import { failResult, okResult, type CommandError, type CommandResult } from "../envelope.js"; +import { buildRelease } from "../../publish/writeRelease.js"; + +export function isCommandError(err: unknown): err is CommandError { + return ( + typeof err === "object" && + err !== null && + "code" in err && + "message" in err && + "resource_id" in err && + "pointer" in err && + "retryable" in err && + "suggested_next" in err + ); +} + +export async function build( + cwd: string, + mode?: "dataset_included" | "results_only" | "dataset_referenced", +): Promise { + const command = "build"; + try { + const data = await buildRelease(cwd, { mode }); + return okResult(command, data); + } catch (err) { + if (isCommandError(err)) { + return failResult(command, err); + } + throw err; + } +} diff --git a/src/cli/commands/capabilities.ts b/src/cli/commands/capabilities.ts new file mode 100644 index 0000000..d8e8f71 --- /dev/null +++ b/src/cli/commands/capabilities.ts @@ -0,0 +1,48 @@ +import { createRequire } from "node:module"; +import { okResult, type CommandResult } from "../envelope.js"; +import { SCHEMA_KINDS } from "./schemaShow.js"; + +const require = createRequire(import.meta.url); +const pkg = require("../../../package.json") as { version: string }; + +export interface CapabilitiesData { + cli_version: string; + schema_kinds: string[]; + commands: string[]; + sources: string[]; + publish_targets: string[]; + chart_types: string[]; + sql_modes: string[]; +} + +export function capabilities(): CommandResult { + return okResult("capabilities", { + cli_version: pkg.version, + schema_kinds: [...SCHEMA_KINDS], + commands: [ + "capabilities", + "schema show", + "templates list", + "init", + "validate", + "dataset describe", + "query", + "test", + "build", + "plan", + "apply", + "refresh", + "runs list", + "runs show", + "runs cancel", + "serve", + "publish", + "doctor", + "fork", + ], + sources: [], + publish_targets: [], + chart_types: ["line", "bar", "area", "kpi", "table"], + sql_modes: ["snapshot"], + }); +} diff --git a/src/cli/commands/describe.ts b/src/cli/commands/describe.ts new file mode 100644 index 0000000..72fb581 --- /dev/null +++ b/src/cli/commands/describe.ts @@ -0,0 +1,31 @@ +import { failResult, okResult, type CommandError, type CommandResult } from "../envelope.js"; +import { describeDataset } from "../../snapshot/describe.js"; + +function isCommandError(err: unknown): err is CommandError { + return ( + typeof err === "object" && + err !== null && + "code" in err && + "message" in err && + "resource_id" in err && + "pointer" in err && + "retryable" in err && + "suggested_next" in err + ); +} + +export async function datasetDescribe( + cwd: string, + datasetId: string, +): Promise { + const command = "dataset describe"; + try { + const data = await describeDataset(cwd, datasetId); + return okResult(command, data); + } catch (err) { + if (isCommandError(err)) { + return failResult(command, err); + } + throw err; + } +} diff --git a/src/cli/commands/doctor.ts b/src/cli/commands/doctor.ts new file mode 100644 index 0000000..63944b2 --- /dev/null +++ b/src/cli/commands/doctor.ts @@ -0,0 +1,14 @@ +import { failResult, okResult, type CommandResult } from "../envelope.js"; +import { runDoctor } from "../../publish/doctor.js"; +import { isCommandError } from "./build.js"; + +export async function doctorCommand(cwd: string): Promise { + try { + const report = await runDoctor(cwd); + const failed = report.checks.some((c) => c.status === "fail"); + return okResult("doctor", { ...report, healthy: !failed }); + } catch (err) { + if (isCommandError(err)) return failResult("doctor", err); + throw err; + } +} diff --git a/src/cli/commands/fork.ts b/src/cli/commands/fork.ts new file mode 100644 index 0000000..124f410 --- /dev/null +++ b/src/cli/commands/fork.ts @@ -0,0 +1,19 @@ +import { failResult, okResult, type CommandResult } from "../envelope.js"; +import { importRelease } from "../../fork/importRelease.js"; +import { isCommandError } from "./build.js"; + +export async function forkCommand( + from: string, + output: string, + allowPrivateNetworks = false, +): Promise { + try { + const { warnings, ...result } = await importRelease(from, output, { + allowPrivateNetworks, + }); + return { ...okResult("fork", result), warnings }; + } catch (err) { + if (isCommandError(err)) return failResult("fork", err); + throw err; + } +} diff --git a/src/cli/commands/init.ts b/src/cli/commands/init.ts new file mode 100644 index 0000000..b707f71 --- /dev/null +++ b/src/cli/commands/init.ts @@ -0,0 +1,76 @@ +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { failResult, okResult, type CommandResult } from "../envelope.js"; +import { listTemplates } from "./templates.js"; + +const TEMPLATES_DIR = path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + "../../../templates", +); + +function validation( + message: string, + pointer: string | null = null, +): CommandResult { + return failResult("init", { + code: "validation", + message, + resource_id: null, + pointer, + retryable: false, + suggested_next: null, + }); +} + +function walkFiles(dir: string, base = dir): string[] { + const out: string[] = []; + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + const full = path.join(dir, entry.name); + if (entry.isDirectory()) { + out.push(...walkFiles(full, base)); + } else { + out.push(path.relative(base, full)); + } + } + return out; +} + +export function initTemplate(id: string, outputDir: string): CommandResult { + const known = listTemplates().some((t) => t.id === id); + if (!known) { + return validation(`unknown template: ${id}`, "/template"); + } + + const templateDir = path.join(TEMPLATES_DIR, id); + const dest = path.resolve(outputDir); + + if (fs.existsSync(dest)) { + const entries = fs.readdirSync(dest); + if (entries.length > 0) { + const colliding = path.join(dest, entries[0]!); + return validation(`output path already exists: ${colliding}`); + } + } + + for (const rel of walkFiles(templateDir)) { + const target = path.join(dest, rel); + if (fs.existsSync(target)) { + return validation(`output path already exists: ${target}`); + } + } + + try { + fs.cpSync(templateDir, dest, { + recursive: true, + errorOnExist: true, + force: false, + }); + } catch (err) { + const message = + err instanceof Error ? err.message : `failed to copy template: ${id}`; + return validation(message); + } + + return okResult("init", { template: id, output: dest }); +} diff --git a/src/cli/commands/plan.ts b/src/cli/commands/plan.ts new file mode 100644 index 0000000..e28b399 --- /dev/null +++ b/src/cli/commands/plan.ts @@ -0,0 +1,68 @@ +import { failResult, okResult, type CommandResult } from "../envelope.js"; +import { loadProject } from "../../project/load.js"; +import { validateProject } from "../../project/validate.js"; +import { createRpcClient } from "../../rpc/client.js"; +import { generatePlan } from "../../plan/generate.js"; +import { isCommandError } from "./build.js"; + +export async function planCommand( + cwd: string, + intent: string, + publishTargetId?: string, +): Promise { + if ( + intent !== "ingest" && + intent !== "refresh" && + intent !== "build" && + intent !== "publish" + ) { + return failResult("plan", { + code: "validation", + message: `unknown intent: ${intent}`, + resource_id: null, + pointer: "/intent", + retryable: false, + suggested_next: "plan --intent ingest|refresh|build|publish", + }); + } + try { + const doc = loadProject(cwd); + const validated = validateProject(doc, cwd); + if (!validated.ok) return failResult("plan", validated.error); + const project = validated.project; + if (intent !== "build" && intent !== "publish") { + if (!project.chain_sources?.length || !project.event_sources?.length) { + return failResult("plan", { + code: "policy_refused", + message: "plan --intent ingest|refresh requires an ingest project (chain_sources + event_sources)", + resource_id: project.id, + pointer: "/chain_sources", + retryable: false, + suggested_next: null, + }); + } + } + const chain = project.chain_sources?.[0]; + const rpcUrl = chain ? (process.env[chain.rpc_secret] ?? "") : ""; + const { plan, planPath } = await generatePlan({ + intent, + cwd, + project, + rpcClient: createRpcClient(rpcUrl), + publishTargetId, + }); + return okResult("plan", { + plan_id: plan.plan_id, + plan_path: planPath, + intent: plan.intent, + chain: plan.chain, + sources: plan.sources, + actions: plan.actions, + limits: plan.limits, + state_assumptions: plan.state_assumptions, + }); + } catch (err) { + if (isCommandError(err)) return failResult("plan", err); + throw err; + } +} diff --git a/src/cli/commands/publish.ts b/src/cli/commands/publish.ts new file mode 100644 index 0000000..69bf03c --- /dev/null +++ b/src/cli/commands/publish.ts @@ -0,0 +1,41 @@ +import { failResult, okResult, type CommandResult } from "../envelope.js"; +import { loadProject } from "../../project/load.js"; +import { validateProject } from "../../project/validate.js"; +import { generatePlan } from "../../plan/generate.js"; +import { applyPlan } from "../../plan/apply.js"; +import { rindexerAdapter } from "../../ingest/rindexer/index.js"; +import { createRpcClient } from "../../rpc/client.js"; +import { isCommandError } from "./build.js"; + +export async function publishCommand( + cwd: string, + targetId?: string, +): Promise { + try { + const doc = loadProject(cwd); + const validated = validateProject(doc, cwd); + if (!validated.ok) return failResult("publish", validated.error); + const project = validated.project; + const chain = project.chain_sources?.[0]; + const { planPath } = await generatePlan({ + intent: "publish", + cwd, + project, + rpcClient: createRpcClient( + chain ? (process.env[chain.rpc_secret] ?? "") : "", + ), + publishTargetId: targetId, + }); + const outcome = await applyPlan({ + cwd, + planRef: planPath, + adapter: rindexerAdapter(), + rindexerBin: process.env.CHAINPLOT_RINDEXER_BIN ?? "rindexer", + rpcClient: createRpcClient(""), + }); + return okResult("publish", outcome); + } catch (err) { + if (isCommandError(err)) return failResult("publish", err); + throw err; + } +} diff --git a/src/cli/commands/query.ts b/src/cli/commands/query.ts new file mode 100644 index 0000000..5e321f7 --- /dev/null +++ b/src/cli/commands/query.ts @@ -0,0 +1,105 @@ +import fs from "node:fs"; +import path from "node:path"; +import { failResult, okResult, type CommandError, type CommandResult } from "../envelope.js"; +import { loadProject } from "../../project/load.js"; +import { validateProject } from "../../project/validate.js"; +import { runQuery } from "../../query/runQuery.js"; +import { rawAmountNames } from "../../project/columns.js"; +import { rowLimitFor } from "../../project/limits.js"; + + +function isCommandError(err: unknown): err is CommandError { + return ( + typeof err === "object" && + err !== null && + "code" in err && + "message" in err && + "resource_id" in err && + "pointer" in err && + "retryable" in err && + "suggested_next" in err + ); +} + +function error( + code: CommandError["code"], + message: string, + opts: { resource_id?: string | null; pointer?: string | null } = {}, +): CommandError { + return { + code, + message, + resource_id: opts.resource_id ?? null, + pointer: opts.pointer ?? null, + retryable: false, + suggested_next: null, + }; +} + +export async function querySnapshot( + cwd: string, + file: string, + snapshotId: string, +): Promise { + const command = "query"; + try { + const doc = loadProject(cwd); + const result = validateProject(doc, cwd); + if (!result.ok) { + return failResult(command, result.error); + } + + const dataset = (result.project.datasets ?? []).find((d) => d.id === snapshotId); + if (!dataset) { + return failResult( + command, + error("validation", `unknown dataset: ${snapshotId}`, { + resource_id: snapshotId, + pointer: "/datasets", + }), + ); + } + + const sqlPath = path.isAbsolute(file) ? file : path.resolve(cwd, file); + if (!fs.existsSync(sqlPath)) { + return failResult(command, error("validation", `missing SQL file: ${file}`)); + } + + // All datasets in scope, matching `build`. + const tables = Object.fromEntries( + (result.project.datasets ?? []).map((d) => [ + d.id, + path.resolve(cwd, d.snapshot), + ]), + ); + const rawAmountColumns = (result.project.queries ?? []) + .filter((q) => q.dataset === snapshotId) + .flatMap((q) => rawAmountNames(q.raw_amount_columns)); + + const models = result.project.models ?? []; + let modelSql: { id: string; sql: string }[] | undefined; + if (models.length > 0) { + const { topoSortModels } = await import("../../project/modelGraph.js"); + const modelOrder = topoSortModels(models); + modelSql = modelOrder.map((id) => { + const m = models.find((mod) => mod.id === id)!; + const f = path.resolve(cwd, m.file); + return { id, sql: fs.readFileSync(f, "utf8") }; + }); + } + + const data = await runQuery({ + sql: fs.readFileSync(sqlPath, "utf8"), + tables, + rawAmountColumns, + rowLimit: rowLimitFor(result.project), + models: modelSql, + }); + return okResult(command, { ...data, snapshot: dataset.snapshot }); + } catch (err) { + if (isCommandError(err)) { + return failResult(command, err); + } + throw err; + } +} diff --git a/src/cli/commands/refresh.ts b/src/cli/commands/refresh.ts new file mode 100644 index 0000000..d50ed83 --- /dev/null +++ b/src/cli/commands/refresh.ts @@ -0,0 +1,132 @@ +import { failResult, okResult, type CommandResult } from "../envelope.js"; +import { loadProject } from "../../project/load.js"; +import { validateProject } from "../../project/validate.js"; +import { generatePlan } from "../../plan/generate.js"; +import { applyPlan } from "../../plan/apply.js"; +import { rindexerAdapter } from "../../ingest/rindexer/index.js"; +import { createRpcClient } from "../../rpc/client.js"; +import { lastSucceededRunKey, readJournalProject } from "../../runtime/journal.js"; +import { publishRelease } from "../../publish/publishRelease.js"; +import { isCommandError } from "./build.js"; + +function refused(message: string, suggestedNext: string | null = null) { + return failResult("refresh", { + code: "policy_refused", + message, + resource_id: null, + pointer: null, + retryable: false, + suggested_next: suggestedNext, + }); +} + +function ingestSignature(project: unknown): string { + const p = project as { + chain_sources?: unknown; + event_sources?: unknown; + publish_targets?: unknown; + }; + return JSON.stringify([ + p.chain_sources ?? null, + p.event_sources ?? null, + p.publish_targets ?? null, + ]); +} + +export async function refreshCommand( + cwd: string, + publishTarget?: string, + jsonl = false, +): Promise { + try { + const doc = loadProject(cwd); + const validated = validateProject(doc, cwd); + if (!validated.ok) return failResult("refresh", validated.error); + const project = validated.project; + + if (!project.chain_sources?.length || !project.event_sources?.length) { + return refused( + "refresh requires an ingest project (chain_sources + event_sources)", + "use build for dataset-only projects", + ); + } + if ( + publishTarget !== undefined && + !project.publish_targets?.some((t) => t.id === publishTarget) + ) { + return refused( + `--publish-target ${publishTarget} is not a target in chainplot.yaml`, + "add the target to chainplot.yaml, then plan --intent publish + apply", + ); + } + + // Authorization: replay only the class the last applied plan authorized. + const lastKey = lastSucceededRunKey(cwd); + if (lastKey !== null) { + const lastProject = readJournalProject(cwd, lastKey); + if (lastProject !== null) { + const before = ingestSignature(lastProject); + const after = ingestSignature(project); + if (before !== after) { + return refused( + "addresses, bounds, chain identity, or publish targets changed since the last applied plan; refresh may not widen scope", + "run plan --intent ingest + apply out-of-band", + ); + } + } + } + // No last applied plan: authorization derives from the project file alone + // (first follow_finalized run from start_block is allowed; nothing else widens). + + const chain = project.chain_sources[0]; + const rpcClient = createRpcClient(process.env[chain.rpc_secret] ?? ""); + const { planPath } = await generatePlan({ + intent: "refresh", + cwd, + project, + rpcClient, + }); + const outcome = await applyPlan({ + cwd, + planRef: planPath, + adapter: rindexerAdapter(), + rindexerBin: process.env.CHAINPLOT_RINDEXER_BIN ?? "rindexer", + rpcClient, + onProgress: jsonl ? progressWriter : undefined, + }); + + if (publishTarget !== undefined) { + const targetDoc = project.publish_targets?.find( + (t) => t.id === publishTarget, + ); + if (!targetDoc) { + return refused(`publish target ${publishTarget} not found`); + } + const published = await publishRelease(cwd, targetDoc); + return okResult("refresh", { ...outcome, publish: published }); + } + return okResult("refresh", outcome); + } catch (err) { + if (isCommandError(err)) return failResult("refresh", err); + throw err; + } +} + +function progressWriter(event: { + run_id: string; + stage: string; + message?: string; + rows?: number; +}): void { + process.stdout.write( + JSON.stringify({ + schema_version: 1, + type: "progress", + run_id: event.run_id, + stage: event.stage, + ...(event.message !== undefined ? { message: event.message } : {}), + ...(event.rows !== undefined ? { rows: event.rows } : {}), + ts: new Date().toISOString(), + }) + "\n", + ); +} diff --git a/src/cli/commands/runs.ts b/src/cli/commands/runs.ts new file mode 100644 index 0000000..f62bebe --- /dev/null +++ b/src/cli/commands/runs.ts @@ -0,0 +1,80 @@ +import fs from "node:fs"; +import path from "node:path"; +import { okResult, failResult, type CommandResult } from "../envelope.js"; +import { + listRuns, + readJournalPlan, + readJournalStatus, + writeJournalStatus, + journalDir, +} from "../../runtime/journal.js"; + +export function runsList(cwd: string): CommandResult { + return okResult("runs list", { runs: listRuns(cwd) }); +} + +export function runsShow(cwd: string, key: string): CommandResult { + const status = readJournalStatus(cwd, key); + if (!status) { + return failResult("runs show", { + code: "validation", + message: `no run with idempotency key ${key}`, + resource_id: key, + pointer: "/idempotency_key", + retryable: false, + suggested_next: "runs list", + }); + } + return okResult("runs show", { + idempotency_key: key, + status, + plan: readJournalPlan(cwd, key), + }); +} + +export function runsCancel(cwd: string, key: string): CommandResult { + const status = readJournalStatus(cwd, key); + if (!status) { + return failResult("runs cancel", { + code: "validation", + message: `no run with idempotency key ${key}`, + resource_id: key, + pointer: "/idempotency_key", + retryable: false, + suggested_next: "runs list", + }); + } + if (status.status === "succeeded") { + return failResult("runs cancel", { + code: "validation", + message: `run ${key} already succeeded; nothing to cancel`, + resource_id: key, + pointer: null, + retryable: false, + suggested_next: null, + }); + } + // Two halves, because there are two situations. + // + // A live apply notices the flag at its next checkpoint and stops there — + // cancellation is cooperative, not a kill. But a run whose process died + // leaves the journal saying "running", and `apply` refuses to start while it + // does, so the flag alone would strand the project with no way back. Moving + // the run to a terminal state here is what unsticks it. + fs.mkdirSync(journalDir(cwd, key), { recursive: true }); + fs.writeFileSync( + path.join(journalDir(cwd, key), "cancel_requested"), + new Date().toISOString(), + ); + writeJournalStatus(cwd, key, { + status: "canceled", + plan_id: status.plan_id, + plan_digest: status.plan_digest, + }); + return okResult("runs cancel", { + idempotency_key: key, + cancel_requested: true, + status: "canceled", + was: status.status, + }); +} diff --git a/src/cli/commands/schemaShow.ts b/src/cli/commands/schemaShow.ts new file mode 100644 index 0000000..a8a094d --- /dev/null +++ b/src/cli/commands/schemaShow.ts @@ -0,0 +1,45 @@ +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { failResult, okResult, type CommandResult } from "../envelope.js"; + +export const SCHEMA_KINDS = [ + "project", + "plan", + "result", + "progress", + "release", + "manifest", + "latest", + "coverage", + "lock", +] as const; +export type SchemaKind = (typeof SCHEMA_KINDS)[number]; + +const SCHEMAS_DIR = path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + "../../../schemas", +); + +function isSchemaKind(kind: string): kind is SchemaKind { + return (SCHEMA_KINDS as readonly string[]).includes(kind); +} + +export function schemaShow(kind: string): CommandResult { + const command = "schema show"; + if (!isSchemaKind(kind)) { + return failResult(command, { + code: "validation", + message: `unknown schema kind: ${kind}`, + resource_id: null, + pointer: "/kind", + retryable: false, + suggested_next: null, + }); + } + + const schemaPath = path.join(SCHEMAS_DIR, `${kind}.schema.json`); + const raw = fs.readFileSync(schemaPath, "utf8"); + const schema = JSON.parse(raw) as unknown; + return okResult(command, schema); +} diff --git a/src/cli/commands/serve.ts b/src/cli/commands/serve.ts new file mode 100644 index 0000000..20b7739 --- /dev/null +++ b/src/cli/commands/serve.ts @@ -0,0 +1,55 @@ +import path from "node:path"; +import { + failResult, + okResult, + type CommandResult, +} from "../envelope.js"; +import { startServe, validateServeDir } from "../../publish/serve.js"; +import { isCommandError } from "./build.js"; + +let activeServer: { close(): void } | null = null; + +/** Shut down the preview server, if one is running. Used by tests. */ +export function closeActiveServer(): void { + activeServer?.close(); + activeServer = null; +} + +export async function serveCommand( + cwd: string, + dir?: string, + port = 0, +): Promise { + try { + const target = path.resolve(cwd, dir ?? "dist/releases/local"); + validateServeDir(target); + const handle = startServe(target, port); + activeServer = handle; + // The assigned port is only knowable once listen() has called back, so + // reporting it before that yields the literal 0 the caller passed in. + await handle.ready; + const url = `http://127.0.0.1:${handle.port}`; + // Diagnostics to stderr; stdout stays reserved for the result envelope. + process.stderr.write(`serving ${target} at ${url} (Ctrl+C to stop)\n`); + const shutdown = () => { + closeActiveServer(); + process.exit(0); + }; + process.on("SIGINT", shutdown); + process.on("SIGTERM", shutdown); + return okResult("serve", { url, dir: target }); + } catch (err) { + if (isCommandError(err)) return failResult("serve", err); + if (err instanceof Error && "code" in err && (err as { code?: string }).code === "ENOENT") { + return failResult("serve", { + code: "validation", + message: err.message, + resource_id: null, + pointer: null, + retryable: false, + suggested_next: "run build first", + }); + } + throw err; + } +} diff --git a/src/cli/commands/templates.ts b/src/cli/commands/templates.ts new file mode 100644 index 0000000..6b3afab --- /dev/null +++ b/src/cli/commands/templates.ts @@ -0,0 +1,29 @@ +import { okResult, type CommandResult } from "../envelope.js"; + +export interface TemplateInfo { + id: string; + required_inputs: string[]; + limitations: string; +} + +const TEMPLATES: TemplateInfo[] = [ + { + id: "fixture-transfers", + required_inputs: [], + limitations: "dataset-only fixture; no RPC; no ingest", + }, + { + id: "ingest-transfers", + required_inputs: ["RPC_URL (archive-capable Ethereum JSON-RPC)", "Docker for compose"], + limitations: + "one chain, explicit addresses, pinned end block; rindexer image is linux/amd64", + }, +]; + +export function listTemplates(): TemplateInfo[] { + return TEMPLATES.map((t) => ({ ...t, required_inputs: [...t.required_inputs] })); +} + +export function templatesList(): CommandResult<{ templates: TemplateInfo[] }> { + return okResult("templates list", { templates: listTemplates() }); +} diff --git a/src/cli/commands/test.ts b/src/cli/commands/test.ts new file mode 100644 index 0000000..d268b48 --- /dev/null +++ b/src/cli/commands/test.ts @@ -0,0 +1,6 @@ +import { runAssertions } from "../../project/assertions.js"; +import type { CommandResult } from "../envelope.js"; + +export async function testProject(cwd: string): Promise { + return runAssertions(cwd); +} diff --git a/src/cli/commands/validate.ts b/src/cli/commands/validate.ts new file mode 100644 index 0000000..d246334 --- /dev/null +++ b/src/cli/commands/validate.ts @@ -0,0 +1,33 @@ +import { failResult, okResult, type CommandError, type CommandResult } from "../envelope.js"; +import { loadProject } from "../../project/load.js"; +import { validateProject } from "../../project/validate.js"; + +function isCommandError(err: unknown): err is CommandError { + return ( + typeof err === "object" && + err !== null && + "code" in err && + "message" in err && + "resource_id" in err && + "pointer" in err && + "retryable" in err && + "suggested_next" in err + ); +} + +export function validate(cwd: string): CommandResult { + const command = "validate"; + try { + const doc = loadProject(cwd); + const result = validateProject(doc, cwd); + if (!result.ok) { + return failResult(command, result.error); + } + return okResult(command, { id: result.project.id }); + } catch (err) { + if (isCommandError(err)) { + return failResult(command, err); + } + throw err; + } +} diff --git a/src/cli/envelope.ts b/src/cli/envelope.ts new file mode 100644 index 0000000..ea5f5d1 --- /dev/null +++ b/src/cli/envelope.ts @@ -0,0 +1,53 @@ +export const SCHEMA_VERSION = 1 as const; + +export type ErrorCode = + | "validation" + | "missing_credentials" + | "unsupported_capability" + | "policy_refused" + | "source_inconsistent" + | "transient_dependency" + | "internal"; + +export interface CommandError { + code: ErrorCode; + message: string; + resource_id: string | null; + pointer: string | null; + retryable: boolean; + suggested_next: string | null; +} + +export interface CommandResult { + schema_version: typeof SCHEMA_VERSION; + ok: boolean; + command: string; + data: T | null; + warnings: string[]; + error: CommandError | null; +} + +export function okResult(command: string, data: T): CommandResult { + return { + schema_version: SCHEMA_VERSION, + ok: true, + command, + data, + warnings: [], + error: null, + }; +} + +export function failResult( + command: string, + error: CommandError, +): CommandResult { + return { + schema_version: SCHEMA_VERSION, + ok: false, + command, + data: null, + warnings: [], + error, + }; +} diff --git a/src/cli/main.ts b/src/cli/main.ts new file mode 100644 index 0000000..9cfd364 --- /dev/null +++ b/src/cli/main.ts @@ -0,0 +1,32 @@ +#!/usr/bin/env node +import { failResult, type CommandResult } from "./envelope.js"; +import { runCli } from "./run.js"; +import { loadDotEnv } from "../config/env.js"; +import { errorMessage } from "../plan/errors.js"; + +// Secrets enter through the environment or a local .env (see .env.example). +loadDotEnv(process.cwd()); + +let result: CommandResult; +try { + result = await runCli(process.argv.slice(2), { cwd: process.cwd() }); +} catch (err) { + const argv = process.argv.slice(2).filter((a) => a !== "--json"); + const command = argv.find((a) => !a.startsWith("-")) ?? ""; + result = failResult(command, { + code: "internal", + message: errorMessage(err), + resource_id: null, + pointer: null, + retryable: false, + suggested_next: null, + }); +} +process.stdout.write(JSON.stringify(result) + "\n"); +// `serve` is long-running: its result (with the URL) is printed once and the +// process stays alive until SIGINT/SIGTERM (handled inside the command). +if (result.ok && result.command === "serve") { + // keep the event loop alive; no exit code until the signal handler runs. +} else { + process.exit(result.ok ? 0 : 1); +} diff --git a/src/cli/run.ts b/src/cli/run.ts new file mode 100644 index 0000000..1ea9704 --- /dev/null +++ b/src/cli/run.ts @@ -0,0 +1,325 @@ +import path from "node:path"; +import { Command, CommanderError } from "commander"; +import { failResult, type CommandResult } from "./envelope.js"; +import { capabilities } from "./commands/capabilities.js"; +import { datasetDescribe } from "./commands/describe.js"; +import { initTemplate } from "./commands/init.js"; +import { querySnapshot } from "./commands/query.js"; +import { schemaShow } from "./commands/schemaShow.js"; +import { templatesList } from "./commands/templates.js"; +import { build } from "./commands/build.js"; +import { testProject } from "./commands/test.js"; +import { validate } from "./commands/validate.js"; +import { planCommand } from "./commands/plan.js"; +import { applyCommand } from "./commands/apply.js"; +import { refreshCommand } from "./commands/refresh.js"; +import { runsList, runsShow, runsCancel } from "./commands/runs.js"; +import { serveCommand } from "./commands/serve.js"; +import { publishCommand } from "./commands/publish.js"; +import { doctorCommand } from "./commands/doctor.js"; +import { forkCommand } from "./commands/fork.js"; + +export interface RunCliOptions { + cwd: string; +} + +export type { CommandResult } from "./envelope.js"; + +function validationError( + command: string, + message: string, + pointer: string | null = null, +): CommandResult { + return failResult(command, { + code: "validation", + message, + resource_id: null, + pointer, + retryable: false, + suggested_next: null, + }); +} + +export async function runCli( + argv: string[], + opts: RunCliOptions, +): Promise { + const json = argv.includes("--json"); + const cmdArgv = argv.filter((arg) => arg !== "--json"); + + // Gate before dispatch, not after. Checking this once the action has run + // means a refused `build` still writes a release and a refused `publish` + // still uploads — a caller that trusts `ok: false` and retries would then + // double the side effects. + if (!json) { + return validationError( + cmdArgv.find((arg) => !arg.startsWith("-")) ?? "", + "--json is required", + "/json", + ); + } + + let result: CommandResult | undefined; + let commandName = ""; + + const program = new Command(); + program + .name("chainplot") + .exitOverride() + .allowExcessArguments(false) + .showHelpAfterError(false) + .configureOutput({ + writeOut: () => {}, + writeErr: () => {}, + }); + + program + .command("capabilities") + .description("report CLI version and supported capabilities") + .action(() => { + commandName = "capabilities"; + result = capabilities(); + }); + + program + .command("validate") + .description("load and validate chainplot.yaml") + .action(() => { + commandName = "validate"; + result = validate(opts.cwd); + }); + + program + .command("init") + .description("scaffold a project from a template") + .requiredOption("--template ", "template id") + .requiredOption("--output ", "output directory") + .action((options: { template: string; output: string }) => { + commandName = "init"; + const output = path.isAbsolute(options.output) + ? options.output + : path.resolve(opts.cwd, options.output); + result = initTemplate(options.template, output); + }); + + const templates = program + .command("templates") + .description("template operations"); + templates + .command("list") + .description("list available project templates") + .action(() => { + commandName = "templates list"; + result = templatesList(); + }); + + const schema = program + .command("schema") + .description("JSON Schema operations"); + schema + .command("show") + .description("print the JSON Schema for a document kind") + .argument("", "schema kind") + .action((kind: string) => { + commandName = "schema show"; + result = schemaShow(kind); + }); + + const dataset = program.command("dataset").description("dataset operations"); + dataset + .command("describe") + .description("describe a dataset snapshot") + .argument("", "dataset id") + .action(async (id: string) => { + commandName = "dataset describe"; + result = await datasetDescribe(opts.cwd, id); + }); + + program + .command("query") + .description("run SQL against a dataset snapshot") + .requiredOption("--file ", "SQL file") + .requiredOption("--snapshot ", "dataset id") + .action(async (options: { file: string; snapshot: string }) => { + commandName = "query"; + result = await querySnapshot(opts.cwd, options.file, options.snapshot); + }); + + program + .command("test") + .description("run local dataset assertions") + .action(async () => { + commandName = "test"; + result = await testProject(opts.cwd); + }); + + program + .command("build") + .description("write the full static release (results, dashboards, viewer, source/)") + .option( + "--mode ", + "dataset_included | results_only | dataset_referenced", + (v: string) => v as "dataset_included" | "results_only" | "dataset_referenced", + ) + .action(async (options: { mode?: "dataset_included" | "results_only" | "dataset_referenced" }) => { + commandName = "build"; + result = await build(opts.cwd, options.mode); + }); + + program + .command("plan") + .description("write a digest-bound plan; mutates nothing") + .requiredOption("--intent ", "ingest|refresh|build|publish") + .option("--publish-target ", "publish target id for --intent publish") + .action(async (options: { intent: string; publishTarget?: string }) => { + commandName = "plan"; + result = await planCommand( + opts.cwd, + options.intent, + options.publishTarget, + ); + }); + + program + .command("publish") + .description("publish an already-built release to a target") + .option("--publish-target ", "publish target id (default: first)") + .action(async (options: { publishTarget?: string }) => { + commandName = "publish"; + result = await publishCommand(opts.cwd, options.publishTarget); + }); + + program + .command("apply") + .description("execute a plan written by `plan`") + .requiredOption("--plan ", "plan path or plan id") + .option("--idempotency-key ", "idempotency key (default: plan digest)") + .action(async (options: { plan: string; idempotencyKey?: string }) => { + commandName = "apply"; + result = await applyCommand( + opts.cwd, + options.plan, + options.idempotencyKey, + argv.includes("--jsonl"), + ); + }); + + program + .command("refresh") + .description("resume indexing, rebuild, optionally publish") + .option("--publish-target ", "publish target id (must exist in chainplot.yaml)") + .action(async (options: { publishTarget?: string }) => { + commandName = "refresh"; + result = await refreshCommand( + opts.cwd, + options.publishTarget, + argv.includes("--jsonl"), + ); + }); + + const runs = program.command("runs").description("run journal operations"); + runs + .command("list") + .description("list journal entries") + .action(() => { + commandName = "runs list"; + result = runsList(opts.cwd); + }); + runs + .command("show") + .description("show one journal entry") + .argument("", "idempotency key") + .action((key: string) => { + commandName = "runs show"; + result = runsShow(opts.cwd, key); + }); + runs + .command("cancel") + .description("request cooperative cancel of a running apply") + .argument("", "idempotency key") + .action((key: string) => { + commandName = "runs cancel"; + result = runsCancel(opts.cwd, key); + }); + + program + .command("serve") + .description("preview a built release over loopback HTTP") + .option("--dir ", "release directory (default: dist/releases/local)") + .option("--port ", "port (default: random)", (v: string) => parseInt(v, 10)) + .action(async (options: { dir?: string; port?: number }) => { + commandName = "serve"; + result = await serveCommand(opts.cwd, options.dir, options.port ?? 0); + if (result.ok) { + // serve keeps the process alive; the envelope is printed by main.ts + // only when it exits, so surface the URL immediately on stderr. + const url = (result.data as { url: string }).url; + process.stderr.write(`${url}\n`); + } + }); + + program + .command("doctor") + .description("check credentials, RPC, Postgres, rindexer, storage, S3") + .action(async () => { + commandName = "doctor"; + result = await doctorCommand(opts.cwd); + }); + + program + .command("fork") + .description("import a published release into a new project") + .requiredOption("--from ", "release directory or https URL") + .requiredOption("--output ", "new project directory") + .option( + "--allow-private-networks", + "escape hatch: allow fetches to private networks (default off)", + false, + ) + .action( + async (options: { + from: string; + output: string; + allowPrivateNetworks: boolean; + }) => { + commandName = "fork"; + result = await forkCommand( + options.from, + options.output, + options.allowPrivateNetworks, + ); + }, + ); + + try { + await program.parseAsync(cmdArgv, { from: "user" }); + } catch (err) { + if (err instanceof CommanderError) { + const cmd = + commandName || + cmdArgv.find((a) => !a.startsWith("-")) || + ""; + if ( + err.code === "commander.helpDisplayed" || + err.code === "commander.help" + ) { + return validationError( + cmd, + cmd ? "help is not available in JSON mode" : "a command is required", + cmd ? null : "/command", + ); + } + if (err.code === "commander.unknownCommand") { + return validationError(cmd, err.message, "/command"); + } + return validationError(cmd, err.message); + } + throw err; + } + + if (!result) { + return validationError(commandName, "a command is required", "/command"); + } + + return result; +} diff --git a/src/config/env.ts b/src/config/env.ts new file mode 100644 index 0000000..bceea6f --- /dev/null +++ b/src/config/env.ts @@ -0,0 +1,27 @@ +// Minimal .env loader: KEY=VALUE lines, # comments, no dependency. +// Existing environment variables always win (never override). +import fs from "node:fs"; +import path from "node:path"; + +export function loadDotEnv(dir: string = process.cwd()): void { + const file = path.join(dir, ".env"); + if (!fs.existsSync(file)) return; + const text = fs.readFileSync(file, "utf8"); + for (const rawLine of text.split("\n")) { + const line = rawLine.trim(); + if (!line || line.startsWith("#")) continue; + const eq = line.indexOf("="); + if (eq <= 0) continue; + const key = line.slice(0, eq).trim(); + let value = line.slice(eq + 1).trim(); + if ( + (value.startsWith('"') && value.endsWith('"')) || + (value.startsWith("'") && value.endsWith("'")) + ) { + value = value.slice(1, -1); + } + if (process.env[key] === undefined) { + process.env[key] = value; + } + } +} diff --git a/src/fork/fetchGuard.ts b/src/fork/fetchGuard.ts new file mode 100644 index 0000000..787cdbf --- /dev/null +++ b/src/fork/fetchGuard.ts @@ -0,0 +1,214 @@ +import { lookup } from "node:dns/promises"; +import https from "node:https"; +import type { RequestOptions } from "node:https"; +import type { IncomingMessage } from "node:http"; +import { commandError } from "../plan/errors.js"; + +export interface FetchGuardOptions { + allowPrivateNetworks?: boolean; // default false; documented escape hatch + maxBytes: number; + timeoutMs?: number; // per request, default 30_000 +} + +export const FORK_LIMITS = { + releaseJsonBytes: 1024 * 1024, + totalBytes: 512 * 1024 * 1024, + perRequestTimeoutMs: 30_000, +}; + +function ipv4ToInt(ip: string): number | null { + const parts = ip.split("."); + if (parts.length !== 4) return null; + let out = 0; + for (const part of parts) { + const n = Number(part); + if (!Number.isInteger(n) || n < 0 || n > 255) return null; + out = out * 256 + n; + } + return out >>> 0; +} + +function inCidr4(ip: string, base: string, bits: number): boolean { + const a = ipv4ToInt(ip); + const b = ipv4ToInt(base); + if (a === null || b === null) return false; + const mask = bits === 0 ? 0 : (0xffffffff << (32 - bits)) >>> 0; + return (a & mask) === (b & mask); +} + +function isBlockedIPv4(ip: string): boolean { + return ( + inCidr4(ip, "0.0.0.0", 8) || + inCidr4(ip, "10.0.0.0", 8) || + inCidr4(ip, "100.64.0.0", 10) || + inCidr4(ip, "127.0.0.0", 8) || + inCidr4(ip, "169.254.0.0", 16) || + inCidr4(ip, "172.16.0.0", 12) || + inCidr4(ip, "192.168.0.0", 16) + ); +} + +function isBlockedIPv6(ip: string): boolean { + const lower = ip.toLowerCase(); + if (lower === "::" || lower === "::1") return true; + // IPv4-mapped ::ffff:a.b.c.d + const mapped = lower.match(/^::ffff:(\d+\.\d+\.\d+\.\d+)$/); + if (mapped) return isBlockedIPv4(mapped[1]!); + const firstGroup = parseInt(lower.split(":")[0] ?? "0", 16); + if (Number.isNaN(firstGroup)) return false; + const firstByte = (firstGroup >> 8) & 0xff; + // fc00::/7 (ULA): first byte 0xfc-0xfd + if ((firstByte & 0xfe) === 0xfc) return true; + // fe80::/10 (link-local): first 10 bits = 1111111010 → fe80..febf in group 0 + if (firstGroup >= 0xfe80 && firstGroup <= 0xfebf) return true; + return false; +} + +// Normalize non-dotted IP literal forms (decimal/hex/octal) to dotted quad. +export function normalizeIpLiteral(host: string): string | null { + if (/^\d+\.\d+\.\d+\.\d+$/.test(host)) { + // reject leading-zero octal components like 0177.0.0.1 + if (/(^|\.)0\d/.test(host)) { + const parts = host.split(".").map((p) => parseInt(p, 8)); + if (parts.every((n) => Number.isInteger(n) && n >= 0 && n <= 255)) { + return parts.join("."); + } + return null; + } + return host; + } + if (/^0x[0-9a-f]+$/i.test(host)) { + const n = Number.parseInt(host, 16); + return n >= 0 && n <= 0xffffffff ? intToIpv4(n >>> 0) : null; + } + if (/^\d+$/.test(host)) { + const n = Number.parseInt(host, 10); + return n >= 0 && n <= 0xffffffff ? intToIpv4(n >>> 0) : null; + } + return null; +} + +function intToIpv4(n: number): string { + return [(n >>> 24) & 255, (n >>> 16) & 255, (n >>> 8) & 255, n & 255].join("."); +} + +export function isBlockedAddress(address: string): boolean { + if (address.includes(":")) return isBlockedIPv6(address); + return isBlockedIPv4(address); +} + +export async function assertAllowedUrl( + rawUrl: string, + opts: FetchGuardOptions, +): Promise<{ url: string; pinnedAddress: string | null }> { + let url: URL; + try { + url = new URL(rawUrl); + } catch { + throw commandError("validation", `invalid URL: ${rawUrl}`); + } + if (url.protocol !== "https:") { + throw commandError("validation", `fork fetch requires https: got ${url.protocol}`); + } + const host = url.hostname.replace(/^\[|\]$/g, ""); + const literal = normalizeIpLiteral(host); + if (literal && isBlockedAddress(literal) && !opts.allowPrivateNetworks) { + throw commandError("policy_refused", `fork fetch blocked address: ${host}`); + } + let pinnedAddress: string | null = literal; + if (!literal) { + const addresses = await lookup(host, { all: true, verbatim: true }); + if (addresses.length === 0) { + throw commandError("transient_dependency", `DNS resolution failed: ${host}`, { + retryable: true, + }); + } + for (const { address } of addresses) { + if (isBlockedAddress(address) && !opts.allowPrivateNetworks) { + throw commandError( + "policy_refused", + `fork fetch blocked: ${host} resolves to private address ${address}`, + ); + } + } + pinnedAddress = addresses[0]!.address; + } + return { url: rawUrl, pinnedAddress }; +} + +// HTTPS GET with: no redirects, pinned DNS, per-request timeout, byte cap. +export function guardedFetch( + rawUrl: string, + opts: FetchGuardOptions, +): Promise<{ body: Buffer; contentType: string | null }> { + const timeoutMs = opts.timeoutMs ?? FORK_LIMITS.perRequestTimeoutMs; + return assertAllowedUrl(rawUrl, opts).then( + ({ pinnedAddress }) => + new Promise((resolve, reject) => { + const url = new URL(rawUrl); + const req = https.request( + { + hostname: pinnedAddress ?? url.hostname, + port: url.port ? Number(url.port) : 443, + path: `${url.pathname}${url.search}`, + method: "GET", + servername: url.hostname, + timeout: timeoutMs, + headers: { host: url.hostname }, + } as import("node:https").RequestOptions, + (res: import("node:http").IncomingMessage) => { + const status = res.statusCode ?? 0; + if (status >= 300 && status < 400) { + res.destroy(); + reject(commandError("policy_refused", `fork fetch: redirect forbidden (${status})`)); + return; + } + if (status !== 200) { + res.destroy(); + reject(commandError("transient_dependency", `fork fetch: HTTP ${status}`, { + retryable: true, + })); + return; + } + const chunks: Buffer[] = []; + let total = 0; + res.on("data", (chunk: Buffer) => { + total += chunk.length; + if (total > opts.maxBytes) { + res.destroy(); + reject( + commandError( + "policy_refused", + `fork fetch exceeded byte cap (${opts.maxBytes})`, + ), + ); + return; + } + chunks.push(chunk); + }); + res.on("end", () => { + resolve({ + body: Buffer.concat(chunks), + contentType: res.headers["content-type"] ?? null, + }); + }); + res.on("error", (err) => + reject(commandError("transient_dependency", `fork fetch failed: ${err.message}`, { + retryable: true, + })), + ); + }, + ); + req.on("timeout", () => { + const err = commandError( + "transient_dependency", + `fork fetch timed out after ${timeoutMs}ms`, + { retryable: true }, + ) as unknown as Error; + req.destroy(err); + }); + req.on("error", (err) => reject(err)); + req.end(); + }), + ); +} diff --git a/src/fork/importRelease.ts b/src/fork/importRelease.ts new file mode 100644 index 0000000..82aaf25 --- /dev/null +++ b/src/fork/importRelease.ts @@ -0,0 +1,321 @@ +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { createHash } from "node:crypto"; +import { Ajv2020 } from "ajv/dist/2020.js"; +import { commandError } from "../plan/errors.js"; +import { + guardedFetch, + FORK_LIMITS, + type FetchGuardOptions, +} from "./fetchGuard.js"; + +const ajv = new Ajv2020({ strict: true }); +const releaseSchema = JSON.parse( + fs.readFileSync( + path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + "../../schemas/release.schema.json", + ), + "utf8", + ), +) as object; +const validateRelease = ajv.compile(releaseSchema); + +export interface ReleaseDoc { + schema_version: number; + project_id: string; + mode: string; + queries: string[]; + dashboards: string[]; + generated_at: string; + snapshots: { dataset_id: string; snapshot_id: string }[]; + coverage: unknown[]; + finality: unknown; + files: { path: string; checksum: string }[]; +} + +export interface ForkLimits { + releaseJsonBytes: number; + totalBytes: number; + perRequestTimeoutMs: number; +} + +export const DEFAULT_LIMITS: ForkLimits = { + releaseJsonBytes: FORK_LIMITS.releaseJsonBytes, + totalBytes: FORK_LIMITS.totalBytes, + perRequestTimeoutMs: FORK_LIMITS.perRequestTimeoutMs, +}; + +export interface ForkSource { + kind: "local" | "url"; + location: string; +} + +export function parseForkSource(from: string): ForkSource { + if (/^https:\/\//.test(from)) return { kind: "url", location: from }; + if (fs.existsSync(from)) return { kind: "local", location: from }; + throw commandError("validation", `fork source not found: ${from}`); +} + +function readLocal(releaseRoot: string, rel: string, maxBytes: number): Buffer { + const resolved = path.resolve(releaseRoot, rel); + if (resolved !== releaseRoot && !resolved.startsWith(releaseRoot + path.sep)) { + throw commandError("policy_refused", `fork: path traversal in ${rel}`); + } + const stat = fs.statSync(resolved); + if (stat.size > maxBytes) { + throw commandError("policy_refused", `fork: ${rel} exceeds byte cap (${maxBytes})`); + } + return fs.readFileSync(resolved); +} + +export async function importRelease( + from: string, + outputDir: string, + opts: { allowPrivateNetworks?: boolean } = {}, +): Promise<{ + project_dir: string; + files: number; + release_prefix: string | null; + mode: string; + datasets_referenced: string[]; + warnings: string[]; +}> { + const source = parseForkSource(from); + const guard: FetchGuardOptions = { + allowPrivateNetworks: opts.allowPrivateNetworks, + maxBytes: DEFAULT_LIMITS.releaseJsonBytes, + }; + + // 1. release.json (≤ 1 MiB), validate against the frozen schema. + let releaseBody: Buffer; + let releaseRoot: string | null = null; + let releasePrefix: string | null = null; + if (source.kind === "local") { + // Accept either a release dir (contains release.json) or a publish root (contains latest.json). + if (fs.existsSync(path.join(source.location, "release.json"))) { + releaseRoot = path.resolve(source.location); + releaseBody = readLocal(releaseRoot, "release.json", DEFAULT_LIMITS.releaseJsonBytes); + } else if (fs.existsSync(path.join(source.location, "latest.json"))) { + const pointer = JSON.parse( + fs.readFileSync(path.join(source.location, "latest.json"), "utf8"), + ) as { release_prefix: string; release_json_checksum: string }; + releaseRoot = path.resolve(source.location); + releasePrefix = pointer.release_prefix; + releaseBody = readLocal( + releaseRoot, + path.join(pointer.release_prefix, "release.json"), + DEFAULT_LIMITS.releaseJsonBytes, + ); + const actual = createHash("sha256").update(releaseBody).digest("hex"); + if (actual !== pointer.release_json_checksum) { + throw commandError( + "policy_refused", + "fork: release.json does not match the latest.json pointer checksum", + ); + } + } else { + throw commandError("validation", `no release.json or latest.json in ${source.location}`); + } + } else { + const base = source.location.replace(/\/$/, ""); + const res = await guardedFetch(`${base}/release.json`, guard); + releaseBody = res.body; + } + if (!validateRelease(JSON.parse(releaseBody.toString("utf8")))) { + throw commandError("validation", "fork: release.json failed schema validation"); + } + const release = JSON.parse(releaseBody.toString("utf8")) as ReleaseDoc; + + // 2. Download/copy declared files with checksums, hard total cap. + const outDir = path.resolve(outputDir); + fs.mkdirSync(outDir, { recursive: true }); + let total = releaseBody.length; + for (const file of release.files) { + if (file.path.includes("..") || path.isAbsolute(file.path)) { + throw commandError("policy_refused", `fork: unsafe file path ${file.path}`); + } + let body: Buffer; + if (source.kind === "local") { + body = readLocal( + releaseRoot!, + releasePrefix ? path.join(releasePrefix, file.path) : file.path, + DEFAULT_LIMITS.totalBytes, + ); + } else { + const base = source.location.replace(/\/$/, ""); + const res = await guardedFetch(`${base}/${file.path}`, { + ...guard, + maxBytes: DEFAULT_LIMITS.totalBytes, + }); + body = res.body; + } + total += body.length; + if (total > DEFAULT_LIMITS.totalBytes) { + throw commandError( + "policy_refused", + `fork: total download exceeds ${DEFAULT_LIMITS.totalBytes} bytes`, + ); + } + const actual = createHash("sha256").update(body).digest("hex"); + if (actual !== file.checksum) { + throw commandError( + "policy_refused", + `fork: checksum mismatch for ${file.path}`, + ); + } + const dest = path.join(outDir, file.path); + fs.mkdirSync(path.dirname(dest), { recursive: true }); + fs.writeFileSync(dest, body); + } + + // 2b. A dataset_referenced release points at its parquet instead of + // carrying it, so the loop above never fetched it. Pull it in and verify + // it against the manifest, or the fork is a recipe with nothing to run + // against. + const referenced: string[] = []; + for (const dataset of manifestDatasets(outDir)) { + const body = await readReferenced(source, releasePrefix, dataset, guard); + total += body.length; + if (total > DEFAULT_LIMITS.totalBytes) { + throw commandError( + "policy_refused", + `fork: total download exceeds ${DEFAULT_LIMITS.totalBytes} bytes`, + ); + } + const actual = createHash("sha256").update(body).digest("hex"); + if (actual !== dataset.checksum) { + throw commandError( + "policy_refused", + `fork: checksum mismatch for referenced dataset ${dataset.path}`, + ); + } + const dest = path.join(outDir, dataset.path); + fs.mkdirSync(path.dirname(dest), { recursive: true }); + fs.writeFileSync(dest, body); + referenced.push(dataset.path); + } + + // 3. Forked project: chainplot.yaml from the sanitized source bundle, + // with dataset snapshots pinned to the forked copies. + const sourceYamlEntry = release.files.find((f) => f.path === "source/chainplot.yaml"); + if (!sourceYamlEntry) { + throw commandError( + "policy_refused", + "fork: release has no source/chainplot.yaml; results-only fork needs a recipe", + ); + } + const { parse: parseYaml, stringify: stringifyYaml } = await import("yaml"); + const projectDoc = parseYaml( + fs.readFileSync(path.join(outDir, "source/chainplot.yaml"), "utf8"), + ) as { + datasets?: { id: string; snapshot: string }[]; + chain_sources?: unknown; + event_sources?: unknown; + }; + for (const dataset of projectDoc.datasets ?? []) { + const basename = path.basename(dataset.snapshot); + dataset.snapshot = `datasets/${dataset.id}/tables/${basename}`; + } + // Referenced datasets land at the very same layout, so nothing special is + // needed here — the assertion is that they did land. + for (const rel of referenced) { + if (!fs.existsSync(path.join(outDir, rel))) { + throw commandError("internal", `fork: referenced dataset ${rel} was not written`); + } + } + // A fork has no chain access (spec §16.4): strip ingest sources so the + // forked project is dataset-only. Expanding history is a new ingest project. + delete projectDoc.chain_sources; + delete projectDoc.event_sources; + fs.writeFileSync(path.join(outDir, "chainplot.yaml"), stringifyYaml(projectDoc)); + + // Recipe directories live at project root for the forked copy. + for (const dir of ["queries", "models", "tests", "abis", "schemas"]) { + const src = path.join(outDir, "source", dir); + if (fs.existsSync(src)) { + fs.cpSync(src, path.join(outDir, dir), { recursive: true }); + } + } + + // A results-only release publishes the recipe and the rendered answers but + // no dataset, so the fork is real and useful yet cannot rebuild until it is + // pointed at a snapshot. Saying so here beats a bare "missing snapshot file" + // from a `build` the forker has no reason to expect to fail. + const warnings: string[] = []; + if (release.mode === "results_only") { + warnings.push( + `release mode is ${release.mode}: the dataset is not part of it, so the ` + + `forked project has the recipe and the published results but no snapshot. ` + + `Point ${datasetPaths(projectDoc)} at your own copy before running build.`, + ); + } + + return { + project_dir: outDir, + files: release.files.length + referenced.length, + release_prefix: releasePrefix, + mode: release.mode, + datasets_referenced: referenced, + warnings, + }; +} + +interface ReferencedDataset { + /** Release-relative path the manifest names. */ + path: string; + checksum: string; +} + +/** Datasets the forked release points at rather than carries. */ +function manifestDatasets(outDir: string): ReferencedDataset[] { + const datasetsDir = path.join(outDir, "datasets"); + if (!fs.existsSync(datasetsDir)) return []; + const out: ReferencedDataset[] = []; + for (const entry of fs.readdirSync(datasetsDir, { withFileTypes: true })) { + if (!entry.isDirectory()) continue; + const manifestPath = path.join(datasetsDir, entry.name, "manifest.json"); + if (!fs.existsSync(manifestPath)) continue; + const manifest = JSON.parse(fs.readFileSync(manifestPath, "utf8")) as { + mode?: string; + external?: { path?: string; checksum?: string }; + }; + if (manifest.mode !== "dataset_referenced") continue; + const rel = manifest.external?.path; + const checksum = manifest.external?.checksum; + if (!rel || !checksum) continue; + if (rel.includes("..") || path.isAbsolute(rel)) { + throw commandError("policy_refused", `fork: unsafe dataset path ${rel}`); + } + out.push({ path: rel, checksum }); + } + return out; +} + +/** Fetch or copy a referenced dataset from wherever the release came from. */ +async function readReferenced( + source: ForkSource, + releasePrefix: string | null, + dataset: ReferencedDataset, + guard: FetchGuardOptions, +): Promise { + if (source.kind === "local") { + return readLocal( + path.resolve(source.location), + releasePrefix ? path.join(releasePrefix, dataset.path) : dataset.path, + DEFAULT_LIMITS.totalBytes, + ); + } + const base = source.location.replace(/\/$/, ""); + const res = await guardedFetch(`${base}/${dataset.path}`, { + ...guard, + maxBytes: DEFAULT_LIMITS.totalBytes, + }); + return res.body; +} + +function datasetPaths(doc: { datasets?: { id: string }[] }): string { + const ids = (doc.datasets ?? []).map((d) => d.id); + return ids.length ? `datasets (${ids.join(", ")})` : "the datasets"; +} diff --git a/src/ingest/adapter.ts b/src/ingest/adapter.ts new file mode 100644 index 0000000..9efb63d --- /dev/null +++ b/src/ingest/adapter.ts @@ -0,0 +1,70 @@ +import { RpcError } from "../rpc/client.js"; + +export type CoverageStatus = + | "not_indexed" + | "incomplete" + | "complete_empty" + | "complete_with_rows"; + +export interface IndexedFilterSpec { + event_name: string; + indexed_1?: string[]; + indexed_2?: string[]; + indexed_3?: string[]; +} + +export interface BoundedJob { + sourceId: string; + contractName: string; + networkName: string; + chainId: number; + addresses: string[]; + abiPath: string; + events: string[]; + jobStart: number; + jobEnd: number; + rpcUrl: string; + databaseUrl: string; + workDir: string; + indexedFilters?: IndexedFilterSpec[]; +} + +const IDENTIFIER = /^[a-z0-9_]+$/; + +export function assertValidJobIdentifiers(job: BoundedJob): void { + for (const [name, value] of [ + ["contractName", job.contractName], + ["networkName", job.networkName], + ] as const) { + if (!IDENTIFIER.test(value)) { + throw new RpcError( + false, + `invalid ${name} identifier: ${JSON.stringify(value)}`, + ); + } + } +} + +export interface RunOptions { + rindexerBin: string; + wallClockMs: number; +} + +export interface RunHandle { + job: BoundedJob; + pid: number; + completedLogSeen: boolean; +} + +export interface CoverageReport { + status: CoverageStatus; + lastSyncedBlock: number | null; + rowCount: number; +} + +export interface IngestAdapter { + renderConfig(job: BoundedJob): string; + runBounded(job: BoundedJob, opts: RunOptions): Promise; + stopAndQuiesce(handle: RunHandle): Promise; + inspectCoverage(job: BoundedJob): Promise; +} diff --git a/src/ingest/coverage.ts b/src/ingest/coverage.ts new file mode 100644 index 0000000..8d78f8c --- /dev/null +++ b/src/ingest/coverage.ts @@ -0,0 +1,84 @@ +import type { EventEnd } from "../project/types.js"; + +export interface CoverageSegment { + start_block: number; + end_block: number; + start_block_hash: string; + end_block_hash: string; + start_block_parent_hash: string; + status: "complete_empty" | "complete_with_rows"; + row_count?: number; + /** Unix seconds of `end_block`. The chain's own clock, not the host's. */ + end_block_timestamp?: number; + /** When this segment was proven, for "last checked" as distinct from "data through". */ + indexed_at?: string; +} + +export interface SourceCoverage { + source_id: string; + segments: CoverageSegment[]; +} + +export interface CoverageFile { + schema_version: 1; + chain_id: number; + sources: SourceCoverage[]; +} + +export function hashJoinOk( + prev: CoverageSegment, + next: CoverageSegment, +): boolean { + return ( + next.start_block === prev.end_block + 1 && + next.start_block_parent_hash === prev.end_block_hash + ); +} + +export function lastProvenCompleteBlock( + segments: CoverageSegment[], + projectStartBlock: number, +): number { + let proven = projectStartBlock - 1; + let prev: CoverageSegment | null = null; + for (const segment of segments) { + if (segment.start_block !== proven + 1) break; + if (prev !== null && !hashJoinOk(prev, segment)) break; + proven = segment.end_block; + prev = segment; + } + return proven; +} + +export function requiredEnd( + end: EventEnd, + segments: CoverageSegment[], +): number | null { + if (end.mode === "pinned") return end.block; + if (segments.length === 0) return null; + return Math.max(...segments.map((s) => s.end_block)); +} + +export function isComplete( + segments: CoverageSegment[], + projectStartBlock: number, + end: EventEnd, +): { complete: boolean; reason: string | null } { + const target = requiredEnd(end, segments); + if (target === null) return { complete: false, reason: "not_indexed" }; + let reached = projectStartBlock - 1; + let prev: CoverageSegment | null = null; + for (const segment of segments) { + if (reached >= target) break; + if (segment.start_block !== reached + 1) { + return { complete: false, reason: "truncated" }; + } + if (prev !== null && !hashJoinOk(prev, segment)) { + return { complete: false, reason: "hash_join_broken" }; + } + reached = segment.end_block; + prev = segment; + } + if (reached < target) return { complete: false, reason: "truncated" }; + return { complete: true, reason: null }; +} diff --git a/src/ingest/coverageStore.ts b/src/ingest/coverageStore.ts new file mode 100644 index 0000000..e7f4304 --- /dev/null +++ b/src/ingest/coverageStore.ts @@ -0,0 +1,44 @@ +import fs from "node:fs"; +import path from "node:path"; +import type { CoverageFile, CoverageSegment } from "./coverage.js"; + +const COVERAGE_PATH = ["chainplot", "coverage.json"]; + +export function coverageFilePath(cwd: string): string { + return path.join(cwd, ".chainplot", "coverage.json"); +} + +export function readCoverageFile(cwd: string): CoverageFile { + const file = coverageFilePath(cwd); + if (!fs.existsSync(file)) { + return { schema_version: 1, chain_id: 0, sources: [] }; + } + return JSON.parse(fs.readFileSync(file, "utf8")) as CoverageFile; +} + +export function writeCoverageFile(cwd: string, file: CoverageFile): void { + const file_ = coverageFilePath(cwd); + fs.mkdirSync(path.dirname(file_), { recursive: true }); + fs.writeFileSync(file_, `${JSON.stringify(file, null, 2)}\n`); +} + +export function segmentsFor( + file: CoverageFile, + sourceId: string, +): CoverageSegment[] { + return file.sources.find((s) => s.source_id === sourceId)?.segments ?? []; +} + +export function appendSegment( + file: CoverageFile, + sourceId: string, + segment: CoverageSegment, +): CoverageFile { + const existing = file.sources.find((s) => s.source_id === sourceId); + if (existing) { + existing.segments.push(segment); + } else { + file.sources.push({ source_id: sourceId, segments: [segment] }); + } + return file; +} diff --git a/src/ingest/exportWorkerMain.ts b/src/ingest/exportWorkerMain.ts new file mode 100644 index 0000000..0d907b9 --- /dev/null +++ b/src/ingest/exportWorkerMain.ts @@ -0,0 +1,76 @@ +import fs from "node:fs"; +import { DuckDBInstance } from "@duckdb/node-api"; +import { buildExportSql, type ExportRequest } from "./exporter.js"; + +async function readRequest(): Promise { + let buf = ""; + for await (const chunk of process.stdin) { + buf += typeof chunk === "string" ? chunk : chunk.toString("utf8"); + const nl = buf.indexOf("\n"); + if (nl !== -1) { + return JSON.parse(buf.slice(0, nl)) as ExportRequest; + } + } + if (!buf) { + throw new Error("empty export worker request"); + } + return JSON.parse(buf) as ExportRequest; +} + +async function execute(req: ExportRequest): Promise { + const instance = await DuckDBInstance.create(":memory:"); + try { + const conn = await instance.connect(); + try { + const statements = buildExportSql(req) + .split(";") + .map((s) => s.trim()) + .filter(Boolean); + for (let i = 0; i < statements.length; i++) { + const sql = statements[i]; + if (i === 3) { + const reader = await conn.runAndReadAll(sql); + const rows = reader.getRowsJson() as unknown as { + total: bigint | number; + distinct_keys: bigint | number; + }[]; + const total = Number(rows[0]?.total ?? 0); + const distinctKeys = Number(rows[0]?.distinct_keys ?? 0); + if (total !== distinctKeys) { + throw new Error( + JSON.stringify({ + code: "source_inconsistent", + message: `duplicate physical keys: total=${total} distinct=${distinctKeys}`, + }), + ); + } + } else if (i === statements.length - 1) { + const reader = await conn.runAndReadAll(sql); + const rows = reader.getRowsJson() as unknown as { n: bigint | number }[]; + return Number(rows[0]?.n ?? 0); + } else { + await conn.run(sql); + } + } + throw new Error("unreachable"); + } finally { + conn.closeSync(); + } + } finally { + instance.closeSync(); + } +} + +function reply(payload: unknown, exitCode: number): void { + fs.writeSync(1, JSON.stringify(payload) + "\n"); + process.exit(exitCode); +} + +try { + const req = await readRequest(); + const rowCount = await execute(req); + reply({ ok: true, rowCount }, 0); +} catch (err) { + const message = err instanceof Error ? err.message : String(err); + reply({ ok: false, message }, 1); +} diff --git a/src/ingest/exporter.ts b/src/ingest/exporter.ts new file mode 100644 index 0000000..e1d5787 --- /dev/null +++ b/src/ingest/exporter.ts @@ -0,0 +1,132 @@ +import { fork } from "node:child_process"; +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { eventTableName } from "./rindexer/inspectCoverage.js"; +import type { BoundedJob } from "./adapter.js"; + +export interface ExportRequest { + databaseUrl: string; + networkName: string; + contractName: string; + event: string; + chainId: number; + outPath: string; +} + +function escapeSingleQuotes(text: string): string { + return text.replaceAll("'", "''"); +} + +export function buildExportSql(req: ExportRequest): string { + const table = eventTableName(req.networkName, req.contractName, req.event); + return [ + "INSTALL postgres;", + "LOAD postgres;", + `ATTACH '${escapeSingleQuotes(req.databaseUrl)}' AS pg (TYPE POSTGRES, READ_ONLY);`, + `SELECT count(*)::bigint AS total, count(DISTINCT (contract_address, block_number, tx_hash, log_index))::bigint AS distinct_keys FROM pg.${table};`, + `COPY (SELECT * REPLACE (CAST(block_number AS BIGINT) AS block_number, CAST(tx_index AS BIGINT) AS tx_index), ${req.chainId} AS chain_id FROM pg.${table}) TO '${escapeSingleQuotes(req.outPath)}' (FORMAT PARQUET);`, + `SELECT count(*)::bigint AS n FROM read_parquet('${escapeSingleQuotes(req.outPath)}');`, + ].join("\n"); +} + +export function buildUniquenessSql( + networkName: string, + contractName: string, + event: string, +): string { + const table = eventTableName(networkName, contractName, event); + return `SELECT count(*)::bigint AS total, count(DISTINCT (contract_address, block_number, tx_hash, log_index))::bigint AS distinct_keys FROM pg.${table}`; +} + +function workerLaunch(): { modulePath: string; execArgv: string[] } { + const self = fileURLToPath(import.meta.url); + const isTs = self.endsWith(".ts"); + const modulePath = fileURLToPath( + new URL(isTs ? "./exportWorkerMain.ts" : "./exportWorkerMain.js", import.meta.url), + ); + const execArgv = + isTs && !process.features.typescript ? ["--experimental-strip-types"] : []; + return { modulePath, execArgv }; +} + +function strippedEnv(): NodeJS.ProcessEnv { + const env: NodeJS.ProcessEnv = {}; + for (const key of ["PATH", "HOME", "LANG", "TMPDIR"] as const) { + const value = process.env[key]; + if (value !== undefined) { + env[key] = value; + } + } + return env; +} + +export interface ExportResult { + parquetPath: string; + rowCount: number; +} + +export async function exportEventTable( + job: BoundedJob, + outDir: string, +): Promise { + fs.mkdirSync(outDir, { recursive: true }); + const req: ExportRequest = { + databaseUrl: job.databaseUrl, + networkName: job.networkName, + contractName: job.contractName, + event: job.events[0], + chainId: job.chainId, + outPath: path.join(outDir, `${job.contractName}_${job.events[0].toLowerCase()}.parquet`), + }; + const { modulePath, execArgv } = workerLaunch(); + return await new Promise((resolve, reject) => { + const child = fork(modulePath, [], { + execArgv, + env: strippedEnv(), + stdio: ["pipe", "pipe", "pipe", "ipc"], + }); + let stdout = ""; + let stderr = ""; + let settled = false; + const timer = setTimeout(() => { + child.kill("SIGKILL"); + finish(new Error("export deadline exceeded")); + }, 120_000); + + function finish(err: Error | null, value?: ExportResult): void { + if (settled) return; + settled = true; + clearTimeout(timer); + if (err) reject(err); + else resolve(value as ExportResult); + } + + child.stdout?.setEncoding("utf8"); + child.stdout?.on("data", (chunk: string) => (stdout += chunk)); + child.stderr?.setEncoding("utf8"); + child.stderr?.on("data", (chunk: string) => (stderr += chunk)); + child.on("error", (err) => finish(err)); + child.on("exit", (code) => { + let payload: { ok: boolean; rowCount?: number; message?: string } | null = + null; + try { + payload = JSON.parse(stdout.trim().split("\n").pop() ?? "null"); + } catch { + payload = null; + } + if (payload?.ok === true && typeof payload.rowCount === "number") { + finish(null, { parquetPath: req.outPath, rowCount: payload.rowCount }); + return; + } + const detail = + payload?.message ?? + stderr.trim() ?? + `export worker exited with code ${code ?? "unknown"}`; + finish(new Error(detail)); + }); + + child.stdin?.write(JSON.stringify(req) + "\n"); + child.stdin?.end(); + }); +} diff --git a/src/ingest/rindexer/index.ts b/src/ingest/rindexer/index.ts new file mode 100644 index 0000000..78be12f --- /dev/null +++ b/src/ingest/rindexer/index.ts @@ -0,0 +1,32 @@ +import { Client } from "pg"; +import type { BoundedJob, IngestAdapter, RunHandle, RunOptions } from "../adapter.js"; +import { inspectCoverage } from "./inspectCoverage.js"; +import { renderConfig } from "./renderConfig.js"; +import { runBounded, stopAndQuiesce } from "./runBounded.js"; +import { errorMessage } from "../../plan/errors.js"; + +export function rindexerAdapter(): IngestAdapter { + return { + renderConfig: (job: BoundedJob) => renderConfig(job), + runBounded: (job: BoundedJob, opts: RunOptions) => runBounded(job, opts), + stopAndQuiesce: (handle: RunHandle) => stopAndQuiesce(handle), + inspectCoverage: async (job: BoundedJob) => { + const client = new Client({ connectionString: job.databaseUrl }); + try { + await client.connect(); + } catch (err) { + throw Object.assign( + new Error( + `cannot reach postgres for coverage inspection: ${errorMessage(err)}`, + ), + { retryable: true }, + ); + } + try { + return await inspectCoverage(job, client); + } finally { + await client.end().catch(() => undefined); + } + }, + }; +} diff --git a/src/ingest/rindexer/inspectCoverage.ts b/src/ingest/rindexer/inspectCoverage.ts new file mode 100644 index 0000000..106619a --- /dev/null +++ b/src/ingest/rindexer/inspectCoverage.ts @@ -0,0 +1,110 @@ +import type { PoolClient } from "pg"; +import { + assertValidJobIdentifiers, + type BoundedJob, + type CoverageReport, + type CoverageStatus, +} from "../adapter.js"; + +// rindexer derives table names from the manifest `name` (not the network): +// event table `{name}_{contract}.{event}`, cursor +// `rindexer_internal.{name}_{contract}_{event}`. renderConfig sets +// name = `chainplot_`. +export function manifestName(networkName: string): string { + return `chainplot_${networkName}`; +} + +export function cursorTableName( + networkName: string, + contractName: string, + event: string, +): string { + return `rindexer_internal.${manifestName(networkName)}_${contractName}_${event.toLowerCase()}`; +} + +export function eventTableName( + networkName: string, + contractName: string, + event: string, +): string { + return `${manifestName(networkName)}_${contractName}.${event.toLowerCase()}`; +} + +export function buildCursorQuery( + networkName: string, + contractName: string, + event: string, +): { text: string; params: string[] } { + return { + text: `SELECT last_synced_block FROM ${cursorTableName(networkName, contractName, event)} WHERE network = $1`, + params: [networkName], + }; +} + +export function buildRowCountQuery( + networkName: string, + contractName: string, + event: string, +): string { + return `SELECT count(*)::bigint AS n FROM ${eventTableName(networkName, contractName, event)}`; +} + +export function classifyCoverage( + lastSyncedBlock: number | null, + rowCount: number, + jobEnd: number, +): CoverageReport { + const status: CoverageStatus = + lastSyncedBlock === null + ? "not_indexed" + : lastSyncedBlock < jobEnd + ? "incomplete" + : rowCount === 0 + ? "complete_empty" + : "complete_with_rows"; + return { status, lastSyncedBlock, rowCount }; +} + +export async function inspectCoverage( + job: BoundedJob, + client: Pick, +): Promise { + assertValidJobIdentifiers(job); + // Every declared event must prove coverage; the aggregate is the worst + // per-event status and the summed row count. + const statuses: CoverageStatus[] = []; + let minCursor: number | null = null; + let totalRows = 0; + for (const event of job.events) { + const cursor = await client.query<{ last_synced_block: string }>( + buildCursorQuery(job.networkName, job.contractName, event).text, + [job.networkName], + ); + const lastSyncedBlock = + cursor.rows.length === 0 + ? null + : Number(BigInt(cursor.rows[0].last_synced_block)); + const countResult = await client.query<{ n: string }>( + buildRowCountQuery(job.networkName, job.contractName, event), + ); + const rowCount = Number(countResult.rows[0]?.n ?? 0); + statuses.push(classifyCoverage(lastSyncedBlock, rowCount, job.jobEnd).status); + totalRows += rowCount; + if (lastSyncedBlock !== null) { + minCursor = + minCursor === null + ? lastSyncedBlock + : Math.min(minCursor, lastSyncedBlock); + } + } + const worst: CoverageStatus = statuses.includes("not_indexed") + ? "not_indexed" + : statuses.includes("incomplete") + ? "incomplete" + : statuses.includes("complete_with_rows") + ? "complete_with_rows" + : "complete_empty"; + return { status: worst, lastSyncedBlock: minCursor, rowCount: totalRows }; +} + +export { assertValidJobIdentifiers }; diff --git a/src/ingest/rindexer/renderConfig.ts b/src/ingest/rindexer/renderConfig.ts new file mode 100644 index 0000000..8b49961 --- /dev/null +++ b/src/ingest/rindexer/renderConfig.ts @@ -0,0 +1,48 @@ +import { stringify } from "yaml"; +import type { BoundedJob } from "../adapter.js"; + +export interface IndexedFilter { + event_name: string; + indexed_1?: string[]; + indexed_2?: string[]; + indexed_3?: string[]; +} + +export function renderConfig(job: BoundedJob): string { + const doc: Record = { + name: `chainplot_${job.networkName}`, + description: "Generated by chainplot; do not edit", + project_type: "no-code", + networks: [ + { + name: job.networkName, + chain_id: job.chainId, + rpc: "${RPC_URL}", + }, + ], + storage: { postgres: { enabled: true } }, + graphql: { enabled: false }, + contracts: [ + { + name: job.contractName, + details: [ + { + network: job.networkName, + address: job.addresses.map((a) => a.toLowerCase()), + start_block: job.jobStart, + end_block: job.jobEnd, + }, + ], + abi: job.abiPath, + include_events: [...job.events], + timestamp: true, + }, + ], + }; + if (job.indexedFilters && job.indexedFilters.length > 0) { + const details = (doc.contracts as Array>)[0]! + .details as Array>; + details[0]!.indexed_filters = job.indexedFilters; + } + return stringify(doc, { lineWidth: 0 }); +} diff --git a/src/ingest/rindexer/runBounded.ts b/src/ingest/rindexer/runBounded.ts new file mode 100644 index 0000000..24fc4a4 --- /dev/null +++ b/src/ingest/rindexer/runBounded.ts @@ -0,0 +1,134 @@ +import { spawn } from "node:child_process"; +import fs from "node:fs"; +import path from "node:path"; +import { RpcError } from "../../rpc/client.js"; +import { renderConfig } from "./renderConfig.js"; +import type { RunHandle, RunOptions } from "../adapter.js"; + +const QUIT_TIMEOUT_MS = 10_000; + +export interface RunExtras { + fakeMode?: string; +} + +export async function runBounded( + job: Parameters[0], + opts: RunOptions, + extras: RunExtras = {}, +): Promise { + fs.mkdirSync(job.workDir, { recursive: true }); + const configPath = path.join(job.workDir, "rindexer.yaml"); + fs.writeFileSync(configPath, renderConfig(job)); + + const [bin, ...binArgs] = opts.rindexerBin.split(" ").filter(Boolean); + const child = spawn(bin, [...binArgs, "start", "-p", job.workDir, "indexer"], { + // Own process group so stopAndQuiesce can signal rindexer AND its children. + detached: process.platform !== "win32", + env: { + PATH: process.env.PATH ?? "", + HOME: process.env.HOME ?? "", + LANG: process.env.LANG ?? "", + TMPDIR: process.env.TMPDIR ?? "", + RPC_URL: job.rpcUrl, + DATABASE_URL: job.databaseUrl, + ...(extras.fakeMode !== undefined + ? { FAKE_RINDEXER_MODE: extras.fakeMode } + : {}), + }, + stdio: ["ignore", "pipe", "pipe"], + }); + + const handle: RunHandle = { + job, + pid: child.pid ?? -1, + completedLogSeen: false, + }; + + return await new Promise((resolve, reject) => { + let settled = false; + let stderrTail = ""; + const finish = (err: RpcError | null) => { + if (settled) return; + settled = true; + clearTimeout(wallClock); + if (err) { + child.removeAllListeners("exit"); + child.kill("SIGTERM"); + if (stderrTail) { + err.message += `; rindexer stderr: ${stderrTail.slice(-400)}`; + } + reject(err); + } else { + resolve(handle); + } + }; + + const wallClock = setTimeout(() => { + finish( + new RpcError( + true, + `rpc job wall clock exceeded (${opts.wallClockMs}ms); job is resumable`, + ), + ); + }, opts.wallClockMs); + + child.stdout?.on("data", (chunk: Buffer) => { + if (chunk.toString().includes("Historical indexing completed")) { + handle.completedLogSeen = true; + finish(null); + } + }); + + child.stderr?.on("data", (chunk: Buffer) => { + stderrTail += chunk.toString(); + if (stderrTail.length > 4000) stderrTail = stderrTail.slice(-4000); + }); + + child.on("error", (err) => { + finish(new RpcError(true, `failed to spawn rindexer: ${err.message}`)); + }); + + child.on("exit", (code, signal) => { + if (handle.completedLogSeen) return; + finish( + new RpcError( + true, + `rindexer exited before historic completion (code=${code} signal=${signal})`, + ), + ); + }); + }); +} + +export async function stopAndQuiesce(handle: RunHandle): Promise { + if (handle.pid <= 0) return; + // Negative pid signals the whole process group (rindexer forks children + // that keep writing after the parent exits). Windows does not support negative PIDs. + const group = process.platform === "win32" ? handle.pid : -handle.pid; + try { + process.kill(group, "SIGTERM"); + } catch { + return; // already gone + } + await new Promise((resolve) => { + const deadline = Date.now() + QUIT_TIMEOUT_MS; + const poll = () => { + try { + process.kill(group, 0); + if (Date.now() > deadline) { + try { + process.kill(group, "SIGKILL"); + } catch { + /* gone */ + } + resolve(); + } else { + setTimeout(poll, 100); + } + } catch { + resolve(); + } + }; + poll(); + }); +} diff --git a/src/plan/apply.ts b/src/plan/apply.ts new file mode 100644 index 0000000..bf2a7d3 --- /dev/null +++ b/src/plan/apply.ts @@ -0,0 +1,422 @@ +import fs from "node:fs"; +import path from "node:path"; +import type { RpcClient } from "../rpc/client.js"; +import { getHeader } from "../rpc/heads.js"; +import type { BoundedJob, IngestAdapter } from "../ingest/adapter.js"; +import type { ExportResult } from "../ingest/exporter.js"; +import { exportEventTable } from "../ingest/exporter.js"; +import { + appendSegment, + readCoverageFile, + segmentsFor, + writeCoverageFile, +} from "../ingest/coverageStore.js"; +import { hashJoinOk, isComplete, lastProvenCompleteBlock } from "../ingest/coverage.js"; +import { loadProject } from "../project/load.js"; +import { validateProject } from "../project/validate.js"; +import type { EventSource, ProjectDocument } from "../project/types.js"; +import { + appendCheckpoint, + readJournalPlan, + readJournalStatus, + writeJournalPlan, + writeJournalProject, + writeJournalStatus, +} from "../runtime/journal.js"; +import { acquireLocalLock } from "../runtime/locks.js"; +import { publishRelease, type PublishResult } from "../publish/publishRelease.js"; +import { commandError, errorMessage } from "./errors.js"; +import { projectDigest, type PlanDocument } from "./generate.js"; + +export interface ApplyOptions { + cwd: string; + planRef: string; // path or plan id + idempotencyKey?: string; + adapter: IngestAdapter; + exportFn?: (job: BoundedJob, outDir: string) => Promise; + buildFn?: (cwd: string) => Promise<{ distDir: string }>; + rindexerBin: string; + wallClockMs?: number; + rpcClient: RpcClient; + onProgress?: (event: { + run_id: string; + stage: string; + message?: string; + rows?: number; + }) => void; +} + +export interface ApplyOutcome { + plan_id: string; + idempotency_key: string; + status: "succeeded" | "canceled"; + reused: boolean; + release_dir: string | null; + publish: PublishResult | null; +} + +const DEFAULT_WALL_CLOCK_MS = 30 * 60 * 1000; + +function resolvePlanFile(cwd: string, planRef: string): string { + return planRef.endsWith(".json") + ? path.resolve(cwd, planRef) + : path.join(cwd, ".chainplot", "plans", `${planRef}.json`); +} + +function cancelRequested(cwd: string, key: string): boolean { + return fs.existsSync( + path.join(cwd, ".chainplot", "runs", key, "cancel_requested"), + ); +} + +function assertNotCanceled(cwd: string, key: string): void { + if (cancelRequested(cwd, key)) { + writeJournalStatus(cwd, key, { + status: "canceled", + plan_id: key, + plan_digest: key, + }); + throw commandError("policy_refused", `run ${key} canceled`, { resource_id: key }); + } +} + +export async function applyPlan(opts: ApplyOptions): Promise { + const planPath = resolvePlanFile(opts.cwd, opts.planRef); + if (!fs.existsSync(planPath)) { + throw commandError("validation", `plan not found: ${opts.planRef}`); + } + const plan = JSON.parse(fs.readFileSync(planPath, "utf8")) as PlanDocument; + if (!plan.plan_id) { + throw commandError("validation", `plan file has no plan_id: ${opts.planRef}`); + } + const key = opts.idempotencyKey ?? plan.plan_id; + + // Idempotency: same key + same digest → reuse outcome; different digest → refuse. + const journalPlan = readJournalPlan(opts.cwd, key); + if (journalPlan !== null && journalPlan.plan_id !== plan.plan_id) { + throw commandError( + "policy_refused", + `idempotency key ${key} already used by a different plan digest`, + { + resource_id: key, + suggested_next: "omit --idempotency-key to default to the plan digest", + }, + ); + } + const existingStatus = readJournalStatus(opts.cwd, key); + if (existingStatus?.status === "succeeded") { + return { + plan_id: plan.plan_id, + idempotency_key: key, + status: "succeeded", + reused: true, + release_dir: + (existingStatus.result as { release_dir?: string } | undefined)?.release_dir ?? null, + publish: + (existingStatus.result as { publish?: PublishResult | null } | undefined)?.publish ?? null, + }; + } + if (existingStatus?.status === "running") { + throw commandError("policy_refused", `run ${key} is already running`, { + resource_id: key, + }); + } + + // Configuration drift: plan digest must match current chainplot.yaml. + if (projectDigest(opts.cwd) !== plan.project_digest) { + throw commandError( + "policy_refused", + "chainplot.yaml changed since this plan was written; write a new plan", + { resource_id: plan.project_id, suggested_next: "plan --intent ingest" }, + ); + } + + const project = loadAndValidate(opts.cwd); + assertStateAssumptionsHold(opts.cwd, plan, project); + + const lock = acquireLocalLock(opts.cwd, "ingest"); + const progress = (stage: string, extra: { message?: string; rows?: number } = {}) => + opts.onProgress?.({ run_id: key, stage, ...extra }); + try { + writeJournalPlan(opts.cwd, key, plan); + writeJournalProject(opts.cwd, key, project); + writeJournalStatus(opts.cwd, key, { + status: "running", + plan_id: plan.plan_id, + plan_digest: plan.plan_id, + }); + progress("plan_verified"); + + const chain = project.chain_sources?.[0]; + const rpcUrl = chain ? (process.env[chain.rpc_secret] ?? "") : ""; + const databaseUrl = process.env.DATABASE_URL ?? ""; + const sourceById = new Map((project.event_sources ?? []).map((s) => [s.id, s])); + + for (const action of plan.actions) { + assertNotCanceled(opts.cwd, key); + if (action.type !== "ingest") continue; + const source = sourceById.get(action.source_id); + if (!source) { + throw commandError("validation", `unknown source ${action.source_id}`); + } + await runIngestAction(opts, plan, source, key, rpcUrl, databaseUrl, progress); + } + + const hasBuildWork = plan.actions.some( + (a) => a.type === "export" || a.type === "build_results", + ); + const releaseDir = hasBuildWork + ? await runExportAndBuild(opts, plan, sourceById, key, databaseUrl, progress) + : (plan.publish_target !== null ? path.join(opts.cwd, "dist", "releases", "local") : null); + + let published: PublishResult | null = null; + for (const action of plan.actions) { + assertNotCanceled(opts.cwd, key); + if (action.type !== "publish") continue; + const targetDoc = (project.publish_targets ?? []).find( + (t) => t.id === action.target_id, + ); + if (!targetDoc) { + throw commandError( + "validation", + `unknown publish target ${action.target_id}`, + ); + } + published = await publishRelease(opts.cwd, targetDoc); + appendCheckpoint(opts.cwd, key, { + stage: "published", + target_id: published.target_id, + release_prefix: published.release_prefix, + }); + } + + writeJournalStatus(opts.cwd, key, { + status: "succeeded", + plan_id: plan.plan_id, + plan_digest: plan.plan_id, + result: { release_dir: releaseDir, publish: published }, + }); + progress("release_written", { message: releaseDir ?? "" }); + return { + plan_id: plan.plan_id, + idempotency_key: key, + status: "succeeded", + reused: false, + release_dir: releaseDir, + publish: published, + }; + } catch (err) { + // A cancellation is not a failure. `assertNotCanceled` already recorded it + // as canceled; overwriting that with "failed" made the `canceled` status + // unreachable from the cooperative path, so `runs list` could never show + // the one thing `runs cancel` exists to produce. + writeJournalStatus(opts.cwd, key, { + status: cancelRequested(opts.cwd, key) ? "canceled" : "failed", + plan_id: plan.plan_id, + plan_digest: plan.plan_id, + result: { message: errorMessage(err) }, + }); + throw err; + } finally { + lock.release(); + } +} + +function loadAndValidate(cwd: string): ProjectDocument { + const doc = loadProject(cwd); + const validated = validateProject(doc, cwd); + if (!validated.ok) throw validated.error; + return validated.project; +} + +function assertStateAssumptionsHold( + cwd: string, + plan: PlanDocument, + project: ProjectDocument, +): void { + const coverage = readCoverageFile(cwd); + for (const source of project.event_sources ?? []) { + const assumption = plan.state_assumptions.sources[source.id]; + if (!assumption) continue; + const proven = lastProvenCompleteBlock( + segmentsFor(coverage, source.id), + source.start_block, + ); + if (proven !== assumption.last_proven_complete_block) { + throw commandError( + "policy_refused", + `coverage for ${source.id} changed since the plan was written (assumed ${assumption.last_proven_complete_block}, now ${proven}); re-plan`, + { resource_id: source.id, suggested_next: "plan --intent ingest" }, + ); + } + } +} + +async function runIngestAction( + opts: ApplyOptions, + plan: PlanDocument, + source: EventSource, + key: string, + rpcUrl: string, + databaseUrl: string, + progress: (stage: string, extra?: { message?: string; rows?: number }) => void, +): Promise { + const planSource = plan.sources.find((s) => s.source_id === source.id); + if (!planSource) { + throw commandError("validation", `plan has no bounds for ${source.id}`); + } + const job: BoundedJob = { + sourceId: source.id, + contractName: source.id.toLowerCase(), + networkName: `chainplot_${plan.chain?.chain_id ?? 0}`, + chainId: plan.chain?.chain_id ?? 0, + addresses: source.addresses, + abiPath: path.resolve(opts.cwd, source.abi), + events: source.events, + indexedFilters: source.indexed_filters, + jobStart: planSource.job_start, + jobEnd: planSource.job_end, + rpcUrl, + databaseUrl, + workDir: path.join(opts.cwd, ".chainplot", "ingest", source.id), + }; + + progress("ingest_started", { message: source.id }); + const handle = await opts.adapter.runBounded(job, { + rindexerBin: opts.rindexerBin, + wallClockMs: opts.wallClockMs ?? DEFAULT_WALL_CLOCK_MS, + }); + await opts.adapter.stopAndQuiesce(handle); + progress("ingest_completed", { message: source.id }); + let report = await opts.adapter.inspectCoverage(job); + if (report.status === "complete_empty") { + // rindexer commits the sync cursor before its final event-row flush; a + // complete_empty read immediately after SIGTERM can be a flush race. + // Re-inspect after a settle window; genuinely empty ranges stay empty (A6). + await new Promise((r) => setTimeout(r, 3_000)); + const second = await opts.adapter.inspectCoverage(job); + if (second.rowCount > report.rowCount) report = second; + } + if (report.status !== "complete_empty" && report.status !== "complete_with_rows") { + throw commandError( + "transient_dependency", + `ingest for ${source.id} did not prove coverage: ${report.status} (last_synced_block=${report.lastSyncedBlock}, job_end=${job.jobEnd})`, + { + resource_id: source.id, + retryable: true, + suggested_next: "apply the same plan again to resume", + }, + ); + } + + const parent = await getHeader(opts.rpcClient, job.jobStart - 1); + const start = await getHeader(opts.rpcClient, job.jobStart); + const end = await getHeader(opts.rpcClient, job.jobEnd); + + const segment = { + start_block: job.jobStart, + end_block: job.jobEnd, + start_block_hash: start.hash, + end_block_hash: end.hash, + start_block_parent_hash: parent.hash, + status: report.status, + row_count: report.rowCount, + end_block_timestamp: end.timestamp, + indexed_at: new Date().toISOString(), + }; + const coverage = readCoverageFile(opts.cwd); + // The coverage file is the proof; it has to say which chain it proves. + coverage.chain_id = plan.chain?.chain_id ?? coverage.chain_id; + const tail = segmentsFor(coverage, source.id).at(-1); + if (tail && !hashJoinOk(tail, segment)) { + throw commandError( + "source_inconsistent", + `new segment for ${source.id} does not hash-join the previous segment (possible reorg); coverage unchanged`, + { resource_id: source.id }, + ); + } + writeCoverageFile(opts.cwd, appendSegment(coverage, source.id, segment)); + progress("coverage_recorded", { + message: source.id, + rows: report.rowCount, + }); + appendCheckpoint(opts.cwd, key, { + stage: "coverage_recorded", + source_id: source.id, + start_block: segment.start_block, + end_block: segment.end_block, + status: segment.status, + }); +} + +async function runExportAndBuild( + opts: ApplyOptions, + plan: PlanDocument, + sourceById: Map, + key: string, + databaseUrl: string, + progress: (stage: string, extra?: { message?: string; rows?: number }) => void, +): Promise { + const coverage = readCoverageFile(opts.cwd); + + // Promotion gate: every source must be complete over [start_block, required_end]. + for (const source of sourceById.values()) { + assertNotCanceled(opts.cwd, key); + const verdict = isComplete( + segmentsFor(coverage, source.id), + source.start_block, + source.end, + ); + if (!verdict.complete) { + throw commandError( + "policy_refused", + `source ${source.id} is incomplete (${verdict.reason ?? "unknown"}); incomplete data cannot be promoted`, + { resource_id: source.id, suggested_next: "plan --intent ingest to continue" }, + ); + } + } + + for (const action of plan.actions) { + if (action.type !== "export") continue; + const source = sourceById.get(action.source_id); + if (!source) continue; + // One parquet per (source, event): dataset snapshots map 1:1 to events. + for (const event of source.events) { + const job: BoundedJob = { + sourceId: source.id, + contractName: source.id.toLowerCase(), + networkName: `chainplot_${plan.chain?.chain_id ?? 0}`, + chainId: plan.chain?.chain_id ?? 0, + addresses: source.addresses, + abiPath: path.resolve(opts.cwd, source.abi), + events: [event], + jobStart: 0, + jobEnd: 0, + rpcUrl: "", + databaseUrl, + workDir: path.join(opts.cwd, ".chainplot", "ingest", source.id), + }; + const outDir = path.join(opts.cwd, ".chainplot", "snapshots", source.id); + const exportFn = opts.exportFn ?? exportEventTable; + const result = await exportFn(job, outDir); + progress("export_completed", { + message: `${source.id}.${event}`, + rows: result.rowCount, + }); + appendCheckpoint(opts.cwd, key, { + stage: "export_completed", + source_id: source.id, + event, + rows: result.rowCount, + parquet: result.parquetPath, + }); + } + } + + const buildFn = + opts.buildFn ?? + ((c: string) => + import("../publish/writeRelease.js").then((m) => m.buildRelease(c))); + const release = await buildFn(opts.cwd); + return release.distDir; +} diff --git a/src/plan/digest.ts b/src/plan/digest.ts new file mode 100644 index 0000000..9985830 --- /dev/null +++ b/src/plan/digest.ts @@ -0,0 +1,22 @@ +import { createHash } from "node:crypto"; + +export function canonicalJson(value: unknown): string { + return JSON.stringify(sortValue(value)); +} + +function sortValue(value: unknown): unknown { + if (Array.isArray(value)) { + return value.map(sortValue); + } + if (value !== null && typeof value === "object") { + const entries = Object.entries(value as Record) + .filter(([, v]) => v !== undefined) + .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)); + return Object.fromEntries(entries.map(([k, v]) => [k, sortValue(v)])); + } + return value; +} + +export function sha256Hex(text: string): string { + return createHash("sha256").update(text).digest("hex"); +} diff --git a/src/plan/errors.ts b/src/plan/errors.ts new file mode 100644 index 0000000..497bd64 --- /dev/null +++ b/src/plan/errors.ts @@ -0,0 +1,54 @@ +import type { CommandError, ErrorCode } from "../cli/envelope.js"; + +export function commandError( + code: ErrorCode, + message: string, + opts: { + resource_id?: string | null; + pointer?: string | null; + retryable?: boolean; + suggested_next?: string | null; + } = {}, +): CommandError { + return { + code, + message, + resource_id: opts.resource_id ?? null, + pointer: opts.pointer ?? null, + retryable: opts.retryable ?? false, + suggested_next: opts.suggested_next ?? null, + }; +} + +export function isCommandError(err: unknown): err is CommandError { + return ( + typeof err === "object" && + err !== null && + "code" in err && + "message" in err && + "resource_id" in err && + "pointer" in err && + "retryable" in err && + "suggested_next" in err + ); +} + +/** + * Readable text for anything thrown. + * + * `CommandError` is a plain object, not an `Error`, so the common + * `err instanceof Error ? err.message : String(err)` renders it as + * "[object Object]" and loses the diagnosis entirely. + */ +export function errorMessage(err: unknown): string { + if (err instanceof Error) return err.message; + if ( + typeof err === "object" && + err !== null && + "message" in err && + typeof (err as { message: unknown }).message === "string" + ) { + return (err as { message: string }).message; + } + return String(err); +} diff --git a/src/plan/generate.ts b/src/plan/generate.ts new file mode 100644 index 0000000..b0fad27 --- /dev/null +++ b/src/plan/generate.ts @@ -0,0 +1,333 @@ +import fs from "node:fs"; +import path from "node:path"; +import type { RpcClient } from "../rpc/client.js"; +import { getFinalizedHead } from "../rpc/heads.js"; +import type { ChainSource, EventSource, ProjectDocument } from "../project/types.js"; +import { + lastProvenCompleteBlock, + requiredEnd, +} from "../ingest/coverage.js"; +import { + readCoverageFile, + segmentsFor, +} from "../ingest/coverageStore.js"; +import { sha256Hex, canonicalJson } from "./digest.js"; +import { commandError } from "./errors.js"; + +export interface PlanSource { + source_id: string; + end_mode: "pinned" | "follow_finalized"; + job_start: number; + job_end: number; + job_target_end: number; + required_end: number | null; + blocks_remaining: number; +} + +export interface PlanDocument { + schema_version: 1; + plan_id?: string; + intent: "ingest" | "refresh" | "build" | "publish"; + project_id: string; + project_digest: string; + created_at: string; + chain: { chain_id: number; finality: ChainSource["finality"] } | null; + sources: PlanSource[]; + actions: Array< + | { type: "ingest"; source_id: string } + | { type: "export"; source_id: string } + | { type: "build_results" } + | { type: "publish"; target_id: string } + >; + limits: { block_budget: number }; + deletes_data: boolean; + makes_data_public: boolean; + state_assumptions: { + sources: Record; + }; + publish_target: string | null; + /** + * content_digest of the built release this publish would upload. + * + * The plan digest otherwise covers only the project files, so rebuilding a + * release with different content produced an identical plan: `apply` then + * reused the previous run and uploaded nothing while reporting success. + */ + release_digest?: string | null; +} + +export interface GeneratePlanOptions { + intent: "ingest" | "refresh" | "build" | "publish"; + cwd: string; + project: ProjectDocument; + rpcClient: RpcClient; + now?: () => Date; + publishTargetId?: string; +} + +const DEFAULT_BLOCK_BUDGET = 100_000; + +export function projectDigest(cwd: string): string { + return sha256Hex(fs.readFileSync(path.join(cwd, "chainplot.yaml"), "utf8")); +} + +export async function generatePlan( + opts: GeneratePlanOptions, +): Promise<{ plan: PlanDocument; planPath: string }> { + const { cwd, project, intent } = opts; + const chain = project.chain_sources?.[0]; + const eventSources = project.event_sources ?? []; + if (intent !== "build" && intent !== "publish") { + if (!chain) throw commandError("validation", "project has no chain_sources"); + if (eventSources.length === 0) { + throw commandError("validation", "project has no event_sources"); + } + } + + const blockBudget = project.policy?.block_budget ?? DEFAULT_BLOCK_BUDGET; + const coverage = readCoverageFile(cwd); + const isBuild = intent === "build"; + + // Credentials are required only when work needs them (spec §11: a refresh + // that performs no ingest performs no RPC; a build plan performs neither). + const needsHead = + (intent === "ingest" || intent === "refresh") && + !!chain && + needsFinalizedProbe(eventSources, coverage); + if (needsHead && chain && !process.env[chain.rpc_secret]) { + throw commandError( + "missing_credentials", + `secret reference ${chain.rpc_secret} is not set in the environment`, + { resource_id: chain.rpc_secret }, + ); + } + + // One finalized-head probe only when some source still needs ingest. + const finalizedHead = needsHead + ? (await getFinalizedHead(opts.rpcClient)).number + : null; + + const planSources: PlanSource[] = []; + const actions: PlanDocument["actions"] = []; + const ingesting: string[] = []; + const stateAssumptions: PlanDocument["state_assumptions"]["sources"] = {}; + + for (const source of eventSources) { + const segments = segmentsFor(coverage, source.id); + const proven = lastProvenCompleteBlock(segments, source.start_block); + stateAssumptions[source.id] = { last_proven_complete_block: proven }; + + let jobTargetEnd: number; + if (isBuild) { + // Build plans never ingest; bounds are informational no-ops. + jobTargetEnd = proven; + } else if (source.end.mode === "pinned") { + jobTargetEnd = source.end.block; + if ((intent === "ingest" || intent === "refresh") && proven < jobTargetEnd && chain) { + assertPinnedSatisfiesFinality( + source as EventSource & { end: { mode: "pinned"; block: number } }, + chain, + finalizedHead, + ); + } + } else { + if (!chain || chain.finality.policy !== "finalized") { + throw commandError( + "unsupported_capability", + "follow_finalized requires chain finality policy finalized", + { resource_id: source.id, pointer: "/event_sources/end" }, + ); + } + if (finalizedHead === null) { + throw commandError( + "transient_dependency", + "cannot resolve finalized head for follow_finalized source", + { resource_id: source.id, retryable: true }, + ); + } + jobTargetEnd = finalizedHead; + if (jobTargetEnd < source.start_block) { + throw commandError( + "policy_refused", + `resolved_safe_end ${jobTargetEnd} is below start_block ${source.start_block}`, + { resource_id: source.id, pointer: "/event_sources/start_block" }, + ); + } + } + + const jobStart = proven + 1; + const jobEnd = Math.min(jobTargetEnd, jobStart + blockBudget - 1); + planSources.push({ + source_id: source.id, + end_mode: source.end.mode, + job_start: jobStart, + job_end: jobStart <= jobEnd ? jobEnd : jobStart - 1, + job_target_end: jobTargetEnd, + required_end: requiredEnd(source.end, segments), + blocks_remaining: Math.max(0, jobTargetEnd - proven), + }); + // Only ingest plans ingest; refresh ingests only follow_finalized sources + // (pinned skips on refresh, spec §9.2); build/publish never ingest. + const skipIngest = + intent === "build" || + intent === "publish" || + (intent === "refresh" && source.end.mode === "pinned"); + if (jobStart <= jobEnd && !skipIngest) { + ingesting.push(source.id); + } + } + + // A backfill wider than the block budget takes several runs. Only the run + // that closes the range exports and builds: the promotion gate refuses + // anything short of complete coverage, so an intermediate run that also + // tried would ingest correctly and then fail, reporting the whole run as + // failed. Intermediate runs ingest and stop. + const closesRange = planSources.every((s) => s.job_end >= s.job_target_end); + for (const sourceId of ingesting) { + actions.push({ type: "ingest", source_id: sourceId }); + if (closesRange) actions.push({ type: "export", source_id: sourceId }); + } + + if (actions.some((a) => a.type === "ingest") && !process.env.DATABASE_URL) { + throw commandError("missing_credentials", "DATABASE_URL is not set", { + resource_id: "DATABASE_URL", + }); + } + + // Publish plans are publication-only (spec §8): they never rebuild. + // + // A backfill that the block budget splits across several runs must not + // promise a build it already knows the promotion gate will refuse: the + // ingest would succeed, the build would fail, and `apply` would report the + // whole run as failed with the coverage it just proved discarded from view. + // Intermediate plans therefore ingest and export only; the plan that closes + // the range carries the build. A `build` intent always builds — an explicit + // request deserves the gate's own error, not a silently empty plan. + if ( + project.queries?.length && + intent !== "publish" && + (ingesting.length === 0 || closesRange) + ) { + actions.push({ type: "build_results" }); + } + + let publishTarget: string | null = null; + if (intent === "publish") { + const targets = project.publish_targets ?? []; + if (targets.length === 0) { + throw commandError( + "policy_refused", + "publish intent requires a publish target in chainplot.yaml", + { pointer: "/publish_targets" }, + ); + } + const target = opts.publishTargetId + ? targets.find((t) => t.id === opts.publishTargetId) + : targets[0]; + if (!target) { + throw commandError( + "policy_refused", + `publish target ${opts.publishTargetId} is not in chainplot.yaml`, + { pointer: "/publish_targets" }, + ); + } + actions.push({ type: "publish", target_id: target.id }); + publishTarget = target.id; + } + + const plan: PlanDocument = { + schema_version: 1, + intent, + project_id: project.id, + project_digest: projectDigest(cwd), + created_at: (opts.now ?? (() => new Date()))().toISOString(), + chain: chain + ? { chain_id: chain.chain_id, finality: chain.finality } + : null, + sources: planSources, + actions, + limits: { block_budget: blockBudget }, + deletes_data: false, + makes_data_public: intent === "publish", + state_assumptions: { sources: stateAssumptions }, + publish_target: publishTarget, + release_digest: intent === "publish" ? builtReleaseDigest(cwd) : null, + }; + plan.plan_id = sha256Hex( + canonicalJson({ ...plan, plan_id: undefined, created_at: undefined }), + ); + + const plansDir = path.join(cwd, ".chainplot", "plans"); + fs.mkdirSync(plansDir, { recursive: true }); + const planPath = path.join(plansDir, `${plan.plan_id}.json`); + fs.writeFileSync(planPath, `${JSON.stringify(plan, null, 2)}\n`); + return { plan, planPath }; +} + +/** content_digest of the release sitting in dist/releases/local, if any. */ +function builtReleaseDigest(cwd: string): string | null { + const releaseJson = path.join(cwd, "dist", "releases", "local", "release.json"); + if (!fs.existsSync(releaseJson)) return null; + try { + const doc = JSON.parse(fs.readFileSync(releaseJson, "utf8")) as { + content_digest?: string; + }; + return doc.content_digest ?? null; + } catch { + return null; + } +} + +function pinnedEnd(source: EventSource): number { + return source.end.mode === "pinned" ? source.end.block : Number.MAX_SAFE_INTEGER; +} + +function provenFor(source: EventSource, coverage: ReturnType): number { + return lastProvenCompleteBlock(segmentsFor(coverage, source.id), source.start_block); +} + +function needsFinalizedProbe( + sources: EventSource[], + coverage: ReturnType, +): boolean { + if (sources.some((s) => s.end.mode === "follow_finalized")) return true; + return sources.some( + (s) => s.end.mode === "pinned" && provenFor(s, coverage) < s.end.block, + ); +} + +function assertPinnedSatisfiesFinality( + source: EventSource & { end: { mode: "pinned"; block: number } }, + chain: ChainSource, + finalizedHead: number | null, +): void { + if (finalizedHead === null) { + throw commandError( + "transient_dependency", + "cannot verify pinned end_block against the chain head", + { resource_id: source.id, retryable: true }, + ); + } + if (chain.finality.policy === "finalized") { + if (source.end.block > finalizedHead) { + throw commandError( + "policy_refused", + `pinned end_block ${source.end.block} is above finalized head ${finalizedHead}; pinning into the unfinalized zone is refused`, + { + resource_id: source.id, + pointer: "/event_sources/end", + suggested_next: "set end_block to a block at or below the finalized head", + }, + ); + } + return; + } + const limit = finalizedHead - chain.finality.depth; + if (source.end.block > limit) { + throw commandError( + "policy_refused", + `pinned end_block ${source.end.block} does not satisfy confirmation_depth ${chain.finality.depth} (limit ${limit})`, + { resource_id: source.id, pointer: "/event_sources/end" }, + ); + } +} diff --git a/src/project/assertions.ts b/src/project/assertions.ts new file mode 100644 index 0000000..8b04385 --- /dev/null +++ b/src/project/assertions.ts @@ -0,0 +1,231 @@ +import fs from "node:fs"; +import path from "node:path"; +import { parse as parseYaml } from "yaml"; +import { + failResult, + okResult, + type CommandError, + type CommandResult, +} from "../cli/envelope.js"; +import { runQuery } from "../query/runQuery.js"; +import { loadProject } from "./load.js"; +import { validateProject } from "./validate.js"; +import { errorMessage } from "../plan/errors.js"; +import { DEFAULT_ROW_LIMIT } from "./limits.js"; + + +export interface AssertionFile { + dataset: string; + query?: string; + expect: { + row_count?: number; + columns?: string[]; + }; +} + +function error( + code: CommandError["code"], + message: string, + opts: { resource_id?: string | null; pointer?: string | null } = {}, +): CommandError { + return { + code, + message, + resource_id: opts.resource_id ?? null, + pointer: opts.pointer ?? null, + retryable: false, + suggested_next: null, + }; +} + +function isCommandError(err: unknown): err is CommandError { + return ( + typeof err === "object" && + err !== null && + "code" in err && + "message" in err && + "resource_id" in err && + "pointer" in err && + "retryable" in err && + "suggested_next" in err + ); +} + +function isPlainObject(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function parseAssertion( + doc: unknown, + pointer: string, +): AssertionFile | CommandError { + if (!isPlainObject(doc)) { + return error("validation", "assertion must be an object", { pointer }); + } + + const allowed = new Set(["dataset", "query", "expect"]); + for (const key of Object.keys(doc)) { + if (!allowed.has(key)) { + return error("validation", `unknown assertion field: ${key}`, { + pointer, + }); + } + } + + if (typeof doc.dataset !== "string" || doc.dataset.length === 0) { + return error("validation", "assertion requires dataset", { pointer }); + } + + if (doc.query !== undefined && typeof doc.query !== "string") { + return error("validation", "query must be a string", { pointer }); + } + + if (!isPlainObject(doc.expect)) { + return error("validation", "assertion requires expect object", { pointer }); + } + + const expectAllowed = new Set(["row_count", "columns"]); + for (const key of Object.keys(doc.expect)) { + if (!expectAllowed.has(key)) { + return error("validation", `unknown expect field: ${key}`, { pointer }); + } + } + + const expect: AssertionFile["expect"] = {}; + if (doc.expect.row_count !== undefined) { + if ( + typeof doc.expect.row_count !== "number" || + !Number.isInteger(doc.expect.row_count) + ) { + return error("validation", "row_count must be an integer", { pointer }); + } + expect.row_count = doc.expect.row_count; + } + if (doc.expect.columns !== undefined) { + if ( + !Array.isArray(doc.expect.columns) || + !doc.expect.columns.every((c) => typeof c === "string") + ) { + return error("validation", "columns must be a string array", { pointer }); + } + expect.columns = doc.expect.columns; + } + + return { + dataset: doc.dataset, + ...(typeof doc.query === "string" ? { query: doc.query } : {}), + expect, + }; +} + +export async function runAssertions( + projectDir: string, +): Promise { + const command = "test"; + try { + const doc = loadProject(projectDir); + const validated = validateProject(doc, projectDir); + if (!validated.ok) { + return failResult(command, validated.error); + } + + const testsDir = path.join(projectDir, "tests"); + const files = fs.existsSync(testsDir) + ? fs + .readdirSync(testsDir) + .filter((name) => name.endsWith(".yaml")) + .sort() + .map((name) => path.join("tests", name)) + : []; + + for (const relPath of files) { + const absPath = path.join(projectDir, relPath); + let raw: unknown; + try { + raw = parseYaml(fs.readFileSync(absPath, "utf8")); + } catch (err) { + return failResult( + command, + error( + "validation", + errorMessage(err), + { pointer: relPath }, + ), + ); + } + const assertion = parseAssertion(raw, relPath); + if ("code" in assertion && "message" in assertion) { + return failResult(command, assertion); + } + + const dataset = (validated.project.datasets ?? []).find( + (d) => d.id === assertion.dataset, + ); + if (!dataset) { + return failResult( + command, + error("validation", `unknown dataset: ${assertion.dataset}`, { + resource_id: assertion.dataset, + pointer: relPath, + }), + ); + } + + const parquetPath = path.resolve(projectDir, dataset.snapshot); + if (!fs.existsSync(parquetPath)) { + return failResult( + command, + error("validation", `missing snapshot file: ${dataset.snapshot}`, { + resource_id: dataset.id, + pointer: relPath, + }), + ); + } + + const result = await runQuery({ + sql: `SELECT * FROM ${dataset.id}`, + tables: { [dataset.id]: parquetPath }, + rawAmountColumns: [], + rowLimit: DEFAULT_ROW_LIMIT, + }); + + if ( + assertion.expect.row_count !== undefined && + result.rows.length !== assertion.expect.row_count + ) { + return failResult( + command, + error( + "validation", + `row_count expected ${assertion.expect.row_count}, got ${result.rows.length}`, + { resource_id: dataset.id, pointer: relPath }, + ), + ); + } + + if (assertion.expect.columns !== undefined) { + const actual = result.columns.map((c) => c.name); + if ( + actual.length !== assertion.expect.columns.length || + actual.some((name, i) => name !== assertion.expect.columns![i]) + ) { + return failResult( + command, + error( + "validation", + `columns expected [${assertion.expect.columns.join(", ")}], got [${actual.join(", ")}]`, + { resource_id: dataset.id, pointer: relPath }, + ), + ); + } + } + } + + return okResult(command, { assertions: files.length }); + } catch (err) { + if (isCommandError(err)) { + return failResult(command, err); + } + throw err; + } +} diff --git a/src/project/columns.ts b/src/project/columns.ts new file mode 100644 index 0000000..e5c19dc --- /dev/null +++ b/src/project/columns.ts @@ -0,0 +1,65 @@ +import type { Query, RawAmountColumn, RawAmountColumnSpec } from "./types.js"; + +/** + * Column metadata as it reaches a release, and from there the viewer. + * + * `raw_amount` marks a column whose value is an exact integer carried as a + * decimal string. `decimals`/`symbol` say how to *display* it; neither ever + * changes what is stored, so a uint256 still round-trips exactly (spec §12). + */ +export interface ResultColumn { + name: string; + logical_type: string; + raw_amount?: true; + decimals?: number; + symbol?: string; + label?: string; +} + +/** Accept both the bare-name and the descriptor form of a raw amount column. */ +export function normalizeRawAmountColumns( + spec: RawAmountColumnSpec[] | undefined, +): RawAmountColumn[] { + return (spec ?? []).map((entry) => + typeof entry === "string" ? { name: entry } : entry, + ); +} + +export function rawAmountNames( + spec: RawAmountColumnSpec[] | undefined, +): string[] { + return normalizeRawAmountColumns(spec).map((column) => column.name); +} + +/** + * Attach a query's declared display metadata to the columns DuckDB reported. + * Matching is case-insensitive because SQL identifiers are. + */ +export function decorateColumns( + columns: { name: string; logical_type: string }[], + spec: RawAmountColumnSpec[] | undefined, +): ResultColumn[] { + const declared = new Map( + normalizeRawAmountColumns(spec).map((column) => [ + column.name.toLowerCase(), + column, + ]), + ); + return columns.map((column) => { + const meta = declared.get(column.name.toLowerCase()); + if (!meta) return { name: column.name, logical_type: column.logical_type }; + return { + name: column.name, + logical_type: column.logical_type, + raw_amount: true as const, + ...(meta.decimals === undefined ? {} : { decimals: meta.decimals }), + ...(meta.symbol === undefined ? {} : { symbol: meta.symbol }), + ...(meta.label === undefined ? {} : { label: meta.label }), + }; + }); +} + +/** Panel heading precedence: panel title, then query title, then query id. */ +export function queryTitle(query: Pick): string { + return query.title ?? query.id; +} diff --git a/src/project/limits.ts b/src/project/limits.ts new file mode 100644 index 0000000..427d467 --- /dev/null +++ b/src/project/limits.ts @@ -0,0 +1,20 @@ +/** + * Limits that more than one command has to agree on. + * + * The row limit lived as four separate copies of `10_000`, with nothing + * keeping them in step — a caller could raise one and leave the others behind. + */ +import type { ProjectDocument } from "./types.js"; + +/** + * Rows a single query may return. + * + * This bounds the viewer, not the pipeline: results are embedded in the + * release and the table renders every row into the DOM. Going over is a typed + * refusal, never a silent truncation. + */ +export const DEFAULT_ROW_LIMIT = 10_000; + +export function rowLimitFor(project: Pick): number { + return project.policy?.row_limit ?? DEFAULT_ROW_LIMIT; +} diff --git a/src/project/load.ts b/src/project/load.ts new file mode 100644 index 0000000..6580379 --- /dev/null +++ b/src/project/load.ts @@ -0,0 +1,29 @@ +import fs from "node:fs"; +import path from "node:path"; +import { parse as parseYaml } from "yaml"; +import type { CommandError } from "../cli/envelope.js"; +import { errorMessage } from "../plan/errors.js"; + +function validation(message: string): CommandError { + return { + code: "validation", + message, + resource_id: null, + pointer: null, + retryable: false, + suggested_next: null, + }; +} + +export function loadProject(projectDir: string): unknown { + const filePath = path.join(projectDir, "chainplot.yaml"); + if (!fs.existsSync(filePath)) { + throw validation(`missing chainplot.yaml in ${projectDir}`); + } + const raw = fs.readFileSync(filePath, "utf8"); + try { + return parseYaml(raw); + } catch (err) { + throw validation(errorMessage(err)); + } +} diff --git a/src/project/modelGraph.ts b/src/project/modelGraph.ts new file mode 100644 index 0000000..b7137bc --- /dev/null +++ b/src/project/modelGraph.ts @@ -0,0 +1,53 @@ +import { commandError } from "../plan/errors.js"; +import type { Model } from "./types.js"; + +export type ModelNode = Model; + +export function topoSortModels(models: ModelNode[]): string[] { + const byId = new Map(models.map((m) => [m.id, m])); + if (byId.size !== models.length) { + const seen = new Set(); + for (const m of models) { + if (seen.has(m.id)) { + throw commandError("validation", `duplicate model id: ${m.id}`, { + resource_id: m.id, + pointer: "/models", + }); + } + seen.add(m.id); + } + } + for (const model of models) { + for (const dep of model.depends_on) { + if (!byId.has(dep)) { + throw commandError( + "validation", + `model ${model.id} depends on unknown model: ${dep}`, + { resource_id: model.id, pointer: `/models/${model.id}/depends_on` }, + ); + } + } + } + + const order: string[] = []; + const state = new Map(); + const visit = (id: string, path: string[]): void => { + const s = state.get(id); + if (s === "done") return; + if (s === "visiting") { + throw commandError( + "validation", + `cycle in model dependencies: ${[...path, id].join(" -> ")}`, + { resource_id: id, pointer: "/models" }, + ); + } + state.set(id, "visiting"); + for (const dep of byId.get(id)!.depends_on) { + visit(dep, [...path, id]); + } + state.set(id, "done"); + order.push(id); + }; + for (const model of models) visit(model.id, []); + return order; +} diff --git a/src/project/types.ts b/src/project/types.ts new file mode 100644 index 0000000..1472277 --- /dev/null +++ b/src/project/types.ts @@ -0,0 +1,117 @@ +export interface Dataset { + id: string; + snapshot: string; + schema?: string; +} + +/** + * A column holding an integer token amount as a decimal string. `decimals` + * and `symbol` are display hints only: the stored value is never rewritten, + * so uint256 precision survives the round trip (spec §12). + */ +export interface RawAmountColumn { + name: string; + decimals?: number; + symbol?: string; + label?: string; +} + +/** Bare name, or the same column with display metadata attached. */ +export type RawAmountColumnSpec = string | RawAmountColumn; + +export interface Query { + id: string; + file: string; + dataset: string; + title?: string; + raw_amount_columns?: RawAmountColumnSpec[]; +} + +export type ChartKind = "line" | "bar" | "area" | "kpi" | "table"; + +export interface DashboardPanel { + query: string; + chart: ChartKind; + title?: string; + description?: string; + span?: "half" | "full"; + hide_columns?: string[]; + unit?: string; +} + +export interface Dashboard { + id: string; + title: string; + description?: string; + panels: DashboardPanel[]; +} + +export interface Model { + id: string; + file: string; + depends_on: string[]; + columns?: string[]; +} + +export type Finality = + | { policy: "finalized" } + | { policy: "confirmation_depth"; depth: number }; + +export interface ChainSource { + id: string; + chain_id: number; + rpc_secret: string; + finality: Finality; +} + +export type EventEnd = + | { mode: "pinned"; block: number } + | { mode: "follow_finalized" }; + +export interface IndexedFilter { + event_name: string; + indexed_1?: string[]; + indexed_2?: string[]; + indexed_3?: string[]; +} + +export interface EventSource { + id: string; + chain: string; + addresses: string[]; + abi: string; + events: string[]; + start_block: number; + end: EventEnd; + indexed_filters?: IndexedFilter[]; +} + +export interface PublishTarget { + id: string; + type: "directory" | "s3"; + path?: string; + bucket?: string; + /** Key prefix inside the target; namespaces one project within a shared bucket. */ + prefix?: string; + dataset_license?: string; + public_base_url?: string; +} + +export interface ProjectDocument { + format_version: 1; + id: string; + datasets?: Dataset[]; + queries?: Query[]; + dashboards?: Dashboard[]; + models?: Model[]; + chain_sources?: ChainSource[]; + event_sources?: EventSource[]; + publish_targets?: PublishTarget[]; + policy?: { + block_budget?: number; + /** Rows a single query may return; see src/project/limits.ts. */ + row_limit?: number; + /** Default release mode for `build`; the CLI flag still wins. */ + release_mode?: "dataset_included" | "results_only" | "dataset_referenced"; + }; +} diff --git a/src/project/validate.ts b/src/project/validate.ts new file mode 100644 index 0000000..7225137 --- /dev/null +++ b/src/project/validate.ts @@ -0,0 +1,152 @@ +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { Ajv2020, type ErrorObject } from "ajv/dist/2020.js"; +import type { CommandError } from "../cli/envelope.js"; +import { topoSortModels } from "./modelGraph.js"; +import type { ProjectDocument } from "./types.js"; + +const SCHEMA_PATH = path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + "../../schemas/project.schema.json", +); + +const ajv = new Ajv2020({ allErrors: true, strict: true }); +const schema = JSON.parse(fs.readFileSync(SCHEMA_PATH, "utf8")) as object; +const validateSchema = ajv.compile(schema); + +function error( + code: CommandError["code"], + message: string, + opts: { resource_id?: string | null; pointer?: string | null } = {}, +): CommandError { + return { + code, + message, + resource_id: opts.resource_id ?? null, + pointer: opts.pointer ?? null, + retryable: false, + suggested_next: null, + }; +} + +function pointerFromAjv(err: ErrorObject): string | null { + if (err.keyword === "additionalProperties") { + const prop = (err.params as { additionalProperty?: string }).additionalProperty; + if (typeof prop === "string") { + return `${err.instancePath}/${prop}`; + } + } + return err.instancePath || null; +} + +export function validateProject( + doc: unknown, + projectDir: string, +): + | { ok: true; project: ProjectDocument } + | { ok: false; error: CommandError } { + if ( + doc !== null && + typeof doc === "object" && + "format_version" in doc && + (doc as { format_version: unknown }).format_version !== 1 + ) { + return { + ok: false, + error: error( + "unsupported_capability", + `unsupported format_version: ${String((doc as { format_version: unknown }).format_version)}`, + { pointer: "/format_version" }, + ), + }; + } + + if (!validateSchema(doc)) { + const first = validateSchema.errors?.[0]; + const pointer = first ? pointerFromAjv(first) : null; + const message = first + ? ajv.errorsText(validateSchema.errors, { dataVar: "project" }) + : "invalid project document"; + return { + ok: false, + error: error("validation", message, { pointer }), + }; + } + + const project = doc as ProjectDocument; + + if ((project.models ?? []).length > 0) { + try { + topoSortModels(project.models ?? []); + } catch (err) { + return { ok: false, error: err as CommandError }; + } + } + + const chains = new Map( + (project.chain_sources ?? []).map((c) => [c.id, c] as const), + ); + for (const [index, source] of (project.event_sources ?? []).entries()) { + if (source.end.mode !== "follow_finalized") continue; + const chain = chains.get(source.chain); + if (chain?.finality.policy === "confirmation_depth") { + return { + ok: false, + error: error( + "unsupported_capability", + "follow_finalized requires finalized chain finality; confirmation_depth is unsupported", + { + resource_id: source.id, + pointer: `/event_sources/${index}/end`, + }, + ), + }; + } + } + + for (const query of project.queries ?? []) { + const filePath = path.resolve(projectDir, query.file); + if (!fs.existsSync(filePath)) { + return { + ok: false, + error: error("validation", `missing query file: ${query.file}`, { + resource_id: query.id, + pointer: "/queries", + }), + }; + } + } + + for (const model of project.models ?? []) { + const filePath = path.resolve(projectDir, model.file); + if (!fs.existsSync(filePath)) { + return { + ok: false, + error: error("validation", `missing model file: ${model.file}`, { + resource_id: model.id, + pointer: "/models", + }), + }; + } + } + + // Snapshot files are authored inputs for dataset-only projects. Ingest + // projects materialize them at apply time, so their absence is not an error. + const isIngest = (project.event_sources ?? []).length > 0; + for (const dataset of project.datasets ?? []) { + if (isIngest) break; + const filePath = path.resolve(projectDir, dataset.snapshot); + if (!fs.existsSync(filePath)) { + return { + ok: false, + error: error("validation", `missing snapshot file: ${dataset.snapshot}`, { + resource_id: dataset.id, + pointer: "/datasets", + }), + }; + } + } + + return { ok: true, project }; +} diff --git a/src/publish/directory.ts b/src/publish/directory.ts new file mode 100644 index 0000000..e88ddbd --- /dev/null +++ b/src/publish/directory.ts @@ -0,0 +1,78 @@ +import fs from "node:fs"; +import path from "node:path"; +import type { LatestPointer } from "./target.js"; +import type { PublishTarget } from "./target.js"; + +const LATEST = "latest.json"; +const TEMP_PREFIX = ".latest-tmp-"; + +export class DirectoryTarget implements PublishTarget { + constructor( + private readonly rootDir: string, + private readonly keyPrefix = "", + ) {} + + /** The pointer path, namespaced when the target declares a prefix. */ + private latestPath(): string { + return path.join(this.rootDir, this.keyPrefix, LATEST); + } + + async uploadFiles( + releaseDir: string, + prefix: string, + files: string[], + ): Promise { + for (const rel of files) { + const src = path.join(releaseDir, rel); + const dest = path.join(this.rootDir, prefix, rel); + fs.mkdirSync(path.dirname(dest), { recursive: true }); + fs.copyFileSync(src, dest); + } + } + + async uploadExternal(localPath: string, key: string): Promise { + const dest = path.join(this.rootDir, key); + fs.mkdirSync(path.dirname(dest), { recursive: true }); + fs.copyFileSync(localPath, dest); + } + + async verifyFiles( + prefix: string, + files: string[], + checksums: Record, + ): Promise { + const { createHash } = await import("node:crypto"); + for (const rel of files) { + const dest = path.join(this.rootDir, prefix, rel); + if (!fs.existsSync(dest)) { + throw new Error(`upload verification failed: missing ${rel}`); + } + const expected = checksums[rel]; + if (expected === undefined) continue; + const actual = createHash("sha256") + .update(fs.readFileSync(dest)) + .digest("hex"); + if (actual !== expected) { + throw new Error(`upload verification failed: checksum mismatch ${rel}`); + } + } + } + + async readLatest(): Promise { + const file = this.latestPath(); + if (!fs.existsSync(file)) return null; + return JSON.parse(fs.readFileSync(file, "utf8")) as LatestPointer; + } + + async promoteLatest(pointer: LatestPointer): Promise { + // Atomic on the same filesystem: write temp, rename over latest.json. + const latest = this.latestPath(); + const temp = path.join( + path.dirname(latest), + `${TEMP_PREFIX}${process.pid}-${Date.now()}`, + ); + fs.mkdirSync(path.dirname(latest), { recursive: true }); + fs.writeFileSync(temp, `${JSON.stringify(pointer, null, 2)}\n`); + fs.renameSync(temp, latest); + } +} diff --git a/src/publish/doctor.ts b/src/publish/doctor.ts new file mode 100644 index 0000000..3aac543 --- /dev/null +++ b/src/publish/doctor.ts @@ -0,0 +1,126 @@ +import fs from "node:fs"; +import path from "node:path"; +import { execFileSync } from "node:child_process"; +import { commandError, errorMessage } from "../plan/errors.js"; +import { loadProject } from "../project/load.js"; +import { validateProject } from "../project/validate.js"; +import type { ProjectDocument } from "../project/types.js"; + +export interface DoctorCheck { + name: string; + status: "ok" | "fail" | "unverified" | "skipped"; + detail: string; +} + +export interface DoctorReport { + checks: DoctorCheck[]; +} + +export async function runDoctor(cwd: string): Promise { + const checks: DoctorCheck[] = []; + + // Project file + let project: ProjectDocument | null = null; + let projectError: string | null = null; + try { + const doc = loadProject(cwd); + const validated = validateProject(doc, cwd); + if (validated.ok) { + project = validated.project; + checks.push({ name: "project", status: "ok", detail: "chainplot.yaml valid" }); + } else { + projectError = validated.error.message; + checks.push({ name: "project", status: "fail", detail: projectError }); + } + } catch (err) { + throw commandError( + "validation", + errorMessage(err), + ); + } + + // Secrets: presence only, never values. + const chain = project?.chain_sources?.[0]; + const rpcEnvName = chain?.rpc_secret ?? "RPC_URL"; + checks.push({ + name: "secrets", + status: process.env[rpcEnvName] ? "ok" : "skipped", + detail: `${rpcEnvName} ${process.env[rpcEnvName] ? "present" : "not set (needed for ingest)"}; value not shown`, + }); + checks.push({ + name: "database", + status: process.env.DATABASE_URL ? "ok" : "skipped", + detail: `DATABASE_URL ${process.env.DATABASE_URL ? "present" : "not set (needed for ingest)"}; value not shown`, + }); + + // RPC: finalized-head probe when the secret is present. + if (chain && process.env[chain.rpc_secret]) { + try { + const { createRpcClient } = await import("../rpc/client.js"); + const { getFinalizedHead } = await import("../rpc/heads.js"); + const head = await getFinalizedHead( + createRpcClient(process.env[chain.rpc_secret]!), + ); + checks.push({ + name: "rpc", + status: "ok", + detail: `finalized head ${head.number}`, + }); + } catch (err) { + checks.push({ + name: "rpc", + status: "fail", + detail: errorMessage(err), + }); + } + } else { + checks.push({ name: "rpc", status: "skipped", detail: "RPC secret not set" }); + } + + // rindexer binary (needed for ingest only). + const rindexerBin = process.env.CHAINPLOT_RINDEXER_BIN ?? "rindexer"; + const rindexerName = rindexerBin.split(" ")[0] ?? rindexerBin; + let rindexerVersion: string | null = null; + try { + rindexerVersion = execFileSync(rindexerName, ["--version"], { + encoding: "utf8", + timeout: 5000, + }).trim(); + } catch { + rindexerVersion = null; + } + checks.push({ + name: "rindexer", + status: rindexerVersion ? "ok" : "skipped", + detail: rindexerVersion ?? `${rindexerName} not found (needed for ingest)`, + }); + + // Writable storage. + try { + const probe = path.join(cwd, ".chainplot", "doctor.tmp"); + fs.mkdirSync(path.dirname(probe), { recursive: true }); + fs.writeFileSync(probe, "ok"); + fs.unlinkSync(probe); + checks.push({ name: "storage", status: "ok", detail: `${cwd}/.chainplot writable` }); + } catch (err) { + checks.push({ + name: "storage", + status: "fail", + detail: errorMessage(err), + }); + } + + // S3: HeadBucket-level presence only; write/promote stays unverified. + if (process.env.CHAINPLOT_S3_ENDPOINT && process.env.AWS_ACCESS_KEY_ID) { + checks.push({ + name: "s3", + status: "unverified", + detail: + "S3 env configured; write/promote capability is proven only at upload time", + }); + } else { + checks.push({ name: "s3", status: "skipped", detail: "no S3 env configured" }); + } + + return { checks }; +} diff --git a/src/publish/latestPointer.ts b/src/publish/latestPointer.ts new file mode 100644 index 0000000..4a14d99 --- /dev/null +++ b/src/publish/latestPointer.ts @@ -0,0 +1,25 @@ +import { createHash } from "node:crypto"; + +export interface LatestPointer { + schema_version: 1; + release_prefix: string; + release_json_checksum: string; +} + +export function latestPointer(releasePrefix: string, releaseJsonBody: string): LatestPointer { + return { + schema_version: 1, + release_prefix: releasePrefix, + release_json_checksum: createHash("sha256").update(releaseJsonBody).digest("hex"), + }; +} + +export function pointerChecksumMatches( + pointer: LatestPointer, + releaseJsonBody: string, +): boolean { + return ( + createHash("sha256").update(releaseJsonBody).digest("hex") === + pointer.release_json_checksum + ); +} diff --git a/src/publish/publishRelease.ts b/src/publish/publishRelease.ts new file mode 100644 index 0000000..0812227 --- /dev/null +++ b/src/publish/publishRelease.ts @@ -0,0 +1,169 @@ +import fs from "node:fs"; +import path from "node:path"; +import { createHash } from "node:crypto"; +import { commandError } from "../plan/errors.js"; +import type { PublishTarget as PublishTargetDoc } from "../project/types.js"; +import type { + LatestPointer, + PublishResult, + PublishTarget, +} from "./target.js"; + +export type { PublishResult, LatestPointer }; + +import { latestPointer } from "./latestPointer.js"; +import { DirectoryTarget } from "./directory.js"; +import { S3Target, s3EnvFromProcess } from "./s3.js"; + +export function resolveTarget( + targetDoc: PublishTargetDoc, + projectDir: string, +): PublishTarget { + if (targetDoc.type === "directory") { + const root = path.resolve(projectDir, targetDoc.path ?? "."); + return new DirectoryTarget(root, targetDoc.prefix ?? ""); + } + const env = s3EnvFromProcess(); + if (!env) { + throw commandError( + "missing_credentials", + "S3 target requires CHAINPLOT_S3_ENDPOINT, CHAINPLOT_S3_BUCKET, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY", + { resource_id: targetDoc.id }, + ); + } + return new S3Target(env, undefined, targetDoc.prefix ?? ""); +} + +function walkFiles(dir: string, base = dir): string[] { + const out: string[] = []; + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + if (entry.name.startsWith(".")) continue; + const full = path.join(dir, entry.name); + if (entry.isDirectory()) out.push(...walkFiles(full, base)); + else out.push(path.relative(base, full)); + } + return out; +} + +export function assertLicense(targetDoc: PublishTargetDoc): void { + if (!targetDoc.dataset_license) { + throw commandError( + "policy_refused", + `publish target ${targetDoc.id} has no dataset_license; software license (MIT) is not a dataset license`, + { resource_id: targetDoc.id, pointer: `/publish_targets/${targetDoc.id}/dataset_license` }, + ); + } +} + +interface ReferencedDataset { + datasetId: string; + /** Release-relative key the manifest points at. */ + path: string; + checksum: string; + /** Where the parquet actually is on this machine. */ + localPath: string; +} + +/** Datasets a release points at rather than carries. */ +function referencedDatasets(releaseDir: string): ReferencedDataset[] { + const datasetsDir = path.join(releaseDir, "datasets"); + if (!fs.existsSync(datasetsDir)) return []; + const out: ReferencedDataset[] = []; + for (const entry of fs.readdirSync(datasetsDir, { withFileTypes: true })) { + if (!entry.isDirectory()) continue; + const manifestPath = path.join(datasetsDir, entry.name, "manifest.json"); + if (!fs.existsSync(manifestPath)) continue; + const manifest = JSON.parse(fs.readFileSync(manifestPath, "utf8")) as { + mode?: string; + source_path?: string; + external?: { path?: string; checksum?: string }; + }; + if (manifest.mode !== "dataset_referenced") continue; + const rel = manifest.external?.path; + const checksum = manifest.external?.checksum; + if (!rel || !checksum || !manifest.source_path) continue; + out.push({ + datasetId: entry.name, + path: rel, + checksum, + // source_path is relative to the project, and the release lives at + // /dist/releases/local. + localPath: path.resolve(releaseDir, "../../..", manifest.source_path), + }); + } + return out; +} + +export async function publishRelease( + projectDir: string, + targetDoc: PublishTargetDoc, +): Promise { + assertLicense(targetDoc); + const releaseDir = path.join(projectDir, "dist", "releases", "local"); + const releaseJsonPath = path.join(releaseDir, "release.json"); + if (!fs.existsSync(releaseJsonPath)) { + throw commandError("validation", "no built release found; run build first", { + resource_id: targetDoc.id, + suggested_next: "build", + }); + } + + const target = resolveTarget(targetDoc, projectDir); + const body = fs.readFileSync(releaseJsonPath, "utf8"); + // Key the prefix on the release's content digest, not on the document + // bytes: the document carries a build timestamp, so hashing it would mint + // a new prefix on every rebuild of identical data. + const release = JSON.parse(body) as { content_digest?: string }; + const releaseId = ( + release.content_digest ?? + createHash("sha256").update(body).digest("hex") + ).slice(0, 16); + // A prefix namespaces this project inside a shared bucket; without one, + // two projects in the same bucket overwrite each other's latest.json. + const base = targetDoc.prefix ? `${targetDoc.prefix}/` : ""; + const prefix = `${base}releases/${releaseId}`; + + const files = walkFiles(releaseDir).filter((f) => f !== "latest.json"); + const checksums: Record = {}; + for (const rel of files) { + checksums[rel] = createHash("sha256") + .update(fs.readFileSync(path.join(releaseDir, rel))) + .digest("hex"); + } + + await target.uploadFiles(releaseDir, prefix, files); + await target.verifyFiles(prefix, files, checksums); + + // A dataset_referenced release keeps its parquet out of the release, so the + // walk above never saw it. Upload it beside the release at the path the + // manifest names, and verify it the same way as everything else — otherwise + // the reference points at nothing and the mode is decorative. + const referenced = referencedDatasets(releaseDir); + for (const ref of referenced) { + if (!fs.existsSync(ref.localPath)) { + throw commandError( + "validation", + `dataset ${ref.datasetId} is referenced by the release but its snapshot is missing at ${ref.localPath}`, + { resource_id: ref.datasetId, suggested_next: "build" }, + ); + } + await target.uploadExternal(ref.localPath, `${prefix}/${ref.path}`); + await target.verifyFiles(prefix, [ref.path], { [ref.path]: ref.checksum }); + } + const pointer: LatestPointer = latestPointer(prefix, body); + await target.promoteLatest(pointer); + + return { + target_id: targetDoc.id, + release_prefix: prefix, + latest_url: targetDoc.public_base_url + ? `${targetDoc.public_base_url.replace(/\/$/, "")}/${base}latest.json` + : null, + dashboard_url: targetDoc.public_base_url + ? `${targetDoc.public_base_url.replace(/\/$/, "")}/${prefix}/index.html` + : null, + files_uploaded: files.length + referenced.length, + datasets_referenced: referenced.map((r) => r.path), + promoted: true, + }; +} diff --git a/src/publish/s3.ts b/src/publish/s3.ts new file mode 100644 index 0000000..2d64895 --- /dev/null +++ b/src/publish/s3.ts @@ -0,0 +1,260 @@ +import { + S3Client, + PutObjectCommand, + GetObjectCommand, + HeadObjectCommand, + DeleteObjectCommand, +} from "@aws-sdk/client-s3"; +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; +import type { LatestPointer, PublishTarget } from "./target.js"; +import { CONTENT_TYPES } from "./serve.js"; +import { commandError } from "../plan/errors.js"; + +export interface S3TargetEnv { + endpoint: string; + bucket: string; + accessKeyId: string; + secretAccessKey: string; + region?: string; +} + +export function s3EnvFromProcess( + env: NodeJS.ProcessEnv = process.env, +): S3TargetEnv | null { + if ( + !env.CHAINPLOT_S3_ENDPOINT || + !env.CHAINPLOT_S3_BUCKET || + !env.AWS_ACCESS_KEY_ID || + !env.AWS_SECRET_ACCESS_KEY + ) { + return null; + } + return { + endpoint: env.CHAINPLOT_S3_ENDPOINT, + bucket: env.CHAINPLOT_S3_BUCKET, + accessKeyId: env.AWS_ACCESS_KEY_ID, + secretAccessKey: env.AWS_SECRET_ACCESS_KEY, + region: env.CHAINPLOT_S3_REGION ?? "auto", + }; +} + +export const LATEST_KEY = "latest.json"; + +// Minimal storage surface so unit tests can mock without the AWS SDK. +export interface S3Ops { + put( + key: string, + body: string | Uint8Array, + conditions?: { ifMatch?: string; ifNoneMatch?: string; contentType?: string }, + ): Promise<{ etag: string }>; + get(key: string): Promise<{ body: Buffer; etag: string } | null>; + head(key: string): Promise<{ size: number; etag: string } | null>; + delete(key: string): Promise; +} + +export function makeS3Ops(env: S3TargetEnv): S3Ops { + const client = new S3Client({ + endpoint: env.endpoint, + region: env.region ?? "auto", + credentials: { + accessKeyId: env.accessKeyId, + secretAccessKey: env.secretAccessKey, + }, + forcePathStyle: true, + }); + return { + async put(key, body, conditions) { + const input: { + Bucket: string; + Key: string; + Body: string | Uint8Array; + IfMatch?: string; + IfNoneMatch?: string; + ContentType?: string; + } = { + Bucket: env.bucket, + Key: key, + Body: body, + }; + if (conditions?.ifMatch !== undefined) input.IfMatch = conditions.ifMatch; + if (conditions?.ifNoneMatch !== undefined) { + input.IfNoneMatch = conditions.ifNoneMatch; + } + if (conditions?.contentType !== undefined) { + input.ContentType = conditions.contentType; + } + try { + const out = await client.send(new PutObjectCommand(input)); + return { etag: String(out.ETag ?? "") }; + } catch (err) { + throw mapS3Error(err); + } + }, + async get(key) { + try { + const out = await client.send(new GetObjectCommand({ Bucket: env.bucket, Key: key })); + return { + body: await streamToBuffer(out.Body), + etag: String(out.ETag ?? ""), + }; + } catch (err) { + if (isNotFound(err)) return null; + throw mapS3Error(err); + } + }, + async head(key) { + try { + const out = await client.send(new HeadObjectCommand({ Bucket: env.bucket, Key: key })); + return { size: Number(out.ContentLength ?? 0), etag: String(out.ETag ?? "") }; + } catch (err) { + if (isNotFound(err)) return null; + throw mapS3Error(err); + } + }, + async delete(key) { + await client.send(new DeleteObjectCommand({ Bucket: env.bucket, Key: key })); + }, + }; +} + +function isNotFound(err: unknown): boolean { + const name = (err as { name?: string })?.name; + const status = (err as { $metadata?: { httpStatusCode?: number } })?.$metadata + ?.httpStatusCode; + return name === "NotFound" || name === "NoSuchKey" || status === 404; +} + +function isPreconditionFailed(err: unknown): boolean { + const status = (err as { $metadata?: { httpStatusCode?: number } })?.$metadata + ?.httpStatusCode; + const name = (err as { name?: string })?.name; + return status === 412 || name === "PreconditionFailed"; +} + +function mapS3Error(err: unknown): unknown { + if (isPreconditionFailed(err)) { + return commandError( + "policy_refused", + "conditional write refused (412): another writer changed the object concurrently", + { retryable: false, suggested_next: "re-read latest.json and retry" }, + ); + } + return err; +} + +async function streamToBuffer(body: unknown): Promise { + if (typeof body === "string") return Buffer.from(body, "utf8"); + const chunks: Uint8Array[] = []; + for await (const chunk of body as AsyncIterable) { + chunks.push(chunk); + } + return Buffer.concat(chunks); +} + +export class S3Target implements PublishTarget { + private readonly ops: S3Ops; + private lastPointerETag: string | null = null; + + private readonly keyPrefix: string; + + constructor(env: S3TargetEnv, ops?: S3Ops, keyPrefix = "") { + this.ops = ops ?? makeS3Ops(env); + this.keyPrefix = keyPrefix; + } + + /** The pointer key, namespaced when the target declares a prefix. */ + private latestKey(): string { + return this.keyPrefix ? `${this.keyPrefix}/${LATEST_KEY}` : LATEST_KEY; + } + + async uploadFiles( + releaseDir: string, + prefix: string, + files: string[], + ): Promise { + for (const rel of files) { + const body = fs.readFileSync(path.join(releaseDir, rel)); + const ext = path.extname(rel).toLowerCase(); + const contentType = CONTENT_TYPES[ext] ?? "application/octet-stream"; + await this.ops.put(`${prefix}/${rel}`, new Uint8Array(body), { + contentType, + }); + } + } + + async uploadExternal(localPath: string, key: string): Promise { + const body = fs.readFileSync(localPath); + const ext = path.extname(localPath).toLowerCase(); + await this.ops.put(key, new Uint8Array(body), { + contentType: CONTENT_TYPES[ext] ?? "application/octet-stream", + }); + } + + async verifyFiles( + prefix: string, + files: string[], + checksums: Record, + ): Promise { + for (const rel of files) { + const expected = checksums[rel]; + if (expected === undefined) continue; + const head = await this.ops.head(`${prefix}/${rel}`); + if (head === null) { + throw commandError("transient_dependency", `verify failed: missing ${rel}`, { + retryable: true, + }); + } + if (head.size <= 5 * 1024 * 1024) { + const obj = await this.ops.get(`${prefix}/${rel}`); + if (obj === null) { + throw commandError("transient_dependency", `verify failed: missing ${rel}`, { + retryable: true, + }); + } + const actual = createHash("sha256").update(obj.body).digest("hex"); + if (actual !== expected) { + throw commandError("transient_dependency", `verify failed: checksum mismatch ${rel}`, { + retryable: true, + }); + } + } + } + } + + async readLatest(): Promise { + const obj = await this.ops.get(this.latestKey()); + if (obj === null) return null; + this.lastPointerETag = obj.etag; + return JSON.parse(obj.body.toString("utf8")) as LatestPointer; + } + + async promoteLatest(pointer: LatestPointer): Promise { + const body = JSON.stringify(pointer, null, 2); + const existing = await this.readLatest(); + try { + if (existing) { + await this.ops.put(this.latestKey(), body, { + ifMatch: this.lastPointerETag ?? undefined, + contentType: "application/json", + }); + } else { + await this.ops.put(this.latestKey(), body, { + ifNoneMatch: "*", + contentType: "application/json", + }); + } + } catch (err) { + if ((err as { code?: string }).code === "policy_refused") throw err; + if (isPreconditionFailed(err)) { + throw commandError( + "policy_refused", + "conditional write refused (412): another writer changed latest.json concurrently", + { retryable: false, suggested_next: "re-read latest.json and retry" }, + ); + } + throw err; + } + } +} diff --git a/src/publish/serve.ts b/src/publish/serve.ts new file mode 100644 index 0000000..37d7adc --- /dev/null +++ b/src/publish/serve.ts @@ -0,0 +1,68 @@ +import fs from "node:fs"; +import http from "node:http"; +import path from "node:path"; +import { commandError } from "../plan/errors.js"; + +export const CONTENT_TYPES: Record = { + ".html": "text/html; charset=utf-8", + ".js": "text/javascript", + ".css": "text/css", + ".json": "application/json", + ".parquet": "application/octet-stream", + ".svg": "image/svg+xml", + ".png": "image/png", + ".ico": "image/x-icon", + ".woff2": "font/woff2", +}; + +export interface ServeHandle { + port: number; + ready: Promise; + close(): void; +} + +export function startServe(rootDir: string, port: number): ServeHandle { + const root = path.resolve(rootDir); + let readyResolve: (() => void) | null = null; + const ready = new Promise((resolve) => { + readyResolve = resolve; + }); + const server = http.createServer((req, res) => { + const url = new URL(req.url ?? "/", "http://127.0.0.1"); + const rel = decodeURIComponent(url.pathname).replace(/^\/+/, ""); + const resolved = path.resolve(root, rel); + if (resolved !== root && !resolved.startsWith(root + path.sep)) { + res.writeHead(404).end("not found"); + return; + } + let filePath = resolved; + if (fs.existsSync(filePath) && fs.statSync(filePath).isDirectory()) { + filePath = path.join(filePath, "index.html"); + } + if (!fs.existsSync(filePath) || !fs.statSync(filePath).isFile()) { + res.writeHead(404).end("not found"); + return; + } + const type = CONTENT_TYPES[path.extname(filePath)] ?? "application/octet-stream"; + res.writeHead(200, { "content-type": type }); + fs.createReadStream(filePath).pipe(res); + }); + server.listen(port, "127.0.0.1", () => readyResolve?.()); + const handle: ServeHandle = { + get port(): number { + const address = server.address(); + return typeof address === "object" && address !== null + ? address.port + : port; + }, + ready, + close: () => server.close(), + }; + return handle; +} + +export function validateServeDir(dir: string): void { + if (!fs.existsSync(dir) || !fs.statSync(dir).isDirectory()) { + throw commandError("validation", `serve directory not found: ${dir}`); + } +} diff --git a/src/publish/sourceBundle.ts b/src/publish/sourceBundle.ts new file mode 100644 index 0000000..48b3c50 --- /dev/null +++ b/src/publish/sourceBundle.ts @@ -0,0 +1,49 @@ +import fs from "node:fs"; +import path from "node:path"; + +// Explicit allowlist (spec §16.1): sanitized recipe files only. +const ALLOWED_ENTRIES = [ + "chainplot.yaml", + "abis", + "models", + "queries", + "tests", + "schemas", +] as const; + +function walkFiles(dir: string): string[] { + const out: string[] = []; + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + if (entry.name.startsWith(".")) continue; + const full = path.join(dir, entry.name); + if (entry.isDirectory()) { + out.push(...walkFiles(full)); + } else { + out.push(full); + } + } + return out; +} + +export function copySourceBundle( + projectDir: string, + releaseDir: string, +): string[] { + const destDir = path.join(releaseDir, "source"); + fs.mkdirSync(destDir, { recursive: true }); + const copied: string[] = []; + for (const entry of ALLOWED_ENTRIES) { + const src = path.join(projectDir, entry); + if (!fs.existsSync(src)) continue; + const dest = path.join(destDir, entry); + fs.cpSync(src, dest, { recursive: true }); + if (fs.statSync(dest).isDirectory()) { + for (const file of walkFiles(dest)) { + copied.push(`source/${path.relative(destDir, file)}`); + } + } else { + copied.push(`source/${entry}`); + } + } + return copied; +} diff --git a/src/publish/target.ts b/src/publish/target.ts new file mode 100644 index 0000000..ff77883 --- /dev/null +++ b/src/publish/target.ts @@ -0,0 +1,39 @@ +export interface LatestPointer { + schema_version: 1; + release_prefix: string; + release_json_checksum: string; +} + +export interface PublishResult { + target_id: string; + release_prefix: string; + latest_url: string | null; + /** Direct link to this exact release's dashboard, for a human to open. */ + dashboard_url: string | null; + /** Release-relative keys of datasets uploaded beside the release. */ + datasets_referenced?: string[]; + files_uploaded: number; + promoted: boolean; +} + +export interface PublishTarget { + uploadFiles( + releaseDir: string, + prefix: string, + files: string[], + ): Promise; + /** + * Upload one file from anywhere on disk to an exact key. + * + * A `dataset_referenced` release keeps its parquet outside the release + * directory, so it cannot be named among `files`. + */ + uploadExternal(localPath: string, key: string): Promise; + verifyFiles( + prefix: string, + files: string[], + checksums: Record, + ): Promise; + readLatest(): Promise; + promoteLatest(pointer: LatestPointer): Promise; +} diff --git a/src/publish/writeRelease.ts b/src/publish/writeRelease.ts new file mode 100644 index 0000000..de98b98 --- /dev/null +++ b/src/publish/writeRelease.ts @@ -0,0 +1,421 @@ +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { createHash } from "node:crypto"; +import type { CommandError } from "../cli/envelope.js"; +import { loadProject } from "../project/load.js"; +import { validateProject } from "../project/validate.js"; +import { topoSortModels } from "../project/modelGraph.js"; +import { runQuery } from "../query/runQuery.js"; +import { isComplete, requiredEnd } from "../ingest/coverage.js"; +import { readCoverageFile, segmentsFor } from "../ingest/coverageStore.js"; +import { lastProvenCompleteBlock } from "../ingest/coverage.js"; +import { copySourceBundle } from "./sourceBundle.js"; +import { decorateColumns, rawAmountNames } from "../project/columns.js"; +import type { CoverageSegment } from "../ingest/coverage.js"; +import { rowLimitFor } from "../project/limits.js"; + + +/** + * How current the data is, and on whose authority. + * + * An ingest project can answer from the chain: the timestamp of the last + * block proven complete. A dataset-only project cannot, and says so rather + * than passing a file's mtime off as freshness — a checkout or a `cp` + * rewrites mtime without the data changing at all. + */ +interface Freshness { + kind: "chain" | "snapshot_mtime"; + data_through: { block: number; timestamp: string | null } | null; + indexed_at: string | null; + snapshot_mtime: string; +} + +function isoFromUnixSeconds(seconds: number | undefined): string | null { + return typeof seconds === "number" && seconds > 0 + ? new Date(seconds * 1000).toISOString() + : null; +} + +function freshnessFor( + segments: CoverageSegment[], + provenBlock: number, + snapshotMtime: string, + isIngest: boolean, +): Freshness { + if (!isIngest) { + return { + kind: "snapshot_mtime", + data_through: null, + indexed_at: null, + snapshot_mtime: snapshotMtime, + }; + } + const proven = segments.filter((s) => s.end_block <= provenBlock); + const last = proven.at(-1); + const indexedAt = proven + .map((s) => s.indexed_at) + .filter((at): at is string => typeof at === "string") + .sort() + .at(-1); + return { + kind: "chain", + data_through: { + block: provenBlock, + timestamp: isoFromUnixSeconds(last?.end_block_timestamp), + }, + indexed_at: indexedAt ?? null, + snapshot_mtime: snapshotMtime, + }; +} + +const VIEWER_DIST = path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + "../../viewer/dist", +); + +function error( + code: CommandError["code"], + message: string, + opts: { resource_id?: string | null; pointer?: string | null } = {}, +): CommandError { + return { + code, + message, + resource_id: opts.resource_id ?? null, + pointer: opts.pointer ?? null, + retryable: false, + suggested_next: null, + }; +} + +function writeJson(filePath: string, value: unknown): void { + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + fs.writeFileSync(filePath, `${JSON.stringify(value, null, 2)}\n`); +} + +function sha256(data: string | Buffer): string { + return createHash("sha256").update(data).digest("hex"); +} + +export async function buildRelease( + projectDir: string, + opts: { mode?: "dataset_included" | "results_only" | "dataset_referenced" } = {}, +): Promise<{ + distDir: string; + files: string[]; +}> { + const doc = loadProject(projectDir); + const validated = validateProject(doc, projectDir); + if (!validated.ok) { + throw validated.error; + } + const project = validated.project; + + // Promotion gate: an ingest project's sources must be complete over + // [start_block, required_end] before any release is written (spec §11). + if ((project.event_sources ?? []).length > 0) { + const coverage = readCoverageFile(projectDir); + for (const source of project.event_sources ?? []) { + const verdict = isComplete( + segmentsFor(coverage, source.id), + source.start_block, + source.end, + ); + if (!verdict.complete) { + throw error( + "policy_refused", + `source ${source.id} is incomplete (${verdict.reason ?? "unknown"}); incomplete data cannot be promoted`, + { resource_id: source.id }, + ); + } + } + } + + const datasets = project.datasets ?? []; + const queries = project.queries ?? []; + const models = project.models ?? []; + const datasetById = new Map(datasets.map((d) => [d.id, d])); + const allTables = Object.fromEntries( + datasets.map((d) => [d.id, path.resolve(projectDir, d.snapshot)]), + ); + + // Size cap before any query work (fail fast, spec §14.1/§16). + const MAX_COPIED_BYTES = 100 * 1024 * 1024; + // An explicit --mode wins, then the project's own declaration, then the + // conservative default: the page and its answers, without the dataset. + // + // Publishing is outward and irreversible, so the default uploads the least + // that still works. Shipping the parquet is a deliberate choice — it is what + // lets someone fork the release and recompute, and it is also what turns an + // 800 KB page into hundreds of megabytes. + const mode = opts.mode ?? project.policy?.release_mode ?? "results_only"; + let copiedBytes = 0; + for (const dataset of datasets) { + const parquetPath = path.resolve(projectDir, dataset.snapshot); + copiedBytes += fs.statSync(parquetPath).size; + } + if (mode === "dataset_included" && copiedBytes > MAX_COPIED_BYTES) { + throw error( + "policy_refused", + `copied dataset size ${copiedBytes} bytes exceeds the ${MAX_COPIED_BYTES}-byte cap; choose --mode results_only, --mode dataset_referenced, or reduce the data`, + { pointer: "/datasets" }, + ); + } + + // Models materialize in dependency order for every query. + const modelOrder = topoSortModels(models); + const modelSql: { id: string; sql: string }[] = modelOrder.map((id) => { + const model = models.find((m) => m.id === id)!; + const file = path.resolve(projectDir, model.file); + if (!fs.existsSync(file)) { + throw error("validation", `missing model file: ${model.file}`, { + resource_id: model.id, + pointer: "/models", + }); + } + return { id, sql: fs.readFileSync(file, "utf8") }; + }); + + const releasesDir = path.join(projectDir, "dist", "releases"); + fs.mkdirSync(releasesDir, { recursive: true }); + const staging = fs.mkdtempSync(path.join(releasesDir, ".tmp-")); + const files: string[] = []; + + try { + // Queries (with models materialized first). + for (const query of queries) { + const dataset = datasetById.get(query.dataset); + if (!dataset) { + throw error("validation", `unknown dataset: ${query.dataset}`, { + resource_id: query.id, + pointer: "/queries", + }); + } + const sqlPath = path.resolve(projectDir, query.file); + const data = await runQuery({ + sql: fs.readFileSync(sqlPath, "utf8"), + // Every dataset is in scope, not just the declared one. Models are + // materialized into each query's session, so loading one table meant a + // model over dataset A failed every query on dataset B — which made + // models unusable in any multi-dataset project. It also lets a query + // join across datasets. `query.dataset` still names the provenance. + tables: allTables, + rawAmountColumns: rawAmountNames(query.raw_amount_columns), + rowLimit: rowLimitFor(project), + models: modelSql, + }); + + const rel = path.join("results", `${query.id}.json`); + writeJson(path.join(staging, rel), { + schema_version: 1, + query_id: query.id, + title: query.title ?? query.id, + // Display metadata rides on the column descriptors, so the viewer + // never has to correlate two lists to format a cell. + columns: decorateColumns(data.columns, query.raw_amount_columns), + rows: data.rows, + snapshot: dataset.snapshot, + query_digest: sha256(fs.readFileSync(sqlPath, "utf8")), + raw_amount_columns: rawAmountNames(query.raw_amount_columns), + }); + files.push(rel); + } + + // Datasets: manifest + parquet copy (mode-dependent). + + const coverageFile = readCoverageFile(projectDir); + const coverageRows = (project.event_sources ?? []).map((source) => { + const segments = segmentsFor(coverageFile, source.id); + const proven = lastProvenCompleteBlock(segments, source.start_block); + const target = requiredEnd(source.end, segments); + return { + source_id: source.id, + start_block: source.start_block, + end_block: proven, + status: + target !== null && proven >= target ? "complete" : "incomplete", + }; + }); + const finality = project.chain_sources?.[0]?.finality ?? null; + const isIngest = (project.event_sources ?? []).length > 0; + const allSegments = (project.event_sources ?? []).flatMap((source) => + segmentsFor(coverageFile, source.id), + ); + const provenBlock = Math.max( + 0, + ...coverageRows.map((row) => row.end_block), + ); + + for (const dataset of datasets) { + const parquetPath = path.resolve(projectDir, dataset.snapshot); + const manifestRel = path.join("datasets", dataset.id, "manifest.json"); + const freshness = freshnessFor( + allSegments, + provenBlock, + fs.statSync(parquetPath).mtime.toISOString(), + isIngest, + ); + const manifest: Record = { + schema_version: 1, + snapshot_id: dataset.id, + mode, + files: [] as string[], + source_path: dataset.snapshot, + coverage: coverageRows, + finality, + freshness, + }; + if (mode === "dataset_included") { + const tableRel = path.join( + "datasets", + dataset.id, + "tables", + path.basename(dataset.snapshot), + ); + fs.mkdirSync(path.dirname(path.join(staging, tableRel)), { + recursive: true, + }); + fs.copyFileSync(parquetPath, path.join(staging, tableRel)); + manifest.files = [`tables/${path.basename(dataset.snapshot)}`]; + files.push(manifestRel, tableRel); + } else if (mode === "dataset_referenced") { + // The reference is relative to the release, not to the machine that + // built it: a consumer resolves it against whatever base URL the + // release is served from, exactly as the page resolves release.json. + // Recording the producer's own path made the reference unusable by + // anyone else. The parquet stays out of `files`, so it is neither + // copied into the release nor covered by its checksums — `publish` + // uploads it alongside, and `fork` fetches it on demand. + manifest.external = { + path: path.join( + "datasets", + dataset.id, + "tables", + path.basename(dataset.snapshot), + ), + checksum: sha256(fs.readFileSync(parquetPath)), + bytes: fs.statSync(parquetPath).size, + }; + files.push(manifestRel); + } else { + files.push(manifestRel); + } + writeJson(path.join(staging, manifestRel), manifest); + } + + // Dashboards data. Panel headings are resolved here — panel title, then + // the query's title, then its id — so the viewer needs no query registry. + const queryTitles = new Map(queries.map((q) => [q.id, q.title ?? q.id])); + const dashboardIds: string[] = []; + for (const dashboard of project.dashboards ?? []) { + const rel = path.join("dashboards", `${dashboard.id}.json`); + writeJson(path.join(staging, rel), { + schema_version: 1, + dashboard_id: dashboard.id, + title: dashboard.title, + description: dashboard.description ?? null, + panels: dashboard.panels.map((panel) => ({ + ...panel, + title: panel.title ?? queryTitles.get(panel.query) ?? panel.query, + span: panel.span ?? "half", + })), + }); + files.push(rel); + dashboardIds.push(dashboard.id); + } + + // Viewer static assets (release-independent bundle). A release without + // the viewer is a directory of JSON nobody can read, so its absence is an + // error rather than a silent omission. + if (!fs.existsSync(path.join(VIEWER_DIST, "index.html"))) { + throw error( + "internal", + `viewer bundle missing at ${VIEWER_DIST}; run \`pnpm build\` to build it`, + ); + } + fs.cpSync(VIEWER_DIST, staging, { recursive: true }); + files.push("index.html"); + for (const asset of walkFiles(path.join(staging, "assets"))) { + files.push(path.relative(staging, asset)); + } + + // Sanitized source bundle. + files.push(...copySourceBundle(projectDir, staging)); + + // Release document (provenance + per-file checksums for fork). + const releaseRel = "release.json"; + files.push("release.json"); + // Checksums cover every file written before release.json itself. + const fileChecksums = files + .filter((f) => f !== "release.json") + .map((rel) => ({ + path: rel, + checksum: createHash("sha256") + .update(fs.readFileSync(path.join(staging, rel))) + .digest("hex"), + })) + .sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0)); + + // Identity of a release is its content, not the clock. Rebuilding the + // same inputs yields the same digest, so re-publishing is idempotent + // instead of littering the bucket with a new prefix every time. + const contentDigest = sha256( + [ + `project:${project.id}`, + `mode:${mode}`, + ...fileChecksums.map((f) => `${f.path}:${f.checksum}`), + ].join("\n"), + ); + + writeJson(path.join(staging, releaseRel), { + schema_version: 1, + project_id: project.id, + mode, + content_digest: contentDigest, + queries: queries.map((q) => q.id), + dashboards: dashboardIds, + generated_at: new Date().toISOString(), + snapshots: datasets.map((d) => ({ + dataset_id: d.id, + snapshot_id: d.id, + })), + coverage: coverageRows, + finality, + freshness: freshnessFor( + allSegments, + provenBlock, + datasets[0] + ? fs.statSync(path.resolve(projectDir, datasets[0].snapshot)).mtime.toISOString() + : new Date().toISOString(), + isIngest, + ), + files: fileChecksums, + }); + + // One build directory, replaced atomically. `serve` and `publish` both + // read `releases/local`; published history lives in the bucket, keyed by + // content digest. + const localDir = path.join(releasesDir, "local"); + if (fs.existsSync(localDir)) { + fs.rmSync(localDir, { recursive: true, force: true }); + } + fs.renameSync(staging, localDir); + return { distDir: localDir, files }; + } catch (err) { + fs.rmSync(staging, { recursive: true, force: true }); + throw err; + } +} + +function walkFiles(dir: string): string[] { + if (!fs.existsSync(dir)) return []; + const out: string[] = []; + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + if (entry.name.startsWith(".")) continue; + const full = path.join(dir, entry.name); + if (entry.isDirectory()) out.push(...walkFiles(full)); + else out.push(full); + } + return out; +} diff --git a/src/query/runQuery.ts b/src/query/runQuery.ts new file mode 100644 index 0000000..eddd707 --- /dev/null +++ b/src/query/runQuery.ts @@ -0,0 +1,215 @@ +import { fork } from "node:child_process"; +import { fileURLToPath } from "node:url"; +import type { CommandError } from "../cli/envelope.js"; + +export interface QueryRequest { + sql: string; + tables: Record; + rawAmountColumns: string[]; + rowLimit: number; + models?: { id: string; sql: string }[]; +} + +export interface QuerySuccess { + columns: { name: string; logical_type: string }[]; + rows: unknown[][]; + snapshot: string; +} + +export interface ParquetColumn { + name: string; + logical_type: string; +} + +const DEADLINE_MS = 60_000; + +function error( + code: CommandError["code"], + message: string, + opts: { retryable?: boolean } = {}, +): CommandError { + return { + code, + message, + resource_id: null, + pointer: null, + retryable: opts.retryable ?? false, + suggested_next: null, + }; +} + +function workerLaunch(): { modulePath: string; execArgv: string[] } { + const self = fileURLToPath(import.meta.url); + const isTs = self.endsWith(".ts"); + const modulePath = fileURLToPath( + new URL(isTs ? "./workerMain.ts" : "./workerMain.js", import.meta.url), + ); + const execArgv = + isTs && !process.features.typescript ? ["--experimental-strip-types"] : []; + return { modulePath, execArgv }; +} + +function strippedEnv(): NodeJS.ProcessEnv { + const env: NodeJS.ProcessEnv = {}; + for (const key of ["PATH", "HOME", "LANG"] as const) { + const value = process.env[key]; + if (value !== undefined) { + env[key] = value; + } + } + return env; +} + +interface WorkerSuccess { + columns: { name: string; logical_type: string }[]; + rows: unknown[][]; + truncated: boolean; +} + +function parseWorkerPayload( + line: string, +): + | { ok: true; value: WorkerSuccess } + | { ok: false; code: CommandError["code"]; message: string } + | null { + let parsed: unknown; + try { + parsed = JSON.parse(line); + } catch { + return null; + } + if (parsed === null || typeof parsed !== "object" || !("ok" in parsed)) { + return null; + } + const body = parsed as { + ok: unknown; + columns?: WorkerSuccess["columns"]; + rows?: WorkerSuccess["rows"]; + truncated?: unknown; + code?: unknown; + message?: unknown; + }; + if (body.ok === true && body.columns !== undefined && body.rows !== undefined) { + return { + ok: true, + value: { + columns: body.columns, + rows: body.rows, + truncated: body.truncated === true, + }, + }; + } + if (body.ok === false) { + const code = typeof body.code === "string" ? body.code : "validation"; + return { + ok: false, + code: code as CommandError["code"], + message: String(body.message ?? "worker failed"), + }; + } + return null; +} + +function invokeWorker(req: { + sql: string; + tables: Record; + rawAmountColumns?: string[]; + rowLimit?: number; + models?: { id: string; sql: string }[]; +}): Promise { + const { modulePath, execArgv } = workerLaunch(); + return new Promise((resolve, reject) => { + const child = fork(modulePath, [], { + execArgv, + env: strippedEnv(), + stdio: ["pipe", "pipe", "pipe", "ipc"], + }); + let stdout = ""; + let stderr = ""; + let settled = false; + + const timer = setTimeout(() => { + child.kill("SIGKILL"); + finish( + error("transient_dependency", "query deadline exceeded", { + retryable: true, + }), + ); + }, DEADLINE_MS); + + function finish(err: CommandError | null, value?: WorkerSuccess): void { + if (settled) { + return; + } + settled = true; + clearTimeout(timer); + if (err) { + reject(err); + } else { + resolve(value as WorkerSuccess); + } + } + + child.stdout?.setEncoding("utf8"); + child.stdout?.on("data", (chunk: string) => { + stdout += chunk; + }); + child.stderr?.setEncoding("utf8"); + child.stderr?.on("data", (chunk: string) => { + stderr += chunk; + }); + child.on("error", (err) => { + finish(error("internal", err.message)); + }); + child.on("exit", (code) => { + const payload = parseWorkerPayload(stdout.trim().split("\n").pop() ?? ""); + if (payload?.ok === true) { + finish(null, payload.value); + return; + } + if (payload?.ok === false) { + finish(error(payload.code, payload.message)); + return; + } + const detail = stderr.trim() || `worker exited with code ${code ?? "unknown"}`; + finish(error("internal", detail)); + }); + + child.stdin?.write( + JSON.stringify({ + sql: req.sql, + tables: req.tables, + rawAmountColumns: req.rawAmountColumns ?? [], + rowLimit: req.rowLimit, + models: req.models ?? [], + }) + "\n", + ); + child.stdin?.end(); + }); +} + +export async function runQuery(req: QueryRequest): Promise { + const { columns, rows, truncated } = await invokeWorker(req); + if (truncated) { + throw error( + "policy_refused", + `query returned more than ${req.rowLimit} rows; add a LIMIT or aggregate instead`, + ); + } + return { + columns, + rows, + snapshot: Object.values(req.tables)[0] ?? "", + }; +} + +export async function describeParquet(parquetPath: string): Promise { + const { rows } = await invokeWorker({ + sql: "DESCRIBE SELECT * FROM snapshot", + tables: { snapshot: parquetPath }, + }); + return rows.map((row) => ({ + name: String(row[0]), + logical_type: String(row[1]), + })); +} diff --git a/src/query/sqlGuard.ts b/src/query/sqlGuard.ts new file mode 100644 index 0000000..94e1e4f --- /dev/null +++ b/src/query/sqlGuard.ts @@ -0,0 +1,181 @@ +// SQL admission control, built on DuckDB's own parser rather than regexes. +// +// The worker serializes every statement with `json_serialize_sql` before it +// runs. That call has two useful properties: +// +// 1. It refuses anything that is not a SELECT ("Only SELECT statements can +// be serialized to json!"), so a successful serialize is itself proof +// that the statement is read-only. No keyword denylist is needed. +// 2. It reports the statement count, so `select 1; drop table t` cannot +// slip a second statement past a check aimed at the first. +// +// On top of that we walk the AST to reject lexicographic ORDER BY on raw +// amount columns (spec §12). uint256 amounts are carried as decimal strings, +// so `ORDER BY value` silently orders "9" after "10". The check resolves +// select-list aliases and ordinals, which a token scan cannot do. + +import type { CommandError } from "../cli/envelope.js"; + +export interface SqlIssue { + code: CommandError["code"]; + message: string; +} + +/** Shape of `json_serialize_sql` output that we depend on. */ +export interface SerializedSql { + error?: boolean; + error_message?: string; + error_subtype?: string; + statements?: unknown[]; +} + +interface ColumnRef { + class: "COLUMN_REF"; + alias?: string; + column_names?: string[]; +} + +interface ConstantRef { + class: "CONSTANT"; + alias?: string; + value?: { value?: unknown }; +} + +type Expr = ColumnRef | ConstantRef | { class: string; alias?: string }; + +function isObject(value: unknown): value is Record { + return typeof value === "object" && value !== null; +} + +function asExpr(value: unknown): Expr | null { + return isObject(value) && typeof value.class === "string" + ? (value as unknown as Expr) + : null; +} + +/** Final identifier of a column reference: `t.value` → `value`. */ +function columnName(expr: Expr | null): string | null { + if (!expr || expr.class !== "COLUMN_REF") return null; + const names = (expr as ColumnRef).column_names; + if (!Array.isArray(names) || names.length === 0) return null; + const last = names[names.length - 1]; + return typeof last === "string" ? last : null; +} + +/** Every SELECT_NODE in the tree, so subqueries and CTEs are covered too. */ +function selectNodes(root: unknown, out: Record[] = []): Record[] { + if (Array.isArray(root)) { + for (const item of root) selectNodes(item, out); + return out; + } + if (!isObject(root)) return out; + if (root.type === "SELECT_NODE") out.push(root); + for (const value of Object.values(root)) selectNodes(value, out); + return out; +} + +/** + * Resolve an ORDER BY term to the expression it actually sorts on. + * + * - `ORDER BY 2` → the second select-list entry. + * - `ORDER BY v` → the entry aliased `v`, if one exists. + * - anything else → itself. + */ +function resolveOrderTerm( + expr: Expr | null, + selectList: unknown[], +): Expr | null { + if (!expr) return null; + + if (expr.class === "CONSTANT") { + const raw = (expr as ConstantRef).value?.value; + const ordinal = typeof raw === "number" ? raw : Number(raw); + if (!Number.isInteger(ordinal) || ordinal < 1 || ordinal > selectList.length) { + return null; + } + return asExpr(selectList[ordinal - 1]); + } + + const name = columnName(expr); + if (name === null) return expr; + + for (const entry of selectList) { + const candidate = asExpr(entry); + if (candidate?.alias && candidate.alias.toLowerCase() === name.toLowerCase()) { + return candidate; + } + } + return expr; +} + +function orderModifiers(node: Record): unknown[] { + const modifiers = node.modifiers; + if (!Array.isArray(modifiers)) return []; + const out: unknown[] = []; + for (const modifier of modifiers) { + if (isObject(modifier) && modifier.type === "ORDER_MODIFIER" && Array.isArray(modifier.orders)) { + out.push(...modifier.orders); + } + } + return out; +} + +/** + * Inspect a parsed statement set. Returns the first problem found, or null. + * + * `serialized` is the parsed output of `json_serialize_sql`; keeping this + * function pure makes the whole policy testable without a DuckDB instance. + */ +export function inspectSerializedSql( + serialized: SerializedSql, + opts: { label: string; rawAmountColumns?: string[] }, +): SqlIssue | null { + const { label } = opts; + + if (serialized.error === true) { + const detail = serialized.error_message ?? "could not be parsed"; + // DuckDB uses this exact message for every non-SELECT statement. + if (detail.includes("Only SELECT statements")) { + return { + code: "policy_refused", + message: `${label} must be a single SELECT statement; statements that read or write outside the snapshot are refused`, + }; + } + return { code: "validation", message: `${label} failed to parse: ${detail}` }; + } + + const statements = serialized.statements; + if (!Array.isArray(statements) || statements.length === 0) { + return { code: "validation", message: `${label} contains no statement` }; + } + if (statements.length > 1) { + return { + code: "policy_refused", + message: `${label} contains ${statements.length} statements; exactly one SELECT is allowed`, + }; + } + + const rawColumns = new Set( + (opts.rawAmountColumns ?? []).map((name) => name.toLowerCase()), + ); + if (rawColumns.size === 0) return null; + + for (const node of selectNodes(statements[0])) { + const selectList = Array.isArray(node.select_list) ? node.select_list : []; + for (const order of orderModifiers(node)) { + const term = isObject(order) ? asExpr(order.expression) : null; + const resolved = resolveOrderTerm(term, selectList); + const name = columnName(resolved); + if (name !== null && rawColumns.has(name.toLowerCase())) { + return { + code: "validation", + message: + `${label}: ORDER BY ${name} would sort the raw amount as text ` + + `("9" after "10"). Use ORDER BY cp_sortkey(${name}) for numeric order.`, + }; + } + } + } + + return null; +} diff --git a/src/query/workerMain.ts b/src/query/workerMain.ts new file mode 100644 index 0000000..ea2ba4c --- /dev/null +++ b/src/query/workerMain.ts @@ -0,0 +1,226 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { DuckDBInstance } from "@duckdb/node-api"; +import type { SerializedSql, SqlIssue } from "./sqlGuard.js"; + +// This module is forked as a bare node process, so it is loaded as .ts under +// vitest and as .js from dist/. Node's type stripping does not rewrite `.js` +// specifiers back to `.ts`, so the sibling import is resolved at runtime. +// The type-only import above is erased and needs no such treatment. +const { inspectSerializedSql } = (await import( + new URL( + import.meta.url.endsWith(".ts") ? "./sqlGuard.ts" : "./sqlGuard.js", + import.meta.url, + ).href +)) as typeof import("./sqlGuard.js"); + +interface WorkerRequest { + sql: string; + tables: Record; + rawAmountColumns?: string[]; + rowLimit?: number; + models?: { id: string; sql: string }[]; +} + +const MODEL_ID = /^[a-z0-9_]+$/; +// Fallback only: every caller passes an explicit limit. This module is +// forked as a bare process and deliberately imports no project code. +const DEFAULT_ROW_LIMIT = 10_000; + +// A forked recipe runs here, so an unbounded query is the host's problem. +// DuckDB spills past this rather than failing, provided a temp directory +// exists — without one it raises an out-of-memory error instead. +const MEMORY_LIMIT = process.env.CHAINPLOT_QUERY_MEMORY_LIMIT ?? "1GB"; + +/** + * Canonical sort key for uint256/int256 amounts carried as decimal strings. + * + * One sign digit then a fixed 78-digit body, so plain lexicographic order is + * signed-numeric order: + * negative → '0' + nines-complement of the zero-padded magnitude + * non-negative → '1' + zero-padded magnitude + * + * Nines-complement (not tens) keeps this pure string work: DuckDB's widest + * integer is 128-bit, so 10^78 arithmetic is not available. Complementing the + * magnitude reverses its order, which is exactly what negative numbers need + * (-10 must sort before -9), and the leading sign digit puts every negative + * ahead of every non-negative. + */ +const SORT_KEY_MACRO = ` +CREATE MACRO cp_sortkey(v) AS ( + CASE + WHEN v IS NULL THEN NULL + WHEN starts_with(CAST(v AS VARCHAR), '-') + THEN '0' || translate( + lpad(substr(CAST(v AS VARCHAR), 2), 78, '0'), + '0123456789', '9876543210') + ELSE '1' || lpad(CAST(v AS VARCHAR), 78, '0') + END +)`; + +function issueError(issue: SqlIssue): Error { + const err = new Error(issue.message) as Error & { chainplotCode?: string }; + err.chainplotCode = issue.code; + return err; +} + +function quoteIdent(name: string): string { + return `"${name.replaceAll('"', '""')}"`; +} + +function quoteString(value: string): string { + return `'${value.replaceAll("'", "''")}'`; +} + +interface Conn { + run(sql: string): Promise; + runAndReadAll(sql: string): Promise<{ getRowsJson(): unknown[][] }>; + streamAndReadUntil( + sql: string, + rows: number, + ): Promise<{ + columnCount: number; + columnName(i: number): string; + columnType(i: number): { toString(): string }; + currentRowCount: number; + done: boolean; + getRowsJson(): unknown[][]; + }>; +} + +/** + * Parse `sql` with DuckDB and apply admission control. Throws on refusal. + * + * `json_serialize_sql` only accepts a literal, so the statement is inlined as + * a quoted string; it is parsed, never executed, by this call. + */ +async function assertAdmissible( + conn: Conn, + sql: string, + opts: { label: string; rawAmountColumns?: string[] }, +): Promise { + let serialized: SerializedSql; + try { + const reader = await conn.runAndReadAll( + `SELECT json_serialize_sql(${quoteString(sql)})`, + ); + serialized = JSON.parse(String(reader.getRowsJson()[0]?.[0] ?? "{}")) as SerializedSql; + } catch (err) { + throw issueError({ + code: "validation", + message: `${opts.label} failed to parse: ${err instanceof Error ? err.message : String(err)}`, + }); + } + const issue = inspectSerializedSql(serialized, opts); + if (issue) throw issueError(issue); +} + +async function readRequest(): Promise { + let buf = ""; + for await (const chunk of process.stdin) { + buf += typeof chunk === "string" ? chunk : chunk.toString("utf8"); + const nl = buf.indexOf("\n"); + if (nl !== -1) { + return JSON.parse(buf.slice(0, nl)) as WorkerRequest; + } + } + if (!buf) { + throw new Error("empty worker request"); + } + return JSON.parse(buf) as WorkerRequest; +} + +async function execute(req: WorkerRequest): Promise<{ + columns: { name: string; logical_type: string }[]; + rows: unknown[][]; + truncated: boolean; +}> { + const rowLimit = req.rowLimit ?? DEFAULT_ROW_LIMIT; + const spillDir = fs.mkdtempSync(path.join(os.tmpdir(), "chainplot-duckdb-")); + const instance = await DuckDBInstance.create(":memory:", { + autoinstall_known_extensions: "false", + autoload_known_extensions: "false", + memory_limit: MEMORY_LIMIT, + temp_directory: spillDir, + }); + try { + const conn = (await instance.connect()) as unknown as Conn; + try { + // Snapshots are the only filesystem reads this process is allowed to + // make, so they happen first... + for (const [name, parquetPath] of Object.entries(req.tables)) { + await conn.run( + `CREATE TABLE ${quoteIdent(name)} AS SELECT * FROM read_parquet(${quoteString(parquetPath)})`, + ); + } + + // ...and the door is shut before any project-supplied SQL runs. Models + // arrive from `source/models/` of a forked release and are no more + // trusted than the query itself, so they must land on this side of it. + // DuckDB does not allow re-enabling external access in a session. + await conn.run("SET enable_external_access=false"); + await conn.run(SORT_KEY_MACRO); + + for (const model of req.models ?? []) { + if (!MODEL_ID.test(model.id)) { + throw issueError({ + code: "validation", + message: `invalid model id: ${model.id}`, + }); + } + await assertAdmissible(conn, model.sql, { label: `model ${model.id}` }); + try { + await conn.run(`CREATE TABLE ${quoteIdent(model.id)} AS (${model.sql})`); + } catch (err) { + throw issueError({ + code: "validation", + message: `model ${model.id} failed: ${err instanceof Error ? err.message : String(err)}`, + }); + } + } + + await assertAdmissible(conn, req.sql, { + label: "query", + rawAmountColumns: req.rawAmountColumns ?? [], + }); + + // Stop reading at the limit instead of materializing everything and + // rejecting afterwards; `done` tells us whether more rows existed. + const reader = await conn.streamAndReadUntil(req.sql, rowLimit + 1); + const columns: { name: string; logical_type: string }[] = []; + for (let i = 0; i < reader.columnCount; i++) { + columns.push({ + name: reader.columnName(i), + logical_type: reader.columnType(i).toString(), + }); + } + const all = reader.getRowsJson(); + const truncated = all.length > rowLimit || !reader.done; + return { columns, rows: all.slice(0, rowLimit), truncated }; + } finally { + (conn as unknown as { closeSync(): void }).closeSync(); + } + } finally { + instance.closeSync(); + fs.rmSync(spillDir, { recursive: true, force: true }); + } +} + +function reply(payload: unknown, exitCode: number): void { + fs.writeSync(1, JSON.stringify(payload) + "\n"); + process.exit(exitCode); +} + +try { + const req = await readRequest(); + const { columns, rows, truncated } = await execute(req); + reply({ ok: true, columns, rows, truncated }, 0); +} catch (err) { + const code = + err !== null && typeof err === "object" && "chainplotCode" in err + ? String((err as { chainplotCode: unknown }).chainplotCode) + : "validation"; + const message = err instanceof Error ? err.message : String(err); + reply({ ok: false, code, message }, 1); +} diff --git a/src/rpc/client.ts b/src/rpc/client.ts new file mode 100644 index 0000000..40e59a2 --- /dev/null +++ b/src/rpc/client.ts @@ -0,0 +1,62 @@ +export class RpcError extends Error { + constructor( + public readonly retryable: boolean, + message: string, + ) { + super(message); + this.name = "RpcError"; + } +} + +export interface RpcClient { + call(method: string, params: unknown[]): Promise; +} + +const REQUEST_TIMEOUT_MS = 30_000; + +export function createRpcClient( + url: string, + fetchImpl: typeof fetch = fetch, +): RpcClient { + let nextId = 1; + return { + async call(method: string, params: unknown[]): Promise { + const id = nextId++; + let response: Response; + try { + response = await fetchImpl(url, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ jsonrpc: "2.0", id, method, params }), + redirect: "error", + signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), + }); + } catch (err) { + throw new RpcError(true, `rpc request failed: ${String(err)}`); + } + if (!response.ok) { + throw new RpcError(true, `rpc http status ${response.status}`); + } + let body: unknown; + try { + body = await response.json(); + } catch (err) { + throw new RpcError(false, `rpc response is not json: ${String(err)}`); + } + const envelope = body as { + result?: unknown; + error?: { code?: number; message?: string }; + }; + if (envelope.error !== undefined && envelope.error !== null) { + throw new RpcError( + true, + `rpc error ${envelope.error.code ?? ""}: ${envelope.error.message ?? "unknown"}`, + ); + } + if (!("result" in envelope)) { + throw new RpcError(false, "rpc response missing result"); + } + return envelope.result as T; + }, + }; +} diff --git a/src/rpc/heads.ts b/src/rpc/heads.ts new file mode 100644 index 0000000..60bc43f --- /dev/null +++ b/src/rpc/heads.ts @@ -0,0 +1,62 @@ +import type { RpcClient } from "./client.js"; +import { RpcError } from "./client.js"; + +export interface BlockHeader { + number: number; + hash: string; + parentHash: string; + /** Unix seconds. The block's own clock is the only honest "how fresh". */ + timestamp: number; +} + +interface RawHeader { + number: string; + hash: string; + parentHash: string; + timestamp?: string; +} + +function toHeader(raw: RawHeader): BlockHeader { + if ( + typeof raw?.number !== "string" || + typeof raw?.hash !== "string" || + typeof raw?.parentHash !== "string" + ) { + throw new RpcError(false, "malformed block header from rpc"); + } + return { + number: Number(BigInt(raw.number)), + hash: raw.hash.toLowerCase(), + parentHash: raw.parentHash.toLowerCase(), + timestamp: + typeof raw.timestamp === "string" ? Number(BigInt(raw.timestamp)) : 0, + }; +} + +export async function getFinalizedHead(client: RpcClient): Promise { + const raw = await client.call("eth_getBlockByNumber", [ + "finalized", + false, + ]); + if (raw === null) { + throw new RpcError( + false, + "node cannot supply finalized block; refusing to fall back", + ); + } + return toHeader(raw); +} + +export async function getHeader( + client: RpcClient, + blockNumber: number, +): Promise { + const raw = await client.call("eth_getBlockByNumber", [ + "0x" + blockNumber.toString(16), + false, + ]); + if (raw === null) { + throw new RpcError(false, `block ${blockNumber} vanished from the chain`); + } + return toHeader(raw); +} diff --git a/src/runtime/journal.ts b/src/runtime/journal.ts new file mode 100644 index 0000000..be3c8b5 --- /dev/null +++ b/src/runtime/journal.ts @@ -0,0 +1,107 @@ +import fs from "node:fs"; +import path from "node:path"; + +export type RunStatus = "running" | "succeeded" | "failed" | "canceled"; + +export interface JournalStatus { + status: RunStatus; + plan_id: string; + plan_digest: string; + updated_at: string; + result?: unknown; +} + +export interface RunSummary { + idempotency_key: string; + plan_id: string; + status: RunStatus; + updated_at: string; +} + +export function journalDir(cwd: string, key: string): string { + return path.join(cwd, ".chainplot", "runs", key); +} + +function writeJson(filePath: string, value: unknown): void { + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + fs.writeFileSync(filePath, `${JSON.stringify(value, null, 2)}\n`); +} + +function readJson(filePath: string): T | null { + if (!fs.existsSync(filePath)) return null; + return JSON.parse(fs.readFileSync(filePath, "utf8")) as T; +} + +export function writeJournalPlan(cwd: string, key: string, plan: unknown): void { + writeJson(path.join(journalDir(cwd, key), "plan.json"), plan); +} + +export function readJournalPlan(cwd: string, key: string): T | null { + return readJson(path.join(journalDir(cwd, key), "plan.json")); +} + +export function writeJournalStatus( + cwd: string, + key: string, + status: Omit & { updated_at?: string }, +): void { + writeJson(path.join(journalDir(cwd, key), "status.json"), { + ...status, + updated_at: status.updated_at ?? new Date().toISOString(), + }); +} + +export function readJournalStatus( + cwd: string, + key: string, +): JournalStatus | null { + return readJson(path.join(journalDir(cwd, key), "status.json")); +} + +export function writeJournalProject( + cwd: string, + key: string, + project: unknown, +): void { + writeJson(path.join(journalDir(cwd, key), "project.json"), project); +} + +export function readJournalProject(cwd: string, key: string): T | null { + return readJson(path.join(journalDir(cwd, key), "project.json")); +} + +export function lastSucceededRunKey(cwd: string): string | null { + const succeeded = listRuns(cwd).filter((r) => r.status === "succeeded"); + return succeeded.at(-1)?.idempotency_key ?? null; +} + +export function appendCheckpoint( + cwd: string, + key: string, + entry: Record, +): void { + const file = path.join(journalDir(cwd, key), "checkpoints.jsonl"); + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.appendFileSync( + file, + `${JSON.stringify({ ts: new Date().toISOString(), ...entry })}\n`, + ); +} + +export function listRuns(cwd: string): RunSummary[] { + const runsDir = path.join(cwd, ".chainplot", "runs"); + if (!fs.existsSync(runsDir)) return []; + const out: RunSummary[] = []; + for (const key of fs.readdirSync(runsDir)) { + const status = readJournalStatus(cwd, key); + if (status) { + out.push({ + idempotency_key: key, + plan_id: status.plan_id, + status: status.status, + updated_at: status.updated_at, + }); + } + } + return out.sort((a, b) => a.updated_at.localeCompare(b.updated_at)); +} diff --git a/src/runtime/locks.ts b/src/runtime/locks.ts new file mode 100644 index 0000000..c9205ac --- /dev/null +++ b/src/runtime/locks.ts @@ -0,0 +1,188 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { randomUUID } from "node:crypto"; +import { commandError, errorMessage } from "../plan/errors.js"; + +export interface LocalLock { + release(): void; +} + +interface LockFile { + token?: string; + pid?: number; + host?: string; + heartbeat_at?: string; +} + +/** + * The holder rewrites its lock file on this interval, so an abandoned lock is + * recognisable in seconds rather than by guessing how long a job might run. + * + * Judging by pid only works on the same host, and the CLI normally runs in a + * container against a bind-mounted project — where every recreated container + * has a different hostname, so pid liveness never applies. A heartbeat is the + * signal that works from either side. + */ +const HEARTBEAT_MS = 10_000; + +/** Missed heartbeats tolerated before the lock is considered abandoned. */ +const STALE_AFTER_MS = HEARTBEAT_MS * 4; + +function lockBody(token: string): string { + return JSON.stringify({ + token, + pid: process.pid, + host: os.hostname(), + heartbeat_at: new Date().toISOString(), + }); +} + +function readLock(lockPath: string): LockFile | null { + try { + return JSON.parse(fs.readFileSync(lockPath, "utf8")) as LockFile; + } catch { + return null; + } +} + +/** + * Is an existing lock file abandoned? + * + * An unreadable file was written by a run that died mid-write. A live holder + * on this host settles it outright. Otherwise the heartbeat decides. + */ +function isStale(lockPath: string, now: number): boolean { + const body = readLock(lockPath); + if (body === null) return true; + + if (body.host === os.hostname() && typeof body.pid === "number") { + try { + process.kill(body.pid, 0); + return false; + } catch { + return true; + } + } + + const beat = body.heartbeat_at ? Date.parse(body.heartbeat_at) : NaN; + if (Number.isNaN(beat)) return true; + return now - beat > STALE_AFTER_MS; +} + +function writeExclusive(lockPath: string, body: string): void { + const fd = fs.openSync(lockPath, "wx"); + try { + fs.writeSync(fd, body); + } finally { + fs.closeSync(fd); + } +} + +export function acquireLocalLock(cwd: string, name: string): LocalLock { + const lockDir = path.join(cwd, ".chainplot", "locks"); + fs.mkdirSync(lockDir, { recursive: true }); + const lockPath = path.join(lockDir, `${name}.lock`); + const token = randomUUID(); + + try { + writeExclusive(lockPath, lockBody(token)); + } catch { + // A killed run cannot clean up after itself, and an ingest is killed by + // design when it exceeds its wall clock. Without takeover the first crash + // would lock the project until someone deleted the file by hand. + if (isStale(lockPath, Date.now())) { + fs.rmSync(lockPath, { force: true }); + try { + writeExclusive(lockPath, lockBody(token)); + return holder(lockPath, token); + } catch { + /* lost the race to another writer; fall through to refusal */ + } + } + throw commandError( + "policy_refused", + `another run holds lock ${name}; one active ingest/publish per project. ` + + `If no run is active, delete ${lockPath}`, + { resource_id: name, suggested_next: "runs list" }, + ); + } + + return holder(lockPath, token); +} + +function holder(lockPath: string, token: string): LocalLock { + let released = false; + + // Refresh while held. If the file no longer carries our token, another + // writer has taken the lock over and we must stop touching it rather than + // clobber theirs. + const beat = setInterval(() => { + if (released) return; + if (readLock(lockPath)?.token !== token) { + clearInterval(beat); + return; + } + try { + fs.writeFileSync(lockPath, lockBody(token)); + } catch { + /* the directory went away; release will handle it */ + } + }, HEARTBEAT_MS); + // Never a reason to keep the process alive. + beat.unref?.(); + + return { + release() { + if (released) return; + released = true; + clearInterval(beat); + // Only remove a lock that is still ours. + if (readLock(lockPath)?.token === token) { + try { + fs.unlinkSync(lockPath); + } catch { + /* already gone */ + } + } + }, + }; +} + +export async function acquireAdvisoryLock( + databaseUrl: string, + key: string, +): Promise<() => void> { + const { Client } = await import("pg"); + const client = new Client({ connectionString: databaseUrl }); + try { + await client.connect(); + } catch (err) { + throw commandError( + "transient_dependency", + `cannot reach postgres for advisory lock: ${errorMessage(err)}`, + { retryable: true }, + ); + } + const result = await client.query<{ locked: boolean }>( + "SELECT pg_try_advisory_lock(hashtext($1)) AS locked", + [key], + ); + if (!result.rows[0]?.locked) { + await client.end(); + throw commandError( + "policy_refused", + "postgres advisory lock held by another writer", + { resource_id: key }, + ); + } + let released = false; + return () => { + if (released) return; + released = true; + void client + .query("SELECT pg_advisory_unlock(hashtext($1))", [key]) + .then(() => client.end()) + .catch(() => undefined); + }; +} diff --git a/src/snapshot/describe.ts b/src/snapshot/describe.ts new file mode 100644 index 0000000..cf5e922 --- /dev/null +++ b/src/snapshot/describe.ts @@ -0,0 +1,57 @@ +import path from "node:path"; +import type { CommandError } from "../cli/envelope.js"; +import { loadProject } from "../project/load.js"; +import { validateProject } from "../project/validate.js"; +import { describeParquet } from "../query/runQuery.js"; + +export interface DatasetDescribeData { + id: string; + mode: "dataset_included"; + snapshot: string; + columns: { name: string; logical_type: string }[]; + coverage: null; +} + +function error( + code: CommandError["code"], + message: string, + opts: { resource_id?: string | null; pointer?: string | null } = {}, +): CommandError { + return { + code, + message, + resource_id: opts.resource_id ?? null, + pointer: opts.pointer ?? null, + retryable: false, + suggested_next: null, + }; +} + +export async function describeDataset( + projectDir: string, + datasetId: string, +): Promise { + const doc = loadProject(projectDir); + const result = validateProject(doc, projectDir); + if (!result.ok) { + throw result.error; + } + + const dataset = (result.project.datasets ?? []).find((d) => d.id === datasetId); + if (!dataset) { + throw error("validation", `unknown dataset: ${datasetId}`, { + resource_id: datasetId, + pointer: "/datasets", + }); + } + + const parquetPath = path.resolve(projectDir, dataset.snapshot); + const columns = await describeParquet(parquetPath); + return { + id: dataset.id, + mode: "dataset_included", + snapshot: dataset.snapshot, + columns, + coverage: null, + }; +} diff --git a/templates/fixture-transfers/chainplot.yaml b/templates/fixture-transfers/chainplot.yaml new file mode 100644 index 0000000..36c4411 --- /dev/null +++ b/templates/fixture-transfers/chainplot.yaml @@ -0,0 +1,25 @@ +format_version: 1 +id: fixture-transfers +datasets: + - id: amounts + snapshot: snapshots/amounts.parquet +queries: + - id: raw_amounts + file: queries/raw_amounts.sql + dataset: amounts + title: Signed int256 range + raw_amount_columns: + - name: amount + label: Amount (raw) +dashboards: + - id: overview + title: Amounts + description: >- + Eight values spanning the full signed 256-bit range, carried as decimal + strings so nothing is rounded on the way to the page. + panels: + - query: raw_amounts + chart: table + title: uint256 round-trip + description: Sorted with cp_sortkey, so -2^255 lands before -1. + span: full diff --git a/templates/fixture-transfers/queries/raw_amounts.sql b/templates/fixture-transfers/queries/raw_amounts.sql new file mode 100644 index 0000000..cb88e96 --- /dev/null +++ b/templates/fixture-transfers/queries/raw_amounts.sql @@ -0,0 +1,6 @@ +-- cp_sortkey() is built in: it maps a decimal-string amount to a fixed-width +-- key whose lexicographic order is signed-numeric order. Ordering by the raw +-- column directly is refused, because "9" would sort after "10". +SELECT amount +FROM amounts +ORDER BY cp_sortkey(amount) diff --git a/templates/fixture-transfers/snapshots/amounts.parquet b/templates/fixture-transfers/snapshots/amounts.parquet new file mode 100644 index 0000000000000000000000000000000000000000..44de342fc06ebb84b5b9cc6be6fe1b264463a477 GIT binary patch literal 1183 zcmd6n&ubG=5XWaX*-~r7Ex5}rqBq5p z-o!t^W6%B}o;-NS!HXV(^Af~TL_Fjm>|5Rpv)`Hdym`C#sE-LGU&_g+QY(pRyOC@F zd<>f*0K`E+Zx%#=L44H4m@LPbLFc%VMl$EbHu>5Q5|}@GLOW%w(@aW@Rx8V#F^I-0 zN%2eyDV!3HYb_)x2*E99exWnm8pj!<0IjsNltG)PUV71oP9Lh)8;B}6d1L8QBT%DL zk_%;+vx+gRNr6!=!L@T%>frK}krYWvWt5dvw?l#4X+dQxYV~R>c^zHr`fb!&b=~v1 z*dhQPz+wQq12Tg_nK^QG2#E~_KfXivCJOSthb}?kmXb-u6-DS^Fn@(?tzdk8%t?-( zml76*M8`O{mZMaZdP$K<&5RJ5;25n98t0c~+#n|wPDd~ADWp?U-@PoJ} z3|Vgt$F#gStAp|_{`WO5ai{aUY24UMKiq<5bBCs2=W*@g@O*kaIjdb@M|D)Kc`!RZ bEoy}w6~mzr&x`wcs|r95JfwTLN57_@?pE-M literal 0 HcmV?d00001 diff --git a/templates/fixture-transfers/tests/amounts.yaml b/templates/fixture-transfers/tests/amounts.yaml new file mode 100644 index 0000000..76dfda3 --- /dev/null +++ b/templates/fixture-transfers/tests/amounts.yaml @@ -0,0 +1,4 @@ +dataset: amounts +expect: + row_count: 8 + columns: [amount, amount_sort] diff --git a/templates/ingest-transfers/.env.example b/templates/ingest-transfers/.env.example new file mode 100644 index 0000000..d9d4714 --- /dev/null +++ b/templates/ingest-transfers/.env.example @@ -0,0 +1,13 @@ +# Archive-capable Ethereum JSON-RPC (mainnet). Required for plan/apply. +# publicnode archive eth_getLogs is 403; use an archive endpoint. +# Copy to .env (gitignored). Never commit real endpoints. +RPC_URL= + +# Postgres for the rindexer adapter. compose.yaml provides this service; +# nothing to install or point at yourself. +DATABASE_URL=postgresql://chainplot:chainplot@postgres:5432/chainplot + +# Producer image tag. Build it once from a chainplot checkout: +# docker build --platform linux/amd64 -t chainplot:local \ +# -f docker/producer.Dockerfile . +# CHAINPLOT_IMAGE=chainplot:local diff --git a/templates/ingest-transfers/README.md b/templates/ingest-transfers/README.md new file mode 100644 index 0000000..64b0334 --- /dev/null +++ b/templates/ingest-transfers/README.md @@ -0,0 +1,45 @@ +# Ingest template + +Scoped onchain events → a proven dataset → a static dashboard. + +Postgres and the pinned rindexer binary both come from `compose.yaml`. The only +thing you supply is an archive-capable RPC endpoint. + +## One-time: build the producer image + +The producer image is the chainplot CLI plus rindexer. It is built from a +chainplot checkout, not from this project — a scaffolded project has no CLI +sources. rindexer ships linux/amd64 only, so build for that platform: + +```bash +docker build --platform linux/amd64 -t chainplot:local \ + -f docker/producer.Dockerfile . +``` + +Set `CHAINPLOT_IMAGE` to use a different tag. + +## Run it + +```bash +cp .env.example .env # then fill in RPC_URL +docker compose up -d + +docker compose exec producer chainplot plan --intent ingest --json +docker compose exec producer chainplot apply --plan --json +docker compose exec producer chainplot build --json +docker compose exec producer chainplot serve --port 4173 --json +``` + +`plan` is read-only and writes a digest-bound plan; `apply` executes that plan +and nothing else. A release is refused unless the whole pinned block range is +proven complete. + +## Files + +| File | Purpose | +|---|---| +| `chainplot.yaml` | Project definition: source scope, queries, dashboard | +| `compose.yaml` | Postgres + producer runtime | +| `abis/ERC20.json` | ABI for the `Transfer` event | +| `queries/transfer_count.sql` | KPI: number of transfers | +| `.env.example` | Copy to `.env`; `RPC_URL` is the only value you provide | diff --git a/templates/ingest-transfers/abis/ERC20.json b/templates/ingest-transfers/abis/ERC20.json new file mode 100644 index 0000000..cef2eb7 --- /dev/null +++ b/templates/ingest-transfers/abis/ERC20.json @@ -0,0 +1,12 @@ +[ + { + "anonymous": false, + "inputs": [ + { "indexed": true, "name": "from", "type": "address" }, + { "indexed": true, "name": "to", "type": "address" }, + { "indexed": false, "name": "value", "type": "uint256" } + ], + "name": "Transfer", + "type": "event" + } +] diff --git a/templates/ingest-transfers/chainplot.yaml b/templates/ingest-transfers/chainplot.yaml new file mode 100644 index 0000000..5b17a32 --- /dev/null +++ b/templates/ingest-transfers/chainplot.yaml @@ -0,0 +1,42 @@ +format_version: 1 +id: ingest-transfers +policy: + block_budget: 100000 +chain_sources: + - id: mainnet + chain_id: 1 + rpc_secret: RPC_URL + finality: + policy: finalized +event_sources: + - id: usdc + chain: mainnet + addresses: + - "0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48" + abi: abis/ERC20.json + events: + - Transfer + start_block: 18600000 + end: + mode: pinned + block: 18600010 +datasets: + - id: usdc + snapshot: .chainplot/snapshots/usdc/usdc_transfer.parquet +queries: + - id: transfer_count + file: queries/transfer_count.sql + dataset: usdc + title: Transfers observed + raw_amount_columns: + - name: value + decimals: 6 + symbol: USDC +dashboards: + - id: overview + title: USDC transfers + panels: + - query: transfer_count + chart: kpi + title: Transfers + unit: transfers diff --git a/templates/ingest-transfers/compose.yaml b/templates/ingest-transfers/compose.yaml new file mode 100644 index 0000000..621bbbe --- /dev/null +++ b/templates/ingest-transfers/compose.yaml @@ -0,0 +1,45 @@ +# Chainplot ingest runtime: our Postgres + producer with the CLI and the +# pinned rindexer binary. No docker.sock anywhere. +services: + postgres: + image: postgres:16-alpine + environment: + POSTGRES_USER: chainplot + POSTGRES_PASSWORD: chainplot + POSTGRES_DB: chainplot + volumes: + - postgres-data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U chainplot -d chainplot"] + interval: 2s + timeout: 5s + retries: 30 + + producer: + # The producer image is the chainplot CLI plus the pinned rindexer binary. + # It is built from the chainplot repo, not from this project — a scaffolded + # project has no CLI sources to build from. Once, from a chainplot checkout: + # + # docker build --platform linux/amd64 \ + # -t chainplot:local -f docker/producer.Dockerfile . + # + # Point CHAINPLOT_IMAGE at your own tag to use a different build. + image: ${CHAINPLOT_IMAGE:-chainplot:local} + platform: linux/amd64 + depends_on: + postgres: + condition: service_healthy + env_file: .env + environment: + DATABASE_URL: postgresql://chainplot:chainplot@postgres:5432/chainplot + volumes: + - ./:/workspace + working_dir: /workspace + # The image entrypoint is the CLI itself, so a bare `command:` would be + # read as CLI arguments. Hold the container open and drive it with + # `docker compose exec producer chainplot --json`. + entrypoint: ["sleep"] + command: ["infinity"] + +volumes: + postgres-data: diff --git a/templates/ingest-transfers/queries/transfer_count.sql b/templates/ingest-transfers/queries/transfer_count.sql new file mode 100644 index 0000000..aad8c53 --- /dev/null +++ b/templates/ingest-transfers/queries/transfer_count.sql @@ -0,0 +1 @@ +select count(*) as transfer_count from usdc diff --git a/tests/cli/a10.e2e.test.ts b/tests/cli/a10.e2e.test.ts new file mode 100644 index 0000000..5615b0b --- /dev/null +++ b/tests/cli/a10.e2e.test.ts @@ -0,0 +1,80 @@ +import { describe, expect, it } from "vitest"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { runCliJson } from "../helpers/run.js"; + +const template = path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + "../../templates/fixture-transfers", +); + +describe("A10: second agent forks published data", () => { + it("new query + dashboard over the forked snapshot, no RPC/Postgres/credentials", async () => { + // Producer: build + publish a directory release. + const parent = fs.mkdtempSync(path.join(os.tmpdir(), "chainplot-a10-")); + const producer = path.join(parent, "producer"); + fs.cpSync(template, producer, { recursive: true }); + fs.writeFileSync( + path.join(producer, "chainplot.yaml"), + `${fs.readFileSync(path.join(producer, "chainplot.yaml"), "utf8")} +publish_targets: + - id: local-dir + type: directory + path: ./published + dataset_license: CC-BY-4.0 +`, + ); + // Shipping the dataset is opt-in; A10 is precisely the case that wants it. + expect( + (await runCliJson(["build", "--mode", "dataset_included", "--json"], producer)).ok, + ).toBe(true); + expect((await runCliJson(["publish", "--json"], producer)).ok).toBe(true); + + // Second agent: fork, write a NEW query + dashboard, build. + const forked = path.join(parent, "forked"); + const fork = await runCliJson( + ["fork", "--from", path.join(producer, "published"), "--output", forked, "--json"], + parent, + ); + expect(fork.ok).toBe(true); + + fs.mkdirSync(path.join(forked, "queries"), { recursive: true }); + fs.writeFileSync( + path.join(forked, "queries/max_amount.sql"), + "select max(amount) as max_amount from amounts", + ); + const yamlPath = path.join(forked, "chainplot.yaml"); + const { parse: parseYaml, stringify: stringifyYaml } = await import("yaml"); + const doc = parseYaml(fs.readFileSync(yamlPath, "utf8")) as { + queries: { id: string; file: string; dataset: string }[]; + dashboards: { id: string; title: string; panels: { query: string; chart: string }[] }[]; + }; + doc.queries.push({ + id: "max_amount", + file: "queries/max_amount.sql", + dataset: "amounts", + }); + doc.dashboards.push({ + id: "second-agent", + title: "Second agent view", + panels: [{ query: "max_amount", chart: "kpi" }], + }); + fs.writeFileSync(yamlPath, stringifyYaml(doc)); + + // No network, no credentials: validate + build must succeed offline. + delete process.env.RPC_URL; + delete process.env.DATABASE_URL; + expect((await runCliJson(["validate", "--json"], forked)).ok).toBe(true); + const built = await runCliJson(["build", "--json"], forked); + expect(built.ok).toBe(true); + const results = JSON.parse( + fs.readFileSync( + path.join(forked, "dist/releases/local/results/max_amount.json"), + "utf8", + ), + ); + expect(results.rows.length).toBeGreaterThan(0); + }, 30_000); +}); diff --git a/tests/cli/a15.e2e.test.ts b/tests/cli/a15.e2e.test.ts new file mode 100644 index 0000000..e954874 --- /dev/null +++ b/tests/cli/a15.e2e.test.ts @@ -0,0 +1,40 @@ +import { describe, expect, it } from "vitest"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { spawnSync } from "node:child_process"; + +describe("A15", () => { + it("init, validate, test, build with network disabled", () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "chainplot-a15-")); + const env = { + ...process.env, + NO_NETWORK: "1", + http_proxy: "http://127.0.0.1:1", + https_proxy: "http://127.0.0.1:1", + HTTP_PROXY: "http://127.0.0.1:1", + HTTPS_PROXY: "http://127.0.0.1:1", + }; + // Compiled entry: pnpm exec tsx fails when cwd is the init output (no package.json). + const bin = path.resolve("dist/cli/main.js"); + expect(fs.existsSync(bin)).toBe(true); + const run = (args: string[], cwd: string) => + spawnSync(process.execPath, [bin, ...args, "--json"], { + cwd, + env, + encoding: "utf8", + }); + const init = run( + ["init", "--template", "fixture-transfers", "--output", dir], + process.cwd(), + ); + expect(init.status).toBe(0); + for (const cmd of [["validate"], ["test"], ["build"]]) { + const r = run(cmd, dir); + expect(r.status, r.stderr).toBe(0); + const json = JSON.parse(r.stdout); + expect(json.ok).toBe(true); + expect(json.schema_version).toBe(1); + } + }, 30_000); +}); diff --git a/tests/cli/a2.e2e.test.ts b/tests/cli/a2.e2e.test.ts new file mode 100644 index 0000000..4eb0fc8 --- /dev/null +++ b/tests/cli/a2.e2e.test.ts @@ -0,0 +1,67 @@ +import { describe, expect, it } from "vitest"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { runCliJson } from "../helpers/run.js"; +import { startServe } from "../../src/publish/serve.js"; +const template = path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + "../../templates/fixture-transfers", +); + +describe("A2: presentation rebuild only", () => { + it("dashboard title change → build reflects it, no ingest, offline", async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "chainplot-a2-")); + fs.cpSync(template, dir, { recursive: true }); + + const first = await runCliJson(["build", "--mode", "dataset_included", "--json"], dir); + expect(first.ok).toBe(true); + + const yamlPath = path.join(dir, "chainplot.yaml"); + const before = fs.readFileSync(yamlPath, "utf8"); + fs.writeFileSync(yamlPath, before.replace("title: Amounts", "title: Amounts v2")); + + const second = await runCliJson(["build", "--mode", "dataset_included", "--json"], dir); + expect(second.ok).toBe(true); + + const dash = JSON.parse( + fs.readFileSync( + path.join(dir, "dist/releases/local/dashboards/overview.json"), + "utf8", + ), + ); + expect(dash.title).toBe("Amounts v2"); + // No ingest artifacts were touched: no .chainplot coverage/plans created. + expect(fs.existsSync(path.join(dir, ".chainplot"))).toBe(false); + }); +}); + +describe("A9 groundwork: served release over plain HTTP", () => { + it("release renders from index.html + release.json alone", async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "chainplot-a9-")); + fs.cpSync(template, dir, { recursive: true }); + await runCliJson(["build", "--mode", "dataset_included", "--json"], dir); + const dist = path.join(dir, "dist/releases/local"); + + const server = startServe(dist, 0); + try { + await server.ready; + const url = `http://127.0.0.1:${server.port}`; + const index = await fetch(`${url}/`); + expect(index.status).toBe(200); + const html = await index.text(); + expect(html).toContain("assets/index"); + const assetMatch = html.match(/src="\.\/(assets\/[^"]+\.js)"/); + expect(assetMatch).not.toBeNull(); + const asset = await fetch(`${url}/${assetMatch![1]}`); + expect(asset.status).toBe(200); + const release = await fetch(`${url}/release.json`); + expect( + ((await release.json()) as { mode: string }).mode, + ).toBe("dataset_included"); + } finally { + server.close(); + } + }); +}); diff --git a/tests/cli/build.test.ts b/tests/cli/build.test.ts new file mode 100644 index 0000000..371f6d3 --- /dev/null +++ b/tests/cli/build.test.ts @@ -0,0 +1,136 @@ +import { describe, expect, it } from "vitest"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { runCliJson } from "../helpers/run.js"; + +const template = path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + "../../templates/fixture-transfers", +); + +describe("build", () => { + it("writes the full static release layout", async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "chainplot-build-")); + fs.cpSync(template, dir, { recursive: true }); + + const result = await runCliJson( + ["build", "--mode", "dataset_included", "--json"], + dir, + ); + expect(result.ok).toBe(true); + const dist = path.join(dir, "dist/releases/local"); + for (const rel of [ + "release.json", + "index.html", + "results/raw_amounts.json", + "dashboards/overview.json", + "datasets/amounts/manifest.json", + "datasets/amounts/tables/amounts.parquet", + "source/chainplot.yaml", + "source/queries/raw_amounts.sql", + ]) { + expect(fs.existsSync(path.join(dist, rel))).toBe(true); + } + expect(fs.existsSync(path.join(dist, "assets"))).toBe(true); + + const release = JSON.parse( + fs.readFileSync(path.join(dist, "release.json"), "utf8"), + ); + expect(release).toMatchObject({ + schema_version: 1, + project_id: "fixture-transfers", + mode: "dataset_included", + queries: ["raw_amounts"], + dashboards: ["overview"], + }); + expect(typeof release.generated_at).toBe("string"); + + const dash = JSON.parse( + fs.readFileSync(path.join(dist, "dashboards/overview.json"), "utf8"), + ); + expect(dash.title).toBe("Amounts"); + + const raw = JSON.parse( + fs.readFileSync(path.join(dist, "results/raw_amounts.json"), "utf8"), + ); + expect(typeof raw.rows[0][0]).toBe("string"); + expect(raw.raw_amount_columns).toEqual(["amount"]); + expect(typeof raw.query_digest).toBe("string"); + + // source/ allowlist: no secrets, no work dirs + expect(fs.existsSync(path.join(dist, "source/.env"))).toBe(false); + expect(fs.existsSync(path.join(dist, "source/.chainplot"))).toBe(false); + }); +}); + +// Models are materialized into every query's session. While each session +// loaded only that query's own dataset, a model over dataset A failed every +// query on dataset B and took the whole build with it — which made `models` +// unusable in any project with more than one dataset, i.e. every real one. +describe("multi-dataset projects", () => { + it("materializes models and joins across datasets", async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "chainplot-multi-")); + fs.mkdirSync(path.join(dir, "queries"), { recursive: true }); + fs.mkdirSync(path.join(dir, "models"), { recursive: true }); + fs.mkdirSync(path.join(dir, "snapshots"), { recursive: true }); + + const fixture = path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + "../../templates/fixture-transfers/snapshots/amounts.parquet", + ); + fs.copyFileSync(fixture, path.join(dir, "snapshots/a.parquet")); + fs.copyFileSync(fixture, path.join(dir, "snapshots/b.parquet")); + + fs.writeFileSync( + path.join(dir, "models/a_rollup.sql"), + "SELECT count(*) AS n FROM a", + ); + fs.writeFileSync(path.join(dir, "queries/from_a.sql"), "SELECT n FROM a_rollup"); + fs.writeFileSync( + path.join(dir, "queries/joined.sql"), + "SELECT ((SELECT count(*) FROM a) + (SELECT count(*) FROM b))::VARCHAR AS total", + ); + fs.writeFileSync( + path.join(dir, "chainplot.yaml"), + `format_version: 1 +id: multi +datasets: + - id: a + snapshot: snapshots/a.parquet + - id: b + snapshot: snapshots/b.parquet +models: + - id: a_rollup + file: models/a_rollup.sql + depends_on: [] +queries: + - id: from_a + file: queries/from_a.sql + dataset: a + - id: joined + file: queries/joined.sql + dataset: b +dashboards: + - id: d + title: D + panels: + - query: from_a + chart: kpi +`, + ); + + const built = await runCliJson(["build", "--json"], dir); + expect(built.ok).toBe(true); + + const joined = JSON.parse( + fs.readFileSync( + path.join(dir, "dist/releases/local/results/joined.json"), + "utf8", + ), + ) as { rows: string[][] }; + // Both fixtures have 8 rows, so a real cross-dataset join yields 16. + expect(joined.rows[0]?.[0]).toBe("16"); + }); +}); diff --git a/tests/cli/capabilities.test.ts b/tests/cli/capabilities.test.ts new file mode 100644 index 0000000..8259bda --- /dev/null +++ b/tests/cli/capabilities.test.ts @@ -0,0 +1,22 @@ +import { describe, expect, it } from "vitest"; +import { runCliJson } from "../helpers/run.js"; +import { fileURLToPath } from "node:url"; +import path from "node:path"; + +const cwd = path.dirname(fileURLToPath(import.meta.url)); + +describe("capabilities", () => { + it("prints one JSON object with integer schema_version", async () => { + const result = await runCliJson(["capabilities", "--json"], cwd); + expect(result.schema_version).toBe(1); + expect(result.ok).toBe(true); + expect(result.command).toBe("capabilities"); + expect(result.error).toBeNull(); + expect(result.data).toMatchObject({ + schema_kinds: expect.arrayContaining(["project", "progress", "latest"]), + sql_modes: ["snapshot"], + sources: [], + publish_targets: [], + }); + }); +}); diff --git a/tests/cli/describe.test.ts b/tests/cli/describe.test.ts new file mode 100644 index 0000000..f37d824 --- /dev/null +++ b/tests/cli/describe.test.ts @@ -0,0 +1,25 @@ +import { describe, expect, it } from "vitest"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { runCliJson } from "../helpers/run.js"; + +const template = path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + "../../templates/fixture-transfers", +); + +describe("dataset describe", () => { + it("describes the fixture snapshot", async () => { + const result = await runCliJson( + ["dataset", "describe", "amounts", "--json"], + template, + ); + expect(result.ok).toBe(true); + expect(result.data).toMatchObject({ + id: "amounts", + mode: "dataset_included", + }); + const cols = (result.data as { columns: { name: string }[] }).columns.map((c) => c.name); + expect(cols).toEqual(expect.arrayContaining(["amount", "amount_sort"])); + }); +}); diff --git a/tests/cli/doctor.test.ts b/tests/cli/doctor.test.ts new file mode 100644 index 0000000..0d75648 --- /dev/null +++ b/tests/cli/doctor.test.ts @@ -0,0 +1,54 @@ +import { describe, expect, it } from "vitest"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { runCliJson } from "../helpers/run.js"; + +const template = path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + "../../templates/fixture-transfers", +); + +describe("doctor", () => { + it("offline: rpc/rindexer/s3 skipped, storage checked, no secrets leaked", async () => { + const saved: Record = {}; + for (const key of [ + "RPC_URL", + "DATABASE_URL", + "CHAINPLOT_RINDEXER_BIN", + "CHAINPLOT_S3_ENDPOINT", + "AWS_ACCESS_KEY_ID", + ]) { + saved[key] = process.env[key]; + delete process.env[key]; + } + try { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "chainplot-doctor-")); + fs.cpSync(template, dir, { recursive: true }); + const result = await runCliJson(["doctor", "--json"], dir); + expect(result.ok).toBe(true); + const byName = Object.fromEntries( + (result.data as { checks: { name: string; status: string }[] }).checks.map( + (c) => [c.name, c.status], + ), + ); + expect(byName).toMatchObject({ project: "ok", storage: "ok" }); + expect(byName["rpc"]).toBe("skipped"); + expect(byName["rindexer"]).toBe("skipped"); + expect(byName["s3"]).toBe("skipped"); + expect(JSON.stringify(result.data)).not.toMatch(/RPC_URL=/); + } finally { + for (const [key, value] of Object.entries(saved)) { + if (value !== undefined) process.env[key] = value; + } + } + }); + + it("missing project file → fail, not hang", async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "chainplot-doctor-")); + const result = await runCliJson(["doctor", "--json"], dir); + expect(result.ok).toBe(false); + expect(result.error?.code).toBe("validation"); + }); +}); diff --git a/tests/cli/exampleCoverage.test.ts b/tests/cli/exampleCoverage.test.ts new file mode 100644 index 0000000..859ce7f --- /dev/null +++ b/tests/cli/exampleCoverage.test.ts @@ -0,0 +1,98 @@ +import { describe, expect, it } from "vitest"; +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { parse as parseYaml } from "yaml"; +import { runCliJson } from "../helpers/run.js"; +import type { ProjectDocument } from "../../src/project/types.js"; + +// Every bug found while taking chainplot to real data had one signature: a +// shipped feature that no example or test ever exercised. This asserts the +// examples keep covering the surface, so a feature cannot quietly lose its +// only demonstration. + +const examplesDir = path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + "../../examples", +); + +function exampleProjects(): { id: string; dir: string; doc: ProjectDocument }[] { + return fs + .readdirSync(examplesDir, { withFileTypes: true }) + .filter((e) => e.isDirectory()) + .map((e) => ({ id: e.name, dir: path.join(examplesDir, e.name) })) + .filter(({ dir }) => fs.existsSync(path.join(dir, "chainplot.yaml"))) + .map(({ id, dir }) => ({ + id, + dir, + doc: parseYaml( + fs.readFileSync(path.join(dir, "chainplot.yaml"), "utf8"), + ) as ProjectDocument, + })); +} + +describe("examples", () => { + const projects = exampleProjects(); + + it("there are examples to check", () => { + expect(projects.length).toBeGreaterThanOrEqual(3); + }); + + for (const { id, dir } of projects) { + it(`${id} validates`, async () => { + const result = await runCliJson(["validate", "--json"], dir); + expect(result.ok).toBe(true); + }); + } + + it("every chart kind is demonstrated somewhere", () => { + const used = new Set( + projects.flatMap((p) => + (p.doc.dashboards ?? []).flatMap((d) => d.panels.map((panel) => panel.chart)), + ), + ); + // The allowlist the schema publishes; if a kind is added, an example owes + // it a panel, because nothing else renders one. + for (const kind of ["line", "bar", "area", "kpi", "table"]) { + expect(used, `no example uses chart: ${kind}`).toContain(kind); + } + }); + + it("indexed filters are demonstrated", () => { + const filtered = projects.filter((p) => + (p.doc.event_sources ?? []).some((s) => (s.indexed_filters ?? []).length > 0), + ); + expect(filtered.length, "no example uses indexed_filters").toBeGreaterThan(0); + }); + + it("models are demonstrated, including across datasets", () => { + const withModels = projects.filter((p) => (p.doc.models ?? []).length > 0); + expect(withModels.length, "no example uses models").toBeGreaterThan(0); + // A model is only interesting once more than one dataset is in scope, + // which is the case that was broken. + expect( + withModels.some((p) => (p.doc.datasets ?? []).length > 1), + "no example runs a model over a multi-dataset project", + ).toBe(true); + }); + + it("every release mode is demonstrated", () => { + const modes = projects.map((p) => p.doc.policy?.release_mode ?? "results_only"); + // Each mode makes a different trade between page weight and whether a fork + // can recompute. A mode nobody demonstrates is a mode nobody exercises. + for (const mode of ["results_only", "dataset_referenced", "dataset_included"]) { + expect(modes, `no example uses release_mode: ${mode}`).toContain(mode); + } + }); + + it("every query and model file an example names exists", () => { + for (const { id, dir, doc } of projects) { + for (const q of doc.queries ?? []) { + expect(fs.existsSync(path.join(dir, q.file)), `${id}: ${q.file}`).toBe(true); + } + for (const m of doc.models ?? []) { + expect(fs.existsSync(path.join(dir, m.file)), `${id}: ${m.file}`).toBe(true); + } + } + }); +}); diff --git a/tests/cli/ingestCommands.test.ts b/tests/cli/ingestCommands.test.ts new file mode 100644 index 0000000..6a57612 --- /dev/null +++ b/tests/cli/ingestCommands.test.ts @@ -0,0 +1,266 @@ +import { describe, expect, it, beforeEach, afterEach } from "vitest"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import http from "node:http"; +import { fileURLToPath } from "node:url"; +import type { AddressInfo } from "node:net"; +import { runCliJson } from "../helpers/run.js"; + +const H = (n: number) => "0x" + n.toString(16).padStart(64, "0"); + +let rpcServer: import("node:http").Server | null = null; + +async function startMockRpc(head: number): Promise { + const server = await new Promise((resolve) => { + const s = http.createServer((req, res) => { + let body = ""; + req.on("data", (c: string) => (body += c)); + req.on("end", () => { + const parsed = JSON.parse(body) as { id: number; params: unknown[] }; + const tag = parsed.params[0] as string; + const n = tag === "finalized" ? head : Number(BigInt(tag)); + res.writeHead(200, { "content-type": "application/json" }); + res.end( + JSON.stringify({ + jsonrpc: "2.0", + id: parsed.id, + result: { + number: "0x" + n.toString(16), + hash: H(n), + parentHash: H(n - 1), + }, + }), + ); + }); + }); + s.listen(0, "127.0.0.1", () => resolve(s)); + }); + rpcServer = server; + return `http://127.0.0.1:${(server.address() as AddressInfo).port}`; +} + +const INGEST_YAML = [ + "format_version: 1", + 'id: "ingest-cli"', + "chain_sources:", + " - id: mainnet", + " chain_id: 1", + " rpc_secret: RPC_URL", + " finality:", + " policy: finalized", + "event_sources:", + " - id: usdc", + " chain: mainnet", + " addresses:", + ' - "0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48"', + " abi: abis/ERC20.json", + " events:", + " - Transfer", + " start_block: 100", + " end:", + " mode: pinned", + " block: 110", + "datasets:", + " - id: usdc", + " snapshot: .chainplot/snapshots/usdc/usdc_transfer.parquet", + "queries:", + " - id: count", + " file: queries/count.sql", + " dataset: usdc", +].join("\n"); + +function setupIngestProject(): string { + const cwd = fs.mkdtempSync(path.join(os.tmpdir(), "chainplot-cli-")); + fs.mkdirSync(path.join(cwd, "abis"), { recursive: true }); + fs.mkdirSync(path.join(cwd, "queries"), { recursive: true }); + fs.writeFileSync(path.join(cwd, "chainplot.yaml"), INGEST_YAML + "\n"); + fs.writeFileSync(path.join(cwd, "abis/ERC20.json"), "[]"); + fs.writeFileSync( + path.join(cwd, "queries/count.sql"), + "select count(*) as transfer_count from usdc", + ); + fs.mkdirSync(path.join(cwd, ".chainplot/snapshots/usdc"), { recursive: true }); + fs.writeFileSync( + path.join(cwd, ".chainplot/snapshots/usdc/usdc_transfer.parquet"), + "PK\x03\x04dummy", + ); + return cwd; +} + +function fixtureParquet(): string { + return path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + "../../templates/fixture-transfers/snapshots/amounts.parquet", + ); +} + +function writeCompleteCoverage(cwd: string): void { + fs.mkdirSync(path.join(cwd, ".chainplot"), { recursive: true }); + fs.writeFileSync( + path.join(cwd, ".chainplot/coverage.json"), + JSON.stringify({ + schema_version: 1, + chain_id: 1, + sources: [ + { + source_id: "usdc", + segments: [ + { + start_block: 100, + end_block: 110, + start_block_hash: H(100), + end_block_hash: H(110), + start_block_parent_hash: H(99), + status: "complete_with_rows", + row_count: 92, + }, + ], + }, + ], + }), + ); +} + +let savedEnv: Record; + +beforeEach(() => { + savedEnv = { + RPC_URL: process.env.RPC_URL, + DATABASE_URL: process.env.DATABASE_URL, + CHAINPLOT_RINDEXER_BIN: process.env.CHAINPLOT_RINDEXER_BIN, + }; + delete process.env.RPC_URL; + delete process.env.DATABASE_URL; + delete process.env.CHAINPLOT_RINDEXER_BIN; +}); + +afterEach(() => { + for (const [k, v] of Object.entries(savedEnv)) { + if (v === undefined) delete process.env[k]; + else process.env[k] = v; + } + rpcServer?.close(); + rpcServer = null; +}); + +describe("plan command", () => { + it("build intent → ok with build_results action", async () => { + const cwd = fs.mkdtempSync(path.join(os.tmpdir(), "chainplot-cli-")); + const result = await runCliJson(["plan", "--intent", "build", "--json"], cwd); + expect(result.ok).toBe(false); // project missing chainplot.yaml → validation, not unsupported + expect(result.error?.code).toBe("validation"); + }); + + it("ingest without credentials → missing_credentials", async () => { + const cwd = setupIngestProject(); + const result = await runCliJson(["plan", "--intent", "ingest", "--json"], cwd); + expect(result.ok).toBe(false); + expect(result.error?.code).toBe("missing_credentials"); + }); + + it("ingest with credentials writes a plan file", async () => { + const cwd = setupIngestProject(); + process.env.RPC_URL = await startMockRpc(200); + process.env.DATABASE_URL = "postgres://test@localhost/db"; + const result = await runCliJson(["plan", "--intent", "ingest", "--json"], cwd); + expect(result.ok).toBe(true); + const data = result.data as { plan_id: string; plan_path: string }; + expect(data.plan_id).toMatch(/^[0-9a-f]{64}$/); + expect(fs.existsSync(data.plan_path)).toBe(true); + }); +}); + +describe("apply command", () => { + it("missing plan → validation", async () => { + const cwd = setupIngestProject(); + const result = await runCliJson(["apply", "--plan", "nope", "--json"], cwd); + expect(result.ok).toBe(false); + expect(result.error?.code).toBe("validation"); + }); +}); + +describe("refresh command", () => { + it("dataset-only project → policy_refused", async () => { + const cwd = fs.mkdtempSync(path.join(os.tmpdir(), "chainplot-refresh-")); + fs.mkdirSync(path.join(cwd, "snapshots"), { recursive: true }); + fs.writeFileSync( + path.join(cwd, "chainplot.yaml"), + [ + "format_version: 1", + "id: fixture", + "datasets:", + " - id: amounts", + " snapshot: snapshots/amounts.parquet", + "queries:", + " - id: raw_amounts", + " file: queries/raw_amounts.sql", + " dataset: amounts", + ].join("\n"), + ); + fs.writeFileSync(path.join(cwd, "snapshots/amounts.parquet"), "PK\x03\x04dummy"); + fs.mkdirSync(path.join(cwd, "queries"), { recursive: true }); + fs.writeFileSync(path.join(cwd, "queries/raw_amounts.sql"), "select 1"); + const result = await runCliJson(["refresh", "--json"], cwd); + expect(result.ok).toBe(false); + expect(result.error?.code).toBe("policy_refused"); + }); + + it("unknown --publish-target → policy_refused", async () => { + const cwd = setupIngestProject(); + const result = await runCliJson( + ["refresh", "--publish-target", "nope", "--json"], + cwd, + ); + expect(result.ok).toBe(false); + expect(result.error?.code).toBe("policy_refused"); + }); + + it("ingest-relevant edit after applied plan → policy_refused (A16)", async () => { + const cwd = setupIngestProject(); + // Seed a succeeded run journal (as if a previous apply had completed) + // instead of running a real ingest, which would need live Postgres. + const { parse } = await import("yaml"); + const { createHash } = await import("node:crypto"); + const project = parse(INGEST_YAML); + const key = "seeded-run"; + const runDir = path.join(cwd, ".chainplot", "runs", key); + fs.mkdirSync(runDir, { recursive: true }); + fs.writeFileSync( + path.join(runDir, "project.json"), + JSON.stringify(project), + ); + fs.writeFileSync( + path.join(runDir, "status.json"), + JSON.stringify({ + status: "succeeded", + plan_id: key, + plan_digest: key, + updated_at: new Date().toISOString(), + }), + ); + + fs.writeFileSync( + path.join(cwd, "chainplot.yaml"), + INGEST_YAML.replace( + "0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48", + "0x000000000000000000000000000000000000dead", + ) + "\n", + ); + const refused = await runCliJson(["refresh", "--json"], cwd); + expect(refused.ok).toBe(false); + expect(refused.error?.code).toBe("policy_refused"); + }); + + it("pinned complete, no ingest-relevant edits → rebuild only, zero RPC (A16)", async () => { + const cwd = setupIngestProject(); + writeCompleteCoverage(cwd); + fs.mkdirSync(path.join(cwd, ".chainplot/snapshots/usdc"), { recursive: true }); + fs.copyFileSync( + fixtureParquet(), + path.join(cwd, ".chainplot/snapshots/usdc/usdc_transfer.parquet"), + ); + const result = await runCliJson(["refresh", "--json"], cwd); + expect(result.ok).toBe(true); + }, 30_000); +}); diff --git a/tests/cli/init.test.ts b/tests/cli/init.test.ts new file mode 100644 index 0000000..7653cd4 --- /dev/null +++ b/tests/cli/init.test.ts @@ -0,0 +1,30 @@ +import { describe, expect, it } from "vitest"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { runCliJson } from "../helpers/run.js"; + +describe("init", () => { + it("lists fixture-transfers", async () => { + const result = await runCliJson(["templates", "list", "--json"], process.cwd()); + expect(result.ok).toBe(true); + const ids = (result.data as { templates: { id: string }[] }).templates.map((t) => t.id); + expect(ids).toContain("fixture-transfers"); + }); + + it("creates a project and fails on collision", async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "chainplot-init-")); + const first = await runCliJson( + ["init", "--template", "fixture-transfers", "--output", dir, "--json"], + process.cwd(), + ); + expect(first.ok).toBe(true); + expect(fs.existsSync(path.join(dir, "chainplot.yaml"))).toBe(true); + const second = await runCliJson( + ["init", "--template", "fixture-transfers", "--output", dir, "--json"], + process.cwd(), + ); + expect(second.ok).toBe(false); + expect(second.error?.code).toBe("validation"); + }); +}); diff --git a/tests/cli/initIngest.test.ts b/tests/cli/initIngest.test.ts new file mode 100644 index 0000000..e581f40 --- /dev/null +++ b/tests/cli/initIngest.test.ts @@ -0,0 +1,70 @@ +import { describe, expect, it } from "vitest"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { runCliJson } from "../helpers/run.js"; + +const repoCwd = path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + "../..", +); + +describe("ingest-transfers template", () => { + it("templates list includes it with required inputs", async () => { + const result = await runCliJson(["templates", "list", "--json"], repoCwd); + expect(result.ok).toBe(true); + const templates = (result.data as { templates: { id: string }[] }).templates; + expect(templates.map((t) => t.id)).toContain("ingest-transfers"); + }); + + it("init scaffolds a runnable compose project; validate passes offline", async () => { + const parent = fs.mkdtempSync(path.join(os.tmpdir(), "chainplot-init-")); + const out = path.join(parent, "my-analytics"); + const init = await runCliJson( + ["init", "--template", "ingest-transfers", "--output", out, "--json"], + parent, + ); + expect(init.ok).toBe(true); + for (const file of [ + "chainplot.yaml", + "compose.yaml", + ".env.example", + "abis/ERC20.json", + "queries/transfer_count.sql", + ]) { + expect(fs.existsSync(path.join(out, file))).toBe(true); + } + + // The producer image is built from the chainplot repo, so the scaffold + // must not carry a Dockerfile whose context it cannot supply: `docker + // compose build` in a scaffolded project failed on the missing CLI sources. + expect(fs.existsSync(path.join(out, "Dockerfile"))).toBe(false); + const compose = fs.readFileSync(path.join(out, "compose.yaml"), "utf8"); + expect(compose).toMatch(/image: \$\{CHAINPLOT_IMAGE/); + expect(compose).not.toMatch(/^\s*build:/m); + // The image entrypoint is the CLI, so a bare `command:` would be parsed as + // CLI arguments and the container would exit before anything could run. + expect(compose).toMatch(/entrypoint: \["sleep"\]/); + // No RPC_URL anywhere in the scaffold. + const envExample = fs.readFileSync(path.join(out, ".env.example"), "utf8"); + expect(envExample).toContain("RPC_URL="); + expect(envExample).not.toMatch(/https?:\/\/(?!127\.0\.0\.1|postgres)/); + + const validated = await runCliJson(["validate", "--json"], out); + expect(validated.ok).toBe(true); + }); + + it("init refuses collisions", async () => { + const parent = fs.mkdtempSync(path.join(os.tmpdir(), "chainplot-init-")); + const out = path.join(parent, "proj"); + fs.mkdirSync(out); + fs.writeFileSync(path.join(out, "occupied.txt"), "x"); + const init = await runCliJson( + ["init", "--template", "ingest-transfers", "--output", out, "--json"], + parent, + ); + expect(init.ok).toBe(false); + expect(init.error?.code).toBe("validation"); + }); +}); diff --git a/tests/cli/query.test.ts b/tests/cli/query.test.ts new file mode 100644 index 0000000..8e1f298 --- /dev/null +++ b/tests/cli/query.test.ts @@ -0,0 +1,79 @@ +import { describe, expect, it } from "vitest"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { runCliJson } from "../helpers/run.js"; + +const template = path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + "../../templates/fixture-transfers", +); + +describe("query", () => { + it("returns 2^256-1 as a decimal string, not a number", async () => { + const sql = path.join(os.tmpdir(), "q-a8.sql"); + fs.writeFileSync(sql, "SELECT amount FROM amounts ORDER BY amount_sort"); + const result = await runCliJson( + ["query", "--file", sql, "--snapshot", "amounts", "--json"], + template, + ); + expect(result.ok).toBe(true); + const rows = (result.data as { rows: string[][] }).rows; + const amounts = rows.map((r) => r[0]); + expect(typeof amounts[0]).toBe("string"); + expect(amounts).toContain( + "115792089237316195423570985008687907853269984665640564039457584007913129639935", + ); + expect(amounts[0]).toBe( + "-57896044618658097711785492504343953926634992332820282019728792003956564819968", + ); + }); + + it("maps worker SQL errors to validation", async () => { + const sql = path.join(os.tmpdir(), "q-invalid.sql"); + fs.writeFileSync(sql, "SELECT definitely_not_a_column FROM amounts"); + const result = await runCliJson( + ["query", "--file", sql, "--snapshot", "amounts", "--json"], + template, + ); + expect(result.ok).toBe(false); + expect(result.error?.code).toBe("validation"); + }); + + it("rejects ORDER BY on a raw amount column", async () => { + const sql = path.join(os.tmpdir(), "q-bad.sql"); + fs.writeFileSync(sql, "SELECT amount FROM amounts ORDER BY amount"); + const result = await runCliJson( + ["query", "--file", sql, "--snapshot", "amounts", "--json"], + template, + ); + expect(result.ok).toBe(false); + expect(result.error?.code).toBe("validation"); + }); + + it("executes a query that consumes a project model", async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "chainplot-query-model-")); + fs.cpSync(template, dir, { recursive: true }); + const yaml = path.join(dir, "chainplot.yaml"); + fs.appendFileSync( + yaml, + "\nmodels:\n - id: total_rows\n file: models/total_rows.sql\n depends_on: []\n", + ); + fs.mkdirSync(path.join(dir, "models"), { recursive: true }); + fs.writeFileSync( + path.join(dir, "models/total_rows.sql"), + "SELECT count(*) AS cnt FROM amounts", + ); + const sql = path.join(dir, "queries/consume_model.sql"); + fs.writeFileSync(sql, "SELECT cnt FROM total_rows"); + const result = await runCliJson( + ["query", "--file", sql, "--snapshot", "amounts", "--json"], + dir, + ); + expect(result.ok).toBe(true); + const rows = (result.data as { rows: unknown[][] }).rows; + expect(rows).toHaveLength(1); + expect(Number(rows[0]![0])).toBeGreaterThan(0); + }); +}); diff --git a/tests/cli/runs.cancel.test.ts b/tests/cli/runs.cancel.test.ts new file mode 100644 index 0000000..fb05b4f --- /dev/null +++ b/tests/cli/runs.cancel.test.ts @@ -0,0 +1,291 @@ +import { describe, expect, it, beforeEach, afterEach } from "vitest"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { runCliJson } from "../helpers/run.js"; +import { runsCancel } from "../../src/cli/commands/runs.js"; +import { generatePlan } from "../../src/plan/generate.js"; +import { applyPlan } from "../../src/plan/apply.js"; +import { journalDir, writeJournalStatus } from "../../src/runtime/journal.js"; +import type { IngestAdapter, CoverageReport } from "../../src/ingest/adapter.js"; +import type { RpcClient } from "../../src/rpc/client.js"; +import type { ProjectDocument } from "../../src/project/types.js"; + +// `runs cancel` is documented as cooperative: it drops a flag and the running +// apply notices at its next checkpoint. Nothing exercised it, in either half — +// neither the flag nor the noticing. + +const H = (n: number) => "0x" + n.toString(16).padStart(64, "0"); + +function mockRpc(head: number): RpcClient { + return { + call: async (method: string, params: unknown[]) => { + if (method === "eth_blockNumber") return "0x" + head.toString(16); + const tag = String((params as string[])[0]); + const n = tag === "finalized" ? head : Number(BigInt(tag)); + return { + number: "0x" + n.toString(16), + hash: H(n), + parentHash: H(n - 1), + timestamp: "0x" + (1_700_000_000 + n).toString(16), + }; + }, + } as unknown as RpcClient; +} + +const YAML = [ + "format_version: 1", + 'id: "cancel-test"', + "chain_sources:", + " - id: mainnet", + " chain_id: 1", + " rpc_secret: RPC_URL", + " finality:", + " policy: finalized", + "event_sources:", + " - id: usdc", + " chain: mainnet", + " addresses:", + ' - "0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48"', + " abi: abis/ERC20.json", + " events:", + " - Transfer", + " start_block: 100", + " end:", + " mode: pinned", + " block: 110", + "datasets:", + " - id: usdc", + " snapshot: .chainplot/snapshots/usdc/usdc_transfer.parquet", + "queries:", + " - id: count", + " file: queries/count.sql", + " dataset: usdc", +].join("\n"); + +function setupProject(): string { + const cwd = fs.mkdtempSync(path.join(os.tmpdir(), "chainplot-cancel-")); + fs.mkdirSync(path.join(cwd, "abis"), { recursive: true }); + fs.mkdirSync(path.join(cwd, "queries"), { recursive: true }); + fs.mkdirSync(path.join(cwd, ".chainplot/snapshots/usdc"), { recursive: true }); + fs.writeFileSync(path.join(cwd, "chainplot.yaml"), `${YAML}\n`); + fs.writeFileSync(path.join(cwd, "abis/ERC20.json"), "[]"); + fs.writeFileSync(path.join(cwd, "queries/count.sql"), "select count(*) as n from usdc"); + fs.writeFileSync( + path.join(cwd, ".chainplot/snapshots/usdc/usdc_transfer.parquet"), + "PK\x03\x04dummy", + ); + return cwd; +} + +function project(cwd: string): ProjectDocument { + void cwd; + return { + format_version: 1, + id: "cancel-test", + chain_sources: [ + { + id: "mainnet", + chain_id: 1, + rpc_secret: "RPC_URL", + finality: { policy: "finalized" }, + }, + ], + event_sources: [ + { + id: "usdc", + chain: "mainnet", + addresses: ["0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48"], + abi: "abis/ERC20.json", + events: ["Transfer"], + start_block: 100, + end: { mode: "pinned", block: 110 }, + }, + ], + datasets: [{ id: "usdc", snapshot: ".chainplot/snapshots/usdc/usdc_transfer.parquet" }], + queries: [{ id: "count", file: "queries/count.sql", dataset: "usdc" }], + }; +} + +const report: CoverageReport = { + status: "complete_with_rows", + lastSyncedBlock: 110, + rowCount: 92, +}; + +/** Records whether ingest was ever reached. */ +function adapter(reached: { ingest: boolean }): IngestAdapter { + return { + renderConfig: () => "fake", + runBounded: async (job) => { + reached.ingest = true; + return { job, pid: -1, completedLogSeen: true }; + }, + stopAndQuiesce: async () => {}, + inspectCoverage: async () => report, + } as unknown as IngestAdapter; +} + +let savedEnv: Record = {}; +beforeEach(() => { + savedEnv = { RPC_URL: process.env.RPC_URL, DATABASE_URL: process.env.DATABASE_URL }; + process.env.RPC_URL = "http://rpc.test"; + process.env.DATABASE_URL = "postgresql://u:u@127.0.0.1:1/u"; +}); +afterEach(() => { + for (const [k, v] of Object.entries(savedEnv)) { + if (v === undefined) delete process.env[k]; + else process.env[k] = v; + } +}); + +describe("runs list / show", () => { + it("lists nothing for a project that has never run", async () => { + const cwd = setupProject(); + const result = await runCliJson(["runs", "list", "--json"], cwd); + expect(result.ok).toBe(true); + expect((result.data as { runs: unknown[] }).runs).toEqual([]); + }); + + it("show on an unknown key is a validation error, not a crash", async () => { + const cwd = setupProject(); + const result = await runCliJson(["runs", "show", "nope", "--json"], cwd); + expect(result.ok).toBe(false); + expect(result.error?.code).toBe("validation"); + expect(result.error?.suggested_next).toBe("runs list"); + }); + + it("lists and shows a run once one exists", async () => { + const cwd = setupProject(); + const { plan, planPath } = await generatePlan({ + intent: "ingest", + cwd, + project: project(cwd), + rpcClient: mockRpc(200), + }); + await applyPlan({ + cwd, + planRef: planPath, + adapter: adapter({ ingest: false }), + exportFn: async () => ({ rowCount: 92, parquetPath: "x.parquet" }), + buildFn: async () => ({ distDir: "dist", files: [] }), + rindexerBin: "rindexer", + rpcClient: mockRpc(200), + }); + + const list = await runCliJson(["runs", "list", "--json"], cwd); + const runs = (list.data as { runs: { idempotency_key: string; status: string }[] }) + .runs; + expect(runs).toHaveLength(1); + expect(runs[0]!.status).toBe("succeeded"); + + const shown = await runCliJson(["runs", "show", runs[0]!.idempotency_key, "--json"], cwd); + expect(shown.ok).toBe(true); + expect((shown.data as { plan: { plan_id: string } }).plan.plan_id).toBe(plan.plan_id); + }); +}); + +describe("runs cancel", () => { + it("refuses an unknown key", () => { + const cwd = setupProject(); + const result = runsCancel(cwd, "nope"); + expect(result.ok).toBe(false); + expect(result.error?.code).toBe("validation"); + }); + + it("refuses a run that already succeeded", () => { + const cwd = setupProject(); + writeJournalStatus(cwd, "k", { + status: "succeeded", + plan_id: "k", + plan_digest: "k", + }); + const result = runsCancel(cwd, "k"); + expect(result.ok).toBe(false); + expect(result.error?.message).toMatch(/already succeeded/); + }); + + it("requests cancellation of a run still in flight", () => { + const cwd = setupProject(); + writeJournalStatus(cwd, "k", { status: "running", plan_id: "k", plan_digest: "k" }); + const result = runsCancel(cwd, "k"); + expect(result.ok).toBe(true); + expect(fs.existsSync(path.join(journalDir(cwd, "k"), "cancel_requested"))).toBe(true); + }); + + // A run whose process died leaves the journal saying "running", and apply + // refuses to start while it does. Without a terminal state the project is + // stranded with no way back. + it("moves an abandoned run to a terminal state so apply can run again", async () => { + const cwd = setupProject(); + const { plan, planPath } = await generatePlan({ + intent: "ingest", + cwd, + project: project(cwd), + rpcClient: mockRpc(200), + }); + // Simulate a killed apply: status left at running, no flag. + writeJournalStatus(cwd, plan.plan_id!, { + status: "running", + plan_id: plan.plan_id!, + plan_digest: plan.plan_id!, + }); + await expect( + applyPlan({ + cwd, + planRef: planPath, + adapter: adapter({ ingest: false }), + exportFn: async () => ({ rowCount: 0, parquetPath: "x.parquet" }), + buildFn: async () => ({ distDir: "dist", files: [] }), + rindexerBin: "rindexer", + rpcClient: mockRpc(200), + }), + ).rejects.toMatchObject({ code: "policy_refused" }); + + const cancelled = runsCancel(cwd, plan.plan_id!); + expect(cancelled.ok).toBe(true); + expect((cancelled.data as { was: string }).was).toBe("running"); + const list = await runCliJson(["runs", "list", "--json"], cwd); + expect( + (list.data as { runs: { status: string }[] }).runs.map((r) => r.status), + ).toContain("canceled"); + }); + + // The cooperative half: a flag is only a cancellation if apply honours it. + it("apply stops before doing any work and records the run as canceled", async () => { + const cwd = setupProject(); + const { plan, planPath } = await generatePlan({ + intent: "ingest", + cwd, + project: project(cwd), + rpcClient: mockRpc(200), + }); + + // The key is the plan id, so cancellation can be requested before apply. + writeJournalStatus(cwd, plan.plan_id!, { + status: "running", + plan_id: plan.plan_id!, + plan_digest: plan.project_digest, + }); + expect(runsCancel(cwd, plan.plan_id!).ok).toBe(true); + + const reached = { ingest: false }; + await expect( + applyPlan({ + cwd, + planRef: planPath, + adapter: adapter(reached), + exportFn: async () => ({ rowCount: 0, parquetPath: "x.parquet" }), + buildFn: async () => ({ distDir: "dist", files: [] }), + rindexerBin: "rindexer", + rpcClient: mockRpc(200), + }), + ).rejects.toMatchObject({ code: "policy_refused" }); + + // No ingest ran, and the journal says canceled rather than failed. + expect(reached.ingest).toBe(false); + const list = await runCliJson(["runs", "list", "--json"], cwd); + const runs = (list.data as { runs: { status: string }[] }).runs; + expect(runs.map((r) => r.status)).toContain("canceled"); + }); +}); diff --git a/tests/cli/runs.test.ts b/tests/cli/runs.test.ts new file mode 100644 index 0000000..83e581b --- /dev/null +++ b/tests/cli/runs.test.ts @@ -0,0 +1,199 @@ +import { describe, expect, it, beforeEach, afterEach } from "vitest"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { runCliJson } from "../helpers/run.js"; +import { applyPlan } from "../../src/plan/apply.js"; +import type { BoundedJob, IngestAdapter, CoverageReport } from "../../src/ingest/adapter.js"; +import type { RpcClient } from "../../src/rpc/client.js"; +import type { ProjectDocument } from "../../src/project/types.js"; + +const H = (n: number) => "0x" + n.toString(16).padStart(64, "0"); + +const INGEST_YAML = [ + "format_version: 1", + 'id: "runs-test"', + "chain_sources:", + " - id: mainnet", + " chain_id: 1", + " rpc_secret: RPC_URL", + " finality:", + " policy: finalized", + "event_sources:", + " - id: usdc", + " chain: mainnet", + " addresses:", + ' - "0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48"', + " abi: abis/ERC20.json", + " events:", + " - Transfer", + " start_block: 100", + " end:", + " mode: pinned", + " block: 110", + "datasets:", + " - id: usdc", + " snapshot: .chainplot/snapshots/usdc/usdc_transfer.parquet", + "queries:", + " - id: count", + " file: queries/count.sql", + " dataset: usdc", +].join("\n"); + +function setupProject(): string { + const cwd = fs.mkdtempSync(path.join(os.tmpdir(), "chainplot-runs-")); + fs.mkdirSync(path.join(cwd, "abis"), { recursive: true }); + fs.mkdirSync(path.join(cwd, "queries"), { recursive: true }); + fs.writeFileSync(path.join(cwd, "chainplot.yaml"), INGEST_YAML + "\n"); + fs.writeFileSync(path.join(cwd, "abis/ERC20.json"), "[]"); + fs.writeFileSync(path.join(cwd, "queries/count.sql"), "select count(*) as n from usdc"); + fs.mkdirSync(path.join(cwd, ".chainplot/snapshots/usdc"), { recursive: true }); + fs.writeFileSync( + path.join(cwd, ".chainplot/snapshots/usdc/usdc_transfer.parquet"), + "PK\x03\x04dummy", + ); + return cwd; +} + +let savedEnv: Record; +beforeEach(() => { + savedEnv = { + RPC_URL: process.env.RPC_URL, + DATABASE_URL: process.env.DATABASE_URL, + }; + delete process.env.RPC_URL; + delete process.env.DATABASE_URL; +}); +afterEach(() => { + for (const [k, v] of Object.entries(savedEnv)) { + if (v === undefined) delete process.env[k]; + else process.env[k] = v; + } +}); + +describe("runs commands", () => { + it("list/show roundtrip on a seeded run", async () => { + const cwd = setupProject(); + const runDir = path.join(cwd, ".chainplot", "runs", "k1"); + fs.mkdirSync(runDir, { recursive: true }); + fs.writeFileSync( + path.join(runDir, "status.json"), + JSON.stringify({ + status: "succeeded", + plan_id: "k1", + plan_digest: "k1", + updated_at: "2026-09-13T00:00:00Z", + }), + ); + fs.writeFileSync(path.join(runDir, "plan.json"), JSON.stringify({ schema_version: 1 })); + + const listed = await runCliJson(["runs", "list", "--json"], cwd); + expect(listed.ok).toBe(true); + expect(listed.data).toMatchObject({ + runs: [{ idempotency_key: "k1", status: "succeeded" }], + }); + + const shown = await runCliJson(["runs", "show", "k1", "--json"], cwd); + expect(shown.ok).toBe(true); + expect(shown.data).toMatchObject({ idempotency_key: "k1" }); + + const missing = await runCliJson(["runs", "show", "nope", "--json"], cwd); + expect(missing.ok).toBe(false); + expect(missing.error?.code).toBe("validation"); + }); + + it("cancel writes the flag; cancel of succeeded run → validation", async () => { + const cwd = setupProject(); + const runDir = path.join(cwd, ".chainplot", "runs", "k2"); + fs.mkdirSync(runDir, { recursive: true }); + fs.writeFileSync( + path.join(runDir, "status.json"), + JSON.stringify({ + status: "running", + plan_id: "k2", + plan_digest: "k2", + updated_at: "2026-09-13T00:00:00Z", + }), + ); + const canceled = await runCliJson(["runs", "cancel", "k2", "--json"], cwd); + expect(canceled.ok).toBe(true); + expect(fs.existsSync(path.join(runDir, "cancel_requested"))).toBe(true); + + fs.writeFileSync( + path.join(runDir, "status.json"), + JSON.stringify({ + status: "succeeded", + plan_id: "k2", + plan_digest: "k2", + updated_at: "2026-09-13T00:00:01Z", + }), + ); + const tooLate = await runCliJson(["runs", "cancel", "k2", "--json"], cwd); + expect(tooLate.ok).toBe(false); + expect(tooLate.error?.code).toBe("validation"); + }); +}); + +describe("progress events", () => { + it("applyPlan emits stage events via onProgress", async () => { + const cwd = setupProject(); + process.env.RPC_URL = "http://rpc.test"; + process.env.DATABASE_URL = "postgres://test@localhost/db"; + const { generatePlan } = await import("../../src/plan/generate.js"); + const { parse } = await import("yaml"); + const project = parse(INGEST_YAML) as ProjectDocument; + const { planPath } = await generatePlan({ + intent: "ingest", + cwd, + project, + rpcClient: { + async call() { + return { + number: "0xc8", + hash: H(200), + parentHash: H(199), + } as T; + }, + }, + }); + const report: CoverageReport = { + status: "complete_with_rows", + lastSyncedBlock: 110, + rowCount: 92, + }; + const adapter: IngestAdapter = { + renderConfig: () => "fake", + runBounded: async (job: BoundedJob) => ({ job, pid: -1, completedLogSeen: true }), + stopAndQuiesce: async () => {}, + inspectCoverage: async () => report, + }; + const stages: string[] = []; + await applyPlan({ + cwd, + planRef: planPath, + adapter, + exportFn: async (_job, outDir) => ({ + parquetPath: path.join(outDir, "usdc_transfer.parquet"), + rowCount: 92, + }), + buildFn: async () => ({ distDir: path.join(cwd, "dist/releases/local") }), + rindexerBin: "rindexer", + rpcClient: { + async call(method: string, params: unknown[]) { + const tag = params[0] as string; + const n = tag === "finalized" ? 200 : Number(BigInt(tag)); + return { number: "0x" + n.toString(16), hash: H(n), parentHash: H(n - 1) } as T; + }, + }, + onProgress: (e) => stages.push(e.stage), + }); + expect(stages).toEqual([ + "plan_verified", + "ingest_started", + "ingest_completed", + "coverage_recorded", + "export_completed", + "release_written", + ]); + }); +}); diff --git a/tests/cli/schemaKinds.test.ts b/tests/cli/schemaKinds.test.ts new file mode 100644 index 0000000..536cb5a --- /dev/null +++ b/tests/cli/schemaKinds.test.ts @@ -0,0 +1,66 @@ +import { describe, expect, it } from "vitest"; +import { runCliJson } from "../helpers/run.js"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const cwd = path.dirname(fileURLToPath(import.meta.url)); + +describe("frozen M2 schema kinds", () => { + for (const kind of ["plan", "coverage", "progress"]) { + it(`${kind} is closed (additionalProperties false)`, async () => { + const result = await runCliJson(["schema", "show", kind, "--json"], cwd); + expect(result.ok).toBe(true); + const schema = result.data as { additionalProperties: boolean }; + expect(schema.additionalProperties).toBe(false); + }); + } + + it("coverage requires segment boundary hashes", async () => { + const result = await runCliJson(["schema", "show", "coverage", "--json"], cwd); + expect(result.ok).toBe(true); + const schema = result.data as { + $defs: Record; + }; + expect(schema.$defs.segment.required).toEqual( + expect.arrayContaining([ + "start_block", + "end_block", + "start_block_hash", + "end_block_hash", + "start_block_parent_hash", + "status", + ]), + ); + }); + + it("progress requires run_id and stage", async () => { + const result = await runCliJson(["schema", "show", "progress", "--json"], cwd); + expect(result.ok).toBe(true); + const schema = result.data as { required: string[] }; + expect(schema.required).toEqual( + expect.arrayContaining(["schema_version", "type", "run_id", "stage"]), + ); + }); + + it("plan freezes the M2 plan shape", async () => { + const result = await runCliJson(["schema", "show", "plan", "--json"], cwd); + expect(result.ok).toBe(true); + const schema = result.data as { + required: string[]; + properties: Record; + }; + expect(schema.required).toEqual( + expect.arrayContaining([ + "schema_version", + "intent", + "project_id", + "project_digest", + "chain", + "sources", + "actions", + "state_assumptions", + ]), + ); + expect(schema.properties.schema_version).toMatchObject({ const: 1 }); + }); +}); diff --git a/tests/cli/schemaShow.test.ts b/tests/cli/schemaShow.test.ts new file mode 100644 index 0000000..3eef5b1 --- /dev/null +++ b/tests/cli/schemaShow.test.ts @@ -0,0 +1,21 @@ +import { describe, expect, it } from "vitest"; +import { runCliJson } from "../helpers/run.js"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const cwd = path.dirname(fileURLToPath(import.meta.url)); + +describe("schema show", () => { + it("returns the project schema with additionalProperties false", async () => { + const result = await runCliJson(["schema", "show", "project", "--json"], cwd); + expect(result.ok).toBe(true); + const schema = result.data as { additionalProperties: boolean }; + expect(schema.additionalProperties).toBe(false); + }); + + it("rejects unknown kind", async () => { + const result = await runCliJson(["schema", "show", "nope", "--json"], cwd); + expect(result.ok).toBe(false); + expect(result.error?.code).toBe("validation"); + }); +}); diff --git a/tests/cli/serve.test.ts b/tests/cli/serve.test.ts new file mode 100644 index 0000000..1178468 --- /dev/null +++ b/tests/cli/serve.test.ts @@ -0,0 +1,84 @@ +import { describe, expect, it } from "vitest"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { runCliJson } from "../helpers/run.js"; +import { runCli } from "../../src/cli/run.js"; +import { closeActiveServer, serveCommand } from "../../src/cli/commands/serve.js"; +import { startServe } from "../../src/publish/serve.js"; + +const template = path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + "../../templates/fixture-transfers", +); + +describe("serve", () => { + it("serves the release over loopback HTTP only", async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "chainplot-serve-")); + fs.cpSync(template, dir, { recursive: true }); + await runCliJson(["build", "--json"], dir); + const dist = path.join(dir, "dist/releases/local"); + + const server = startServe(dist, 0); + try { + await server.ready; + const url = `http://127.0.0.1:${server.port}`; + const index = await fetch(`${url}/index.html`); + expect(index.status).toBe(200); + expect(await index.text()).toContain("
"); + const release = await fetch(`${url}/release.json`); + expect(release.status).toBe(200); + const rel = (await release.json()) as { project_id: string }; + expect(rel.project_id).toBe("fixture-transfers"); + const missing = await fetch(`${url}/nope.json`); + expect(missing.status).toBe(404); + const traversal = await fetch(`${url}/../chainplot.yaml`); + expect(traversal.status).toBe(404); + } finally { + server.close(); + } + }); + + it("missing directory → validation error", async () => { + const result = await runCliJson( + ["serve", "--dir", "/nonexistent-xyz", "--json"], + os.tmpdir(), + ); + expect(result.ok).toBe(false); + expect(result.error?.code).toBe("validation"); + }); +}); + +describe("serve command", () => { + it("reports the port it actually bound, not the 0 it was asked for", async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "chainplot-serve-port-")); + fs.cpSync(template, dir, { recursive: true }); + await runCliJson(["build", "--json"], dir); + + const result = await serveCommand(dir); + try { + expect(result.ok).toBe(true); + const { url } = result.data as { url: string }; + expect(new URL(url).port).not.toBe("0"); + const response = await fetch(`${url}/release.json`); + expect(response.status).toBe(200); + } finally { + closeActiveServer(); + } + }); +}); + +describe("cli gating", () => { + // A refusal that has already done the work is not a refusal: a caller that + // trusts `ok: false` and retries would build or publish twice. + it("refuses a missing --json before the command runs", async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "chainplot-gate-")); + fs.cpSync(template, dir, { recursive: true }); + + const result = await runCli(["build"], { cwd: dir }); + expect(result.ok).toBe(false); + expect(result.error?.message).toContain("--json is required"); + expect(fs.existsSync(path.join(dir, "dist", "releases"))).toBe(false); + }); +}); diff --git a/tests/cli/testCmd.test.ts b/tests/cli/testCmd.test.ts new file mode 100644 index 0000000..e83b71f --- /dev/null +++ b/tests/cli/testCmd.test.ts @@ -0,0 +1,16 @@ +import { describe, expect, it } from "vitest"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { runCliJson } from "../helpers/run.js"; + +const template = path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + "../../templates/fixture-transfers", +); + +describe("test", () => { + it("passes fixture assertions", async () => { + const result = await runCliJson(["test", "--json"], template); + expect(result.ok).toBe(true); + }); +}); diff --git a/tests/cli/validate.test.ts b/tests/cli/validate.test.ts new file mode 100644 index 0000000..785f383 --- /dev/null +++ b/tests/cli/validate.test.ts @@ -0,0 +1,58 @@ +import { describe, expect, it } from "vitest"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { runCliJson } from "../helpers/run.js"; + +const fixtures = path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + "../fixtures/projects", +); +const template = path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + "../../templates/fixture-transfers", +); + +describe("validate", () => { + it("accepts a dataset-only project", async () => { + const result = await runCliJson(["validate", "--json"], template); + expect(result.ok).toBe(true); + expect(result.command).toBe("validate"); + }); + + it("rejects malformed YAML with a validation envelope", async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "chainplot-bad-yaml-")); + fs.writeFileSync(path.join(dir, "chainplot.yaml"), "id: [unterminated\n"); + const result = await runCliJson(["validate", "--json"], dir); + expect(result.ok).toBe(false); + expect(result.schema_version).toBe(1); + expect(result.error?.code).toBe("validation"); + }); + + it("rejects unknown fields", async () => { + const result = await runCliJson(["validate", "--json"], path.join(fixtures, "unknown-field")); + expect(result.ok).toBe(false); + expect(result.error?.code).toBe("validation"); + }); + + it("rejects follow_finalized plus confirmation_depth", async () => { + const result = await runCliJson(["validate", "--json"], path.join(fixtures, "follow-plus-depth")); + expect(result.ok).toBe(false); + expect(result.error?.code).toBe("unsupported_capability"); + }); + + it("rejects missing model file", async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "chainplot-missing-model-")); + fs.cpSync(template, dir, { recursive: true }); + const yaml = path.join(dir, "chainplot.yaml"); + fs.appendFileSync( + yaml, + "\nmodels:\n - id: non_existent\n file: models/non_existent.sql\n depends_on: []\n", + ); + const result = await runCliJson(["validate", "--json"], dir); + expect(result.ok).toBe(false); + expect(result.error?.code).toBe("validation"); + expect(result.error?.message).toContain("missing model file"); + }); +}); diff --git a/tests/fixtures/projects/follow-plus-depth/chainplot.yaml b/tests/fixtures/projects/follow-plus-depth/chainplot.yaml new file mode 100644 index 0000000..272f09b --- /dev/null +++ b/tests/fixtures/projects/follow-plus-depth/chainplot.yaml @@ -0,0 +1,33 @@ +format_version: 1 +id: fixture-follow-depth +datasets: + - id: amounts + snapshot: snapshots/amounts.parquet +queries: + - id: raw_amounts + file: queries/raw_amounts.sql + dataset: amounts +dashboards: + - id: overview + title: Amounts + panels: + - query: raw_amounts + chart: table +chain_sources: + - id: eth + chain_id: 1 + rpc_secret: ETH_RPC + finality: + policy: confirmation_depth + depth: 12 +event_sources: + - id: transfers + chain: eth + addresses: + - "0x0000000000000000000000000000000000000001" + abi: abis/erc20.json + events: + - Transfer + start_block: 0 + end: + mode: follow_finalized diff --git a/tests/fixtures/projects/follow-plus-depth/queries/raw_amounts.sql b/tests/fixtures/projects/follow-plus-depth/queries/raw_amounts.sql new file mode 100644 index 0000000..7965465 --- /dev/null +++ b/tests/fixtures/projects/follow-plus-depth/queries/raw_amounts.sql @@ -0,0 +1 @@ +SELECT amount FROM amounts diff --git a/tests/fixtures/projects/unknown-field/chainplot.yaml b/tests/fixtures/projects/unknown-field/chainplot.yaml new file mode 100644 index 0000000..be543ba --- /dev/null +++ b/tests/fixtures/projects/unknown-field/chainplot.yaml @@ -0,0 +1,17 @@ +format_version: 1 +id: fixture-transfers +datasets: + - id: amounts + snapshot: snapshots/amounts.parquet +queries: + - id: raw_amounts + file: queries/raw_amounts.sql + dataset: amounts + raw_amount_columns: [amount] +dashboards: + - id: overview + title: Amounts + panels: + - query: raw_amounts + chart: table +extra: true diff --git a/tests/fixtures/projects/valid-dataset-only/chainplot.yaml b/tests/fixtures/projects/valid-dataset-only/chainplot.yaml new file mode 100644 index 0000000..f34154f --- /dev/null +++ b/tests/fixtures/projects/valid-dataset-only/chainplot.yaml @@ -0,0 +1,16 @@ +format_version: 1 +id: fixture-transfers +datasets: + - id: amounts + snapshot: snapshots/amounts.parquet +queries: + - id: raw_amounts + file: queries/raw_amounts.sql + dataset: amounts + raw_amount_columns: [amount] +dashboards: + - id: overview + title: Amounts + panels: + - query: raw_amounts + chart: table diff --git a/tests/fixtures/projects/valid-dataset-only/queries/raw_amounts.sql b/tests/fixtures/projects/valid-dataset-only/queries/raw_amounts.sql new file mode 100644 index 0000000..7965465 --- /dev/null +++ b/tests/fixtures/projects/valid-dataset-only/queries/raw_amounts.sql @@ -0,0 +1 @@ +SELECT amount FROM amounts diff --git a/tests/fork/fetchGuard.test.ts b/tests/fork/fetchGuard.test.ts new file mode 100644 index 0000000..02e12c0 --- /dev/null +++ b/tests/fork/fetchGuard.test.ts @@ -0,0 +1,88 @@ +import { describe, expect, it } from "vitest"; +import { + assertAllowedUrl, + isBlockedAddress, + normalizeIpLiteral, +} from "../../src/fork/fetchGuard.js"; + +describe("normalizeIpLiteral", () => { + it("decimal integer → dotted quad", () => { + expect(normalizeIpLiteral("2130706433")).toBe("127.0.0.1"); + }); + it("hex → dotted quad", () => { + expect(normalizeIpLiteral("0x7f000001")).toBe("127.0.0.1"); + }); + it("octal components → dotted quad", () => { + expect(normalizeIpLiteral("0177.0.0.1")).toBe("127.0.0.1"); + }); + it("dotted quad unchanged", () => { + expect(normalizeIpLiteral("192.168.1.1")).toBe("192.168.1.1"); + }); + it("hostname → null", () => { + expect(normalizeIpLiteral("example.com")).toBeNull(); + }); +}); + +describe("isBlockedAddress", () => { + it("blocks loopback, RFC1918, CGNAT, link-local, ULA, unspecified", () => { + for (const ip of [ + "127.0.0.1", + "10.0.0.5", + "172.16.0.1", + "192.168.1.1", + "169.254.1.1", + "100.64.0.1", + "0.0.0.0", + "::1", + "::", + "fe80::1", + "fc00::1", + "fd00::1", + "::ffff:127.0.0.1", + ]) { + expect(isBlockedAddress(ip), ip).toBe(true); + } + }); + + it("allows public addresses", () => { + for (const ip of ["8.8.8.8", "1.1.1.1", "2606:4700::1111"]) { + expect(isBlockedAddress(ip), ip).toBe(false); + } + }); +}); + +describe("assertAllowedUrl", () => { + it("refuses http", async () => { + await expect( + assertAllowedUrl("http://example.com/release.json", {}), + ).rejects.toMatchObject({ code: "validation" }); + }); + + it("refuses loopback literal and decimal form", async () => { + await expect( + assertAllowedUrl("https://127.0.0.1/release.json", {}), + ).rejects.toMatchObject({ code: "policy_refused" }); + await expect( + assertAllowedUrl("https://2130706433/release.json", {}), + ).rejects.toMatchObject({ code: "policy_refused" }); + }); + + it("refuses private DNS resolution", async () => { + await expect( + assertAllowedUrl("https://localhost/release.json", {}), + ).rejects.toMatchObject({ code: "policy_refused" }); + }); + + it("escape hatch allows private networks", async () => { + const result = await assertAllowedUrl("https://127.0.0.1/x", { + allowPrivateNetworks: true, + }); + expect(result.pinnedAddress).toBe("127.0.0.1"); + }); + + it("public URL resolves and pins an address", async () => { + const result = await assertAllowedUrl("https://example.com/release.json", {}); + expect(result.pinnedAddress).toBeTruthy(); + expect(isBlockedAddress(result.pinnedAddress!)).toBe(false); + }); +}); diff --git a/tests/fork/hostileRelease.test.ts b/tests/fork/hostileRelease.test.ts new file mode 100644 index 0000000..1cf5001 --- /dev/null +++ b/tests/fork/hostileRelease.test.ts @@ -0,0 +1,167 @@ +import { describe, expect, it } from "vitest"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { runCliJson } from "../helpers/run.js"; + +// A12, the honest version: a fork imports a stranger's recipe and the next +// `build` executes it. These publish a release whose SQL attacks the host, +// fork it, and build — the whole path an attacker actually has. + +const template = path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + "../../templates/fixture-transfers", +); + +interface Hostile { + published: string; + canary: string; +} + +/** Publish a release whose recipe carries `modelSql` as a model. */ +async function publishHostileRelease(modelSql: string): Promise { + const parent = fs.mkdtempSync(path.join(os.tmpdir(), "chainplot-hostile-")); + const project = path.join(parent, "proj"); + fs.cpSync(template, project, { recursive: true }); + + const canary = path.join(parent, "victim-secret.txt"); + fs.writeFileSync(canary, "AWS_SECRET_ACCESS_KEY=canary"); + + fs.mkdirSync(path.join(project, "models"), { recursive: true }); + fs.writeFileSync( + path.join(project, "models", "exfil.sql"), + modelSql.replace("__CANARY__", canary), + ); + fs.writeFileSync(path.join(project, "queries", "exfil.sql"), "SELECT * FROM exfil"); + fs.writeFileSync( + path.join(project, "chainplot.yaml"), + `format_version: 1 +id: fixture-transfers +datasets: + - id: amounts + snapshot: snapshots/amounts.parquet +models: + - id: exfil + file: models/exfil.sql + depends_on: [] +queries: + - id: exfil + file: queries/exfil.sql + dataset: amounts +dashboards: + - id: overview + title: Amounts + panels: + - query: exfil + chart: table +publish_targets: + - id: local-dir + type: directory + path: ./published + dataset_license: CC-BY-4.0 +`, + ); + + // The hostile project must not be buildable either — the publisher is + // running the same engine. Build it with the model neutralised so there is + // a release to fork, then swap the hostile model into the published bundle. + const benign = path.join(project, "models", "exfil.sql"); + const hostileSql = fs.readFileSync(benign, "utf8"); + fs.writeFileSync(benign, "SELECT amount AS leaked FROM amounts"); + expect( + (await runCliJson(["build", "--mode", "dataset_included", "--json"], project)).ok, + ).toBe(true); + expect((await runCliJson(["publish", "--json"], project)).ok).toBe(true); + + // Rewrite the published recipe and its checksum, exactly as whoever + // controls the bucket could. + const published = path.join(project, "published"); + const pointer = JSON.parse( + fs.readFileSync(path.join(published, "latest.json"), "utf8"), + ) as { release_prefix: string }; + const releaseDir = path.join(published, pointer.release_prefix); + const modelPath = path.join(releaseDir, "source", "models", "exfil.sql"); + fs.writeFileSync(modelPath, hostileSql); + + const { createHash } = await import("node:crypto"); + const releaseJsonPath = path.join(releaseDir, "release.json"); + const release = JSON.parse(fs.readFileSync(releaseJsonPath, "utf8")) as { + files: { path: string; checksum: string }[]; + }; + for (const file of release.files) { + if (file.path === "source/models/exfil.sql") { + file.checksum = createHash("sha256").update(hostileSql).digest("hex"); + } + } + fs.writeFileSync(releaseJsonPath, `${JSON.stringify(release, null, 2)}\n`); + const body = fs.readFileSync(releaseJsonPath, "utf8"); + fs.writeFileSync( + path.join(published, "latest.json"), + `${JSON.stringify( + { + schema_version: 1, + release_prefix: pointer.release_prefix, + release_json_checksum: createHash("sha256").update(body).digest("hex"), + }, + null, + 2, + )}\n`, + ); + + return { published, canary }; +} + +async function forkAndBuild(published: string): Promise<{ + ok: boolean; + code: string | null; + out: string; +}> { + const parent = fs.mkdtempSync(path.join(os.tmpdir(), "chainplot-victim-")); + const out = path.join(parent, "forked"); + const fork = await runCliJson( + ["fork", "--from", published, "--output", out, "--json"], + parent, + ); + expect(fork.ok).toBe(true); + const build = await runCliJson(["build", "--json"], out); + return { ok: build.ok, code: build.error?.code ?? null, out }; +} + +describe("forking a hostile release (A12)", () => { + it("a model cannot read the victim's filesystem", async () => { + const { published, canary } = await publishHostileRelease( + "SELECT content AS leaked FROM read_text('__CANARY__')", + ); + const { ok, out } = await forkAndBuild(published); + + expect(ok).toBe(false); + const leaked = path.join(out, "dist", "releases", "local", "results", "exfil.json"); + expect(fs.existsSync(leaked)).toBe(false); + // Belt and braces: nothing anywhere in the forked build echoes the secret. + expect(fs.readFileSync(canary, "utf8")).toContain("canary"); + const dist = path.join(out, "dist"); + const found = fs.existsSync(dist) + ? fs + .readdirSync(dist, { recursive: true, encoding: "utf8" }) + .some((entry) => entry.includes("exfil.json")) + : false; + expect(found).toBe(false); + }); + + // Belt and braces: httpfs cannot autoload either, so this is refused twice + // over. The filesystem case above is the one that pins the access flag. + it("a model cannot reach the network", async () => { + const { published } = await publishHostileRelease( + "SELECT * FROM read_csv('https://example.invalid/steal.csv')", + ); + expect((await forkAndBuild(published)).ok).toBe(false); + }); + + it("a non-SELECT model is refused as policy", async () => { + const { published } = await publishHostileRelease( + "CREATE TABLE pwned AS SELECT 1", + ); + expect((await forkAndBuild(published)).code).toBe("policy_refused"); + }); +}); diff --git a/tests/fork/importRelease.test.ts b/tests/fork/importRelease.test.ts new file mode 100644 index 0000000..c7019b3 --- /dev/null +++ b/tests/fork/importRelease.test.ts @@ -0,0 +1,225 @@ +import { describe, expect, it } from "vitest"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { runCliJson } from "../helpers/run.js"; + +const template = path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + "../../templates/fixture-transfers", +); + +async function buildPublishedRelease( + mode?: "results_only" | "dataset_referenced", +): Promise<{ project: string; published: string }> { + const parent = fs.mkdtempSync(path.join(os.tmpdir(), "chainplot-fork-src-")); + const project = path.join(parent, "proj"); + fs.cpSync(template, project, { recursive: true }); + fs.writeFileSync( + path.join(project, "chainplot.yaml"), + `${fs.readFileSync(path.join(project, "chainplot.yaml"), "utf8")} +publish_targets: + - id: local-dir + type: directory + path: ./published + dataset_license: CC-BY-4.0 +`, + ); + // Forking to rebuild needs the dataset, which is now opt-in. + const buildArgs = ["build", "--mode", mode ?? "dataset_included", "--json"]; + expect((await runCliJson(buildArgs, project)).ok).toBe(true); + expect((await runCliJson(["publish", "--json"], project)).ok).toBe(true); + return { project, published: path.join(project, "published") }; +} + +describe("fork", () => { + it("local fork from publish root → new project validates + builds offline (A10)", async () => { + const { published } = await buildPublishedRelease(); + const parent = fs.mkdtempSync(path.join(os.tmpdir(), "chainplot-fork-out-")); + const out = path.join(parent, "forked"); + + const result = await runCliJson( + ["fork", "--from", published, "--output", out, "--json"], + parent, + ); + expect(result.ok).toBe(true); + + // No secrets, no run state copied. + expect(fs.existsSync(path.join(out, ".env"))).toBe(false); + expect(fs.existsSync(path.join(out, ".chainplot"))).toBe(false); + + // Forked project validates and builds offline over the pinned snapshot. + const validated = await runCliJson(["validate", "--json"], out); + expect(validated.ok).toBe(true); + const built = await runCliJson(["build", "--json"], out); + expect(built.ok).toBe(true); + const release = JSON.parse( + fs.readFileSync(path.join(out, "dist/releases/local/release.json"), "utf8"), + ) as { mode: string; queries: string[] }; + // What A10 asserts is that the fork recomputes from the imported snapshot + // with no RPC, credentials or reindexing. Whether it then republishes that + // snapshot is the forker's own decision, so its release takes the default + // rather than inheriting the producer's. + expect(release.mode).toBe("results_only"); + expect(release.queries).toContain("raw_amounts"); + const results = JSON.parse( + fs.readFileSync( + path.join(out, "dist/releases/local/results/raw_amounts.json"), + "utf8", + ), + ) as { rows: unknown[][] }; + expect(results.rows.length).toBe(8); + }, 30_000); + + it("checksum mismatch → policy_refused", async () => { + const { published } = await buildPublishedRelease(); + // Tamper with a published file. + const pointer = JSON.parse( + fs.readFileSync(path.join(published, "latest.json"), "utf8"), + ); + const tamperPath = path.join(published, pointer.release_prefix, "release.json"); + const body = JSON.parse(fs.readFileSync(tamperPath, "utf8")); + body.generated_at = "tampered"; + fs.writeFileSync(tamperPath, JSON.stringify(body, null, 2)); + const parent = fs.mkdtempSync(path.join(os.tmpdir(), "chainplot-fork-out-")); + const result = await runCliJson( + ["fork", "--from", published, "--output", path.join(parent, "f"), "--json"], + parent, + ); + expect(result.ok).toBe(false); + expect(result.error?.code).toBe("policy_refused"); + }); + + it("http source refused", async () => { + const parent = fs.mkdtempSync(path.join(os.tmpdir(), "chainplot-fork-out-")); + const result = await runCliJson( + ["fork", "--from", "http://example.com/release.json", "--output", path.join(parent, "f"), "--json"], + parent, + ); + expect(result.ok).toBe(false); + expect(result.error?.code).toBe("validation"); + }); + + it("missing source → validation", async () => { + const parent = fs.mkdtempSync(path.join(os.tmpdir(), "chainplot-fork-out-")); + const result = await runCliJson( + ["fork", "--from", "/nonexistent-xyz", "--output", path.join(parent, "f"), "--json"], + parent, + ); + expect(result.ok).toBe(false); + expect(result.error?.code).toBe("validation"); + }); +}); + +// A results-only release carries the recipe and the rendered answers but no +// dataset. Forking one succeeded silently and the next `build` then failed +// with a bare "missing snapshot file" — a broken project and no explanation. +describe("forking a release without its dataset", () => { + it("warns that there is no snapshot to build from", async () => { + const { published } = await buildPublishedRelease("results_only"); + const parent = fs.mkdtempSync(path.join(os.tmpdir(), "chainplot-fork-ro-")); + const out = path.join(parent, "forked"); + + const result = await runCliJson( + ["fork", "--from", published, "--output", out, "--json"], + parent, + ); + expect(result.ok).toBe(true); + expect((result.data as { mode: string }).mode).toBe("results_only"); + expect(result.warnings.join(" ")).toMatch(/no snapshot/i); + // The recipe and the published answers do come across; only the dataset + // is absent, which is exactly what the warning has to convey. + expect(fs.existsSync(path.join(out, "chainplot.yaml"))).toBe(true); + expect(fs.existsSync(path.join(out, "results"))).toBe(true); + expect(fs.existsSync(path.join(out, "datasets/amounts/tables"))).toBe(false); + }); + + it("stays silent when the dataset is included", async () => { + const { published } = await buildPublishedRelease(); + const parent = fs.mkdtempSync(path.join(os.tmpdir(), "chainplot-fork-di-")); + const out = path.join(parent, "forked"); + const result = await runCliJson( + ["fork", "--from", published, "--output", out, "--json"], + parent, + ); + expect(result.ok).toBe(true); + expect(result.warnings).toEqual([]); + }); +}); + +// dataset_referenced used to record the producer's own filesystem path, which +// meant nothing to anyone else, and `fork` ignored the field entirely — so the +// mode was decorative. The reference is now release-relative: `publish` puts +// the parquet beside the release and `fork` fetches it from the same base. +describe("a referenced dataset round-trips", () => { + it("publishes beside the release, and a fork pulls it in and rebuilds", async () => { + const { project, published } = await buildPublishedRelease("dataset_referenced"); + + // The release itself does not carry the parquet... + const pointer = JSON.parse( + fs.readFileSync(path.join(published, "latest.json"), "utf8"), + ) as { release_prefix: string }; + const releaseDir = path.join(published, pointer.release_prefix); + const release = JSON.parse( + fs.readFileSync(path.join(releaseDir, "release.json"), "utf8"), + ) as { mode: string; files: { path: string }[] }; + expect(release.mode).toBe("dataset_referenced"); + expect(release.files.some((f) => f.path.endsWith(".parquet"))).toBe(false); + + // ...but the manifest names it, release-relative, and publish put it there. + const manifest = JSON.parse( + fs.readFileSync(path.join(releaseDir, "datasets/amounts/manifest.json"), "utf8"), + ) as { external: { path: string; checksum: string; bytes: number } }; + expect(manifest.external.path).toBe("datasets/amounts/tables/amounts.parquet"); + expect(manifest.external.bytes).toBeGreaterThan(0); + expect(fs.existsSync(path.join(releaseDir, manifest.external.path))).toBe(true); + + // A fork fetches it, verifies it, and can recompute offline. + const parent = fs.mkdtempSync(path.join(os.tmpdir(), "chainplot-fork-ref-")); + const out = path.join(parent, "forked"); + const forked = await runCliJson( + ["fork", "--from", published, "--output", out, "--json"], + parent, + ); + expect(forked.ok).toBe(true); + expect((forked.data as { datasets_referenced: string[] }).datasets_referenced).toEqual([ + "datasets/amounts/tables/amounts.parquet", + ]); + // Nothing to warn about: the data did come across. + expect(forked.warnings).toEqual([]); + + const built = await runCliJson(["build", "--json"], out); + expect(built.ok).toBe(true); + const results = JSON.parse( + fs.readFileSync( + path.join(out, "dist/releases/local/results/raw_amounts.json"), + "utf8", + ), + ) as { rows: unknown[][] }; + expect(results.rows).toHaveLength(8); + void project; + }, 60_000); + + it("refuses a referenced dataset whose bytes do not match the manifest", async () => { + const { published } = await buildPublishedRelease("dataset_referenced"); + const pointer = JSON.parse( + fs.readFileSync(path.join(published, "latest.json"), "utf8"), + ) as { release_prefix: string }; + const planted = path.join( + published, + pointer.release_prefix, + "datasets/amounts/tables/amounts.parquet", + ); + fs.writeFileSync(planted, "not the dataset you published"); + + const parent = fs.mkdtempSync(path.join(os.tmpdir(), "chainplot-fork-bad-")); + const result = await runCliJson( + ["fork", "--from", published, "--output", path.join(parent, "forked"), "--json"], + parent, + ); + expect(result.ok).toBe(false); + expect(result.error?.code).toBe("policy_refused"); + expect(result.error?.message).toMatch(/checksum mismatch for referenced dataset/); + }, 60_000); +}); diff --git a/tests/helpers/fakeRindexer.mjs b/tests/helpers/fakeRindexer.mjs new file mode 100644 index 0000000..a35525d --- /dev/null +++ b/tests/helpers/fakeRindexer.mjs @@ -0,0 +1,16 @@ +#!/usr/bin/env node +// Fake rindexer for tests. Modes via FAKE_RINDEXER_ENV (JSON in argv[2]). +// Usage: node fakeRindexer.mjs start -p indexer +const mode = process.env.FAKE_RINDEXER_MODE ?? "complete"; + +if (mode === "complete") { + process.stdout.write("Historical indexing completed\n"); + // health-server behavior: never exits on its own + setInterval(() => {}, 1000); +} else if (mode === "exit") { + process.stdout.write("starting up\n"); + process.exit(3); +} else { + // hang: never prints the completed line + setInterval(() => {}, 1000); +} diff --git a/tests/helpers/run.ts b/tests/helpers/run.ts new file mode 100644 index 0000000..3331ba6 --- /dev/null +++ b/tests/helpers/run.ts @@ -0,0 +1,14 @@ +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { loadDotEnv } from "../../src/config/env.js"; +import { runCli, type CommandResult } from "../../src/cli/run.js"; + +// Load the repo-root .env once so env-gated tests see real values. +loadDotEnv(path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..")); + +export async function runCliJson( + argv: string[], + cwd: string, +): Promise { + return runCli(argv, { cwd }); +} diff --git a/tests/ingest/coverage.test.ts b/tests/ingest/coverage.test.ts new file mode 100644 index 0000000..4a9ebd2 --- /dev/null +++ b/tests/ingest/coverage.test.ts @@ -0,0 +1,141 @@ +import { describe, expect, it } from "vitest"; +import { + hashJoinOk, + isComplete, + lastProvenCompleteBlock, + requiredEnd, + type CoverageSegment, +} from "../../src/ingest/coverage.js"; + +const H = (n: number) => "0x" + n.toString(16).padStart(64, "0"); + +function seg( + start: number, + end: number, + opts: { + parent?: string; + startHash?: string; + endHash?: string; + status?: "complete_empty" | "complete_with_rows"; + } = {}, +): CoverageSegment { + return { + start_block: start, + end_block: end, + start_block_hash: opts.startHash ?? H(start), + end_block_hash: opts.endHash ?? H(end), + start_block_parent_hash: opts.parent ?? H(start - 1), + status: opts.status ?? "complete_with_rows", + }; +} + +describe("lastProvenCompleteBlock", () => { + it("no segments → start_block - 1", () => { + expect(lastProvenCompleteBlock([], 100)).toBe(99); + }); + + it("single segment starting at project start", () => { + expect(lastProvenCompleteBlock([seg(100, 110)], 100)).toBe(110); + }); + + it("segment starting after project start (front gap) → start - 1", () => { + expect(lastProvenCompleteBlock([seg(105, 110)], 100)).toBe(99); + }); + + it("two hash-joined segments → second end", () => { + const a = seg(100, 110); + const b = seg(111, 120, { parent: a.end_block_hash }); + expect(lastProvenCompleteBlock([a, b], 100)).toBe(120); + }); + + it("number-adjacent but parent mismatch → first end only", () => { + const a = seg(100, 110); + const b = seg(111, 120, { parent: H(999999) }); + expect(lastProvenCompleteBlock([a, b], 100)).toBe(110); + }); + + it("gap between segments → first end", () => { + const a = seg(100, 110); + const b = seg(115, 120, { parent: H(114) }); + expect(lastProvenCompleteBlock([a, b], 100)).toBe(110); + }); +}); + +describe("hashJoinOk", () => { + it("requires number adjacency and parent link", () => { + const a = seg(100, 110); + expect(hashJoinOk(a, seg(111, 120, { parent: a.end_block_hash }))).toBe( + true, + ); + expect(hashJoinOk(a, seg(112, 120, { parent: a.end_block_hash }))).toBe( + false, + ); + expect(hashJoinOk(a, seg(111, 120, { parent: H(1) }))).toBe(false); + }); +}); + +describe("requiredEnd", () => { + it("pinned → declared block", () => { + expect(requiredEnd({ mode: "pinned", block: 200 }, [seg(100, 110)])).toBe( + 200, + ); + }); + + it("follow_finalized with no segments → null", () => { + expect(requiredEnd({ mode: "follow_finalized" }, [])).toBeNull(); + }); + + it("follow_finalized → max segment end", () => { + const a = seg(100, 110); + const b = seg(111, 150, { parent: a.end_block_hash }); + expect(requiredEnd({ mode: "follow_finalized" }, [a, b])).toBe(150); + }); +}); + +describe("isComplete", () => { + it("pinned truncated → incomplete", () => { + const result = isComplete([seg(100, 150)], 100, { + mode: "pinned", + block: 200, + }); + expect(result.complete).toBe(false); + expect(result.reason).toBe("truncated"); + }); + + it("pinned covered → complete", () => { + expect( + isComplete([seg(100, 200)], 100, { mode: "pinned", block: 200 }).complete, + ).toBe(true); + }); + + it("follow_finalized with no segments → not_indexed", () => { + const result = isComplete([], 100, { mode: "follow_finalized" }); + expect(result.complete).toBe(false); + expect(result.reason).toBe("not_indexed"); + }); + + it("follow_finalized contiguous → complete", () => { + const a = seg(100, 110); + const b = seg(111, 150, { parent: a.end_block_hash }); + expect( + isComplete([a, b], 100, { mode: "follow_finalized" }).complete, + ).toBe(true); + }); + + it("front gap → incomplete", () => { + const result = isComplete([seg(105, 200)], 100, { + mode: "pinned", + block: 200, + }); + expect(result.complete).toBe(false); + expect(result.reason).toBe("truncated"); + }); + + it("hash-join break → incomplete with reason", () => { + const a = seg(100, 110); + const b = seg(111, 200, { parent: H(999999) }); + const result = isComplete([a, b], 100, { mode: "pinned", block: 200 }); + expect(result.complete).toBe(false); + expect(result.reason).toBe("hash_join_broken"); + }); +}); diff --git a/tests/ingest/exporter.test.ts b/tests/ingest/exporter.test.ts new file mode 100644 index 0000000..acc14e9 --- /dev/null +++ b/tests/ingest/exporter.test.ts @@ -0,0 +1,55 @@ +import { describe, expect, it } from "vitest"; +import { + buildExportSql, + buildUniquenessSql, +} from "../../src/ingest/exporter.js"; + +const req = { + databaseUrl: "postgres://u:p@localhost:5432/chainplot", + networkName: "chainplot_1", + contractName: "usdc", + event: "Transfer", + chainId: 1, + outPath: "/out/transfer.parquet", +}; + +describe("exporter SQL", () => { + const sql = buildExportSql(req); + + it("attaches postgres read-only", () => { + expect(sql).toContain("ATTACH 'postgres://u:p@localhost:5432/chainplot'"); + expect(sql).toContain("TYPE POSTGRES"); + expect(sql).toContain("READ_ONLY"); + }); + + it("casts numeric columns to BIGINT (scanner maps numeric to DOUBLE)", () => { + expect(sql).toContain("CAST(block_number AS BIGINT)"); + expect(sql).toContain("CAST(tx_index AS BIGINT)"); + }); + + it("adds chain_id as a literal column", () => { + expect(sql).toContain("1 AS chain_id"); + }); + + it("copies to parquet at the requested path", () => { + expect(sql).toContain("TO '/out/transfer.parquet'"); + expect(sql).toContain("FORMAT PARQUET"); + }); + + it("uniqueness gate uses the physical unique key", () => { + const u = buildUniquenessSql("chainplot_1", "usdc", "Transfer"); + expect(u).toContain("count(*)::bigint AS total"); + expect(u).toContain( + "count(DISTINCT (contract_address, block_number, tx_hash, log_index))", + ); + }); + + it("escapes single quotes in the connection string", () => { + const escaped = buildExportSql({ ...req, outPath: "/o'brien.parquet" }); + expect(escaped).toContain("'/o''brien.parquet'"); + }); + + it("is deterministic", () => { + expect(buildExportSql(req)).toBe(sql); + }); +}); diff --git a/tests/ingest/inspectCoverage.test.ts b/tests/ingest/inspectCoverage.test.ts new file mode 100644 index 0000000..98f7419 --- /dev/null +++ b/tests/ingest/inspectCoverage.test.ts @@ -0,0 +1,116 @@ +import { describe, expect, it } from "vitest"; +import { + assertValidJobIdentifiers, + type BoundedJob, +} from "../../src/ingest/adapter.js"; +import { + cursorTableName, + eventTableName, + classifyCoverage, + buildCursorQuery, + buildRowCountQuery, +} from "../../src/ingest/rindexer/inspectCoverage.js"; + +function job(overrides: Partial = {}): BoundedJob { + return { + sourceId: "src", + contractName: "usdc", + networkName: "chainplot_1", + chainId: 1, + addresses: ["0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48"], + abiPath: "/p/abis/x.json", + events: ["Transfer"], + jobStart: 100, + jobEnd: 110, + rpcUrl: "http://r", + databaseUrl: "postgres://u:p@h/db", + workDir: "/w", + ...overrides, + }; +} + +describe("coverage evidence naming", () => { + it("cursor table follows rindexer_internal.{manifest_name}_{contract}_{event}", () => { + expect(cursorTableName("chainplot_1", "usdc", "Transfer")).toBe( + "rindexer_internal.chainplot_chainplot_1_usdc_transfer", + ); + expect(eventTableName("chainplot_1", "usdc", "Transfer")).toBe( + "chainplot_chainplot_1_usdc.transfer", + ); + }); + + it("rejects identifiers that are not [a-z0-9_]", () => { + expect(() => + assertValidJobIdentifiers(job({ contractName: "Usdc; DROP" })), + ).toThrow(); + expect(() => + assertValidJobIdentifiers(job({ networkName: "chainplot 1" })), + ).toThrow(); + expect(() => assertValidJobIdentifiers(job())).not.toThrow(); + }); + + it("cursor table follows rindexer manifest-name convention", () => { + expect(cursorTableName("chainplot_1", "usdc", "Transfer")).toBe( + "rindexer_internal.chainplot_chainplot_1_usdc_transfer", + ); + expect(eventTableName("chainplot_1", "usdc", "Transfer")).toBe( + "chainplot_chainplot_1_usdc.transfer", + ); + }); + + it("rejects identifiers that are not [a-z0-9_]", () => { + expect(() => + assertValidJobIdentifiers(job({ contractName: "Usdc; DROP" })), + ).toThrow(); + expect(() => assertValidJobIdentifiers(job())).not.toThrow(); + }); + + it("queries are parameterized or identifier-safe", () => { + const q = buildCursorQuery("chainplot_1", "usdc", "Transfer"); + expect(q.text).toBe( + "SELECT last_synced_block FROM rindexer_internal.chainplot_chainplot_1_usdc_transfer WHERE network = $1", + ); + expect(q.params).toEqual(["chainplot_1"]); + expect(buildRowCountQuery("chainplot_1", "usdc", "Transfer")).toBe( + "SELECT count(*)::bigint AS n FROM chainplot_chainplot_1_usdc.transfer", + ); + }); +}); + +describe("classifyCoverage", () => { + it("no cursor row → not_indexed", () => { + expect(classifyCoverage(null, 0, 110)).toEqual({ + status: "not_indexed", + lastSyncedBlock: null, + rowCount: 0, + }); + }); + + it("cursor below job end → incomplete", () => { + expect(classifyCoverage(109, 500, 110)).toEqual({ + status: "incomplete", + lastSyncedBlock: 109, + rowCount: 500, + }); + }); + + it("cursor at end with zero rows → complete_empty (A6)", () => { + expect(classifyCoverage(110, 0, 110)).toEqual({ + status: "complete_empty", + lastSyncedBlock: 110, + rowCount: 0, + }); + }); + + it("cursor at end with rows → complete_with_rows", () => { + expect(classifyCoverage(110, 92, 110)).toEqual({ + status: "complete_with_rows", + lastSyncedBlock: 110, + rowCount: 92, + }); + }); + + it("cursor beyond end with rows → complete_with_rows", () => { + expect(classifyCoverage(120, 7, 110).status).toBe("complete_with_rows"); + }); +}); diff --git a/tests/ingest/live/e2e.live.test.ts b/tests/ingest/live/e2e.live.test.ts new file mode 100644 index 0000000..061b29f --- /dev/null +++ b/tests/ingest/live/e2e.live.test.ts @@ -0,0 +1,214 @@ +import { describe, expect, it, afterEach } from "vitest"; +import { execFile } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { promisify } from "node:util"; +import { fileURLToPath } from "node:url"; +import { loadDotEnv } from "../../../src/config/env.js"; + +const exec = promisify(execFile); + +// Live-gated on the two things that genuinely cannot be provisioned here: an +// archive-capable RPC_URL, and Docker. Postgres and the pinned rindexer binary +// both come from the template's own compose.yaml, so there is nothing else to +// install or point an env var at. +// +// Everything runs inside the producer container, which is where rindexer lives +// (it is linux/amd64-only and never on the host PATH). That also makes this the +// same path a user follows from the template README. + +const repoRoot = path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + "../../..", +); + +// Load the repo-root .env explicitly rather than relying on another helper's +// import side effect. +loadDotEnv(path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../../..")); + +const rpcUrl = process.env.RPC_URL; +const IMAGE = process.env.CHAINPLOT_IMAGE ?? "chainplot:local"; + +async function dockerAvailable(): Promise { + try { + await exec("docker", ["info"], { timeout: 30_000 }); + return true; + } catch { + return false; + } +} + +const hasDocker = await dockerAvailable(); +const d = rpcUrl && hasDocker ? it : it.skip; + +async function compose(cwd: string, args: string[], timeout = 900_000) { + return exec("docker", ["compose", ...args], { + cwd, + timeout, + maxBuffer: 32 * 1024 * 1024, + env: { ...process.env, CHAINPLOT_IMAGE: IMAGE }, + }); +} + +/** Run the CLI inside the producer container and parse its envelope. */ +async function cli(cwd: string, args: string[]): Promise<{ + ok: boolean; + data: unknown; + error: { code: string; message: string } | null; +}> { + // The same invocation the template README documents. The CLI exits non-zero + // on a refusal, which is a result here rather than a failure, so the envelope + // is read either way. + let stdout: string; + try { + ({ stdout } = await compose(cwd, [ + "exec", + "-T", + "producer", + "chainplot", + ...args, + "--json", + ])); + } catch (err) { + stdout = (err as { stdout?: string }).stdout ?? ""; + if (!stdout.trim()) throw err; + } + const last = stdout.trim().split("\n").pop() ?? ""; + return JSON.parse(last) as { + ok: boolean; + data: unknown; + error: { code: string; message: string } | null; + }; +} + +describe("live ingest end-to-end (M0 replay through the product)", () => { + let cwd = ""; + + afterEach(async () => { + if (!cwd) return; + await compose(cwd, ["down", "-v"], 120_000).catch(() => undefined); + cwd = ""; + }); + + d( + "plan → apply → coverage → idempotent re-apply → no-op plan → gate", + async () => { + // The producer image is the CLI plus rindexer, built from this repo. + // rindexer ships linux/amd64 only, so the whole image must be that + // platform; an arm64 host builds it under emulation. + await exec( + "docker", + [ + "build", + "--platform", + "linux/amd64", + "-t", + IMAGE, + "-f", + path.join(repoRoot, "docker/producer.Dockerfile"), + repoRoot, + ], + { timeout: 900_000, maxBuffer: 32 * 1024 * 1024 }, + ); + + const parent = fs.mkdtempSync(path.join(os.tmpdir(), "chainplot-live-")); + cwd = path.join(parent, "proj"); + const init = await exec("node", [ + path.join(repoRoot, "dist/cli/main.js"), + "init", + "--template", + "ingest-transfers", + "--output", + cwd, + "--json", + ]); + expect(JSON.parse(init.stdout.trim()).ok).toBe(true); + + // The only secret the project needs; Postgres comes from compose. + fs.writeFileSync( + path.join(cwd, ".env"), + `RPC_URL=${rpcUrl}\nDATABASE_URL=postgresql://chainplot:chainplot@postgres:5432/chainplot\n`, + ); + await compose(cwd, ["up", "-d"]); + + const plan = await cli(cwd, ["plan", "--intent", "ingest"]); + expect(plan.ok).toBe(true); + const planData = plan.data as { + plan_path: string; + sources: { job_start: number; job_end: number }[]; + }; + expect(planData.sources[0]).toMatchObject({ + job_start: 18600000, + job_end: 18600010, + }); + + const apply = await cli(cwd, ["apply", "--plan", planData.plan_path]); + expect(apply.ok).toBe(true); + expect((apply.data as { reused: boolean }).reused).toBe(false); + + const coverage = JSON.parse( + fs.readFileSync(path.join(cwd, ".chainplot/coverage.json"), "utf8"), + ) as { + chain_id: number; + sources: { + segments: { + start_block: number; + end_block: number; + status: string; + row_count: number; + end_block_timestamp?: number; + indexed_at?: string; + }[]; + }[]; + }; + expect(coverage.chain_id).toBe(1); + expect(coverage.sources[0].segments).toHaveLength(1); + const segment = coverage.sources[0].segments[0]!; + expect(segment).toMatchObject({ + start_block: 18600000, + end_block: 18600010, + status: "complete_with_rows", + }); + expect(segment.row_count).toBeGreaterThan(0); + // Freshness has to come from the chain, not from a file's mtime. + expect(segment.end_block_timestamp).toBeGreaterThan(0); + expect(segment.indexed_at).toBeTruthy(); + + // Re-apply same plan + key → reused, no duplicate rows (A5). + const again = await cli(cwd, ["apply", "--plan", planData.plan_path]); + expect(again.ok).toBe(true); + expect((again.data as { reused: boolean }).reused).toBe(true); + + // Second plan → job_start > job_end → no-op, coverage unchanged. + const plan2 = await cli(cwd, ["plan", "--intent", "ingest"]); + expect(plan2.ok).toBe(true); + const plan2Data = plan2.data as { + sources: { job_start: number; job_end: number }[]; + actions: { type: string }[]; + }; + expect(plan2Data.sources[0].job_start).toBe(18600011); + expect(plan2Data.sources[0].job_end).toBe(18600010); + expect(plan2Data.actions.some((a) => a.type === "ingest")).toBe(false); + + // A build over proven coverage produces a release with chain freshness. + const built = await cli(cwd, ["build"]); + expect(built.ok).toBe(true); + const release = JSON.parse( + fs.readFileSync( + path.join(cwd, "dist/releases/local/release.json"), + "utf8", + ), + ) as { freshness: { kind: string; data_through: { block: number } } }; + expect(release.freshness.kind).toBe("chain"); + expect(release.freshness.data_through.block).toBe(18600010); + + // Truncation probe: drop coverage → build refuses (M2 gate). + fs.rmSync(path.join(cwd, ".chainplot/coverage.json")); + const refused = await cli(cwd, ["build"]); + expect(refused.ok).toBe(false); + expect(refused.error?.code).toBe("policy_refused"); + }, + 1_800_000, + ); +}); diff --git a/tests/ingest/locks.test.ts b/tests/ingest/locks.test.ts new file mode 100644 index 0000000..618f066 --- /dev/null +++ b/tests/ingest/locks.test.ts @@ -0,0 +1,123 @@ +import { describe, expect, it } from "vitest"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { acquireLocalLock } from "../../src/runtime/locks.js"; + +// An ingest is killed by design when it exceeds its wall clock, and a killed +// run cannot clean up after itself. Before staleness detection the first such +// kill locked the project permanently, with an error that did not even name +// the file to delete. + +function project(): string { + return fs.mkdtempSync(path.join(os.tmpdir(), "chainplot-lock-")); +} + +function lockPath(cwd: string, name = "ingest"): string { + return path.join(cwd, ".chainplot", "locks", `${name}.lock`); +} + +describe("local lock", () => { + it("excludes a concurrent holder", () => { + const cwd = project(); + const held = acquireLocalLock(cwd, "ingest"); + expect(() => acquireLocalLock(cwd, "ingest")).toThrow(/another run holds lock/); + held.release(); + expect(() => acquireLocalLock(cwd, "ingest").release()).not.toThrow(); + }); + + it("names the file to delete when it refuses", () => { + const cwd = project(); + acquireLocalLock(cwd, "ingest"); + try { + acquireLocalLock(cwd, "ingest"); + throw new Error("expected a refusal"); + } catch (err) { + expect((err as { message: string }).message).toContain(lockPath(cwd)); + } + }); + + it("takes over a lock whose owner is gone on this host", () => { + const cwd = project(); + acquireLocalLock(cwd, "ingest"); + // A pid that cannot be running; same host, so liveness is decisive. + fs.writeFileSync( + lockPath(cwd), + JSON.stringify({ pid: 2 ** 22, host: os.hostname(), acquired_at: new Date().toISOString() }), + ); + expect(() => acquireLocalLock(cwd, "ingest").release()).not.toThrow(); + }); + + // The CLI normally runs in a container against a bind-mounted project, and + // each recreated container has a different hostname — so pid liveness never + // applies and the heartbeat is the only usable signal. Judging by total age + // instead would have to assume how long a job might legitimately run. + it("takes over a lock whose heartbeat has stopped", () => { + const cwd = project(); + acquireLocalLock(cwd, "ingest"); + fs.writeFileSync( + lockPath(cwd), + JSON.stringify({ + token: "someone-else", + pid: 1, + host: "some-container", + heartbeat_at: new Date(Date.now() - 60_000).toISOString(), + }), + ); + expect(() => acquireLocalLock(cwd, "ingest").release()).not.toThrow(); + }); + + it("leaves a lock alone while its heartbeat is current", () => { + const cwd = project(); + acquireLocalLock(cwd, "ingest"); + fs.writeFileSync( + lockPath(cwd), + JSON.stringify({ + token: "someone-else", + pid: 1, + host: "some-container", + heartbeat_at: new Date().toISOString(), + }), + ); + expect(() => acquireLocalLock(cwd, "ingest")).toThrow(/another run holds lock/); + }); + + // A holder that has lost its lock to a takeover must not delete the new + // holder's file on the way out. + it("release does not remove a lock that now belongs to someone else", () => { + const cwd = project(); + const mine = acquireLocalLock(cwd, "ingest"); + const theirs = JSON.stringify({ + token: "someone-else", + pid: 1, + host: "some-container", + heartbeat_at: new Date().toISOString(), + }); + fs.writeFileSync(lockPath(cwd), theirs); + mine.release(); + expect(fs.existsSync(lockPath(cwd))).toBe(true); + expect(fs.readFileSync(lockPath(cwd), "utf8")).toBe(theirs); + }); + + it("a fresh holder writes a heartbeat and its own identity", () => { + const cwd = project(); + const held = acquireLocalLock(cwd, "ingest"); + const body = JSON.parse(fs.readFileSync(lockPath(cwd), "utf8")) as { + token: string; + pid: number; + host: string; + heartbeat_at: string; + }; + expect(body.token).toMatch(/^[0-9a-f-]{36}$/); + expect(body.pid).toBe(process.pid); + expect(Date.now() - Date.parse(body.heartbeat_at)).toBeLessThan(5_000); + held.release(); + }); + + it("takes over a truncated lock file", () => { + const cwd = project(); + acquireLocalLock(cwd, "ingest"); + fs.writeFileSync(lockPath(cwd), "{ not json"); + expect(() => acquireLocalLock(cwd, "ingest").release()).not.toThrow(); + }); +}); diff --git a/tests/ingest/planApply.test.ts b/tests/ingest/planApply.test.ts new file mode 100644 index 0000000..d2c7f9b --- /dev/null +++ b/tests/ingest/planApply.test.ts @@ -0,0 +1,476 @@ +import { describe, expect, it, beforeEach, afterEach } from "vitest"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { generatePlan } from "../../src/plan/generate.js"; +import { applyPlan } from "../../src/plan/apply.js"; +import type { + BoundedJob, + IngestAdapter, + CoverageReport, +} from "../../src/ingest/adapter.js"; +import type { RpcClient } from "../../src/rpc/client.js"; +import { RpcError } from "../../src/rpc/client.js"; +import type { ProjectDocument } from "../../src/project/types.js"; + +const H = (n: number) => "0x" + n.toString(16).padStart(64, "0"); + +function mockRpc(head: number, calls: string[]): RpcClient { + return { + async call(method: string, params: unknown[]): Promise { + calls.push(`${method}:${String(params[0])}`); + const tag = params[0] as string; + const n = tag === "finalized" ? head : Number(BigInt(tag)); + return { + number: "0x" + n.toString(16), + hash: H(n), + parentHash: H(n - 1), + } as T; + }, + }; +} + +interface FakeState { + runs: BoundedJob[]; + exports: string[]; + coverageReport: CoverageReport; +} + +function fakeAdapter(state: FakeState): IngestAdapter { + return { + renderConfig: () => "fake-config", + runBounded: async (job) => { + state.runs.push(job); + return { job, pid: -1, completedLogSeen: true }; + }, + stopAndQuiesce: async () => {}, + inspectCoverage: async (job) => ({ + ...state.coverageReport, + lastSyncedBlock: Math.max( + state.coverageReport.lastSyncedBlock ?? 0, + job.jobEnd, + ), + }), + }; +} + +const PROJECT_YAML = [ + "format_version: 1", + 'id: "ingest-test"', + "chain_sources:", + " - id: mainnet", + " chain_id: 1", + " rpc_secret: RPC_URL", + " finality:", + " policy: finalized", + "event_sources:", + " - id: usdc", + " chain: mainnet", + " addresses:", + ' - "0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48"', + " abi: abis/ERC20.json", + " events:", + " - Transfer", + " start_block: 100", + " end:", + " mode: pinned", + " block: 110", + "datasets:", + " - id: usdc", + " snapshot: .chainplot/snapshots/usdc/usdc_transfer.parquet", + "queries:", + " - id: count", + " file: queries/count.sql", + " dataset: usdc", +].join("\n"); + +function makeProject(cwd: string): ProjectDocument { + void cwd; + return { + format_version: 1, + id: "ingest-test", + chain_sources: [ + { + id: "mainnet", + chain_id: 1, + rpc_secret: "RPC_URL", + finality: { policy: "finalized" }, + }, + ], + event_sources: [ + { + id: "usdc", + chain: "mainnet", + addresses: ["0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48"], + abi: "abis/ERC20.json", + events: ["Transfer"], + start_block: 100, + end: { mode: "pinned", block: 110 }, + }, + ], + datasets: [ + { + id: "usdc", + snapshot: ".chainplot/snapshots/usdc/usdc_transfer.parquet", + }, + ], + queries: [{ id: "count", file: "queries/count.sql", dataset: "usdc" }], + }; +} + +function setupProject(): string { + const cwd = fs.mkdtempSync(path.join(os.tmpdir(), "chainplot-plan-")); + fs.mkdirSync(path.join(cwd, "abis"), { recursive: true }); + fs.mkdirSync(path.join(cwd, "queries"), { recursive: true }); + fs.writeFileSync(path.join(cwd, "chainplot.yaml"), PROJECT_YAML + "\n"); + fs.writeFileSync(path.join(cwd, "abis/ERC20.json"), "[]"); + fs.writeFileSync(path.join(cwd, "queries/count.sql"), "select count(*) as n from usdc"); + fs.mkdirSync(path.join(cwd, ".chainplot/snapshots/usdc"), { recursive: true }); + fs.writeFileSync( + path.join(cwd, ".chainplot/snapshots/usdc/usdc_transfer.parquet"), + "dummy", + ); + return cwd; +} + +function fakeExport(state: FakeState) { + return async (_job: BoundedJob, outDir: string) => { + state.exports.push(outDir); + return { + parquetPath: path.join(outDir, "usdc_transfer.parquet"), + rowCount: 92, + }; + }; +} + +describe("generatePlan", () => { + let savedEnv: Record; + + beforeEach(() => { + savedEnv = { RPC_URL: process.env.RPC_URL, DATABASE_URL: process.env.DATABASE_URL }; + process.env.RPC_URL = "http://rpc.test"; + process.env.DATABASE_URL = "postgres://test:test@localhost:5432/test"; + }); + + afterEach(() => { + for (const [k, v] of Object.entries(savedEnv)) { + if (v === undefined) delete process.env[k]; + else process.env[k] = v; + } + }); + + it("pinned happy path: bounds, actions, plan file", async () => { + const cwd = setupProject(); + const calls: string[] = []; + const { plan, planPath } = await generatePlan({ + intent: "ingest", + cwd, + project: makeProject(cwd), + rpcClient: mockRpc(200, calls), + }); + expect(plan.sources[0]).toMatchObject({ + source_id: "usdc", + job_start: 100, + job_end: 110, + job_target_end: 110, + blocks_remaining: 11, + }); + expect(plan.actions).toEqual([ + { type: "ingest", source_id: "usdc" }, + { type: "export", source_id: "usdc" }, + { type: "build_results" }, + ]); + expect(fs.existsSync(planPath)).toBe(true); + expect(plan.plan_id).toMatch(/^[0-9a-f]{64}$/); + }); + + // A backfill wider than the block budget is split across runs. While an + // intermediate plan still carried build_results, `apply` ingested + // successfully and then failed on the promotion gate, reporting the whole + // run as failed and making a long backfill look like it was going nowhere. + it("a plan that cannot close the range ingests only", async () => { + const cwd = setupProject(); + const project = makeProject(cwd); + project.policy = { block_budget: 3 }; + const { plan } = await generatePlan({ + intent: "ingest", + cwd, + project, + rpcClient: mockRpc(200, []), + }); + expect(plan.sources[0]!.job_end).toBe(102); + expect(plan.sources[0]!.job_target_end).toBe(110); + // Export and build both sit behind the promotion gate, so an intermediate + // run must not claim them: it would ingest correctly and then fail. + expect(plan.actions).toEqual([{ type: "ingest", source_id: "usdc" }]); + }); + + it("a build intent still builds even when coverage is short", async () => { + const cwd = setupProject(); + const { plan } = await generatePlan({ + intent: "build", + cwd, + project: makeProject(cwd), + rpcClient: mockRpc(200, []), + }); + // An explicit build deserves the promotion gate's own error, not a plan + // that quietly does nothing. + expect(plan.actions).toEqual([{ type: "build_results" }]); + }); + + it("pinned end above finalized head → policy_refused", async () => { + const cwd = setupProject(); + const project = makeProject(cwd); + project.event_sources![0].end = { mode: "pinned", block: 250 }; + await expect( + generatePlan({ + intent: "ingest", + cwd, + project, + rpcClient: mockRpc(200, []), + }), + ).rejects.toMatchObject({ code: "policy_refused" }); + }); + + it("block budget caps job_end", async () => { + const cwd = setupProject(); + const project = makeProject(cwd); + project.policy = { block_budget: 5 }; + const { plan } = await generatePlan({ + intent: "ingest", + cwd, + project, + rpcClient: mockRpc(200, []), + }); + expect(plan.sources[0].job_end).toBe(104); + expect(plan.sources[0].blocks_remaining).toBe(11); + }); + + it("follow_finalized resolves the finalized head", async () => { + const cwd = setupProject(); + const project = makeProject(cwd); + project.event_sources![0].end = { mode: "follow_finalized" }; + const { plan } = await generatePlan({ + intent: "ingest", + cwd, + project, + rpcClient: mockRpc(200, []), + }); + expect(plan.sources[0].job_target_end).toBe(200); + expect(plan.sources[0].job_end).toBe(200); + }); + + it("complete pinned source → no RPC, no ingest action", async () => { + const cwd = setupProject(); + const calls: string[] = []; + writeCoverage(cwd, [segment(100, 110)]); + const { plan } = await generatePlan({ + intent: "ingest", + cwd, + project: makeProject(cwd), + rpcClient: mockRpc(200, calls), + }); + expect(calls).toEqual([]); + expect(plan.actions).toEqual([{ type: "build_results" }]); + expect(plan.sources[0].job_start).toBe(111); + expect(plan.sources[0].job_end).toBe(110); + }); +}); + +describe("applyPlan", () => { + let savedEnv: Record; + + beforeEach(() => { + savedEnv = { RPC_URL: process.env.RPC_URL, DATABASE_URL: process.env.DATABASE_URL }; + process.env.RPC_URL = "http://rpc.test"; + process.env.DATABASE_URL = "postgres://test:test@localhost:5432/test"; + }); + + afterEach(() => { + for (const [k, v] of Object.entries(savedEnv)) { + if (v === undefined) delete process.env[k]; + else process.env[k] = v; + } + }); + + function completeReport(): CoverageReport { + return { status: "complete_with_rows", lastSyncedBlock: 110, rowCount: 92 }; + } + + it("happy path: ingest, export, coverage recorded", async () => { + const cwd = setupProject(); + const state: FakeState = { runs: [], exports: [], coverageReport: completeReport() }; + const { planPath } = await generatePlan({ + intent: "ingest", + cwd, + project: makeProject(cwd), + rpcClient: mockRpc(200, []), + }); + const outcome = await applyPlan({ + cwd, + planRef: planPath, + adapter: fakeAdapter(state), + exportFn: fakeExport(state), + buildFn: fakeBuild(), + rindexerBin: "rindexer", + rpcClient: mockRpc(200, []), + }); + expect(outcome.status).toBe("succeeded"); + expect(outcome.reused).toBe(false); + expect(state.runs).toHaveLength(1); + expect(state.runs[0]).toMatchObject({ jobStart: 100, jobEnd: 110 }); + expect(state.exports).toHaveLength(1); + const coverage = readCoverage(cwd); + expect(coverage.sources[0].segments).toEqual([ + expect.objectContaining({ + start_block: 100, + end_block: 110, + status: "complete_with_rows", + row_count: 92, + }), + ]); + }); + + it("same key re-apply → reused outcome, no second run", async () => { + const cwd = setupProject(); + const state: FakeState = { runs: [], exports: [], coverageReport: completeReport() }; + const { planPath } = await generatePlan({ + intent: "ingest", + cwd, + project: makeProject(cwd), + rpcClient: mockRpc(200, []), + }); + const opts = { + cwd, + planRef: planPath, + adapter: fakeAdapter(state), + exportFn: fakeExport(state), + buildFn: fakeBuild(), + rindexerBin: "rindexer", + rpcClient: mockRpc(200, []), + }; + await applyPlan(opts); + const second = await applyPlan(opts); + expect(second.reused).toBe(true); + expect(state.runs).toHaveLength(1); + }); + + it("configuration drift → policy_refused", async () => { + const cwd = setupProject(); + const state: FakeState = { runs: [], exports: [], coverageReport: completeReport() }; + const { planPath } = await generatePlan({ + intent: "ingest", + cwd, + project: makeProject(cwd), + rpcClient: mockRpc(200, []), + }); + fs.appendFileSync(path.join(cwd, "chainplot.yaml"), "\n"); + await expect( + applyPlan({ + cwd, + planRef: planPath, + adapter: fakeAdapter(state), + buildFn: fakeBuild(), + rindexerBin: "rindexer", + rpcClient: mockRpc(200, []), + }), + ).rejects.toMatchObject({ code: "policy_refused" }); + }); + + it("state drift (coverage changed) → policy_refused", async () => { + const cwd = setupProject(); + writeCoverage(cwd, [segment(100, 110)]); + const { planPath } = await generatePlan({ + intent: "ingest", + cwd, + project: makeProject(cwd), + rpcClient: mockRpc(200, []), + }); + fs.rmSync(path.join(cwd, ".chainplot/coverage.json")); + const state: FakeState = { runs: [], exports: [], coverageReport: completeReport() }; + await expect( + applyPlan({ + cwd, + planRef: planPath, + adapter: fakeAdapter(state), + buildFn: fakeBuild(), + rindexerBin: "rindexer", + rpcClient: mockRpc(200, []), + }), + ).rejects.toMatchObject({ code: "policy_refused" }); + }); + + it("hash-join break → source_inconsistent, coverage unchanged", async () => { + const cwd = setupProject(); + const state: FakeState = { runs: [], exports: [], coverageReport: completeReport() }; + // Seed a prior segment ending at 99 BEFORE planning (assumption: proven=99, + // same as no coverage — but gives apply a tail to hash-join against). + writeCoverage(cwd, [segment(0, 99)]); + const { planPath } = await generatePlan({ + intent: "ingest", + cwd, + project: makeProject(cwd), + rpcClient: mockRpc(200, []), + }); + // Breaking rpc: hashes do not chain (block n hash ≠ H(n)). + const breakingRpc: RpcClient = { + async call(method: string, params: unknown[]): Promise { + const tag = params[0] as string; + const n = tag === "finalized" ? 200 : Number(BigInt(tag)); + return { + number: "0x" + n.toString(16), + hash: H(n * 7 + 1), + parentHash: H(n * 7), + } as T; + }, + }; + await expect( + applyPlan({ + cwd, + planRef: planPath, + adapter: fakeAdapter(state), + exportFn: fakeExport(state), + buildFn: fakeBuild(), + rindexerBin: "rindexer", + rpcClient: breakingRpc, + }), + ).rejects.toMatchObject({ code: "source_inconsistent" }); + const coverage = readCoverage(cwd); + expect(coverage.sources[0].segments).toHaveLength(1); + expect(coverage.sources[0].segments[0].end_block).toBe(99); + }); +}); + +function fakeBuild() { + return async (cwd: string) => ({ distDir: path.join(cwd, "dist/releases/local") }); +} + +function segment(start: number, end: number) { + return { + start_block: start, + end_block: end, + start_block_hash: H(start), + end_block_hash: H(end), + start_block_parent_hash: H(start - 1), + status: "complete_with_rows" as const, + row_count: 5, + }; +} + +function writeCoverage(cwd: string, segments: ReturnType[]): void { + fs.mkdirSync(path.join(cwd, ".chainplot"), { recursive: true }); + fs.writeFileSync( + path.join(cwd, ".chainplot/coverage.json"), + JSON.stringify({ + schema_version: 1, + chain_id: 1, + sources: [{ source_id: "usdc", segments }], + }), + ); +} + +function readCoverage(cwd: string): { sources: { source_id: string; segments: { end_block: number }[] }[] } { + return JSON.parse( + fs.readFileSync(path.join(cwd, ".chainplot/coverage.json"), "utf8"), + ); +} diff --git a/tests/ingest/renderConfig.test.ts b/tests/ingest/renderConfig.test.ts new file mode 100644 index 0000000..149deaa --- /dev/null +++ b/tests/ingest/renderConfig.test.ts @@ -0,0 +1,67 @@ +import { describe, expect, it } from "vitest"; +import { renderConfig } from "../../src/ingest/rindexer/renderConfig.js"; +import type { BoundedJob } from "../../src/ingest/adapter.js"; + +const job: BoundedJob = { + sourceId: "usdc-transfers", + contractName: "usdc", + networkName: "chainplot_1", + chainId: 1, + addresses: [ + "0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48", + "0x000000000000000000000000000000000000dead", + ], + abiPath: "/proj/abis/ERC20.json", + events: ["Transfer", "Approval"], + jobStart: 18600000, + jobEnd: 18600010, + rpcUrl: "http://rpc.example", + databaseUrl: "postgres://u:p@localhost:5432/chainplot", + workDir: "/proj/.chainplot/ingest/usdc-transfers", +}; + +describe("renderConfig", () => { + const yaml = renderConfig(job); + + it("sets no-code project type and single network", () => { + expect(yaml).toContain("project_type: no-code"); + expect(yaml).toContain("name: chainplot_1"); + expect(yaml).toContain("chain_id: 1"); + expect(yaml).toContain("rpc: ${RPC_URL}"); + expect(yaml).toMatch(/^name: chainplot_chainplot_1$/m); + expect(yaml.match(/networks:/g)?.length).toBe(1); + }); + + it("enables postgres storage, disables graphql", () => { + expect(yaml).toContain("postgres:"); + expect(yaml).toContain("enabled: true"); + expect(yaml).toContain("graphql:"); + expect(yaml).toContain("enabled: false"); + }); + + it("never contains forbidden surfaces", () => { + expect(yaml).not.toMatch(/streams|chatbots|csv|docker/i); + expect(yaml).not.toContain("docker.sock"); + }); + + it("writes explicit bounded blocks and timestamp true", () => { + expect(yaml).toContain("start_block: 18600000"); + expect(yaml).toContain("end_block: 18600010"); + expect(yaml).toContain("timestamp: true"); + }); + + it("limits events to declared signatures and lowercases addresses", () => { + expect(yaml).toContain("- Transfer"); + expect(yaml).toContain("- Approval"); + expect(yaml).toContain("0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48"); + expect(yaml).toContain("0x000000000000000000000000000000000000dead"); + }); + + it("does not leak the rpc url into the config", () => { + expect(yaml).not.toContain("http://rpc.example"); + }); + + it("is deterministic", () => { + expect(renderConfig(job)).toBe(yaml); + }); +}); diff --git a/tests/ingest/runBounded.test.ts b/tests/ingest/runBounded.test.ts new file mode 100644 index 0000000..aea758c --- /dev/null +++ b/tests/ingest/runBounded.test.ts @@ -0,0 +1,75 @@ +import { describe, expect, it } from "vitest"; +import path from "node:path"; +import os from "node:os"; +import fs from "node:fs"; +import { fileURLToPath } from "node:url"; +import { + runBounded, + stopAndQuiesce, +} from "../../src/ingest/rindexer/runBounded.js"; +import type { BoundedJob } from "../../src/ingest/adapter.js"; + +const fakeBin = path.join( + path.dirname(fileURLToPath(import.meta.url)), + "../helpers/fakeRindexer.mjs", +); + +function makeJob(workDir: string): BoundedJob { + return { + sourceId: "src", + contractName: "usdc", + networkName: "chainplot_1", + chainId: 1, + addresses: ["0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48"], + abiPath: path.join(workDir, "abis/ERC20.json"), + events: ["Transfer"], + jobStart: 100, + jobEnd: 110, + rpcUrl: "http://rpc.example", + databaseUrl: "postgres://u:p@localhost:5432/chainplot", + workDir, + }; +} + +function tempDir(): string { + return fs.mkdtempSync(path.join(os.tmpdir(), "chainplot-run-")); +} + +describe("runBounded", () => { + it("writes the generated config and detects historic complete", async () => { + const workDir = tempDir(); + const job = makeJob(workDir); + const handle = await runBounded( + { ...job, rpcUrl: "", databaseUrl: "" }, + { rindexerBin: `node ${fakeBin}`, wallClockMs: 10_000 }, + { fakeMode: "complete" }, + ); + expect(handle.completedLogSeen).toBe(true); + expect(fs.existsSync(path.join(workDir, "rindexer.yaml"))).toBe(true); + await stopAndQuiesce(handle); + }, 15_000); + + it("rejects a child that exits before the completed line", async () => { + const workDir = tempDir(); + const job = makeJob(workDir); + await expect( + runBounded( + { ...job, rpcUrl: "", databaseUrl: "" }, + { rindexerBin: `node ${fakeBin}`, wallClockMs: 10_000 }, + { fakeMode: "exit" }, + ), + ).rejects.toMatchObject({ retryable: true }); + }, 15_000); + + it("wall clock exceeded → retryable failure, child killed", async () => { + const workDir = tempDir(); + const job = makeJob(workDir); + await expect( + runBounded( + { ...job, rpcUrl: "", databaseUrl: "" }, + { rindexerBin: `node ${fakeBin}`, wallClockMs: 300 }, + { fakeMode: "hang" }, + ), + ).rejects.toMatchObject({ retryable: true }); + }, 10_000); +}); diff --git a/tests/plan/buildIntent.test.ts b/tests/plan/buildIntent.test.ts new file mode 100644 index 0000000..ea03f36 --- /dev/null +++ b/tests/plan/buildIntent.test.ts @@ -0,0 +1,150 @@ +import { describe, expect, it, beforeEach, afterEach } from "vitest"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { runCliJson } from "../helpers/run.js"; + +const H = (n: number) => "0x" + n.toString(16).padStart(64, "0"); + +const INGEST_YAML = [ + "format_version: 1", + 'id: "build-intent"', + "chain_sources:", + " - id: mainnet", + " chain_id: 1", + " rpc_secret: RPC_URL", + " finality:", + " policy: finalized", + "event_sources:", + " - id: usdc", + " chain: mainnet", + " addresses:", + ' - "0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48"', + " abi: abis/ERC20.json", + " events:", + " - Transfer", + " start_block: 100", + " end:", + " mode: pinned", + " block: 110", + "datasets:", + " - id: usdc", + " snapshot: .chainplot/snapshots/usdc/usdc_transfer.parquet", + "queries:", + " - id: count", + " file: queries/count.sql", + " dataset: usdc", +].join("\n"); + +function setupProject(): string { + const cwd = fs.mkdtempSync(path.join(os.tmpdir(), "chainplot-buildplan-")); + fs.mkdirSync(path.join(cwd, "abis"), { recursive: true }); + fs.mkdirSync(path.join(cwd, "queries"), { recursive: true }); + fs.writeFileSync(path.join(cwd, "chainplot.yaml"), INGEST_YAML + "\n"); + fs.writeFileSync(path.join(cwd, "abis/ERC20.json"), "[]"); + fs.writeFileSync(path.join(cwd, "queries/count.sql"), "select count(*) as n from usdc"); + fs.mkdirSync(path.join(cwd, ".chainplot/snapshots/usdc"), { recursive: true }); + fs.writeFileSync( + path.join(cwd, ".chainplot/snapshots/usdc/usdc_transfer.parquet"), + "PK\x03\x04dummy", + ); + return cwd; +} + +function writeCompleteCoverage(cwd: string): void { + fs.mkdirSync(path.join(cwd, ".chainplot"), { recursive: true }); + fs.writeFileSync( + path.join(cwd, ".chainplot/coverage.json"), + JSON.stringify({ + schema_version: 1, + chain_id: 1, + sources: [ + { + source_id: "usdc", + segments: [ + { + start_block: 100, + end_block: 110, + start_block_hash: H(100), + end_block_hash: H(110), + start_block_parent_hash: H(99), + status: "complete_with_rows", + row_count: 92, + }, + ], + }, + ], + }), + ); +} + +let savedEnv: Record; +beforeEach(() => { + savedEnv = { + RPC_URL: process.env.RPC_URL, + DATABASE_URL: process.env.DATABASE_URL, + }; + delete process.env.RPC_URL; + delete process.env.DATABASE_URL; +}); +afterEach(() => { + for (const [k, v] of Object.entries(savedEnv)) { + if (v === undefined) delete process.env[k]; + else process.env[k] = v; + } +}); + +describe("plan --intent build", () => { + it("no credentials, no RPC, build_results only", async () => { + const cwd = setupProject(); + writeCompleteCoverage(cwd); + const result = await runCliJson(["plan", "--intent", "build", "--json"], cwd); + expect(result.ok).toBe(true); + const data = result.data as { + actions: { type: string }[]; + sources: { job_start: number; job_end: number }[]; + }; + expect(data.actions).toEqual([{ type: "build_results" }]); + expect(data.sources[0].job_start).toBe(111); + expect(data.sources[0].job_end).toBe(110); + }); + + it("publish intent without a target → policy_refused", async () => { + const cwd = setupProject(); + const result = await runCliJson(["plan", "--intent", "publish", "--json"], cwd); + expect(result.ok).toBe(false); + expect(result.error?.code).toBe("policy_refused"); + }); + + it("dataset-only project → build plan with null chain", async () => { + const cwd = fs.mkdtempSync(path.join(os.tmpdir(), "chainplot-buildplan-")); + fs.mkdirSync(path.join(cwd, "queries"), { recursive: true }); + fs.writeFileSync( + path.join(cwd, "chainplot.yaml"), + [ + "format_version: 1", + 'id: "fixture"', + "datasets:", + " - id: amounts", + " snapshot: snapshots/amounts.parquet", + "queries:", + " - id: raw_amounts", + " file: queries/raw_amounts.sql", + " dataset: amounts", + ].join("\n"), + ); + fs.mkdirSync(path.join(cwd, "snapshots"), { recursive: true }); + fs.copyFileSync( + path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + "../../templates/fixture-transfers/snapshots/amounts.parquet", + ), + path.join(cwd, "snapshots/amounts.parquet"), + ); + fs.writeFileSync(path.join(cwd, "queries/raw_amounts.sql"), "select 1 as x"); + const result = await runCliJson(["plan", "--intent", "build", "--json"], cwd); + expect(result.ok).toBe(true); + expect((result.data as { chain: unknown }).chain).toBeNull(); + }); +}); diff --git a/tests/project/modelGraph.test.ts b/tests/project/modelGraph.test.ts new file mode 100644 index 0000000..85e015e --- /dev/null +++ b/tests/project/modelGraph.test.ts @@ -0,0 +1,55 @@ +import { describe, expect, it } from "vitest"; +import { topoSortModels } from "../../src/project/modelGraph.js"; +import type { ModelNode } from "../../src/project/modelGraph.js"; + +function m(id: string, depends_on: string[]): ModelNode { + return { id, file: `models/${id}.sql`, depends_on }; +} + +describe("topoSortModels", () => { + it("empty input → empty order", () => { + expect(topoSortModels([])).toEqual([]); + }); + + it("linear chain → deps first", () => { + const order = topoSortModels([m("c", ["b"]), m("a", []), m("b", ["a"])]); + expect(order).toEqual(["a", "b", "c"]); + }); + + it("diamond → shared dep once, before both dependents", () => { + const order = topoSortModels([ + m("left", ["base"]), + m("right", ["base"]), + m("top", ["left", "right"]), + m("base", []), + ]); + expect(order.indexOf("base")).toBeLessThan(order.indexOf("left")); + expect(order.indexOf("base")).toBeLessThan(order.indexOf("right")); + expect(order.indexOf("left")).toBeLessThan(order.indexOf("top")); + expect(order.indexOf("right")).toBeLessThan(order.indexOf("top")); + expect(order).toHaveLength(4); + }); + + it("unknown dependency → validation error naming it", () => { + expect(() => topoSortModels([m("a", ["ghost"])])).toThrowError(/ghost/); + try { + topoSortModels([m("a", ["ghost"])]); + } catch (err) { + expect((err as { code?: string }).code).toBe("validation"); + } + }); + + it("self dependency → cycle", () => { + expect(() => topoSortModels([m("a", ["a"])])).toThrowError(/cycle/i); + }); + + it("two-node cycle → cycle", () => { + expect(() => + topoSortModels([m("a", ["b"]), m("b", ["a"])]), + ).toThrowError(/cycle/i); + }); + + it("duplicate model ids → validation", () => { + expect(() => topoSortModels([m("a", []), m("a", [])])).toThrowError(/a/); + }); +}); diff --git a/tests/publish/directory.test.ts b/tests/publish/directory.test.ts new file mode 100644 index 0000000..acdbeab --- /dev/null +++ b/tests/publish/directory.test.ts @@ -0,0 +1,132 @@ +import { describe, expect, it } from "vitest"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { createHash } from "node:crypto"; +import { latestPointer, pointerChecksumMatches } from "../../src/publish/latestPointer.js"; +import { DirectoryTarget } from "../../src/publish/directory.js"; + +function tmp(): string { + return fs.mkdtempSync(path.join(os.tmpdir(), "chainplot-pub-")); +} + +function makeRelease(root: string, id: string, body: string): string { + const releaseDir = path.join(root, "releases", id); + fs.mkdirSync(releaseDir, { recursive: true }); + fs.writeFileSync(path.join(releaseDir, "release.json"), body); + fs.writeFileSync(path.join(releaseDir, "index.html"), ""); + return releaseDir; +} + +describe("latestPointer", () => { + it("checksum is sha256 of the release.json bytes", () => { + const body = '{"schema_version":1}'; + const pointer = latestPointer("releases/r1", body); + expect(pointer).toEqual({ + schema_version: 1, + release_prefix: "releases/r1", + release_json_checksum: createHash("sha256").update(body).digest("hex"), + }); + }); + + it("checksumMatches verifies bytes", () => { + const body = "abc"; + expect(pointerChecksumMatches(latestPointer("p", body), body)).toBe(true); + expect(pointerChecksumMatches(latestPointer("p", body), "abd")).toBe(false); + }); +}); + +describe("DirectoryTarget", () => { + it("publishes immutable files then promotes latest atomically", async () => { + const root = tmp(); + const target = new DirectoryTarget(root); + const releaseDir = makeRelease(root, "r1", '{"v":1}'); + const body = fs.readFileSync(path.join(releaseDir, "release.json"), "utf8"); + + await target.uploadFiles(releaseDir, "releases/r1", [ + "release.json", + "index.html", + ]); + // immutable files exist under the prefix + expect( + fs.existsSync(path.join(root, "releases/r1/release.json")), + ).toBe(true); + // no pointer yet + expect(await target.readLatest()).toBeNull(); + + await target.promoteLatest(latestPointer("releases/r1", body)); + const pointer = await target.readLatest(); + expect(pointer?.release_prefix).toBe("releases/r1"); + // latest.json is at the root, never inside releases// + expect(fs.existsSync(path.join(root, "latest.json"))).toBe(true); + expect( + fs.existsSync(path.join(root, "releases/r1/latest.json")), + ).toBe(false); + }); + + it("second publish flips the pointer; previous release untouched", async () => { + const root = tmp(); + const target = new DirectoryTarget(root); + const r1 = makeRelease(root, "r1", '{"v":1}'); + const r2 = makeRelease(root, "r2", '{"v":2}'); + const body1 = fs.readFileSync(path.join(r1, "release.json"), "utf8"); + const body2 = fs.readFileSync(path.join(r2, "release.json"), "utf8"); + + await target.uploadFiles(r1, "releases/r1", ["release.json", "index.html"]); + await target.promoteLatest(latestPointer("releases/r1", body1)); + await target.uploadFiles(r2, "releases/r2", ["release.json", "index.html"]); + await target.promoteLatest(latestPointer("releases/r2", body2)); + + const pointer = await target.readLatest(); + expect(pointer?.release_prefix).toBe("releases/r2"); + // r1 still complete and usable + expect( + JSON.parse(fs.readFileSync(path.join(root, "releases/r1/release.json"), "utf8")), + ).toEqual({ v: 1 }); + }); + + it("pointer checksum mismatch detected on read", async () => { + const root = tmp(); + const target = new DirectoryTarget(root); + const releaseDir = makeRelease(root, "r1", "real-body"); + await target.uploadFiles(releaseDir, "releases/r1", ["release.json"]); + await target.promoteLatest(latestPointer("releases/r1", "real-body")); + // tamper with the release body after promotion + fs.writeFileSync(path.join(root, "releases/r1/release.json"), "tampered"); + const pointer = await target.readLatest(); + const body = fs.readFileSync( + path.join(root, "releases/r1/release.json"), + "utf8", + ); + expect(pointerChecksumMatches(pointer!, body)).toBe(false); + }); +}); + +// Two projects sharing one target previously overwrote each other's pointer: +// latest.json was a fixed key at the root, so whichever published last won and +// the other project's consumers silently followed the wrong release. +describe("prefixed targets", () => { + it("keeps each project's latest.json separate", async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "chainplot-prefix-")); + const a = new DirectoryTarget(root, "project-a"); + const b = new DirectoryTarget(root, "project-b"); + + await a.promoteLatest({ + schema_version: 1, + release_prefix: "project-a/releases/aaaa", + release_json_checksum: "a".repeat(64), + }); + await b.promoteLatest({ + schema_version: 1, + release_prefix: "project-b/releases/bbbb", + release_json_checksum: "b".repeat(64), + }); + + expect((await a.readLatest())?.release_prefix).toBe("project-a/releases/aaaa"); + expect((await b.readLatest())?.release_prefix).toBe("project-b/releases/bbbb"); + expect(fs.existsSync(path.join(root, "project-a", "latest.json"))).toBe(true); + expect(fs.existsSync(path.join(root, "project-b", "latest.json"))).toBe(true); + // No stray pointer at the root to mislead a consumer. + expect(fs.existsSync(path.join(root, "latest.json"))).toBe(false); + }); +}); diff --git a/tests/publish/live/s3.live.test.ts b/tests/publish/live/s3.live.test.ts new file mode 100644 index 0000000..c2d1864 --- /dev/null +++ b/tests/publish/live/s3.live.test.ts @@ -0,0 +1,124 @@ +import { describe, expect, it, beforeEach, afterEach } from "vitest"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { runCliJson } from "../../helpers/run.js"; +import { + S3Target, + makeS3Ops, + s3EnvFromProcess, + LATEST_KEY, +} from "../../../src/publish/s3.js"; +import { latestPointer } from "../../../src/publish/latestPointer.js"; + +// Live-gated: needs CHAINPLOT_S3_ENDPOINT, CHAINPLOT_S3_BUCKET, +// AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY (optionally CHAINPLOT_S3_REGION). +// Skips when absent. Never commit endpoint URLs or keys. +const env = s3EnvFromProcess(); +const d = env ? it : it.skip; + +describe("S3 conditional-write probe (M0 open question)", () => { + const key = `probe/${Date.now()}-initial`; + + d("read-after-write, If-None-Match *, If-Match current, 412 on stale", async () => { + const ops = makeS3Ops(env!); + // 1. plain put + read-after-write + await ops.put(key, "probe-body"); + const first = await ops.get(key); + expect(first).not.toBeNull(); + expect(first!.body.toString("utf8")).toBe("probe-body"); + + // 2. If-None-Match: * on an EXISTING key must be refused (412 raw or mapped) + let ifNoneMatchOnExisting = "allowed"; + try { + await ops.put(key, "probe-body-2", { ifNoneMatch: "*" }); + } catch (err) { + const status = (err as { $metadata?: { httpStatusCode?: number } }) + .$metadata?.httpStatusCode; + ifNoneMatchOnExisting = + status === 412 || (err as { code?: string }).code === "policy_refused" + ? "refused" + : "other-error"; + } + expect(ifNoneMatchOnExisting).toBe("refused"); + + // 3. If-Match with current ETag must succeed + const current = await ops.get(key); + const put = await ops.put(key, "probe-body-3", { + ifMatch: current!.etag, + }); + expect(put.etag).toBeTruthy(); + + // 4. If-Match with a STALE ETag must be refused + let ifMatchStale = "allowed"; + try { + await ops.put(key, "probe-body-4", { ifMatch: current!.etag }); + } catch (err) { + const status = (err as { $metadata?: { httpStatusCode?: number } }) + .$metadata?.httpStatusCode; + ifMatchStale = + status === 412 || (err as { code?: string }).code === "policy_refused" + ? "refused" + : "other-error"; + } + expect(ifMatchStale).toBe("refused"); + + // cleanup + await ops.delete(key); + }); + + d("publish → re-publish flips the pointer; failed upload leaves previous intact", async () => { + const parent = fs.mkdtempSync(path.join(os.tmpdir(), "chainplot-s3pub-")); + const dir = path.join(parent, "proj"); + const init = await runCliJson( + ["init", "--template", "fixture-transfers", "--output", dir, "--json"], + parent, + ); + expect(init.ok).toBe(true); + const yamlPath = path.join(dir, "chainplot.yaml"); + fs.writeFileSync( + yamlPath, + `${fs.readFileSync(yamlPath, "utf8")} +publish_targets: + - id: r2 + type: s3 + bucket: ${env!.bucket} + dataset_license: CC-BY-4.0 +`, + ); + + const build = await runCliJson(["build", "--json"], dir); + expect(build.ok).toBe(true); + const publish1 = await runCliJson(["publish", "--json"], dir); + expect(publish1.ok).toBe(true); + const prefix1 = (publish1.data as { publish: { release_prefix: string } }).publish + .release_prefix; + + const ops = makeS3Ops(env!); + const pointer1 = await ops.get(LATEST_KEY); + expect(pointer1).not.toBeNull(); + expect(JSON.parse(pointer1!.body).release_prefix).toBe(prefix1); + + // second release flips the pointer + fs.writeFileSync( + yamlPath, + fs.readFileSync(yamlPath, "utf8").replace("title: Amounts", "title: Amounts v2"), + ); + await runCliJson(["build", "--json"], dir); + const publish2 = await runCliJson(["publish", "--json"], dir); + expect(publish2.ok).toBe(true); + const pointer2 = await ops.get(LATEST_KEY); + const prefix2 = JSON.parse(pointer2!.body).release_prefix as string; + expect(prefix2).not.toBe(prefix1); + // previous release still readable + expect((await ops.head(`${prefix1}/release.json`)) !== null).toBe(true); + + // checksum in pointer matches release.json bytes + const releaseBody = (await ops.get(`${prefix2}/release.json`))!.body; + const expected = latestPointer(prefix2, releaseBody); + expect(JSON.parse(pointer2!.body).release_json_checksum).toBe( + expected.release_json_checksum, + ); + }, 120_000); +}); diff --git a/tests/publish/modes.test.ts b/tests/publish/modes.test.ts new file mode 100644 index 0000000..7011242 --- /dev/null +++ b/tests/publish/modes.test.ts @@ -0,0 +1,212 @@ +import { describe, expect, it } from "vitest"; +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { runCliJson } from "../helpers/run.js"; + +const template = path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + "../../templates/fixture-transfers", +); + +function setup(): string { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "chainplot-modes-")); + fs.cpSync(template, dir, { recursive: true }); + return dir; +} + +async function makeBigParquet(dir: string): Promise { + const big = path.join(dir, "snapshots/amounts.parquet"); + fs.mkdirSync(path.dirname(big), { recursive: true }); + // Valid parquet > 100 MiB: random strings compress poorly. + const { execFileSync } = await import("node:child_process"); + execFileSync( + process.execPath, + [ + "--input-type=module", + "-e", + `import { DuckDBInstance } from '@duckdb/node-api'; + const db = await DuckDBInstance.create(':memory:'); + const c = await db.connect(); + await c.run("COPY (SELECT i AS amount, md5(i::VARCHAR) AS amount_sort FROM range(3200000) t(i)) TO '${big.replace(/'/g, "''")}' (FORMAT PARQUET)");`, + ], + { stdio: "pipe" }, + ); + // Aggregate query so the row limit does not trip before the size cap. + fs.writeFileSync(path.join(dir, "queries/raw_amounts.sql"), "select count(*) as n from amounts"); +} + +describe("dataset modes and size caps", () => { + // Publishing is outward and irreversible, so the default ships the page and + // its answers and leaves the dataset behind. Shipping the parquet is what + // lets a fork recompute, and it is opt-in. + it("default omits the dataset", async () => { + const dir = setup(); + const result = await runCliJson(["build", "--json"], dir); + expect(result.ok).toBe(true); + const dist = path.join(dir, "dist/releases/local"); + const manifest = JSON.parse( + fs.readFileSync(path.join(dist, "datasets/amounts/manifest.json"), "utf8"), + ); + expect(manifest.mode).toBe("results_only"); + expect(fs.existsSync(path.join(dist, "datasets/amounts/tables"))).toBe(false); + // The page and its answers are still whole. + expect(fs.existsSync(path.join(dist, "index.html"))).toBe(true); + expect(fs.existsSync(path.join(dist, "results/raw_amounts.json"))).toBe(true); + }); + + it("--mode dataset_included ships the parquet", async () => { + const dir = setup(); + const result = await runCliJson( + ["build", "--mode", "dataset_included", "--json"], + dir, + ); + expect(result.ok).toBe(true); + const dist = path.join(dir, "dist/releases/local"); + const manifest = JSON.parse( + fs.readFileSync(path.join(dist, "datasets/amounts/manifest.json"), "utf8"), + ); + expect(manifest.mode).toBe("dataset_included"); + expect( + fs.existsSync(path.join(dist, "datasets/amounts/tables/amounts.parquet")), + ).toBe(true); + }); + + it("an oversized dataset is refused only when it was asked for", async () => { + const dir = setup(); + await makeBigParquet(dir); + // The default no longer trips the cap at all: nothing is copied. + expect((await runCliJson(["build", "--json"], dir)).ok).toBe(true); + + const result = await runCliJson( + ["build", "--mode", "dataset_included", "--json"], + dir, + ); + expect(result.ok).toBe(false); + expect(result.error?.code).toBe("policy_refused"); + expect(result.error?.message).toMatch(/results_only|dataset_referenced|reduce/i); + }); + + it("build --mode results_only omits tables", async () => { + const dir = setup(); + await makeBigParquet(dir); + const result = await runCliJson(["build", "--mode", "results_only", "--json"], dir); + expect(result.ok).toBe(true); + const dist = path.join(dir, "dist/releases/local"); + const manifest = JSON.parse( + fs.readFileSync(path.join(dist, "datasets/amounts/manifest.json"), "utf8"), + ); + expect(manifest.mode).toBe("results_only"); + expect(fs.existsSync(path.join(dist, "datasets/amounts/tables"))).toBe(false); + const release = JSON.parse( + fs.readFileSync(path.join(dist, "release.json"), "utf8"), + ); + expect(release.mode).toBe("results_only"); + }); + + it("build --mode dataset_referenced writes a pointer manifest", async () => { + const dir = setup(); + const result = await runCliJson(["build", "--mode", "dataset_referenced", "--json"], dir); + expect(result.ok).toBe(true); + const dist = path.join(dir, "dist/releases/local"); + const manifest = JSON.parse( + fs.readFileSync(path.join(dist, "datasets/amounts/manifest.json"), "utf8"), + ); + expect(manifest.mode).toBe("dataset_referenced"); + expect(fs.existsSync(path.join(dist, "datasets/amounts/tables"))).toBe(false); + const expectedChecksum = createHash("sha256") + .update(fs.readFileSync(path.join(dir, "snapshots/amounts.parquet"))) + .digest("hex"); + expect(manifest.external.checksum).toBe(expectedChecksum); + }); +}); + +// `apply` and `refresh` build with no way to pass --mode, so a project whose +// dataset exceeds the copy cap could never finish either: the ingest +// succeeded and the build was refused every time, with nowhere to say +// otherwise. +describe("project-declared release mode", () => { + it("uses policy.release_mode when the CLI passes no mode", async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "chainplot-relmode-")); + fs.cpSync(template, dir, { recursive: true }); + fs.writeFileSync( + path.join(dir, "chainplot.yaml"), + `${fs.readFileSync(path.join(dir, "chainplot.yaml"), "utf8")} +policy: + release_mode: results_only +`, + ); + + const built = await runCliJson(["build", "--json"], dir); + expect(built.ok).toBe(true); + const release = JSON.parse( + fs.readFileSync(path.join(dir, "dist/releases/local/release.json"), "utf8"), + ) as { mode: string }; + expect(release.mode).toBe("results_only"); + expect( + fs.existsSync(path.join(dir, "dist/releases/local/datasets/amounts/tables")), + ).toBe(false); + }); + + it("an explicit --mode still wins over the project", async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "chainplot-relmode2-")); + fs.cpSync(template, dir, { recursive: true }); + fs.writeFileSync( + path.join(dir, "chainplot.yaml"), + `${fs.readFileSync(path.join(dir, "chainplot.yaml"), "utf8")} +policy: + release_mode: results_only +`, + ); + + const built = await runCliJson( + ["build", "--mode", "dataset_included", "--json"], + dir, + ); + expect(built.ok).toBe(true); + const release = JSON.parse( + fs.readFileSync(path.join(dir, "dist/releases/local/release.json"), "utf8"), + ) as { mode: string }; + expect(release.mode).toBe("dataset_included"); + }); +}); + +// The row limit bounds the viewer, which renders every row into the DOM. The +// value lived as four separate copies of 10_000 with nothing keeping them in +// step, and no project could raise it — so a legitimate wide table was simply +// impossible to publish. +describe("row limit", () => { + it("is refused, not truncated, and the project can raise it", async () => { + const dir = setup(); + fs.writeFileSync( + path.join(dir, "queries/raw_amounts.sql"), + "SELECT i::VARCHAR AS n FROM range(25) t(i)", + ); + const yaml = fs.readFileSync(path.join(dir, "chainplot.yaml"), "utf8"); + + fs.writeFileSync( + path.join(dir, "chainplot.yaml"), + `${yaml}\npolicy:\n row_limit: 10\n`, + ); + const refused = await runCliJson(["build", "--json"], dir); + expect(refused.ok).toBe(false); + expect(refused.error?.code).toBe("policy_refused"); + expect(refused.error?.message).toMatch(/more than 10 rows/); + + fs.writeFileSync( + path.join(dir, "chainplot.yaml"), + `${yaml}\npolicy:\n row_limit: 100\n`, + ); + const allowed = await runCliJson(["build", "--json"], dir); + expect(allowed.ok).toBe(true); + const result = JSON.parse( + fs.readFileSync( + path.join(dir, "dist/releases/local/results/raw_amounts.json"), + "utf8", + ), + ) as { rows: unknown[][] }; + expect(result.rows).toHaveLength(25); + }); +}); diff --git a/tests/publish/publishCommand.test.ts b/tests/publish/publishCommand.test.ts new file mode 100644 index 0000000..0e0b561 --- /dev/null +++ b/tests/publish/publishCommand.test.ts @@ -0,0 +1,162 @@ +import { describe, expect, it, beforeEach, afterEach } from "vitest"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { runCliJson } from "../helpers/run.js"; + +const template = path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + "../../templates/fixture-transfers", +); + +const TARGETS = [ + "publish_targets:", + " - id: local-dir", + " type: directory", + " path: ./published", + " dataset_license: CC-BY-4.0", +].join("\n"); + +function setupProject(withLicense = true): string { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "chainplot-publish-")); + fs.cpSync(template, dir, { recursive: true }); + const yaml = fs.readFileSync(path.join(dir, "chainplot.yaml"), "utf8"); + fs.writeFileSync( + path.join(dir, "chainplot.yaml"), + yaml + "\n" + (withLicense ? TARGETS : TARGETS.replace(" dataset_license: CC-BY-4.0", "")) + "\n", + ); + return dir; +} + +let savedEnv: Record; +beforeEach(() => { + savedEnv = { RPC_URL: process.env.RPC_URL, DATABASE_URL: process.env.DATABASE_URL }; + delete process.env.RPC_URL; + delete process.env.DATABASE_URL; +}); +afterEach(() => { + for (const [k, v] of Object.entries(savedEnv)) { + if (v === undefined) delete process.env[k]; + else process.env[k] = v; + } +}); + +describe("publish", () => { + it("build + publish writes the release and latest.json under the target", async () => { + const dir = setupProject(); + expect((await runCliJson(["build", "--json"], dir)).ok).toBe(true); + const result = await runCliJson(["publish", "--json"], dir); + expect(result.ok).toBe(true); + const targetRoot = path.join(dir, "published"); + expect(fs.existsSync(path.join(targetRoot, "latest.json"))).toBe(true); + const pointer = JSON.parse( + fs.readFileSync(path.join(targetRoot, "latest.json"), "utf8"), + ); + expect(pointer.release_prefix).toMatch(/^releases\//); + expect( + fs.existsSync( + path.join(targetRoot, pointer.release_prefix, "release.json"), + ), + ).toBe(true); + expect( + fs.existsSync(path.join(targetRoot, "releases/local/release.json")), + ).toBe(false); + }, 30_000); + + it("missing dataset_license → policy_refused", async () => { + const dir = setupProject(false); + await runCliJson(["build", "--json"], dir); + const result = await runCliJson(["publish", "--json"], dir); + expect(result.ok).toBe(false); + expect(result.error?.code).toBe("policy_refused"); + expect(result.error?.message).toMatch(/dataset_license/); + }); + + it("publish without a prior build → validation", async () => { + const dir = setupProject(); + const result = await runCliJson(["publish", "--json"], dir); + expect(result.ok).toBe(false); + expect(result.error?.code).toBe("validation"); + expect(result.error?.message).toMatch(/build/i); + }); + + it("unknown --publish-target → policy_refused", async () => { + const dir = setupProject(); + await runCliJson(["build", "--json"], dir); + const result = await runCliJson( + ["publish", "--publish-target", "nope", "--json"], + dir, + ); + expect(result.ok).toBe(false); + expect(result.error?.code).toBe("policy_refused"); + }); + + it("plan --intent publish makes no RPC and is digest-bound", async () => { + const dir = setupProject(); + await runCliJson(["build", "--json"], dir); + const plan = await runCliJson(["plan", "--intent", "publish", "--json"], dir); + expect(plan.ok).toBe(true); + expect((plan.data as { actions: { type: string }[] }).actions).toEqual([ + { type: "publish", target_id: "local-dir" }, + ]); + // apply the plan → published + const planPath = (plan.data as { plan_path: string }).plan_path; + const applied = await runCliJson(["apply", "--plan", planPath, "--json"], dir); + expect(applied.ok).toBe(true); + expect(fs.existsSync(path.join(dir, "published/latest.json"))).toBe(true); + }); +}); + +// The publish plan digest once covered only the project files, so rebuilding a +// release with different content produced an identical plan: `apply` reused the +// previous run and uploaded nothing, while still reporting files_uploaded and +// promoted from the cached result. Binding the plan to the release's +// content_digest is what makes a changed release a different plan. +describe("publish plan binds to release content", () => { + it("re-publishes after the built release changes", async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "chainplot-repub-")); + fs.cpSync(template, dir, { recursive: true }); + fs.appendFileSync( + path.join(dir, "chainplot.yaml"), + `publish_targets: + - id: local-dir + type: directory + path: ./published + dataset_license: CC-BY-4.0 +`, + ); + + expect((await runCliJson(["build", "--json"], dir)).ok).toBe(true); + const first = await runCliJson(["publish", "--json"], dir); + expect(first.ok).toBe(true); + + // Same bytes: reuse is correct, and the prefix must not move. + expect((await runCliJson(["build", "--json"], dir)).ok).toBe(true); + const again = await runCliJson(["publish", "--json"], dir); + const firstPrefix = (first.data as PublishEnvelope).publish.release_prefix; + expect((again.data as PublishEnvelope).publish.release_prefix).toBe(firstPrefix); + + // Different bytes: a new release must actually be uploaded. + fs.writeFileSync( + path.join(dir, "queries", "raw_amounts.sql"), + "SELECT amount FROM amounts ORDER BY cp_sortkey(amount) DESC", + ); + expect((await runCliJson(["build", "--json"], dir)).ok).toBe(true); + const third = await runCliJson(["publish", "--json"], dir); + expect(third.ok).toBe(true); + const thirdData = third.data as PublishEnvelope; + expect(thirdData.reused).toBe(false); + expect(thirdData.publish.release_prefix).not.toBe(firstPrefix); + expect( + fs.existsSync( + path.join(dir, "published", thirdData.publish.release_prefix, "release.json"), + ), + ).toBe(true); + }); +}); + +interface PublishEnvelope { + reused: boolean; + publish: { release_prefix: string }; +} diff --git a/tests/publish/s3unit.test.ts b/tests/publish/s3unit.test.ts new file mode 100644 index 0000000..9f02370 --- /dev/null +++ b/tests/publish/s3unit.test.ts @@ -0,0 +1,123 @@ +import { describe, expect, it } from "vitest"; +import { createHash } from "node:crypto"; +import { S3Target, type S3Ops } from "../../src/publish/s3.js"; + +interface Stored { + body: string | Uint8Array; + etag: string; +} + +function preconditionError(): Error & { $metadata: { httpStatusCode: number } } { + const err = new Error("precondition failed") as Error & { + $metadata: { httpStatusCode: number }; + }; + err.name = "PreconditionFailed"; + err.$metadata = { httpStatusCode: 412 }; + return err; +} + +function mockOps(): S3Ops & { failNextPut412: boolean } { + const store = new Map(); + let counter = 0; + const self = { + failNextPut412: false, + async put(key, body, conditions) { + if (self.failNextPut412) { + self.failNextPut412 = false; + throw preconditionError(); + } + const existing = store.get(key); + if (conditions?.ifNoneMatch === "*" && existing) throw preconditionError(); + if ( + conditions?.ifMatch !== undefined && + existing && + existing.etag !== conditions.ifMatch + ) { + throw preconditionError(); + } + const etag = `"etag-${++counter}"`; + store.set(key, { body, etag }); + return { etag }; + }, + async get(key) { + const hit = store.get(key); + if (!hit) return null; + return { + body: + typeof hit.body === "string" + ? hit.body + : Buffer.from(hit.body).toString("utf8"), + etag: hit.etag, + }; + }, + async head(key) { + const hit = store.get(key); + if (!hit) return null; + return { + size: typeof hit.body === "string" ? hit.body.length : hit.body.length, + etag: hit.etag, + }; + }, + async delete(key) { + store.delete(key); + }, + }; + return self; +} + +const ENV = { + endpoint: "http://localhost", + bucket: "b", + accessKeyId: "a", + secretAccessKey: "s", +}; + +describe("S3Target with mocked ops", () => { + it("first promote uses If-None-Match *, second uses If-Match", async () => { + const ops = mockOps(); + const target = new S3Target(ENV, ops); + expect(await target.readLatest()).toBeNull(); + await target.promoteLatest(latestPointerOf("releases/r1", "body1")); + const pointer = await target.readLatest(); + expect(pointer?.release_prefix).toBe("releases/r1"); + await target.promoteLatest(latestPointerOf("releases/r2", "body2")); + expect((await target.readLatest())?.release_prefix).toBe("releases/r2"); + }); + + it("412 on conditional put → policy_refused", async () => { + const ops = mockOps(); + const target = new S3Target(ENV, ops); + await target.promoteLatest(latestPointerOf("releases/r1", "body1")); + // Simulate a concurrent writer between our read and our conditional put. + ops.failNextPut412 = true; + await expect( + target.promoteLatest(latestPointerOf("releases/r2", "body2")), + ).rejects.toMatchObject({ code: "policy_refused" }); + }); + + it("verifyFiles detects checksum mismatch", async () => { + const ops = mockOps(); + const target = new S3Target(ENV, ops); + await ops.put("releases/r1/release.json", "body"); + await expect( + target.verifyFiles("releases/r1", ["release.json"], { + "release.json": createHash("sha256").update("other").digest("hex"), + }), + ).rejects.toMatchObject({ code: "transient_dependency" }); + }); + + it("verifyFiles passes on matching checksum", async () => { + const ops = mockOps(); + const target = new S3Target(ENV, ops); + await ops.put("releases/r1/release.json", "body"); + await target.verifyFiles("releases/r1", ["release.json"], { + "release.json": createHash("sha256").update("body").digest("hex"), + }); + }); +}); + +import { latestPointer } from "../../src/publish/latestPointer.js"; + +function latestPointerOf(prefix: string, body: string) { + return latestPointer(prefix, body); +} diff --git a/tests/query/models.test.ts b/tests/query/models.test.ts new file mode 100644 index 0000000..aa2fb2f --- /dev/null +++ b/tests/query/models.test.ts @@ -0,0 +1,221 @@ +import { describe, expect, it } from "vitest"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { runQuery } from "../../src/query/runQuery.js"; + +function tempCopyOfFixture(): string { + const src = path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + "../../templates/fixture-transfers/snapshots/amounts.parquet", + ); + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "chainplot-models-")); + const dest = path.join(dir, "amounts.parquet"); + fs.copyFileSync(src, dest); + return dest; +} + +describe("model materialization", () => { + it("query consumes a materialized model", async () => { + const parquet = tempCopyOfFixture(); + const result = await runQuery({ + sql: "SELECT n FROM daily_totals ORDER BY n", + tables: { amounts: parquet }, + rawAmountColumns: [], + rowLimit: 10_000, + models: [ + { + id: "daily_totals", + sql: "SELECT amount_sort, COUNT(*) AS n FROM amounts GROUP BY amount_sort", + }, + ], + }); + expect(result.columns.map((c) => c.name)).toEqual(["n"]); + expect(result.rows.length).toBeGreaterThan(0); + }); + + it("model on top of model, in array order", async () => { + const parquet = tempCopyOfFixture(); + const result = await runQuery({ + sql: "SELECT total FROM grand_total", + tables: { amounts: parquet }, + rawAmountColumns: [], + rowLimit: 10_000, + models: [ + { + id: "daily_totals", + sql: "SELECT amount_sort, COUNT(*) AS n FROM amounts GROUP BY amount_sort", + }, + { + id: "grand_total", + sql: "SELECT SUM(n) AS total FROM daily_totals", + }, + ], + }); + expect(result.rows).toHaveLength(1); + }); + + it("non-SELECT model SQL → validation error", async () => { + const parquet = tempCopyOfFixture(); + await expect( + runQuery({ + sql: "SELECT 1", + tables: { amounts: parquet }, + rawAmountColumns: [], + rowLimit: 10_000, + models: [{ id: "evil", sql: "CREATE TABLE x AS SELECT 1" }], + }), + ).rejects.toMatchObject({ code: "policy_refused" }); + }); + + // The snapshot is the only file this process may read, and models are no + // more trusted than the query: `fork` writes a stranger's models straight + // into the project. External access must already be off by the time they run. + it("model cannot read the filesystem", async () => { + const parquet = tempCopyOfFixture(); + const canary = path.join(path.dirname(parquet), "canary.txt"); + fs.writeFileSync(canary, "SECRET"); + await expect( + runQuery({ + sql: "SELECT leaked FROM exfil", + tables: { amounts: parquet }, + rawAmountColumns: [], + rowLimit: 10_000, + models: [ + { + id: "exfil", + sql: `SELECT content AS leaked FROM read_text('${canary}')`, + }, + ], + }), + ).rejects.toThrow(/file system operations are disabled/); + }); + + // httpfs also cannot autoload here; the filesystem test above is what + // actually pins enable_external_access. + it("model cannot reach the network", async () => { + const parquet = tempCopyOfFixture(); + await expect( + runQuery({ + sql: "SELECT * FROM exfil", + tables: { amounts: parquet }, + rawAmountColumns: [], + rowLimit: 10_000, + models: [ + { + id: "exfil", + sql: "SELECT * FROM read_csv('https://example.invalid/x.csv')", + }, + ], + }), + ).rejects.toMatchObject({ code: "validation" }); + }); + + it("multi-statement query is refused", async () => { + const parquet = tempCopyOfFixture(); + await expect( + runQuery({ + sql: "SELECT 1 AS a; SELECT 2 AS b", + tables: { amounts: parquet }, + rawAmountColumns: [], + rowLimit: 10_000, + }), + ).rejects.toMatchObject({ code: "policy_refused" }); + }); + + it("failing model surfaces the model id", async () => { + const parquet = tempCopyOfFixture(); + await expect( + runQuery({ + sql: "SELECT * FROM broken", + tables: { amounts: parquet }, + rawAmountColumns: [], + rowLimit: 10_000, + models: [{ id: "broken", sql: "SELECT nope FROM amounts" }], + }), + ).rejects.toThrow(/broken/); + }); + + describe("raw amount ORDER BY guard", () => { + const refused = [ + ["bare column", "SELECT amount FROM amounts ORDER BY amount"], + ["select-list alias", "SELECT amount AS v FROM amounts ORDER BY v"], + ["positional ordinal", "SELECT amount FROM amounts ORDER BY 1"], + ] as const; + + for (const [label, sql] of refused) { + it(`refuses ${label}`, async () => { + await expect( + runQuery({ + sql, + tables: { amounts: tempCopyOfFixture() }, + rawAmountColumns: ["amount"], + rowLimit: 10_000, + }), + ).rejects.toMatchObject({ code: "validation" }); + }); + } + + it("allows an explicit sort key and orders it numerically", async () => { + const result = await runQuery({ + sql: "SELECT amount FROM amounts ORDER BY cp_sortkey(amount)", + tables: { amounts: tempCopyOfFixture() }, + rawAmountColumns: ["amount"], + rowLimit: 10_000, + }); + const got = result.rows.map((row) => String(row[0])); + const want = [...got].sort((a, b) => + BigInt(a) < BigInt(b) ? -1 : BigInt(a) > BigInt(b) ? 1 : 0, + ); + expect(got).toEqual(want); + // The fixture spans the full signed range, negatives included. + expect(got[0]!.startsWith("-")).toBe(true); + expect(got.at(-1)).toBe( + "115792089237316195423570985008687907853269984665640564039457584007913129639935", + ); + }); + }); + + // The fixture carries a precomputed amount_sort; scripts/write-fixture-parquet.ts + // and the cp_sortkey macro must stay one definition, or a snapshot column and + // an in-query call would order the same data differently. + it("the precomputed sort column equals cp_sortkey exactly", async () => { + const result = await runQuery({ + sql: + "SELECT count(*) FILTER (WHERE amount_sort <> cp_sortkey(amount))::VARCHAR AS mismatches," + + " count(*)::VARCHAR AS total FROM amounts", + tables: { amounts: tempCopyOfFixture() }, + rawAmountColumns: ["amount"], + rowLimit: 10_000, + }); + expect(result.rows[0]?.[0]).toBe("0"); + expect(result.rows[0]?.[1]).toBe("8"); + }); + + // security.md lists resource exhaustion by a forked recipe as a real risk; + // the memory cap is the control, and it must spill rather than fail. + it("caps query memory and spills instead of failing", async () => { + const result = await runQuery({ + sql: + "SELECT current_setting('memory_limit') AS limit_setting," + + " (current_setting('temp_directory') <> '')::VARCHAR AS has_spill_dir", + tables: { amounts: tempCopyOfFixture() }, + rawAmountColumns: [], + rowLimit: 10, + }); + expect(String(result.rows[0]?.[0])).toMatch(/MiB|GiB/); + expect(String(result.rows[0]?.[1])).toBe("true"); + }); + + it("refuses to truncate: over the row limit is a typed refusal", async () => { + await expect( + runQuery({ + sql: "SELECT i FROM range(50) t(i)", + tables: { amounts: tempCopyOfFixture() }, + rawAmountColumns: [], + rowLimit: 10, + }), + ).rejects.toMatchObject({ code: "policy_refused" }); + }); +}); diff --git a/tests/query/sqlGuard.test.ts b/tests/query/sqlGuard.test.ts new file mode 100644 index 0000000..47afba1 --- /dev/null +++ b/tests/query/sqlGuard.test.ts @@ -0,0 +1,137 @@ +import { describe, expect, it } from "vitest"; +import { inspectSerializedSql } from "../../src/query/sqlGuard.js"; + +// These exercise the policy against hand-written ASTs, so they run without a +// DuckDB instance. tests/query/models.test.ts covers the same policy against +// DuckDB's real parser output. + +function columnRef(name: string, alias = ""): unknown { + return { class: "COLUMN_REF", alias, column_names: [name] }; +} + +function selectNode( + selectList: unknown[], + orders: unknown[] = [], +): Record { + return { + node: { + type: "SELECT_NODE", + select_list: selectList, + modifiers: orders.length + ? [{ type: "ORDER_MODIFIER", orders: orders.map((e) => ({ expression: e })) }] + : [], + }, + }; +} + +describe("sql admission control", () => { + it("accepts a plain SELECT", () => { + expect( + inspectSerializedSql({ error: false, statements: [selectNode([columnRef("a")])] }, { + label: "query", + }), + ).toBeNull(); + }); + + it("refuses a non-SELECT statement as policy, not malformed input", () => { + const issue = inspectSerializedSql( + { error: true, error_message: "Only SELECT statements can be serialized to json!" }, + { label: "model m" }, + ); + expect(issue?.code).toBe("policy_refused"); + expect(issue?.message).toContain("model m"); + }); + + it("reports a genuine syntax error as validation", () => { + const issue = inspectSerializedSql( + { error: true, error_message: 'syntax error at or near "where"' }, + { label: "query" }, + ); + expect(issue?.code).toBe("validation"); + }); + + it("refuses more than one statement", () => { + const issue = inspectSerializedSql( + { error: false, statements: [selectNode([]), selectNode([])] }, + { label: "query" }, + ); + expect(issue?.code).toBe("policy_refused"); + expect(issue?.message).toMatch(/2 statements/); + }); + + it("refuses no statement at all", () => { + expect( + inspectSerializedSql({ error: false, statements: [] }, { label: "query" })?.code, + ).toBe("validation"); + }); + + describe("raw amount ordering", () => { + const raw = { label: "query", rawAmountColumns: ["value"] }; + + it("refuses a bare raw column", () => { + const ast = selectNode([columnRef("value")], [columnRef("value")]); + expect(inspectSerializedSql({ statements: [ast] }, raw)?.code).toBe("validation"); + }); + + it("refuses through a select-list alias", () => { + const ast = selectNode([columnRef("value", "v")], [columnRef("v")]); + const issue = inspectSerializedSql({ statements: [ast] }, raw); + expect(issue?.code).toBe("validation"); + expect(issue?.message).toContain("cp_sortkey(value)"); + }); + + it("refuses through a positional ordinal", () => { + const ast = selectNode( + [columnRef("tx_hash"), columnRef("value")], + [{ class: "CONSTANT", value: { value: 2 } }], + ); + expect(inspectSerializedSql({ statements: [ast] }, raw)?.code).toBe("validation"); + }); + + it("refuses a qualified reference", () => { + const ast = selectNode( + [columnRef("value")], + [{ class: "COLUMN_REF", column_names: ["t", "value"] }], + ); + expect(inspectSerializedSql({ statements: [ast] }, raw)?.code).toBe("validation"); + }); + + it("refuses inside a subquery", () => { + const inner = selectNode([columnRef("value")], [columnRef("value")]).node; + const outer = { node: { type: "SELECT_NODE", select_list: [columnRef("a")], modifiers: [], from_table: inner } }; + expect(inspectSerializedSql({ statements: [outer] }, raw)?.code).toBe("validation"); + }); + + it("allows an expression over the raw column", () => { + const sortKey = { class: "FUNCTION", function_name: "cp_sortkey", children: [columnRef("value")] }; + const ast = selectNode([columnRef("value")], [sortKey]); + expect(inspectSerializedSql({ statements: [ast] }, raw)).toBeNull(); + }); + + it("allows an alias bound to an expression", () => { + const sortKey = { + class: "FUNCTION", + alias: "k", + function_name: "cp_sortkey", + children: [columnRef("value")], + }; + const ast = selectNode([columnRef("value"), sortKey], [columnRef("k")]); + expect(inspectSerializedSql({ statements: [ast] }, raw)).toBeNull(); + }); + + it("allows ordering a column that is not a raw amount", () => { + const ast = selectNode([columnRef("value")], [columnRef("block_number")]); + expect(inspectSerializedSql({ statements: [ast] }, raw)).toBeNull(); + }); + + it("ignores an out-of-range ordinal rather than throwing", () => { + const ast = selectNode([columnRef("value")], [{ class: "CONSTANT", value: { value: 9 } }]); + expect(inspectSerializedSql({ statements: [ast] }, raw)).toBeNull(); + }); + + it("matches column names case-insensitively", () => { + const ast = selectNode([columnRef("VALUE")], [columnRef("VALUE")]); + expect(inspectSerializedSql({ statements: [ast] }, raw)?.code).toBe("validation"); + }); + }); +}); diff --git a/tests/rpc/client.test.ts b/tests/rpc/client.test.ts new file mode 100644 index 0000000..14e2004 --- /dev/null +++ b/tests/rpc/client.test.ts @@ -0,0 +1,116 @@ +import { describe, expect, it, afterEach } from "vitest"; +import http from "node:http"; +import type { AddressInfo } from "node:net"; +import { createRpcClient, RpcError } from "../../src/rpc/client.js"; +import { getFinalizedHead, getHeader } from "../../src/rpc/heads.js"; + +let server: http.Server | null = null; +let port = 0; + +async function startRpc( + handler: (method: string, params: unknown[]) => unknown, +): Promise { + server = http.createServer((req, res) => { + let body = ""; + req.on("data", (chunk) => (body += chunk)); + req.on("end", () => { + const parsed = JSON.parse(body) as { + id: number; + method: string; + params: unknown[]; + }; + try { + const result = handler(parsed.method, parsed.params ?? []); + res.writeHead(200, { "content-type": "application/json" }); + res.end(JSON.stringify({ jsonrpc: "2.0", id: parsed.id, result })); + } catch (err) { + res.writeHead(200, { "content-type": "application/json" }); + res.end( + JSON.stringify({ + jsonrpc: "2.0", + id: parsed.id, + error: { code: -32000, message: String(err) }, + }), + ); + } + }); + }); + await new Promise((resolve) => server!.listen(0, "127.0.0.1", resolve)); + port = (server.address() as AddressInfo).port; + return `http://127.0.0.1:${port}`; +} + +afterEach(() => { + server?.close(); + server = null; +}); + +describe("rpc client", () => { + it("returns result and parses hex quantities", async () => { + const url = await startRpc((method) => + method === "eth_getBlockByNumber" && JSON.stringify(method) + ? { + number: "0x10", + hash: "0x" + "ab".repeat(32), + parentHash: "0x" + "cd".repeat(32), + } + : null, + ); + const client = createRpcClient(url); + const header = await getHeader(client, 16); + expect(header.number).toBe(16); + expect(header.hash).toBe("0x" + "ab".repeat(32)); + expect(header.parentHash).toBe("0x" + "cd".repeat(32)); + }); + + it("maps JSON-RPC error to retryable RpcError", async () => { + const url = await startRpc((method) => { + if (method === "eth_getBlockByNumber") throw new Error("boom"); + return null; + }); + const client = createRpcClient(url); + await expect(getHeader(client, 1)).rejects.toMatchObject({ + retryable: true, + }); + }); + + it("maps connection refused to retryable RpcError", async () => { + const client = createRpcClient("http://127.0.0.1:1"); + await expect(getHeader(client, 1)).rejects.toBeInstanceOf(RpcError); + await expect(getHeader(client, 1)).rejects.toMatchObject({ + retryable: true, + }); + }); + + it("finalized head returns block", async () => { + const url = await startRpc((method, params) => { + expect(method).toBe("eth_getBlockByNumber"); + expect(params[0]).toBe("finalized"); + expect(params[1]).toBe(false); + return { + number: "0x64", + hash: "0x" + "11".repeat(32), + parentHash: "0x" + "22".repeat(32), + }; + }); + const client = createRpcClient(url); + const head = await getFinalizedHead(client); + expect(head.number).toBe(100); + }); + + it("finalized null result is non-retryable (no fallback)", async () => { + const url = await startRpc(() => null); + const client = createRpcClient(url); + await expect(getFinalizedHead(client)).rejects.toMatchObject({ + retryable: false, + }); + }); + + it("missing block header is non-retryable (block vanished)", async () => { + const url = await startRpc(() => null); + const client = createRpcClient(url); + await expect(getHeader(client, 5)).rejects.toMatchObject({ + retryable: false, + }); + }); +}); diff --git a/tests/viewer/format.test.ts b/tests/viewer/format.test.ts new file mode 100644 index 0000000..fe6c929 --- /dev/null +++ b/tests/viewer/format.test.ts @@ -0,0 +1,283 @@ +import { describe, expect, it } from "vitest"; +import { + asBigInt, + columnLabel, + compareValues, + displayAmount, + formatCell, + groupDigits, + relativeTime, + rowWindow, + scaleAmount, + shortHex, + toChartNumber, +} from "../../viewer/src/format.js"; + +const UINT256_MAX = + "115792089237316195423570985008687907853269984665640564039457584007913129639935"; +const INT256_MIN = + "-57896044618658097711785492504343953926634992332820282019728792003956564819968"; + +describe("groupDigits", () => { + it("groups in threes from the right", () => { + expect(groupDigits("1")).toBe("1"); + expect(groupDigits("1000")).toBe("1,000"); + expect(groupDigits("18600010")).toBe("18,600,010"); + }); + + it("keeps the sign outside the grouping", () => { + expect(groupDigits("-1234567")).toBe("-1,234,567"); + }); + + it("survives a full uint256 without a float in sight", () => { + expect(groupDigits(UINT256_MAX).replace(/,/g, "")).toBe(UINT256_MAX); + }); +}); + +describe("scaleAmount", () => { + it("applies token decimals exactly", () => { + expect(scaleAmount(983644533552n, 6)).toBe("983,644.533552"); + }); + + it("drops trailing fraction zeros but never rounds the integer part", () => { + expect(scaleAmount(309175980000n, 6)).toBe("309,175.98"); + expect(scaleAmount(1500000n, 6)).toBe("1.5"); + expect(scaleAmount(1000000n, 6)).toBe("1"); + }); + + it("pads a value smaller than one unit", () => { + expect(scaleAmount(1n, 18)).toBe("0.000000000000000001"); + }); + + it("handles negatives", () => { + expect(scaleAmount(-1500000n, 6)).toBe("-1.5"); + }); + + it("is a no-op at zero decimals", () => { + expect(scaleAmount(42n, 0)).toBe("42"); + }); + + it("does not lose precision on uint256", () => { + expect(scaleAmount(BigInt(UINT256_MAX), 0).replace(/,/g, "")).toBe(UINT256_MAX); + }); +}); + +describe("asBigInt", () => { + it("accepts decimal strings and rejects everything else", () => { + expect(asBigInt("123")).toBe(123n); + expect(asBigInt("-123")).toBe(-123n); + expect(asBigInt(" 7 ")).toBe(7n); + expect(asBigInt("0x1f")).toBeNull(); + expect(asBigInt("1.5")).toBeNull(); + expect(asBigInt("")).toBeNull(); + expect(asBigInt(null)).toBeNull(); + }); +}); + +describe("formatCell", () => { + const usdc = { + name: "value", + logical_type: "VARCHAR", + raw_amount: true, + decimals: 6, + symbol: "USDC", + }; + + it("scales and labels a raw amount, keeping the exact value available", () => { + const cell = formatCell("983644533552", usdc); + // Display rounds past one unit; the exact figure rides on the title. + expect(cell.text).toBe("983,644.53 USDC"); + expect(cell.exact).toContain("983,644.533552"); + expect(cell.exact).toContain("983644533552"); + expect(cell.numeric).toBe(true); + }); + + it("groups a plain integer column", () => { + expect( + formatCell(18600010, { name: "block_number", logical_type: "INTEGER" }).text, + ).toBe("18,600,010"); + }); + + it("middle-truncates a long hash but keeps the full value for the title", () => { + const hash = `0x${"ab".repeat(32)}`; + const cell = formatCell(hash, { name: "tx_hash", logical_type: "VARCHAR" }); + expect(cell.text).toContain("…"); + expect(cell.text.length).toBeLessThan(hash.length); + expect(cell.exact).toBe(hash); + }); + + it("renders null as a dash rather than an empty cell", () => { + expect(formatCell(null, { name: "x", logical_type: "VARCHAR" }).text).toBe("—"); + }); + + it("falls back to the raw text when an amount is not an integer", () => { + expect(formatCell("n/a", usdc).text).toBe("n/a"); + }); +}); + +describe("toChartNumber", () => { + it("scales in BigInt space before touching a double", () => { + expect( + toChartNumber("983644533552", { + name: "v", + logical_type: "VARCHAR", + raw_amount: true, + decimals: 6, + }), + ).toBeCloseTo(983644.533552, 5); + }); + + it("plots a value far beyond Number.MAX_SAFE_INTEGER without throwing", () => { + const plotted = toChartNumber(UINT256_MAX, { + name: "v", + logical_type: "VARCHAR", + raw_amount: true, + decimals: 18, + }); + expect(Number.isFinite(plotted)).toBe(true); + expect(plotted).toBeGreaterThan(0); + }); +}); + +describe("compareValues", () => { + it("orders the full signed range numerically, not lexicographically", () => { + const values = ["10", "9", "-1", INT256_MIN, UINT256_MAX, "0"]; + expect([...values].sort(compareValues)).toEqual([ + INT256_MIN, + "-1", + "0", + "9", + "10", + UINT256_MAX, + ]); + }); + + it("falls back to text for non-numeric values", () => { + expect(compareValues("0xbb", "0xaa")).toBe(1); + }); +}); + +describe("columnLabel", () => { + it("prefers the declared label", () => { + expect(columnLabel({ name: "value", logical_type: "VARCHAR", label: "Amount" })).toBe( + "Amount", + ); + }); + + it("humanises a snake_case column name otherwise", () => { + expect(columnLabel({ name: "block_number", logical_type: "INTEGER" })).toBe( + "Block number", + ); + }); +}); + +describe("relativeTime", () => { + const now = Date.parse("2026-09-15T12:00:00Z"); + + it("describes the recent past", () => { + expect(relativeTime("2026-09-15T10:00:00Z", now)).toMatch(/2 hours ago/); + }); + + it("returns null for missing or unparseable input", () => { + expect(relativeTime(null, now)).toBeNull(); + expect(relativeTime("not a date", now)).toBeNull(); + }); +}); + +describe("shortHex", () => { + it("leaves short values alone", () => { + expect(shortHex("0xabc")).toBe("0xabc"); + }); +}); + +// Six decimals on a figure in the billions is noise, and it wrapped the +// headline number onto two lines. Rounding is display-only: the exact figure +// and the raw integer both stay reachable. +describe("display rounding", () => { + const usdc = { + name: "v", + logical_type: "VARCHAR", + raw_amount: true, + decimals: 6, + symbol: "USDC", + }; + + it("shows two decimals once past one unit", () => { + const cell = formatCell("21972372081082838", usdc); + expect(cell.text).toBe("21,972,372,081.08 USDC"); + }); + + it("keeps the exact figure and the raw integer on hover", () => { + const cell = formatCell("21972372081082838", usdc); + expect(cell.exact).toContain("21,972,372,081.082838"); + expect(cell.exact).toContain("21972372081082838"); + }); + + it("rounds half away from zero rather than truncating", () => { + expect(displayAmount(1_999_999n, 6)).toBe("2"); + expect(displayAmount(-1_999_999n, 6)).toBe("-2"); + expect(displayAmount(1_005_000n, 6)).toBe("1.01"); + }); + + it("keeps full precision below one unit, where the fraction is the value", () => { + expect(displayAmount(1n, 18)).toBe("0.000000000000000001"); + expect(displayAmount(-1n, 18)).toBe("-0.000000000000000001"); + }); + + it("leaves a value that needs no rounding untouched", () => { + const cell = formatCell("1500000", usdc); + expect(cell.text).toBe("1.5 USDC"); + expect(cell.exact).toBe("1500000"); + }); + + it("does not round integer columns that are not amounts", () => { + expect(formatCell(18600010, { name: "b", logical_type: "BIGINT" }).text).toBe( + "18,600,010", + ); + }); +}); + +// Results ride inside the release, so a wide table would otherwise put every +// row in the DOM. The window is the part worth testing without a browser. +describe("rowWindow", () => { + const opts = { threshold: 200, overscan: 10 }; + + it("renders everything below the threshold", () => { + const w = rowWindow(150, 0, 400, 33, opts); + expect(w).toEqual({ virtual: false, first: 0, last: 150 }); + }); + + it("windows a large result and keeps the slice inside it", () => { + const w = rowWindow(5000, 0, 458, 34, opts); + expect(w.virtual).toBe(true); + expect(w.first).toBe(0); + expect(w.last).toBeLessThan(50); + }); + + it("tracks the scroll position", () => { + const w = rowWindow(5000, 34 * 2500, 458, 34, opts); + expect(w.first).toBe(2490); + expect(w.last).toBeGreaterThan(2500); + expect(w.last).toBeLessThan(2530); + }); + + it("clamps to the end of the result", () => { + const w = rowWindow(5000, 34 * 5000, 458, 34, opts); + expect(w.last).toBe(5000); + expect(w.first).toBeLessThan(5000); + }); + + it("never returns a negative start", () => { + expect(rowWindow(5000, 0, 458, 34, opts).first).toBe(0); + }); + + it("renders the overscan before the container has been measured", () => { + // viewport 0 is the first paint; rendering nothing would look empty. + const w = rowWindow(5000, 0, 0, 34, opts); + expect(w.last).toBeGreaterThan(w.first); + }); + + it("falls back to rendering everything if a row height is unknown", () => { + expect(rowWindow(5000, 0, 400, 0, opts).virtual).toBe(false); + }); +}); diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000..8113e90 --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,14 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "types": ["node"], + "strict": true, + "outDir": "dist", + "rootDir": "src", + "declaration": true, + "skipLibCheck": true + }, + "include": ["src"] +} diff --git a/viewer/index.html b/viewer/index.html new file mode 100644 index 0000000..257c292 --- /dev/null +++ b/viewer/index.html @@ -0,0 +1,12 @@ + + + + + + Chainplot + + +
+ + + diff --git a/viewer/package.json b/viewer/package.json new file mode 100644 index 0000000..dd9ba67 --- /dev/null +++ b/viewer/package.json @@ -0,0 +1,23 @@ +{ + "name": "chainplot-viewer", + "private": true, + "version": "0.1.0", + "type": "module", + "scripts": { + "build": "tsc -p tsconfig.json && vite build", + "typecheck": "tsc -p tsconfig.json", + "dev": "vite" + }, + "dependencies": { + "echarts": "^6.0.0", + "react": "^19.2.0", + "react-dom": "^19.2.0" + }, + "devDependencies": { + "@types/react": "^19.2.0", + "@types/react-dom": "^19.2.0", + "@vitejs/plugin-react": "^5.1.0", + "typescript": "^7.0.2", + "vite": "^7.1.0" + } +} diff --git a/viewer/pnpm-lock.yaml b/viewer/pnpm-lock.yaml new file mode 100644 index 0000000..fa6f87b --- /dev/null +++ b/viewer/pnpm-lock.yaml @@ -0,0 +1,1369 @@ +lockfileVersion: '9.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +importers: + + .: + dependencies: + echarts: + specifier: ^6.0.0 + version: 6.1.0 + react: + specifier: ^19.2.0 + version: 19.3.0 + react-dom: + specifier: ^19.2.0 + version: 19.3.0(react@19.3.0) + devDependencies: + '@types/react': + specifier: ^19.2.0 + version: 19.3.0 + '@types/react-dom': + specifier: ^19.2.0 + version: 19.3.0(@types/react@19.3.0) + '@vitejs/plugin-react': + specifier: ^5.1.0 + version: 5.2.0(vite@7.3.6) + typescript: + specifier: ^7.0.2 + version: 7.0.2 + vite: + specifier: ^7.1.0 + version: 7.3.6 + +packages: + + '@babel/code-frame@7.29.7': + resolution: {integrity: sha512-Aup7aUOfpbAUg2ROOJN6Iw5f9DMBlzu0mIkm/malLQFN/YQgO48wCj0Kxa3sEHJvPVFg7siR+qRInwXd2qhQKw==} + engines: {node: '>=6.9.0'} + + '@babel/compat-data@7.29.7': + resolution: {integrity: sha512-locTkQyKvwIEgBzVrn8693ebc97F2U8ZHjbXwDXJ5Fn2TCpNwTlKcaKLkdHop5c/icOFE7qt7Q9JC5hnKNa6Gg==} + engines: {node: '>=6.9.0'} + + '@babel/core@7.29.7': + resolution: {integrity: sha512-RgHBCvtjbOK2gXSNBNIkNoEc9qoVEtau3hj8gEqKQuL3HZAibKarWFEI3Lfm6EYKkLalOh8eSrj9b+ch9H/VBA==} + engines: {node: '>=6.9.0'} + + '@babel/generator@7.29.8': + resolution: {integrity: sha512-gZbepsdh3WDtgZKWL+vTPh71LSBrm/Y4/QDZBVCcYfmeTEEuoOYwlSy+G1StfJg+/Zy550u/3TATbm7qDbbMtg==} + engines: {node: '>=6.9.0'} + + '@babel/helper-compilation-targets@7.29.7': + resolution: {integrity: sha512-wem6WaBj4NaVYVdNhLPPVacES6ZJ+KBBfSkTMD3YZxbP3rm3Di85tJU5ljaUNhaOynt+Aj0xruhYuzQBt8n71g==} + engines: {node: '>=6.9.0'} + + '@babel/helper-globals@7.29.7': + resolution: {integrity: sha512-3nQVUAtvkKH9zahfWgw96Jc/uFOmjACE1kQz82E2lqWmHBgjzbNlsC22nuQTfahmWeQtTq5nQ/4Nnd2A1wj4zA==} + engines: {node: '>=6.9.0'} + + '@babel/helper-module-imports@7.29.7': + resolution: {integrity: sha512-ejHwrQQYcm9xnTivShn2IDOlIzInN34AXskvq9QicvCtEzq1Vzclu/tKF8Jq1Cg8JG2GL6/EmjgsCT7lXepE3g==} + engines: {node: '>=6.9.0'} + + '@babel/helper-module-transforms@7.29.7': + resolution: {integrity: sha512-UPUVSyXbOh627KiCIGQSgwWzGeBKLkaJ9PJEdrngIwMSzxLR4jS4+f1f1jb7VzBbg8nFLaYotvVPFCTqdrmTAg==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0 + + '@babel/helper-plugin-utils@7.29.7': + resolution: {integrity: sha512-G7sHYigPY17oO5SYWnfD/0MTBwVR781S/JI643e/JhUYgVgWE/61SoW3NH9KWUKyKq5LVh3npif99Wkt6j86Jw==} + engines: {node: '>=6.9.0'} + + '@babel/helper-string-parser@7.29.7': + resolution: {integrity: sha512-Pb5ijPrZ89GDH8223L4UP8i6QApWxs04RbPQJTeWDV0/keR2E36MeKnyr6LYmUUvqRRI+Iv87SuF1W6ErINzYw==} + engines: {node: '>=6.9.0'} + + '@babel/helper-validator-identifier@7.29.7': + resolution: {integrity: sha512-qehxGkRj55h/ff8EMaJ+cYhyaKlHIxqYDn682wQD7RNp9UujOQsHog2uS0r2vzr4pW+sXf90NeeayjcNaX3fFg==} + engines: {node: '>=6.9.0'} + + '@babel/helper-validator-option@7.29.7': + resolution: {integrity: sha512-N9ZErrD+yW5geCDtBqnOoxmR8+tNKiGuxKlDpuJxfsqpa2dFcexaziGAE/qoHLiDDreVNMupxGmSoNlyvsA3gw==} + engines: {node: '>=6.9.0'} + + '@babel/helpers@7.29.7': + resolution: {integrity: sha512-1k2lAGRMfHTcwuNYcCNUmaUffmQv8KWMfh2iJUUeRlwlwH4FdNG7mfPI10NPfLHJFThE4Tyr4mv7kTNZOiPuBg==} + engines: {node: '>=6.9.0'} + + '@babel/parser@7.29.8': + resolution: {integrity: sha512-E8lTAYNB1KW+FH+VGJuZM1ioAx2E6oVlvQFRrf5P8ZZmsiJXYAD9vTFV7yyEURNzgh1dFqMZuO6tUwcARbqFCA==} + engines: {node: '>=6.0.0'} + hasBin: true + + '@babel/plugin-transform-react-jsx-self@7.29.7': + resolution: {integrity: sha512-TL0hMc9xzy86VD31nUiwzd5otRAcyEPcsegCxolO0PvcXuH1v0kECe/UIznYFihpkvU5wg/jk4v0TTEFfm53fw==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + + '@babel/plugin-transform-react-jsx-source@7.29.7': + resolution: {integrity: sha512-06IyK09H3wi4cGbhDBwp5gUGo0IKtnYa8tyTiephirPCK6fbobVGiXMMI5zLQ4aKEYP3wZ3ArU44o+8KMrSG/Q==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + + '@babel/template@7.29.7': + resolution: {integrity: sha512-puq+Gf35oI24FeN11LkoUQFqv9uwNeWpxXZi/Ji3rRIoKAzKnxRaZ+Gkj0vKS9ZCiTESfng1N9LyOyXvo+m+Gg==} + engines: {node: '>=6.9.0'} + + '@babel/traverse@7.29.8': + resolution: {integrity: sha512-I5z7H3bf/41ktsNVLtpN0wAa336HkqIHQ5BuPLEhTkt1jVSyZpeNKIzTgEWmlxjdg81R0IgUCcaE+Ok3NvrfZg==} + engines: {node: '>=6.9.0'} + + '@babel/types@7.29.8': + resolution: {integrity: sha512-Vj1jF3cPfxg7OAfoI7QnVKLoILlm2JF9pnVHrX8qx7AHMiYWT+NDAA7jChlNgRS4WTLc/fD1lXLmPixluj+3Gg==} + engines: {node: '>=6.9.0'} + + '@esbuild/aix-ppc64@0.28.2': + resolution: {integrity: sha512-XExcO+dvLKvVtNTibSTBej1NCAbaGhWn9Ww1ZPx80qsahhPFe/8jgWP0IchNe0F3HwkU7n8ejhH8bjonqht8mQ==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [aix] + + '@esbuild/android-arm64@0.28.2': + resolution: {integrity: sha512-5YfKeeI8qWfBZIX+u2xZC3Zlb3Os/gLS2sbEKM+I4ZOcsWmHS2WLysCcQZDAFRslDUU5Oiq44gf6PYN1vGwG5A==} + engines: {node: '>=18'} + cpu: [arm64] + os: [android] + + '@esbuild/android-arm@0.28.2': + resolution: {integrity: sha512-kXXoiPVVGQcnIYGOeaovwOURpniDBpSq4A03qkQ+BMQqtGG6HYap3xne9C1O1yo4TR3qxlCX5IqqmX6fFo2Lqg==} + engines: {node: '>=18'} + cpu: [arm] + os: [android] + + '@esbuild/android-x64@0.28.2': + resolution: {integrity: sha512-O387ite7SzUyCcy3JQX4P4bLtEA7bLLkx+esve5JHnyYfNTxcVpXZo9jhdB0lTKN44gztELTdU7nS8Nr16Fs1Q==} + engines: {node: '>=18'} + cpu: [x64] + os: [android] + + '@esbuild/darwin-arm64@0.28.2': + resolution: {integrity: sha512-n4KqkOQrraxHJcgjM1RvwbigfQKIKJVpM7xp+KsxiyUSrRdIXnt73VhrPAx0fV44hgfmIVKjxMN9J1t5jySVkw==} + engines: {node: '>=18'} + cpu: [arm64] + os: [darwin] + + '@esbuild/darwin-x64@0.28.2': + resolution: {integrity: sha512-uq6suIWYP37qzGddBKPw5QEQPi6HiLGsO7UmkpfyaYNQ3D+rN6w6WfwH+nuqcGXWvawGwxOEroO4YGnFh95azw==} + engines: {node: '>=18'} + cpu: [x64] + os: [darwin] + + '@esbuild/freebsd-arm64@0.28.2': + resolution: {integrity: sha512-n+I0BTSRIoy+d6RPKnEVwql5UwBJolytvY4mAOIEJorKlqgPII8ix6slVVrfZ5Tnj7glIZvloylbB/EJPMWEXw==} + engines: {node: '>=18'} + cpu: [arm64] + os: [freebsd] + + '@esbuild/freebsd-x64@0.28.2': + resolution: {integrity: sha512-78XJTJkvPs0kz2w61301PJjXl4g7q3JqiYMZ/M/yVI73EHBrCRTgkhu9oqG7vPqq+a/yadEW8aD+agKlk5xrmg==} + engines: {node: '>=18'} + cpu: [x64] + os: [freebsd] + + '@esbuild/linux-arm64@0.28.2': + resolution: {integrity: sha512-pW4AC0P3it8c7do9MVM4p51FzHzdM/TZrerurgRcHJ2WTa1VQ1CIq18xncfpBJw4ojkiZZrKW2yIBWBP92j6Ug==} + engines: {node: '>=18'} + cpu: [arm64] + os: [linux] + + '@esbuild/linux-arm@0.28.2': + resolution: {integrity: sha512-XlDnu2q5yoqems+xay6wSAcg9DDD7K9RLKZEBOMZm3ckNpJBvOX20tSfby8KfrrhINDyv9V2YVZKY/SpoGJI8w==} + engines: {node: '>=18'} + cpu: [arm] + os: [linux] + + '@esbuild/linux-ia32@0.28.2': + resolution: {integrity: sha512-CYbnj78HsIeA+DhgUKgFCfvNsTHFhMMrinUrMZpDXJXKN8T3XViTZ/+wtHeVxEWY8ewSzTFN+nRmSwO2tZaLUQ==} + engines: {node: '>=18'} + cpu: [ia32] + os: [linux] + + '@esbuild/linux-loong64@0.28.2': + resolution: {integrity: sha512-buwkd8nsph4R+ajRvw0qM5Hja/TXQow3ptzWO2EbG/cqcIkHloRrdlBtQlshyYGTNFvfkfJ5tpPLVkY4DtsPfQ==} + engines: {node: '>=18'} + cpu: [loong64] + os: [linux] + + '@esbuild/linux-mips64el@0.28.2': + resolution: {integrity: sha512-ZVykbDyk7519VwiNb9Lcj9m8XM6v5V9uKPvrEMkkEedVewf+0itkhahp4HDpgERXhwLRpWFypsGbG/J8s0QjJA==} + engines: {node: '>=18'} + cpu: [mips64el] + os: [linux] + + '@esbuild/linux-ppc64@0.28.2': + resolution: {integrity: sha512-CAXl+Dtd9UUuJd8pKKdwh6MLm3MUMiqMPmhZ3tTSXPqfyQ3vDl6R5hZdZ/kYojK4ofXtdfSv1tFq8XzWx3heNQ==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [linux] + + '@esbuild/linux-riscv64@0.28.2': + resolution: {integrity: sha512-GeXCej4IQtU1B+QlDV8W/RRvbzI3O/Stss+/bCXv4lZls5WGRtu2a+3JkA3i4qIUlMXpcHebWpF8AkJhATowuA==} + engines: {node: '>=18'} + cpu: [riscv64] + os: [linux] + + '@esbuild/linux-s390x@0.28.2': + resolution: {integrity: sha512-3H1weTYZPxt/WOhByszQZybS9w5lKzUn1FDMsgEChbHWQwHYQQRfBxgCcZvPhjHfKyJjIievvMmEUawJrdY9Dg==} + engines: {node: '>=18'} + cpu: [s390x] + os: [linux] + + '@esbuild/linux-x64@0.28.2': + resolution: {integrity: sha512-4xTZr1FUmSoQW4XIWmit3tzQrUTZM+N3P0XV8xROKYF50XfI7xeO90+1bZvNwxIufQ9hDQVRJH5YhgPVF8A/HQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [linux] + + '@esbuild/netbsd-arm64@0.28.2': + resolution: {integrity: sha512-sSATRjPeDBg3pdgHoQfoYBob11Kk1FGa9lui5RIHZCoCkJa9QKlvl3/vKz2usCmYYjs7ymJR/2Nnsqe+Hjt5nw==} + engines: {node: '>=18'} + cpu: [arm64] + os: [netbsd] + + '@esbuild/netbsd-x64@0.28.2': + resolution: {integrity: sha512-lqnzCV+mM0gIADaKihiCg6ifgfU2L3h5E33rNQBN1Y4MaVGnzryzmvvf7UHxprpQdE8hpqLolJ9Rl+SkIRDpyw==} + engines: {node: '>=18'} + cpu: [x64] + os: [netbsd] + + '@esbuild/openbsd-arm64@0.28.2': + resolution: {integrity: sha512-AL2qJILH7lNjrDmCQDvdxMfAUIv8KMNZOvrwAQ8i8//ntL9FflhOyMJ8OZSMBb8/AWXe3/5v5S20y3zCoZWKoQ==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openbsd] + + '@esbuild/openbsd-x64@0.28.2': + resolution: {integrity: sha512-QtiuPytchRyC4rwUKhexJdQKvDuZ6hWloi3igqPQNUJCS1/v9EiO3UTOXR6A3FoMo4fnAKbWJdqaIwhOzh8qEw==} + engines: {node: '>=18'} + cpu: [x64] + os: [openbsd] + + '@esbuild/openharmony-arm64@0.28.2': + resolution: {integrity: sha512-WkhYDmpTjLvGlScA1rwjRUmhl4k8oXR3cIbtqWmELgU/dFeHHlEllxDvdWcNJV9rbzCexB5vz8gtNewWLgCT7Q==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openharmony] + + '@esbuild/sunos-x64@0.28.2': + resolution: {integrity: sha512-GPMSkTOtMnv2U2F8gxe4Io6qmVs+YKyp832Etqqxr0hFngmXQ3rzwytelm3GIn7T4VviRUlf3sOgBOiTdvaf7g==} + engines: {node: '>=18'} + cpu: [x64] + os: [sunos] + + '@esbuild/win32-arm64@0.28.2': + resolution: {integrity: sha512-PIhhEkE9uPBleRBrQEJpUn7MBnibZzbGzYWPmY3x+YoVg/95zbjB4CxPPOQ8l5tYYM4mMaCthF8/1DIfBQQyWQ==} + engines: {node: '>=18'} + cpu: [arm64] + os: [win32] + + '@esbuild/win32-ia32@0.28.2': + resolution: {integrity: sha512-YmJbfTlvU7Sdn9BB+4PRES4oB6pxgS37MAONj+hBr/cpXS1aBPKXxNnDbu+QCWPj0o9dgyxeq79g6c5P8KeuYA==} + engines: {node: '>=18'} + cpu: [ia32] + os: [win32] + + '@esbuild/win32-x64@0.28.2': + resolution: {integrity: sha512-5ebpxr3nWMzrL/rnUI755Jkuee0bHL/Gq0WTF9lvcpv73wAp5eu8MfBUgWK9bhWvZjj7yX8etf/8tI8Ney695g==} + engines: {node: '>=18'} + cpu: [x64] + os: [win32] + + '@jridgewell/gen-mapping@0.3.13': + resolution: {integrity: sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==} + + '@jridgewell/remapping@2.3.5': + resolution: {integrity: sha512-LI9u/+laYG4Ds1TDKSJW2YPrIlcVYOwi2fUC6xB43lueCjgxV4lffOCZCtYFiH6TNOX+tQKXx97T4IKHbhyHEQ==} + + '@jridgewell/resolve-uri@3.1.2': + resolution: {integrity: sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==} + engines: {node: '>=6.0.0'} + + '@jridgewell/sourcemap-codec@1.6.0': + resolution: {integrity: sha512-T7jf+5zgsZHwNJ4lvQ7/aezbyk0nNX+zJVWpmHA7VYsEx7a7qr5Rg5IbtJFqkgze5Y2sruq1RUY8Q837Od7iFw==} + + '@jridgewell/trace-mapping@0.3.31': + resolution: {integrity: sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==} + + '@napi-rs/lzma-linux-x64-gnu@1.5.1': + resolution: {integrity: sha512-oTXEIha4SsuXdTA4Iyskj0kpdx2yVXdhd75c2v3xGrHFfVMsbhTPZU/nMPL4sWKo4pBHm3aucLaqGlF696dTyQ==} + engines: {node: ^22.20 || ^24.12 || >=25} + cpu: [x64] + os: [linux] + libc: [glibc] + + '@rolldown/pluginutils@1.0.0-rc.3': + resolution: {integrity: sha512-eybk3TjzzzV97Dlj5c+XrBFW57eTNhzod66y9HrBlzJ6NsCrWCp/2kaPS3K9wJmurBC0Tdw4yPjXKZqlznim3Q==} + + '@rollup/rollup-android-arm-eabi@4.63.2': + resolution: {integrity: sha512-Xa6RDoWa+hNiX6PgsljlH6W75RaONx3y6PVlbLhkEWW+GaPQ3dP5gwbL/erAzQHWwkvW5UxdD5l87Qx2FAQ/4A==} + cpu: [arm] + os: [android] + + '@rollup/rollup-android-arm64@4.63.2': + resolution: {integrity: sha512-vNASxsghMfQ5s+v3PrpnJd+ryL/26lxCCaGI+sDJ7VzmHiYXIrrVltsDhaawxLM1WcoMU2oYlbPHLaYQtBzhcg==} + cpu: [arm64] + os: [android] + + '@rollup/rollup-darwin-arm64@4.63.2': + resolution: {integrity: sha512-0dWDjmlrpZAgjPD/aPzUDhBW8APLRjAni5bOrM76wiiZm+E+KTMVKNhAzaTBohz8UyO2fKNAl0+fygbe2HZXOA==} + cpu: [arm64] + os: [darwin] + + '@rollup/rollup-darwin-x64@4.63.2': + resolution: {integrity: sha512-N58uktcwzk3+qT4KHEuNdIxX1N01RWrkfVoml69EAbSaNDL+sbNVLx2RMl4Qd23lpA0fgPvyh5hHb4weD5WKmg==} + cpu: [x64] + os: [darwin] + + '@rollup/rollup-freebsd-arm64@4.63.2': + resolution: {integrity: sha512-HWF2zH8EAp2scWRpt2PGe6iUGz7zi04waXsdRr3zb4DWCk2ImIo5FZu0jjmD53nP/DGSvnW0e7/1ToCNZs2lZw==} + cpu: [arm64] + os: [freebsd] + + '@rollup/rollup-freebsd-x64@4.63.2': + resolution: {integrity: sha512-MkvcwHMnzPSMOQEwB6wHnLzmc+hT8BGc5bW/Mhmjjgx3wbj6VBnlc47XsK74kD0K9MikFfXpQqyz4NUXaUW62A==} + cpu: [x64] + os: [freebsd] + + '@rollup/rollup-linux-arm-gnueabihf@4.63.2': + resolution: {integrity: sha512-xe1bCKPJaKsD0tfd7Rb6bGfUogJTpKbTEEthsfdb7hTfTRNJVQTdirabQx0o6ERVba/smkM720soMY+0QnrlSQ==} + cpu: [arm] + os: [linux] + libc: [glibc] + + '@rollup/rollup-linux-arm-musleabihf@4.63.2': + resolution: {integrity: sha512-yOM7LdK0p6gk6+Q773OEwtlsikT1TL3yMmYsTtRlDRPha5vV2DC5x7LqRWDr6f3cSYNMKVqxzffXv8ivxNBIFQ==} + cpu: [arm] + os: [linux] + libc: [musl] + + '@rollup/rollup-linux-arm64-gnu@4.63.2': + resolution: {integrity: sha512-qiWuJJV3DybA2IfzvRimeKXGrGuVPv1zobSY/26KnP3HbV0VcNb3ECzgvtbvF3xjSMkcooou6HASXZuLdjnhpQ==} + cpu: [arm64] + os: [linux] + libc: [glibc] + + '@rollup/rollup-linux-arm64-musl@4.63.2': + resolution: {integrity: sha512-akcZquRzCY/KpUoZAMBhGf7oi4LmXq1BzRA5CPAC3rkUf28Y/sAYV3jSL+JKd7cwEyFvR5G0XVZ0gaMedP+60A==} + cpu: [arm64] + os: [linux] + libc: [musl] + + '@rollup/rollup-linux-loong64-gnu@4.63.2': + resolution: {integrity: sha512-fNwYHrPyYyxauPzX/cpYw8Z7LQpp+DGA0KCoswA0aVFBpmdMil9XgjB8V3Ny64Ihu797+GKcuJqnsOKEmor7fA==} + cpu: [loong64] + os: [linux] + libc: [glibc] + + '@rollup/rollup-linux-loong64-musl@4.63.2': + resolution: {integrity: sha512-XfvsgzR7DZqREdst7K1Mj3ilSUM5xLAHJcIMDFPKdxTs9q5VHOT8aMA+a683fqBu7DQl8+Sd9HCsQYL8EMY9qA==} + cpu: [loong64] + os: [linux] + libc: [musl] + + '@rollup/rollup-linux-ppc64-gnu@4.63.2': + resolution: {integrity: sha512-Pp7gVZggEFlbcuztay+/U0gVG9S1XAh8i7I1Re/htbAzo43P5wHZHw6pTyzotISqlKohoh9RpIfnOz3RbemK1w==} + cpu: [ppc64] + os: [linux] + libc: [glibc] + + '@rollup/rollup-linux-ppc64-musl@4.63.2': + resolution: {integrity: sha512-zkgL2xff6i7u5hau/m6FGeS8gRkLEdgLw522WGmdWWlLd9btmNl3S80mcEjtGq+kvgUekQ3+BOYLLLcPlS2LIA==} + cpu: [ppc64] + os: [linux] + libc: [musl] + + '@rollup/rollup-linux-riscv64-gnu@4.63.2': + resolution: {integrity: sha512-qOheJomrkVCbbHFJ7L3J97cnhfogKqguAQphv26+3ZsAQIF1L19b+dArl//s8rjJHJLz9byykyM8NBP4nmSa1g==} + cpu: [riscv64] + os: [linux] + libc: [glibc] + + '@rollup/rollup-linux-riscv64-musl@4.63.2': + resolution: {integrity: sha512-XlxLD54wQhH3FciCgMofxBw27NzUe818gJH410qWvc41UT0ZFcgxVjyX5/EK8MPTupjeVWqN5oy+9pCA9mqfCA==} + cpu: [riscv64] + os: [linux] + libc: [musl] + + '@rollup/rollup-linux-s390x-gnu@4.63.2': + resolution: {integrity: sha512-vdryWeRb2bLJZf0Fv/W8se6nvsHe2PkTCxV0meheK3nQE+G90VCJcke51Miy1yQRsfm2uqIyjXOu4wmUzbTtkQ==} + cpu: [s390x] + os: [linux] + libc: [glibc] + + '@rollup/rollup-linux-x64-gnu@4.63.2': + resolution: {integrity: sha512-bcq2h2pkKmH2po4cZV8VWzO4lL40STyu/nLoFpYMQp9C2tCVNTdcVv86MwSsn3D5s1FBe2Ty1atqvVAUTMimNg==} + cpu: [x64] + os: [linux] + libc: [glibc] + + '@rollup/rollup-linux-x64-musl@4.63.2': + resolution: {integrity: sha512-EGoo5DMVMRkTId8fuTDaoxVlR5ZTsKULUezRjd9gCw5eeY+DjCvDpZAOlNUvKPGX+7rS1RWx6j+yOpNPx0cUgQ==} + cpu: [x64] + os: [linux] + libc: [musl] + + '@rollup/rollup-openbsd-x64@4.63.2': + resolution: {integrity: sha512-MErl12k7BFHZG1TI9QF/3lSSZARzq9KgNy/FjnqFMCkv+N4RSSzoUCA5h2mqHX4Mox3WaTVKblyzhQ1zRb2ZuQ==} + cpu: [x64] + os: [openbsd] + + '@rollup/rollup-openharmony-arm64@4.63.2': + resolution: {integrity: sha512-ILs8k07Wh4p0PsNY4wYLEaXZKMOpVhrG5QDB0yHhGhuzOfDlnyHN6sflL4El/MpUP1y8uY2lUZrv4oBS6pTT3g==} + cpu: [arm64] + os: [openharmony] + + '@rollup/rollup-win32-arm64-msvc@4.63.2': + resolution: {integrity: sha512-hKgB3nz/TKD3Wv78XEsyXzQsNjvhOHmwKQTvXADGOyU/cIClZDO7DsoggbdmJDPGp5V80tA3Vfv61PaKTLH3LA==} + cpu: [arm64] + os: [win32] + + '@rollup/rollup-win32-ia32-msvc@4.63.2': + resolution: {integrity: sha512-T4wf1mudIDxN8Q/CWIBJC1u5gQUc+r5mPvlwoSbIvNkyVTP2TAFeobEmst5AQ4gMyAz4sSByVdoTDfvTmGK/8g==} + cpu: [ia32] + os: [win32] + + '@rollup/rollup-win32-x64-gnu@4.63.2': + resolution: {integrity: sha512-tC3IY7qoaD9Ll3/8WJQn49j5V2f/NuI9S41NOE2iM5MPs3sPIvOkVToLcz/7Bz4pyF7PSvrtwu8I/pUrGOSecQ==} + cpu: [x64] + os: [win32] + + '@rollup/rollup-win32-x64-msvc@4.63.2': + resolution: {integrity: sha512-6NHnk/K3eq2ZFYcU1X8g67s9qIJRCOTT92gwLMVBp08dB2uuuwI1/Q/empzL2Bfr2f2WRLJVwpp90RmacQyFkw==} + cpu: [x64] + os: [win32] + + '@types/babel__core@7.20.5': + resolution: {integrity: sha512-qoQprZvz5wQFJwMDqeseRXWv3rqMvhgpbXFfVyWhbx9X47POIA6i/+dXefEmZKoAgOaTdaIgNSMqMIU61yRyzA==} + + '@types/babel__generator@7.27.0': + resolution: {integrity: sha512-ufFd2Xi92OAVPYsy+P4n7/U7e68fex0+Ee8gSG9KX7eo084CWiQ4sdxktvdl0bOPupXtVJPY19zk6EwWqUQ8lg==} + + '@types/babel__template@7.4.4': + resolution: {integrity: sha512-h/NUaSyG5EyxBIp8YRxo4RMe2/qQgvyowRwVMzhYhBCONbW8PUsg4lkFMrhgZhUe5z3L3MiLDuvyJ/CaPa2A8A==} + + '@types/babel__traverse@7.28.0': + resolution: {integrity: sha512-8PvcXf70gTDZBgt9ptxJ8elBeBjcLOAcOtoO/mPJjtji1+CdGbHgm77om1GrsPxsiE+uXIpNSK64UYaIwQXd4Q==} + + '@types/estree@1.0.9': + resolution: {integrity: sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==} + + '@types/react-dom@19.3.0': + resolution: {integrity: sha512-ZI7bU42mZXXKHn/qNLEw2IrbiINU7X5+vfgdixBHkCNpYWXjKgfQ/P+uyGb5CjOLB9UcnTeg3rylQtV2hym44Q==} + peerDependencies: + '@types/react': ^19.3.0 + + '@types/react@19.3.0': + resolution: {integrity: sha512-N0rFCuH9YoxG9/m61l9MfpJKfmLOVU0em7ipIz6TRgSSkvReLB9vL85GB+yr8Bs5leqpvg96JSwF4ZS1s4viQg==} + + '@typescript/typescript-aix-ppc64@7.0.2': + resolution: {integrity: sha512-MTKKkWB7p/0E9xi1d1tHtZ5PiLkGEMIq88pK2CubZjOsLtYTLqhgIgi6zepFa+9GHZ6h05NMCkQxGKiPXMxXtQ==} + engines: {node: '>=16.20.0'} + cpu: [ppc64] + os: [aix] + + '@typescript/typescript-darwin-arm64@7.0.2': + resolution: {integrity: sha512-gowzar9MwS/aRWp6f3a4KUqzRjAZjOsmGNCM6LcTgXum+dBfgsBVMN+AgvOCCbguXyick6LJhpBszxMebJ8syA==} + engines: {node: '>=16.20.0'} + cpu: [arm64] + os: [darwin] + + '@typescript/typescript-darwin-x64@7.0.2': + resolution: {integrity: sha512-SZ9xZInqApNlNGc9s0W1VSsktYSOe9cFqNOIqmN1Gs8SmkjKZYFt017G4VwPxASInODuAdbTW7sXiFUf893RgA==} + engines: {node: '>=16.20.0'} + cpu: [x64] + os: [darwin] + + '@typescript/typescript-freebsd-arm64@7.0.2': + resolution: {integrity: sha512-W5NH4y/J0plIIS5b2xvTEkU7JFxyqdMAOgf+Ilhl0vHQXKO5dZoxd+C/jEtq56c4F3wk71RB4BMRQ2XdI+bwYQ==} + engines: {node: '>=16.20.0'} + cpu: [arm64] + os: [freebsd] + + '@typescript/typescript-freebsd-x64@7.0.2': + resolution: {integrity: sha512-UMGDx5sTpzNw3WiPebH7l90IWfJggEd+egHt/q6p7/Cm3zqoV7VxkGXt+3DxPIw8CcmvAB0j3sVVfbhX+M4Tpw==} + engines: {node: '>=16.20.0'} + cpu: [x64] + os: [freebsd] + + '@typescript/typescript-linux-arm64@7.0.2': + resolution: {integrity: sha512-Qh4eU4/y3yDjnfjjyPYihMj5/ODIlmt+Bzu17OI+fiSRDW57QmU5SiN63exPRNJPKUzcc1INa1NXdrJ+MqHjUQ==} + engines: {node: '>=16.20.0'} + cpu: [arm64] + os: [linux] + + '@typescript/typescript-linux-arm@7.0.2': + resolution: {integrity: sha512-gffT3xPz9sR7j/YJExkyPntrI0P2EP9XbOyWzth2/Gs0RstK+90RBcO0ncXoXy/beYll1SXw846Nf2zdnEz0QQ==} + engines: {node: '>=16.20.0'} + cpu: [arm] + os: [linux] + + '@typescript/typescript-linux-loong64@7.0.2': + resolution: {integrity: sha512-uEHck9i8hoAzXPiYRib1O7miOnz23SxIeVl6F4LXox+qov1K35jHcEW6VHKvZI+pyvl7fZEP4MCU5LYvIq1GuQ==} + engines: {node: '>=16.20.0'} + cpu: [loong64] + os: [linux] + + '@typescript/typescript-linux-mips64el@7.0.2': + resolution: {integrity: sha512-R4KvAMnE43W5Qeqb0Ly56O3mWMWIAgsMyz36DCaycd5nbg/9kzm0liw3JocfRqyJY0KPmzFjbswozXyW0DnIYA==} + engines: {node: '>=16.20.0'} + cpu: [mips64el] + os: [linux] + + '@typescript/typescript-linux-ppc64@7.0.2': + resolution: {integrity: sha512-DORx5b3sd/4S7eayxm4FQv+A7CrkUIGRaHiwI8oiHTAI1fAPWhF4J0vAlkC8biAlHSVVwxMQ3tjZ2/DVbnQiiA==} + engines: {node: '>=16.20.0'} + cpu: [ppc64] + os: [linux] + + '@typescript/typescript-linux-riscv64@7.0.2': + resolution: {integrity: sha512-wf0jqEDOjrPRnKwYRyyJDRo11KMbvMFrU+q4zqKyChODBzvlkbhNQfKvLxQCcwTpdDaXSHZTVuh0JoCrKCUMHQ==} + engines: {node: '>=16.20.0'} + cpu: [riscv64] + os: [linux] + + '@typescript/typescript-linux-s390x@7.0.2': + resolution: {integrity: sha512-IkwJc3L7yhytWd/ewjyxNDfOmswCm9GWMJT/ue/dU4aZNbwZeYAetq42VyLmsmSjvoX7z74X6ZaYCtzAr0EuGw==} + engines: {node: '>=16.20.0'} + cpu: [s390x] + os: [linux] + + '@typescript/typescript-linux-x64@7.0.2': + resolution: {integrity: sha512-EYdf2cNg7rgCWJnxCdJ+F3V39O8ihb37eHAu1LK8oAFizgTQbPOK7zHHXbPt8rX24COqODXeI3sIf0fCXG7H/A==} + engines: {node: '>=16.20.0'} + cpu: [x64] + os: [linux] + + '@typescript/typescript-netbsd-arm64@7.0.2': + resolution: {integrity: sha512-+polYF4MF04aPpO5FTkHran9yUQDSXqy5GiSDKpsll5jy3l3+g9QLhpf39T+ePtefhXLOGrLl0QIjkQP6VnelA==} + engines: {node: '>=16.20.0'} + cpu: [arm64] + os: [netbsd] + + '@typescript/typescript-netbsd-x64@7.0.2': + resolution: {integrity: sha512-8YIT0EHM/3dq10ZOVF/A7pc/YSMtbcecct4rWtexrnSCHOPcpC2KTLXfTCR6vDpnSiY12heNb1GiN/wu+T/FyA==} + engines: {node: '>=16.20.0'} + cpu: [x64] + os: [netbsd] + + '@typescript/typescript-openbsd-arm64@7.0.2': + resolution: {integrity: sha512-APT8+ClYnuYm1u9+kgGXoMj2VzWzcymwh2gNSQVySHfkRDGOTVkoWLjCmOQSaO+PoqQ57B0flRp9SA+7GnnkzQ==} + engines: {node: '>=16.20.0'} + cpu: [arm64] + os: [openbsd] + + '@typescript/typescript-openbsd-x64@7.0.2': + resolution: {integrity: sha512-yX7s+Q0Dln0Dt9tEzZsAjXXR/+ytBM7AlglaqyeMPxQszJ1JhlJdZ6jLA+IzldHtflX81em7lDao1xXu+aRRkg==} + engines: {node: '>=16.20.0'} + cpu: [x64] + os: [openbsd] + + '@typescript/typescript-sunos-x64@7.0.2': + resolution: {integrity: sha512-dLJDGaLZ1D4HPQn62u1n8mBDkJREwMsAkCdkwd4Ieqw+x3TUyTsqY0YiBCtE6H6OzzgGk3iuZ3vFWRS+E8/d1g==} + engines: {node: '>=16.20.0'} + cpu: [x64] + os: [sunos] + + '@typescript/typescript-win32-arm64@7.0.2': + resolution: {integrity: sha512-Gyl1Vy6OsWesLzmq+EP0Fb7b4Nid5232AvcA2SFcdYreldpNtYFFofPjnt62y9hQy7VTaZp65ICJjuAQRaVcIQ==} + engines: {node: '>=16.20.0'} + cpu: [arm64] + os: [win32] + + '@typescript/typescript-win32-x64@7.0.2': + resolution: {integrity: sha512-0BQ3HkAHHlKLSp1qRvf3SUhGpGsDuhB/jgFw75guyqbxJqEaS0Cw/VFO8i2nHglJUzQCRtMMR/IBAKE3ETMC4g==} + engines: {node: '>=16.20.0'} + cpu: [x64] + os: [win32] + + '@vitejs/plugin-react@5.2.0': + resolution: {integrity: sha512-YmKkfhOAi3wsB1PhJq5Scj3GXMn3WvtQ/JC0xoopuHoXSdmtdStOpFrYaT1kie2YgFBcIe64ROzMYRjCrYOdYw==} + engines: {node: ^20.19.0 || >=22.12.0} + peerDependencies: + vite: ^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0 + + baseline-browser-mapping@2.11.22: + resolution: {integrity: sha512-pWc4w51fBFd7mav43/zKRC+RI6f4yfzQoVlfvE8dECePyfkn1bzLp01Fj0QACcyCZyFhiEMyD2qScfKRWgWibA==} + engines: {node: '>=6.0.0'} + hasBin: true + + browserslist@4.28.9: + resolution: {integrity: sha512-EWazOblFYUvlGZcfGhPUPmYh3nikUxBVb+y9MJun5f3hBi812X+8MSQTujLBtgK3cf51fJWbWfOjyeO954d+Eg==} + engines: {node: ^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7} + hasBin: true + + caniuse-lite@1.0.30001810: + resolution: {integrity: sha512-TITQPUkaz+aVk5GL6NhOdwk1aEaNTSDPsGFWrTuhKGtjTF70jL/Oht2W4c6rXUe5fu7Ie19VIahAXHIIiWWNeg==} + + convert-source-map@2.0.0: + resolution: {integrity: sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==} + + csstype@3.2.3: + resolution: {integrity: sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==} + + debug@4.4.3: + resolution: {integrity: sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==} + engines: {node: '>=6.0'} + peerDependencies: + supports-color: '*' + peerDependenciesMeta: + supports-color: + optional: true + + echarts@6.1.0: + resolution: {integrity: sha512-q0yaFPggC9FUdsWH4blavRWFmxdrIodbkoKNAjJudAI6CA9gNPxHtV2RcZNEepZVlk4yvBYkOkbk6HIVpIyHZA==} + + electron-to-chromium@1.5.427: + resolution: {integrity: sha512-n14zb3FdsChZ2BNobqNHAJMcP3ifFv4paox2LvCrfVAQcqGiSURgbJl+PfMpHVCNFkStnNc+RRVtPBTVW5PDgw==} + + esbuild@0.28.2: + resolution: {integrity: sha512-HKVLS8dvII+xoKW9kmqxbRKrnWEXfJJr/FZhhJmiqIB0e053QNYFqOBouTMO/k5sID4MvCiUCvv8b9M4h32wIA==} + engines: {node: '>=18'} + hasBin: true + + escalade@3.2.0: + resolution: {integrity: sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==} + engines: {node: '>=6'} + + fdir@6.5.0: + resolution: {integrity: sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==} + engines: {node: '>=12.0.0'} + peerDependencies: + picomatch: ^3 || ^4 + peerDependenciesMeta: + picomatch: + optional: true + + fsevents@2.3.3: + resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} + engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} + os: [darwin] + + gensync@1.0.0-beta.2: + resolution: {integrity: sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==} + engines: {node: '>=6.9.0'} + + js-tokens@4.0.0: + resolution: {integrity: sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==} + + jsesc@3.1.0: + resolution: {integrity: sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA==} + engines: {node: '>=6'} + hasBin: true + + json5@2.2.3: + resolution: {integrity: sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==} + engines: {node: '>=6'} + hasBin: true + + lru-cache@5.1.1: + resolution: {integrity: sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==} + + ms@2.1.3: + resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + + nanoid@3.3.19: + resolution: {integrity: sha512-Y2tUNy4ouw6tq5oDSKeQYGOyhkUBhNOcGV/02KC+6kd9eDGqdZd++mjMiIDilrBYvjEnCYvVtsuHCuP+okSfug==} + engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} + hasBin: true + + node-releases@2.0.55: + resolution: {integrity: sha512-mIrE/Cw9y+9Au6dS5vDKDhQza9YvG6w+ZrS6X+ZzA7yFW/soAeaups4Qzn1bL6g5FVy8WtP79+0j82oPIbqRjQ==} + engines: {node: '>=18'} + + picocolors@1.1.1: + resolution: {integrity: sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==} + + picomatch@4.0.7: + resolution: {integrity: sha512-qcJu88Q2IWqJsDD529JKMdwGm/dvInW4HvQnRwiH9JtihJvzGOscDtHE3x1pBKeUOTysQ8kVmLnJ2kJu7yhcGA==} + engines: {node: '>=12'} + + postcss@8.5.28: + resolution: {integrity: sha512-RRuzqDtt5Y9h3quz5hWhK+TPnsmVs6WwSU6LkJMeY4HstUEDuYTG8UJSdawMRzmzAtV+KEoG8N3Qg2qLy5vM/A==} + engines: {node: ^10 || ^12 || >=14} + + react-dom@19.3.0: + resolution: {integrity: sha512-JDk8dgif51OjFoDE70+OT9ICyYr+69HlmihNwp1+Nsfbna3t5sIiCa9ZJktDmQ4/1b/rn26hIAR2uYXDMr5r0Q==} + peerDependencies: + react: ^19.3.0 + + react-refresh@0.18.0: + resolution: {integrity: sha512-QgT5//D3jfjJb6Gsjxv0Slpj23ip+HtOpnNgnb2S5zU3CB26G/IDPGoy4RJB42wzFE46DRsstbW6tKHoKbhAxw==} + engines: {node: '>=0.10.0'} + + react@19.3.0: + resolution: {integrity: sha512-E8LUcbtBWt20bbl2YoHfx4ZDBdxVTfOKtCZn9cDSJ4l6/nuoApcpIBcj47t2wZoVX8g2ZHuMHbiShgCR1T5Sog==} + engines: {node: '>=0.10.0'} + + rollup@4.63.2: + resolution: {integrity: sha512-l5eyksV4tPBj6lJyEa37YzIOCSOV7lkZzEHUdpjWZbtD7wTcFYmEYXSgm5bT4vV+dZLb9rBG1W9GROOG4NS4Ew==} + engines: {node: '>=18.0.0', npm: '>=8.0.0'} + hasBin: true + + scheduler@0.28.0: + resolution: {integrity: sha512-juorfCmIkIw8tT+p5BXSm6PJjQF/ycEYmKyzURCIt/RaZIhL+PulbQ9Yu2z1HdOJDdqDTlxA1+xKBmHXJsczAw==} + + semver@6.3.1: + resolution: {integrity: sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==} + hasBin: true + + source-map-js@1.2.1: + resolution: {integrity: sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==} + engines: {node: '>=0.10.0'} + + tinyglobby@0.2.17: + resolution: {integrity: sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==} + engines: {node: '>=12.0.0'} + + tslib@2.3.0: + resolution: {integrity: sha512-N82ooyxVNm6h1riLCoyS9e3fuJ3AMG2zIZs2Gd1ATcSFjSA23Q0fzjjZeh0jbJvWVDZ0cJT8yaNNaaXHzueNjg==} + + typescript@7.0.2: + resolution: {integrity: sha512-8FYau96o3NKOhbjKi/qNvG/W5jhzxkbdm5sj9AbZ/5T5sWqn3hJgLfGx27sRKZWTvyzCP8dLRBTf5tBTSRVUNA==} + engines: {node: '>=16.20.0'} + hasBin: true + + update-browserslist-db@1.3.3: + resolution: {integrity: sha512-pJ2sYawQS0R/WI928Gj5GlPhTGzbMelq0+4INtSYNDV9ErKJcX6xjGWkoG/VnB3dpUm00zALaqkrUD77pO5TDQ==} + hasBin: true + peerDependencies: + browserslist: '>= 4.21.0' + + vite@7.3.6: + resolution: {integrity: sha512-4XP60spRGjSZFf1qYH+dJIkK2znL3zQfl9KkOV9MkkRR/3Dls0dxaBsQPTloEc5BLXWPL9vsOxopxyKoMmDueg==} + engines: {node: ^20.19.0 || >=22.12.0} + hasBin: true + peerDependencies: + '@types/node': ^20.19.0 || >=22.12.0 + jiti: '>=1.21.0' + less: ^4.0.0 + lightningcss: ^1.21.0 + sass: ^1.70.0 + sass-embedded: ^1.70.0 + stylus: '>=0.54.8' + sugarss: ^5.0.0 + terser: ^5.16.0 + tsx: ^4.8.1 + yaml: ^2.4.2 + peerDependenciesMeta: + '@types/node': + optional: true + jiti: + optional: true + less: + optional: true + lightningcss: + optional: true + sass: + optional: true + sass-embedded: + optional: true + stylus: + optional: true + sugarss: + optional: true + terser: + optional: true + tsx: + optional: true + yaml: + optional: true + + yallist@3.1.1: + resolution: {integrity: sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==} + + zrender@6.1.0: + resolution: {integrity: sha512-oEGMDB6pOP2S6OwRR4PdVv610zrjnA3Bh+JnSG12fYJlBKjtNAoEb5fSUoCOOINlH96I2fU38/A2UpRKs67xYQ==} + +snapshots: + + '@babel/code-frame@7.29.7': + dependencies: + '@babel/helper-validator-identifier': 7.29.7 + js-tokens: 4.0.0 + picocolors: 1.1.1 + + '@babel/compat-data@7.29.7': {} + + '@babel/core@7.29.7': + dependencies: + '@babel/code-frame': 7.29.7 + '@babel/generator': 7.29.8 + '@babel/helper-compilation-targets': 7.29.7 + '@babel/helper-module-transforms': 7.29.7(@babel/core@7.29.7) + '@babel/helpers': 7.29.7 + '@babel/parser': 7.29.8 + '@babel/template': 7.29.7 + '@babel/traverse': 7.29.8 + '@babel/types': 7.29.8 + '@jridgewell/remapping': 2.3.5 + convert-source-map: 2.0.0 + debug: 4.4.3 + gensync: 1.0.0-beta.2 + json5: 2.2.3 + semver: 6.3.1 + transitivePeerDependencies: + - supports-color + + '@babel/generator@7.29.8': + dependencies: + '@babel/parser': 7.29.8 + '@babel/types': 7.29.8 + '@jridgewell/gen-mapping': 0.3.13 + '@jridgewell/trace-mapping': 0.3.31 + jsesc: 3.1.0 + + '@babel/helper-compilation-targets@7.29.7': + dependencies: + '@babel/compat-data': 7.29.7 + '@babel/helper-validator-option': 7.29.7 + browserslist: 4.28.9 + lru-cache: 5.1.1 + semver: 6.3.1 + + '@babel/helper-globals@7.29.7': {} + + '@babel/helper-module-imports@7.29.7': + dependencies: + '@babel/traverse': 7.29.8 + '@babel/types': 7.29.8 + transitivePeerDependencies: + - supports-color + + '@babel/helper-module-transforms@7.29.7(@babel/core@7.29.7)': + dependencies: + '@babel/core': 7.29.7 + '@babel/helper-module-imports': 7.29.7 + '@babel/helper-validator-identifier': 7.29.7 + '@babel/traverse': 7.29.8 + transitivePeerDependencies: + - supports-color + + '@babel/helper-plugin-utils@7.29.7': {} + + '@babel/helper-string-parser@7.29.7': {} + + '@babel/helper-validator-identifier@7.29.7': {} + + '@babel/helper-validator-option@7.29.7': {} + + '@babel/helpers@7.29.7': + dependencies: + '@babel/template': 7.29.7 + '@babel/types': 7.29.8 + + '@babel/parser@7.29.8': + dependencies: + '@babel/types': 7.29.8 + + '@babel/plugin-transform-react-jsx-self@7.29.7(@babel/core@7.29.7)': + dependencies: + '@babel/core': 7.29.7 + '@babel/helper-plugin-utils': 7.29.7 + + '@babel/plugin-transform-react-jsx-source@7.29.7(@babel/core@7.29.7)': + dependencies: + '@babel/core': 7.29.7 + '@babel/helper-plugin-utils': 7.29.7 + + '@babel/template@7.29.7': + dependencies: + '@babel/code-frame': 7.29.7 + '@babel/parser': 7.29.8 + '@babel/types': 7.29.8 + + '@babel/traverse@7.29.8': + dependencies: + '@babel/code-frame': 7.29.7 + '@babel/generator': 7.29.8 + '@babel/helper-globals': 7.29.7 + '@babel/parser': 7.29.8 + '@babel/template': 7.29.7 + '@babel/types': 7.29.8 + debug: 4.4.3 + transitivePeerDependencies: + - supports-color + + '@babel/types@7.29.8': + dependencies: + '@babel/helper-string-parser': 7.29.7 + '@babel/helper-validator-identifier': 7.29.7 + + '@esbuild/aix-ppc64@0.28.2': + optional: true + + '@esbuild/android-arm64@0.28.2': + optional: true + + '@esbuild/android-arm@0.28.2': + optional: true + + '@esbuild/android-x64@0.28.2': + optional: true + + '@esbuild/darwin-arm64@0.28.2': + optional: true + + '@esbuild/darwin-x64@0.28.2': + optional: true + + '@esbuild/freebsd-arm64@0.28.2': + optional: true + + '@esbuild/freebsd-x64@0.28.2': + optional: true + + '@esbuild/linux-arm64@0.28.2': + optional: true + + '@esbuild/linux-arm@0.28.2': + optional: true + + '@esbuild/linux-ia32@0.28.2': + optional: true + + '@esbuild/linux-loong64@0.28.2': + optional: true + + '@esbuild/linux-mips64el@0.28.2': + optional: true + + '@esbuild/linux-ppc64@0.28.2': + optional: true + + '@esbuild/linux-riscv64@0.28.2': + optional: true + + '@esbuild/linux-s390x@0.28.2': + optional: true + + '@esbuild/linux-x64@0.28.2': + optional: true + + '@esbuild/netbsd-arm64@0.28.2': + optional: true + + '@esbuild/netbsd-x64@0.28.2': + optional: true + + '@esbuild/openbsd-arm64@0.28.2': + optional: true + + '@esbuild/openbsd-x64@0.28.2': + optional: true + + '@esbuild/openharmony-arm64@0.28.2': + optional: true + + '@esbuild/sunos-x64@0.28.2': + optional: true + + '@esbuild/win32-arm64@0.28.2': + optional: true + + '@esbuild/win32-ia32@0.28.2': + optional: true + + '@esbuild/win32-x64@0.28.2': + optional: true + + '@jridgewell/gen-mapping@0.3.13': + dependencies: + '@jridgewell/sourcemap-codec': 1.6.0 + '@jridgewell/trace-mapping': 0.3.31 + + '@jridgewell/remapping@2.3.5': + dependencies: + '@jridgewell/gen-mapping': 0.3.13 + '@jridgewell/trace-mapping': 0.3.31 + + '@jridgewell/resolve-uri@3.1.2': {} + + '@jridgewell/sourcemap-codec@1.6.0': {} + + '@jridgewell/trace-mapping@0.3.31': + dependencies: + '@jridgewell/resolve-uri': 3.1.2 + '@jridgewell/sourcemap-codec': 1.6.0 + + '@napi-rs/lzma-linux-x64-gnu@1.5.1': + optional: true + + '@rolldown/pluginutils@1.0.0-rc.3': {} + + '@rollup/rollup-android-arm-eabi@4.63.2': + optional: true + + '@rollup/rollup-android-arm64@4.63.2': + optional: true + + '@rollup/rollup-darwin-arm64@4.63.2': + optional: true + + '@rollup/rollup-darwin-x64@4.63.2': + optional: true + + '@rollup/rollup-freebsd-arm64@4.63.2': + optional: true + + '@rollup/rollup-freebsd-x64@4.63.2': + optional: true + + '@rollup/rollup-linux-arm-gnueabihf@4.63.2': + optional: true + + '@rollup/rollup-linux-arm-musleabihf@4.63.2': + optional: true + + '@rollup/rollup-linux-arm64-gnu@4.63.2': + optional: true + + '@rollup/rollup-linux-arm64-musl@4.63.2': + optional: true + + '@rollup/rollup-linux-loong64-gnu@4.63.2': + optional: true + + '@rollup/rollup-linux-loong64-musl@4.63.2': + optional: true + + '@rollup/rollup-linux-ppc64-gnu@4.63.2': + optional: true + + '@rollup/rollup-linux-ppc64-musl@4.63.2': + optional: true + + '@rollup/rollup-linux-riscv64-gnu@4.63.2': + optional: true + + '@rollup/rollup-linux-riscv64-musl@4.63.2': + optional: true + + '@rollup/rollup-linux-s390x-gnu@4.63.2': + optional: true + + '@rollup/rollup-linux-x64-gnu@4.63.2': + optional: true + + '@rollup/rollup-linux-x64-musl@4.63.2': + optional: true + + '@rollup/rollup-openbsd-x64@4.63.2': + optional: true + + '@rollup/rollup-openharmony-arm64@4.63.2': + optional: true + + '@rollup/rollup-win32-arm64-msvc@4.63.2': + optional: true + + '@rollup/rollup-win32-ia32-msvc@4.63.2': + optional: true + + '@rollup/rollup-win32-x64-gnu@4.63.2': + optional: true + + '@rollup/rollup-win32-x64-msvc@4.63.2': + optional: true + + '@types/babel__core@7.20.5': + dependencies: + '@babel/parser': 7.29.8 + '@babel/types': 7.29.8 + '@types/babel__generator': 7.27.0 + '@types/babel__template': 7.4.4 + '@types/babel__traverse': 7.28.0 + + '@types/babel__generator@7.27.0': + dependencies: + '@babel/types': 7.29.8 + + '@types/babel__template@7.4.4': + dependencies: + '@babel/parser': 7.29.8 + '@babel/types': 7.29.8 + + '@types/babel__traverse@7.28.0': + dependencies: + '@babel/types': 7.29.8 + + '@types/estree@1.0.9': {} + + '@types/react-dom@19.3.0(@types/react@19.3.0)': + dependencies: + '@types/react': 19.3.0 + + '@types/react@19.3.0': + dependencies: + csstype: 3.2.3 + + '@typescript/typescript-aix-ppc64@7.0.2': + optional: true + + '@typescript/typescript-darwin-arm64@7.0.2': + optional: true + + '@typescript/typescript-darwin-x64@7.0.2': + optional: true + + '@typescript/typescript-freebsd-arm64@7.0.2': + optional: true + + '@typescript/typescript-freebsd-x64@7.0.2': + optional: true + + '@typescript/typescript-linux-arm64@7.0.2': + optional: true + + '@typescript/typescript-linux-arm@7.0.2': + optional: true + + '@typescript/typescript-linux-loong64@7.0.2': + optional: true + + '@typescript/typescript-linux-mips64el@7.0.2': + optional: true + + '@typescript/typescript-linux-ppc64@7.0.2': + optional: true + + '@typescript/typescript-linux-riscv64@7.0.2': + optional: true + + '@typescript/typescript-linux-s390x@7.0.2': + optional: true + + '@typescript/typescript-linux-x64@7.0.2': + optional: true + + '@typescript/typescript-netbsd-arm64@7.0.2': + optional: true + + '@typescript/typescript-netbsd-x64@7.0.2': + optional: true + + '@typescript/typescript-openbsd-arm64@7.0.2': + optional: true + + '@typescript/typescript-openbsd-x64@7.0.2': + optional: true + + '@typescript/typescript-sunos-x64@7.0.2': + optional: true + + '@typescript/typescript-win32-arm64@7.0.2': + optional: true + + '@typescript/typescript-win32-x64@7.0.2': + optional: true + + '@vitejs/plugin-react@5.2.0(vite@7.3.6)': + dependencies: + '@babel/core': 7.29.7 + '@babel/plugin-transform-react-jsx-self': 7.29.7(@babel/core@7.29.7) + '@babel/plugin-transform-react-jsx-source': 7.29.7(@babel/core@7.29.7) + '@rolldown/pluginutils': 1.0.0-rc.3 + '@types/babel__core': 7.20.5 + react-refresh: 0.18.0 + vite: 7.3.6 + transitivePeerDependencies: + - supports-color + + baseline-browser-mapping@2.11.22: {} + + browserslist@4.28.9: + dependencies: + baseline-browser-mapping: 2.11.22 + caniuse-lite: 1.0.30001810 + electron-to-chromium: 1.5.427 + node-releases: 2.0.55 + update-browserslist-db: 1.3.3(browserslist@4.28.9) + + caniuse-lite@1.0.30001810: {} + + convert-source-map@2.0.0: {} + + csstype@3.2.3: {} + + debug@4.4.3: + dependencies: + ms: 2.1.3 + + echarts@6.1.0: + dependencies: + tslib: 2.3.0 + zrender: 6.1.0 + + electron-to-chromium@1.5.427: {} + + esbuild@0.28.2: + optionalDependencies: + '@esbuild/aix-ppc64': 0.28.2 + '@esbuild/android-arm': 0.28.2 + '@esbuild/android-arm64': 0.28.2 + '@esbuild/android-x64': 0.28.2 + '@esbuild/darwin-arm64': 0.28.2 + '@esbuild/darwin-x64': 0.28.2 + '@esbuild/freebsd-arm64': 0.28.2 + '@esbuild/freebsd-x64': 0.28.2 + '@esbuild/linux-arm': 0.28.2 + '@esbuild/linux-arm64': 0.28.2 + '@esbuild/linux-ia32': 0.28.2 + '@esbuild/linux-loong64': 0.28.2 + '@esbuild/linux-mips64el': 0.28.2 + '@esbuild/linux-ppc64': 0.28.2 + '@esbuild/linux-riscv64': 0.28.2 + '@esbuild/linux-s390x': 0.28.2 + '@esbuild/linux-x64': 0.28.2 + '@esbuild/netbsd-arm64': 0.28.2 + '@esbuild/netbsd-x64': 0.28.2 + '@esbuild/openbsd-arm64': 0.28.2 + '@esbuild/openbsd-x64': 0.28.2 + '@esbuild/openharmony-arm64': 0.28.2 + '@esbuild/sunos-x64': 0.28.2 + '@esbuild/win32-arm64': 0.28.2 + '@esbuild/win32-ia32': 0.28.2 + '@esbuild/win32-x64': 0.28.2 + + escalade@3.2.0: {} + + fdir@6.5.0(picomatch@4.0.7): + optionalDependencies: + picomatch: 4.0.7 + + fsevents@2.3.3: + optional: true + + gensync@1.0.0-beta.2: {} + + js-tokens@4.0.0: {} + + jsesc@3.1.0: {} + + json5@2.2.3: {} + + lru-cache@5.1.1: + dependencies: + yallist: 3.1.1 + + ms@2.1.3: {} + + nanoid@3.3.19: {} + + node-releases@2.0.55: {} + + picocolors@1.1.1: {} + + picomatch@4.0.7: {} + + postcss@8.5.28: + dependencies: + nanoid: 3.3.19 + picocolors: 1.1.1 + source-map-js: 1.2.1 + + react-dom@19.3.0(react@19.3.0): + dependencies: + react: 19.3.0 + scheduler: 0.28.0 + + react-refresh@0.18.0: {} + + react@19.3.0: {} + + rollup@4.63.2: + dependencies: + '@types/estree': 1.0.9 + optionalDependencies: + '@napi-rs/lzma-linux-x64-gnu': 1.5.1 + '@rollup/rollup-android-arm-eabi': 4.63.2 + '@rollup/rollup-android-arm64': 4.63.2 + '@rollup/rollup-darwin-arm64': 4.63.2 + '@rollup/rollup-darwin-x64': 4.63.2 + '@rollup/rollup-freebsd-arm64': 4.63.2 + '@rollup/rollup-freebsd-x64': 4.63.2 + '@rollup/rollup-linux-arm-gnueabihf': 4.63.2 + '@rollup/rollup-linux-arm-musleabihf': 4.63.2 + '@rollup/rollup-linux-arm64-gnu': 4.63.2 + '@rollup/rollup-linux-arm64-musl': 4.63.2 + '@rollup/rollup-linux-loong64-gnu': 4.63.2 + '@rollup/rollup-linux-loong64-musl': 4.63.2 + '@rollup/rollup-linux-ppc64-gnu': 4.63.2 + '@rollup/rollup-linux-ppc64-musl': 4.63.2 + '@rollup/rollup-linux-riscv64-gnu': 4.63.2 + '@rollup/rollup-linux-riscv64-musl': 4.63.2 + '@rollup/rollup-linux-s390x-gnu': 4.63.2 + '@rollup/rollup-linux-x64-gnu': 4.63.2 + '@rollup/rollup-linux-x64-musl': 4.63.2 + '@rollup/rollup-openbsd-x64': 4.63.2 + '@rollup/rollup-openharmony-arm64': 4.63.2 + '@rollup/rollup-win32-arm64-msvc': 4.63.2 + '@rollup/rollup-win32-ia32-msvc': 4.63.2 + '@rollup/rollup-win32-x64-gnu': 4.63.2 + '@rollup/rollup-win32-x64-msvc': 4.63.2 + fsevents: 2.3.3 + + scheduler@0.28.0: {} + + semver@6.3.1: {} + + source-map-js@1.2.1: {} + + tinyglobby@0.2.17: + dependencies: + fdir: 6.5.0(picomatch@4.0.7) + picomatch: 4.0.7 + + tslib@2.3.0: {} + + typescript@7.0.2: + optionalDependencies: + '@typescript/typescript-aix-ppc64': 7.0.2 + '@typescript/typescript-darwin-arm64': 7.0.2 + '@typescript/typescript-darwin-x64': 7.0.2 + '@typescript/typescript-freebsd-arm64': 7.0.2 + '@typescript/typescript-freebsd-x64': 7.0.2 + '@typescript/typescript-linux-arm': 7.0.2 + '@typescript/typescript-linux-arm64': 7.0.2 + '@typescript/typescript-linux-loong64': 7.0.2 + '@typescript/typescript-linux-mips64el': 7.0.2 + '@typescript/typescript-linux-ppc64': 7.0.2 + '@typescript/typescript-linux-riscv64': 7.0.2 + '@typescript/typescript-linux-s390x': 7.0.2 + '@typescript/typescript-linux-x64': 7.0.2 + '@typescript/typescript-netbsd-arm64': 7.0.2 + '@typescript/typescript-netbsd-x64': 7.0.2 + '@typescript/typescript-openbsd-arm64': 7.0.2 + '@typescript/typescript-openbsd-x64': 7.0.2 + '@typescript/typescript-sunos-x64': 7.0.2 + '@typescript/typescript-win32-arm64': 7.0.2 + '@typescript/typescript-win32-x64': 7.0.2 + + update-browserslist-db@1.3.3(browserslist@4.28.9): + dependencies: + browserslist: 4.28.9 + escalade: 3.2.0 + picocolors: 1.1.1 + + vite@7.3.6: + dependencies: + esbuild: 0.28.2 + fdir: 6.5.0(picomatch@4.0.7) + picomatch: 4.0.7 + postcss: 8.5.28 + rollup: 4.63.2 + tinyglobby: 0.2.17 + optionalDependencies: + fsevents: 2.3.3 + + yallist@3.1.1: {} + + zrender@6.1.0: + dependencies: + tslib: 2.3.0 diff --git a/viewer/pnpm-workspace.yaml b/viewer/pnpm-workspace.yaml new file mode 100644 index 0000000..5ed0b5a --- /dev/null +++ b/viewer/pnpm-workspace.yaml @@ -0,0 +1,2 @@ +allowBuilds: + esbuild: true diff --git a/viewer/src/App.tsx b/viewer/src/App.tsx new file mode 100644 index 0000000..582a3cc --- /dev/null +++ b/viewer/src/App.tsx @@ -0,0 +1,533 @@ +import { useEffect, useMemo, useRef, useState } from "react"; +import { + fetchJson, + type ChartKind, + type ColumnMeta, + type DashboardDoc, + type DashboardPanelDoc, + type QueryResultDoc, + type ReleaseDoc, +} from "./data.js"; +import { + columnLabel, + compareValues, + formatCell, + relativeTime, + rowWindow, + toChartNumber, +} from "./format.js"; + +/** Columns the panel asked to compute but not show, e.g. an explicit sort key. */ +function visibleColumns( + columns: ColumnMeta[], + hidden: string[] | undefined, +): number[] { + const drop = new Set((hidden ?? []).map((name) => name.toLowerCase())); + return columns + .map((column, index) => (drop.has(column.name.toLowerCase()) ? -1 : index)) + .filter((index) => index >= 0); +} + +function Chip({ + label, + value, + tone = "neutral", + title, +}: { + label: string; + value: string; + tone?: "neutral" | "good" | "warn"; + title?: string; +}) { + return ( + + {label} + {value} + + ); +} + +function Chart({ + result, + kind, + columns, +}: { + result: QueryResultDoc; + kind: ChartKind; + columns: number[]; +}) { + const [el, setEl] = useState(null); + const instanceRef = useRef<{ dispose(): void; resize(): void } | null>(null); + + useEffect(() => { + if (!el) return; + let disposed = false; + + // Loaded on demand and tree-shaken: a release with only KPI and table + // panels never downloads the charting library at all. + void import("./echarts.js").then(({ init }) => { + if (disposed) return; + const dark = matchMedia("(prefers-color-scheme: dark)").matches; + const instance = init(el, undefined, { renderer: "canvas" }); + instanceRef.current = instance; + + const [categoryIndex, ...seriesIndexes] = columns; + if (categoryIndex === undefined) return; + const axis = result.rows.map((row) => String(row[categoryIndex] ?? "")); + // cp-ui-kit border-static / text-secondary, per theme. + const grid = dark ? "#2e3338" : "#e4ebf1"; + const text = dark ? "#8d95a5" : "#606772"; + + instance.setOption({ + animationDuration: 400, + // Chainstack brand blue leading, then the kit's status contrasts. + color: dark + ? ["#007bff", "#2dd272", "#25a4ff", "#ffdd33", "#ff294c"] + : ["#007bff", "#25b15f", "#0095ff", "#ffd102", "#ff1a40"], + grid: { left: 8, right: 16, top: 24, bottom: 8, containLabel: true }, + tooltip: { + trigger: "axis", + axisPointer: { type: kind === "bar" ? "shadow" : "line" }, + }, + legend: + seriesIndexes.length > 1 + ? { top: 0, textStyle: { color: text }, icon: "roundRect" } + : undefined, + xAxis: { + type: "category", + data: axis, + boundaryGap: kind === "bar", + axisLine: { lineStyle: { color: grid } }, + axisLabel: { color: text, hideOverlap: true }, + }, + yAxis: { + type: "value", + splitLine: { lineStyle: { color: grid } }, + axisLabel: { color: text }, + }, + series: seriesIndexes.map((index) => { + const column = result.columns[index]!; + return { + name: columnLabel(column), + type: kind === "bar" ? "bar" : "line", + smooth: kind !== "bar", + showSymbol: result.rows.length <= 60, + areaStyle: kind === "area" ? { opacity: 0.18 } : undefined, + barMaxWidth: 36, + itemStyle: { borderRadius: kind === "bar" ? [4, 4, 0, 0] : 0 }, + data: result.rows.map((row) => toChartNumber(row[index], column)), + }; + }), + }); + }); + + const observer = new ResizeObserver(() => instanceRef.current?.resize()); + observer.observe(el); + return () => { + disposed = true; + observer.disconnect(); + instanceRef.current?.dispose(); + instanceRef.current = null; + }; + }, [result, kind, columns, el]); + + return
; +} + +// Below this, rendering every row costs nothing and avoids the measurement +// dance entirely. +const VIRTUALIZE_ABOVE = 200; +// Rows rendered beyond the viewport, so a fast scroll does not show gaps. +const OVERSCAN = 12; +// Only a starting guess: the real height is measured from the first render, +// so this constant and the stylesheet cannot drift apart. +const ASSUMED_ROW_HEIGHT = 33; + +function DataTable({ + result, + columns, +}: { + result: QueryResultDoc; + columns: number[]; +}) { + const [sortCol, setSortCol] = useState(null); + const [asc, setAsc] = useState(true); + const [scrollTop, setScrollTop] = useState(0); + const [viewport, setViewport] = useState(0); + const [rowHeight, setRowHeight] = useState(ASSUMED_ROW_HEIGHT); + const bodyRef = useRef(null); + + const sorted = useMemo(() => { + if (sortCol === null) return result.rows; + const rows = [...result.rows]; + rows.sort((a, b) => { + const cmp = compareValues(a[sortCol], b[sortCol]); + return asc ? cmp : -cmp; + }); + return rows; + }, [result, sortCol, asc]); + + const virtual = sorted.length > VIRTUALIZE_ABOVE; + + // Measure a real row rather than trusting a constant to match the CSS. + useEffect(() => { + if (!virtual) return; + const row = bodyRef.current?.querySelector("tr[data-row]"); + const measured = row instanceof HTMLElement ? row.offsetHeight : 0; + if (measured > 0 && measured !== rowHeight) setRowHeight(measured); + }, [virtual, rowHeight, sorted]); + + const { first, last } = rowWindow(sorted.length, scrollTop, viewport, rowHeight, { + threshold: VIRTUALIZE_ABOVE, + overscan: OVERSCAN, + }); + const visible = sorted.slice(first, last); + + if (result.rows.length === 0) { + return

No rows.

; + } + + return ( +
{ + const el = event.currentTarget; + setScrollTop(el.scrollTop); + setViewport(el.clientHeight); + } + : undefined + } + ref={ + virtual + ? (el) => { + if (el && viewport === 0) setViewport(el.clientHeight); + } + : undefined + } + > +
+ + + {columns.map((index) => { + const column = result.columns[index]!; + const active = sortCol === index; + return ( + + ); + })} + + + + {/* Spacers stand in for the rows above and below the window, so the + scrollbar reflects the whole result while the DOM holds a screenful. */} + {virtual && first > 0 ? ( + + ) : null} + {visible.map((row, i) => ( + + {columns.map((index) => { + const column = result.columns[index]!; + const cell = formatCell(row[index], column); + return ( + + ); + })} + + ))} + {virtual && last < sorted.length ? ( + + ) : null} + +
+ +
+ {cell.text} +
+ + ); +} + +function Kpi({ + result, + columns, + unit, +}: { + result: QueryResultDoc; + columns: number[]; + unit?: string; +}) { + const index = columns[0]; + const row = result.rows[0]; + if (index === undefined || row === undefined) { + return

No value.

; + } + const cell = formatCell(row[index], result.columns[index]!); + return ( +
+
+ {cell.text} +
+ {unit ?
{unit}
: null} +
+ ); +} + +function Panel({ + panel, + result, +}: { + panel: DashboardPanelDoc; + result: QueryResultDoc | null | undefined; +}) { + const columns = useMemo( + () => (result ? visibleColumns(result.columns, panel.hide_columns) : []), + [result, panel.hide_columns], + ); + + const body = (): React.ReactNode => { + if (result === undefined) return

Loading…

; + if (result === null) { + return

No result published for “{panel.query}”.

; + } + if (result.error) { + return ( +

+ {result.error.code}: {result.error.message} +

+ ); + } + if (columns.length === 0) { + return

Every column is hidden.

; + } + if (panel.chart === "kpi") { + return ; + } + if (panel.chart === "table") { + return ; + } + return ; + }; + + return ( +
+
+

{panel.title ?? panel.query}

+ {panel.description ?

{panel.description}

: null} +
+ {body()} +
+ ); +} + +function Provenance({ release }: { release: ReleaseDoc }) { + const chips: React.ReactNode[] = []; + const freshness = release.freshness; + + if (freshness?.kind === "chain" && freshness.data_through) { + const { block, timestamp } = freshness.data_through; + const ago = relativeTime(timestamp); + chips.push( + , + ); + const checked = relativeTime(freshness.indexed_at); + if (checked) { + chips.push( + , + ); + } + } else if (freshness) { + // No chain provenance to offer, so name what this timestamp actually is + // rather than dressing a file mtime up as freshness. + chips.push( + , + ); + } + + if (release.finality) { + chips.push( + , + ); + } + + for (const coverage of release.coverage) { + chips.push( + , + ); + } + + chips.push(); + if (release.content_digest) { + chips.push( + , + ); + } + return
{chips}
; +} + +export function App() { + const [release, setRelease] = useState(null); + const [dashboards, setDashboards] = useState([]); + const [results, setResults] = useState>({}); + const [loadError, setLoadError] = useState(null); + + useEffect(() => { + let cancelled = false; + void (async () => { + try { + const rel = await fetchJson("release.json"); + if (cancelled) return; + setRelease(rel); + + const [docs, entries] = await Promise.all([ + Promise.all( + rel.dashboards.map((id) => + fetchJson(`dashboards/${id}.json`), + ), + ), + Promise.all( + rel.queries.map(async (id) => { + try { + return [id, await fetchJson(`results/${id}.json`)] as const; + } catch { + return [id, null] as const; + } + }), + ), + ]); + if (cancelled) return; + setDashboards(docs); + setResults(Object.fromEntries(entries)); + } catch (err) { + if (!cancelled) { + setLoadError(err instanceof Error ? err.message : String(err)); + } + } + })(); + return () => { + cancelled = true; + }; + }, []); + + if (loadError) { + return ( +
+

Failed to load release: {loadError}

+
+ ); + } + if (!release) { + // A cold object-store edge can take seconds to answer; a bare line of text + // reads as a broken page while it does. + return ( +
+
+
+
+
+
+
+
+
+
+
+

Loading release…

+
+ ); + } + + return ( +
+
+

{release.project_id}

+ +
+ + {dashboards.length === 0 ? ( +

This release has no dashboards.

+ ) : null} + + {dashboards.map((dash) => ( +
+
+

{dash.title}

+ {dash.description ?

{dash.description}

: null} +
+
+ {dash.panels.map((panel, i) => ( + + ))} +
+
+ ))} + +
+ Built by chainplot · {release.mode.replace(/_/g, " ")} ·{" "} + +
+
+ ); +} diff --git a/viewer/src/data.ts b/viewer/src/data.ts new file mode 100644 index 0000000..9727280 --- /dev/null +++ b/viewer/src/data.ts @@ -0,0 +1,68 @@ +import type { ColumnMeta } from "./format.js"; + +export type { ColumnMeta }; + +export interface QueryResultDoc { + schema_version: number; + query_id: string; + title?: string; + columns: ColumnMeta[]; + rows: unknown[][]; + snapshot: string; + raw_amount_columns?: string[]; + error?: { code: string; message: string } | null; +} + +export interface FreshnessDoc { + kind: "chain" | "snapshot_mtime"; + data_through: { block: number; timestamp: string | null } | null; + indexed_at: string | null; + snapshot_mtime: string; +} + +export interface ReleaseDoc { + schema_version: number; + project_id: string; + mode: string; + queries: string[]; + dashboards: string[]; + generated_at: string; + content_digest?: string; + snapshots: { dataset_id: string; snapshot_id: string }[]; + coverage: { + source_id: string; + start_block: number; + end_block: number; + status: string; + }[]; + finality: { policy: string; depth?: number } | null; + freshness?: FreshnessDoc; +} + +export type ChartKind = "line" | "bar" | "area" | "kpi" | "table"; + +export interface DashboardPanelDoc { + query: string; + chart: ChartKind; + title?: string; + description?: string; + span?: "half" | "full"; + hide_columns?: string[]; + unit?: string; +} + +export interface DashboardDoc { + schema_version: number; + dashboard_id: string; + title: string; + description?: string | null; + panels: DashboardPanelDoc[]; +} + +export async function fetchJson(path: string): Promise { + const response = await fetch(path); + if (!response.ok) { + throw new Error(`${path}: HTTP ${response.status}`); + } + return (await response.json()) as T; +} diff --git a/viewer/src/echarts.ts b/viewer/src/echarts.ts new file mode 100644 index 0000000..65bf6b3 --- /dev/null +++ b/viewer/src/echarts.ts @@ -0,0 +1,21 @@ +// Only the pieces the viewer actually renders. Importing the `echarts` +// barrel instead pulls in every chart type and costs ~1.1 MB. +import * as echarts from "echarts/core"; +import { BarChart, LineChart } from "echarts/charts"; +import { + GridComponent, + LegendComponent, + TooltipComponent, +} from "echarts/components"; +import { CanvasRenderer } from "echarts/renderers"; + +echarts.use([ + BarChart, + LineChart, + GridComponent, + LegendComponent, + TooltipComponent, + CanvasRenderer, +]); + +export const init = echarts.init; diff --git a/viewer/src/env.d.ts b/viewer/src/env.d.ts new file mode 100644 index 0000000..ecad9c2 --- /dev/null +++ b/viewer/src/env.d.ts @@ -0,0 +1,2 @@ +// Vite resolves these at build time; TypeScript needs to be told they exist. +declare module "*.css"; diff --git a/viewer/src/format.ts b/viewer/src/format.ts new file mode 100644 index 0000000..9d022b2 --- /dev/null +++ b/viewer/src/format.ts @@ -0,0 +1,241 @@ +// Display formatting for values that must not lose precision. +// +// Amounts arrive as decimal strings because a uint256 does not fit in a +// double. Every transform here stays in BigInt or string space; `Number` is +// used only for chart geometry, where a pixel is the unit anyway and the +// exact value is still shown on hover. + +export interface ColumnMeta { + name: string; + logical_type: string; + raw_amount?: boolean; + decimals?: number; + symbol?: string; + label?: string; +} + +const DECIMAL = /^-?\d+$/; + +export function asBigInt(value: unknown): bigint | null { + if (typeof value === "bigint") return value; + if (typeof value === "number") { + return Number.isInteger(value) ? BigInt(value) : null; + } + if (typeof value !== "string") return null; + const trimmed = value.trim(); + if (!DECIMAL.test(trimmed)) return null; + try { + return BigInt(trimmed); + } catch { + return null; + } +} + +/** Group the integer part in threes without going through Number. */ +export function groupDigits(digits: string): string { + const negative = digits.startsWith("-"); + const body = negative ? digits.slice(1) : digits; + let out = ""; + for (let i = body.length; i > 0; i -= 3) { + const start = Math.max(0, i - 3); + out = body.slice(start, i) + (out ? "," + out : ""); + } + return (negative ? "-" : "") + out; +} + +/** + * Scale an integer amount by `decimals`, exactly. + * + * 983644533552 with 6 decimals → "983,644.533552". Trailing zeros in the + * fraction are dropped; the integer part is never rounded. + */ +export function scaleAmount(value: bigint, decimals: number): string { + if (decimals <= 0) return groupDigits(value.toString()); + const negative = value < 0n; + const digits = (negative ? -value : value).toString().padStart(decimals + 1, "0"); + const whole = digits.slice(0, digits.length - decimals); + const fraction = digits.slice(digits.length - decimals).replace(/0+$/, ""); + const body = groupDigits(whole) + (fraction ? `.${fraction}` : ""); + return (negative ? "-" : "") + body; +} + +/** + * Round an implied-decimal integer to `keep` decimal places, half away from + * zero, entirely in BigInt. Used for display only — the exact value is always + * still available, and `scaleAmount` remains the lossless rendering. + */ +export function roundAmount(value: bigint, decimals: number, keep: number): bigint { + if (keep >= decimals) return value; + const drop = BigInt(decimals - keep); + const divisor = 10n ** drop; + const half = divisor / 2n; + return value < 0n ? (value - half) / divisor : (value + half) / divisor; +} + +/** + * How many decimal places are worth showing. + * + * Six decimals on a figure in the billions is noise, and it pushes a headline + * number onto two lines. Below one unit the fraction is the whole story, so it + * is kept in full. + */ +export function displayDecimals(value: bigint, decimals: number): number { + if (decimals <= 0) return 0; + const magnitude = (value < 0n ? -value : value) / 10n ** BigInt(decimals); + return magnitude === 0n ? decimals : 2; +} + +/** Display rendering of a raw amount: exact below one unit, 2dp above. */ +export function displayAmount(value: bigint, decimals: number): string { + const keep = displayDecimals(value, decimals); + return scaleAmount(roundAmount(value, decimals, keep), keep); +} + +/** Middle-truncate a 0x hash so a table column stays readable. */ +export function shortHex(value: string): string { + return value.length > 18 ? `${value.slice(0, 10)}…${value.slice(-8)}` : value; +} + +export function isHex(value: unknown): value is string { + return typeof value === "string" && /^0x[0-9a-fA-F]{16,}$/.test(value); +} + +export interface Formatted { + /** What the cell shows. */ + text: string; + /** The exact value, for a title attribute and copy. */ + exact: string; + numeric: boolean; +} + +export function formatCell(value: unknown, column: ColumnMeta): Formatted { + if (value === null || value === undefined) { + return { text: "—", exact: "", numeric: false }; + } + const exact = String(value); + + if (column.raw_amount) { + const amount = asBigInt(value); + if (amount !== null) { + const scaled = displayAmount(amount, column.decimals ?? 0); + const precise = scaleAmount(amount, column.decimals ?? 0); + return { + text: column.symbol ? `${scaled} ${column.symbol}` : scaled, + // Hovering a rounded figure should reveal the full one, then the raw + // integer it came from. + exact: + precise === scaled + ? exact + : `${precise}${column.symbol ? ` ${column.symbol}` : ""} (${exact})`, + numeric: true, + }; + } + } + + const asInt = asBigInt(value); + if (asInt !== null) { + return { text: groupDigits(asInt.toString()), exact, numeric: true }; + } + if (typeof value === "number") { + return { text: String(value), exact, numeric: true }; + } + if (isHex(value)) { + return { text: shortHex(value), exact, numeric: false }; + } + return { text: exact, exact, numeric: false }; +} + +export function columnLabel(column: ColumnMeta): string { + if (column.label) return column.label; + const base = column.name.replace(/_/g, " "); + return base.charAt(0).toUpperCase() + base.slice(1); +} + +/** + * Chart-space value. Scaling happens in BigInt, so only the final magnitude + * touches a double — an amount too large for one still plots at the right + * height, and the tooltip carries the exact figure. + */ +export function toChartNumber(value: unknown, column: ColumnMeta): number { + const amount = asBigInt(value); + if (amount === null) { + const parsed = Number(value); + return Number.isFinite(parsed) ? parsed : 0; + } + const decimals = column.raw_amount ? (column.decimals ?? 0) : 0; + if (decimals <= 0) return Number(amount); + const divisor = 10n ** BigInt(decimals); + const whole = amount / divisor; + const remainder = amount % divisor; + return Number(whole) + Number(remainder) / Number(divisor); +} + +/** Sign-aware numeric compare, falling back to text for non-numbers. */ +export function compareValues(a: unknown, b: unknown): number { + const left = asBigInt(a); + const right = asBigInt(b); + if (left !== null && right !== null) { + return left < right ? -1 : left > right ? 1 : 0; + } + const as = String(a ?? ""); + const bs = String(b ?? ""); + return as < bs ? -1 : as > bs ? 1 : 0; +} + +/** "3 minutes ago" / "in 2 hours", or null when the input is unusable. */ +export function relativeTime(iso: string | null, now = Date.now()): string | null { + if (!iso) return null; + const then = Date.parse(iso); + if (Number.isNaN(then)) return null; + const seconds = Math.round((then - now) / 1000); + const units: [Intl.RelativeTimeFormatUnit, number][] = [ + ["year", 31_536_000], + ["month", 2_592_000], + ["day", 86_400], + ["hour", 3600], + ["minute", 60], + ]; + const formatter = new Intl.RelativeTimeFormat(undefined, { numeric: "auto" }); + for (const [unit, size] of units) { + if (Math.abs(seconds) >= size) { + return formatter.format(Math.round(seconds / size), unit); + } + } + return formatter.format(seconds, "second"); +} + +/** + * Which slice of rows a virtualised table should render. + * + * Results ride inside the release, so a wide table would otherwise put every + * row in the DOM. Spacer rows above and below stand in for what is not + * rendered, so the scrollbar still spans the whole result. + */ +export interface RowWindow { + /** Whether windowing applies at all. */ + virtual: boolean; + /** First row index to render, inclusive. */ + first: number; + /** Last row index to render, exclusive. */ + last: number; +} + +export function rowWindow( + total: number, + scrollTop: number, + viewport: number, + rowHeight: number, + opts: { threshold: number; overscan: number }, +): RowWindow { + if (total <= opts.threshold || rowHeight <= 0) { + return { virtual: false, first: 0, last: total }; + } + const first = Math.max(0, Math.floor(scrollTop / rowHeight) - opts.overscan); + const last = Math.min( + total, + Math.ceil((scrollTop + viewport) / rowHeight) + opts.overscan, + ); + // A viewport of zero happens on the first paint, before the container has + // been measured; render the overscan rather than nothing. + return { virtual: true, first, last: Math.max(last, first + opts.overscan) }; +} diff --git a/viewer/src/main.tsx b/viewer/src/main.tsx new file mode 100644 index 0000000..515b069 --- /dev/null +++ b/viewer/src/main.tsx @@ -0,0 +1,13 @@ +import { StrictMode } from "react"; +import { createRoot } from "react-dom/client"; +import { App } from "./App.js"; +import "./styles.css"; + +const root = document.getElementById("root"); +if (root) { + createRoot(root).render( + + + , + ); +} diff --git a/viewer/src/styles.css b/viewer/src/styles.css new file mode 100644 index 0000000..914b2bd --- /dev/null +++ b/viewer/src/styles.css @@ -0,0 +1,431 @@ +/* Chainstack design tokens, transcribed from cp-ui-kit (src/styles/tailwind.css + and src/styles/theme.ts) rather than imported — a release is a static page + with no build step of its own, so the kit's Tailwind pipeline cannot run here. + Names mirror the kit's so the two stay comparable. + + Fonts are named, not bundled: Suisse Intl is licensed, and these pages are + published to public buckets. Anyone with it installed gets it; everyone else + gets the system stack. */ +:root { + color-scheme: light dark; + + /* Background */ + --bg: #f5f8fc; + --bg-elevated: #ffffff; + --bg-row: #fbfdff; + --bg-header: #f3f8fc; + + /* Text */ + --text-primary: #22252a; + --text-secondary: #606772; + --text-tertiary: #9aa3ac; + + /* Border */ + --border-static: #e4ebf1; + --border-default: #d7e0e9; + --border-hover: #b3bfcc; + + /* Brand */ + --blue-brand: #007bff; + --blue-light: #cce9ff; + --blue-dark: #003399; + + /* Status: surface + contrast pairs */ + --success-surface: #d6f5e0; + --success-contrast: #25b15f; + --warning-surface: #fff6cc; + --warning-contrast: #ffd102; + --error-surface: #fce8eb; + --error-contrast: #ff1a40; + --info-surface: #e5f4ff; + --info-contrast: #0095ff; + + --shadow: 0 1px 2px rgb(18 24 33 / 4%), 0 1px 3px rgb(18 24 33 / 6%); + + /* Radius scale: xs 4, s 6, m 8, l 12, xl 16 */ + --radius-xs: 4px; + --radius-s: 6px; + --radius-m: 8px; + --radius-l: 12px; + --radius-full: 1000px; + + --sans: "Suisse Intl", ui-sans-serif, system-ui, -apple-system, "Segoe UI", + Roboto, "Helvetica Neue", Arial, sans-serif; + --mono: "Source Code Pro", "Courier Prime", ui-monospace, SFMono-Regular, + Menlo, Consolas, "Liberation Mono", monospace; + + font-family: var(--sans); + font-size: 15px; + line-height: 1.5; +} + +@media (prefers-color-scheme: dark) { + :root { + --bg: #090a0b; + --bg-elevated: #1f2228; + --bg-row: #0f1114; + --bg-header: #14171b; + + --text-primary: #f6f9fd; + --text-secondary: #8d95a5; + --text-tertiary: #656e80; + + --border-static: #2e3338; + --border-default: #40474e; + --border-hover: #5e666e; + + --blue-brand: #007bff; + --blue-light: #004c9e; + --blue-dark: #4dafff; + + --success-surface: #124a24; + --success-contrast: #2dd272; + --warning-surface: #563906; + --warning-contrast: #ffdd33; + --error-surface: #591721; + --error-contrast: #ff294c; + --info-surface: #0b3f65; + --info-contrast: #25a4ff; + + --shadow: none; + } +} + +* { + box-sizing: border-box; +} + +body { + margin: 0; + background: var(--bg); + color: var(--text-primary); + -webkit-font-smoothing: antialiased; +} + +.app { + max-width: 1180px; + margin: 0 auto; + padding: 2.5rem 1.25rem 4rem; +} + +/* ---- masthead ---- */ + +.masthead h1 { + margin: 0 0 0.75rem; + font-size: 1.375rem; /* font-xl */ + font-weight: 600; + letter-spacing: -0.015em; +} + +.chips { + display: flex; + flex-wrap: wrap; + gap: 0.4rem; +} + +.chip { + display: inline-flex; + align-items: baseline; + gap: 0.4rem; + padding: 0.25rem 0.65rem; + border: 1px solid var(--border-static); + border-radius: var(--radius-full); + background: var(--bg-elevated); + font-size: 0.75rem; + white-space: nowrap; +} + +.chip-label { + color: var(--text-tertiary); + text-transform: uppercase; + letter-spacing: 0.05em; + font-size: 0.625rem; + font-weight: 600; +} + +.chip-value { + font-variant-numeric: tabular-nums; +} + +.chip-good { + background: var(--success-surface); + border-color: transparent; + color: var(--success-contrast); +} + +.chip-warn { + background: var(--warning-surface); + border-color: transparent; + color: var(--warning-contrast); +} + +/* ---- dashboards ---- */ + +.dashboard { + margin-top: 2.25rem; +} + +.dashboard-head { + margin-bottom: 1rem; +} + +.dashboard-head h2 { + margin: 0; + font-size: 1.125rem; /* font-l */ + font-weight: 600; + letter-spacing: -0.01em; +} + +.dashboard-head p { + margin: 0.35rem 0 0; + color: var(--text-secondary); + font-size: 0.875rem; +} + +.panel-grid { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: 1rem; + align-items: start; +} + +.panel { + /* min-width:0 lets a grid child shrink, which is what keeps a wide table + inside its card instead of stretching the whole row. */ + min-width: 0; + padding: 1rem 1.25rem 1.25rem; + border: 1px solid var(--border-static); + border-radius: var(--radius-l); + background: var(--bg-elevated); + box-shadow: var(--shadow); +} + +.panel.span-full { + grid-column: 1 / -1; +} + +.panel-head { + margin-bottom: 0.85rem; +} + +.panel-head h3 { + margin: 0; + font-size: 0.8125rem; + font-weight: 600; + color: var(--text-secondary); +} + +.panel-head p { + margin: 0.3rem 0 0; + font-size: 0.8125rem; + color: var(--text-tertiary); +} + +/* ---- kpi ---- */ + +.kpi { + display: flex; + align-items: baseline; + gap: 0.55rem; + flex-wrap: wrap; +} + +.kpi-value { + font-size: clamp(1.75rem, 4.5vw, 2rem); /* font-xxl ceiling */ + font-weight: 600; + letter-spacing: -0.02em; + font-variant-numeric: tabular-nums; + /* Long uint256 values wrap instead of widening the grid column. */ + overflow-wrap: anywhere; +} + +.kpi-unit { + font-size: 0.75rem; + color: var(--text-tertiary); + text-transform: uppercase; + letter-spacing: 0.05em; +} + +/* ---- table ---- */ + +.table-scroll { + overflow-x: auto; + max-height: 460px; + overflow-y: auto; + border: 1px solid var(--border-static); + border-radius: var(--radius-m); +} + +table.data-table { + border-collapse: separate; + border-spacing: 0; + width: 100%; + font-size: 0.8125rem; +} + +table.data-table th { + position: sticky; + top: 0; + z-index: 1; + height: 40px; + padding: 0; + background: var(--bg-header); + border-bottom: 1px solid var(--border-static); + white-space: nowrap; +} + +table.data-table th button { + width: 100%; + padding: 0 0.75rem; + border: 0; + background: none; + color: var(--text-secondary); + font: inherit; + font-weight: 600; + cursor: pointer; + /* `th` defaults to centre; headings must line up with their own cells. */ + text-align: left; +} + +table.data-table th button:hover { + color: var(--blue-brand); +} + +table.data-table th.numeric button { + text-align: right; +} + +.sort-arrow { + display: inline-block; + width: 0.9em; + color: var(--blue-brand); +} + +table.data-table td { + padding: 0.5rem 0.75rem; + border-bottom: 1px solid var(--border-static); + background: var(--bg-elevated); + white-space: nowrap; +} + +table.data-table tbody tr:last-child td { + border-bottom: 0; +} + +table.data-table tbody tr:hover td { + background: var(--bg-row); +} + +table.data-table .numeric { + text-align: right; + font-variant-numeric: tabular-nums; +} + +table.data-table td.text { + font-family: var(--mono); + font-size: 0.78rem; + color: var(--text-secondary); +} + +table.data-table th.raw button::after { + content: "·"; + margin-left: 0.3rem; + color: var(--blue-brand); +} + +/* ---- charts, states, footer ---- */ + +.chart { + width: 100%; + height: 280px; +} + +.panel.span-full .chart { + height: 340px; +} + +/* ---- loading skeleton ---- */ + +.skeleton { + border-radius: var(--radius-m); + background: linear-gradient( + 90deg, + var(--bg-elevated) 25%, + var(--bg-header) 37%, + var(--bg-elevated) 63% + ); + background-size: 400% 100%; + animation: skeleton-sheen 1.4s ease-in-out infinite; +} + +.skeleton-title { + width: min(18rem, 60%); + height: 1.6rem; + margin-bottom: 0.9rem; +} + +.skeleton-chip { + width: 9rem; + height: 1.55rem; + border-radius: var(--radius-full); +} + +.skeleton-panel { + height: 9rem; + border: 1px solid var(--border-static); +} + +@keyframes skeleton-sheen { + from { + background-position: 100% 50%; + } + to { + background-position: 0 50%; + } +} + +@media (prefers-reduced-motion: reduce) { + .skeleton { + animation: none; + } +} + +.empty { + margin: 0; + padding: 1.25rem 0; + color: var(--text-tertiary); + font-size: 0.85rem; +} + +.error-note { + margin: 0; + padding: 0.65rem 0.8rem; + border-radius: var(--radius-s); + background: var(--error-surface); + color: var(--error-contrast); + font-family: var(--mono); + font-size: 0.78rem; + white-space: pre-wrap; +} + +.colophon { + margin-top: 3rem; + padding-top: 1rem; + border-top: 1px solid var(--border-static); + color: var(--text-tertiary); + font-size: 0.75rem; +} + +.colophon span { + font-weight: 600; + color: var(--text-secondary); +} + +@media (max-width: 760px) { + .app { + padding: 1.5rem 0.9rem 3rem; + } + .panel-grid { + grid-template-columns: minmax(0, 1fr); + } +} diff --git a/viewer/tsconfig.json b/viewer/tsconfig.json new file mode 100644 index 0000000..9714625 --- /dev/null +++ b/viewer/tsconfig.json @@ -0,0 +1,12 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "ESNext", + "moduleResolution": "bundler", + "jsx": "react-jsx", + "strict": true, + "skipLibCheck": true, + "noEmit": true + }, + "include": ["src"] +} diff --git a/viewer/vite.config.ts b/viewer/vite.config.ts new file mode 100644 index 0000000..64be6ff --- /dev/null +++ b/viewer/vite.config.ts @@ -0,0 +1,11 @@ +import { defineConfig } from "vite"; +import react from "@vitejs/plugin-react"; + +export default defineConfig({ + base: "./", + plugins: [react()], + build: { + outDir: "dist", + emptyOutDir: true, + }, +}); diff --git a/vitest.config.ts b/vitest.config.ts new file mode 100644 index 0000000..17a318c --- /dev/null +++ b/vitest.config.ts @@ -0,0 +1,10 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + // Build/ingest e2e tests spawn DuckDB children and Docker-side work; + // 5s default is too tight under full-suite parallelism. + testTimeout: 30_000, + hookTimeout: 30_000, + }, +}); From e60b4285fcae9ecb2036158cc8f0f6f9d5aedc9a Mon Sep 17 00:00:00 2001 From: Eugene Aseev Date: Fri, 18 Sep 2026 10:46:31 +0800 Subject: [PATCH 02/19] Derive rindexer's table names the way rindexer does MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit rindexer snake_cases the manifest, contract and event names into Postgres identifiers; chainplot lowercased them. The two agree on `Transfer` — the only event any template uses — and disagree on everything longer: `RelayERC20Deposit` becomes `relay_erc_20_deposit` upstream and `relayerc20deposit` here, so rows landed in a table `apply` could not find. naming.ts ports rindexer's `camel_to_snake` byte for byte, and both the coverage lookup and the export read through it. The snapshot file name keeps chainplot's own convention, which the templates' `snapshot:` paths document. Past 63 characters rindexer compacts the cursor table with a hash and Postgres truncates the schema; neither can be derived, so `validate` now refuses that combination before a block is indexed, naming the source and event that make up the excess. The live compose e2e gains a second source whose event is multi-word (stETH `TransferShares`) and asserts it indexes, proves coverage and exports — the case that was invisible while every fixture used `Transfer`. Co-Authored-By: Claude Fable 5.1 --- src/ingest/exporter.ts | 3 + src/ingest/rindexer/inspectCoverage.ts | 31 ++----- src/ingest/rindexer/naming.ts | 118 +++++++++++++++++++++++++ src/project/validate.ts | 22 ++++- tests/cli/validate.test.ts | 26 ++++++ tests/ingest/live/e2e.live.test.ts | 70 ++++++++++++++- tests/ingest/naming.test.ts | 77 ++++++++++++++++ 7 files changed, 322 insertions(+), 25 deletions(-) create mode 100644 src/ingest/rindexer/naming.ts create mode 100644 tests/ingest/naming.test.ts diff --git a/src/ingest/exporter.ts b/src/ingest/exporter.ts index e1d5787..35b128b 100644 --- a/src/ingest/exporter.ts +++ b/src/ingest/exporter.ts @@ -77,6 +77,9 @@ export async function exportEventTable( contractName: job.contractName, event: job.events[0], chainId: job.chainId, + // The file name is chainplot's own convention (documented in the + // templates' `snapshot:` paths); only the table it reads from follows + // rindexer's snake_case naming, via eventTableName. outPath: path.join(outDir, `${job.contractName}_${job.events[0].toLowerCase()}.parquet`), }; const { modulePath, execArgv } = workerLaunch(); diff --git a/src/ingest/rindexer/inspectCoverage.ts b/src/ingest/rindexer/inspectCoverage.ts index 106619a..71df739 100644 --- a/src/ingest/rindexer/inspectCoverage.ts +++ b/src/ingest/rindexer/inspectCoverage.ts @@ -6,29 +6,14 @@ import { type CoverageStatus, } from "../adapter.js"; -// rindexer derives table names from the manifest `name` (not the network): -// event table `{name}_{contract}.{event}`, cursor -// `rindexer_internal.{name}_{contract}_{event}`. renderConfig sets -// name = `chainplot_`. -export function manifestName(networkName: string): string { - return `chainplot_${networkName}`; -} - -export function cursorTableName( - networkName: string, - contractName: string, - event: string, -): string { - return `rindexer_internal.${manifestName(networkName)}_${contractName}_${event.toLowerCase()}`; -} - -export function eventTableName( - networkName: string, - contractName: string, - event: string, -): string { - return `${manifestName(networkName)}_${contractName}.${event.toLowerCase()}`; -} +// Table names are derived in naming.ts, which ports rindexer's own snake_case +// rule; re-exported here because this is where callers look for them. +export { + cursorTableName, + eventTableName, + manifestName, +} from "./naming.js"; +import { cursorTableName, eventTableName } from "./naming.js"; export function buildCursorQuery( networkName: string, diff --git a/src/ingest/rindexer/naming.ts b/src/ingest/rindexer/naming.ts new file mode 100644 index 0000000..fd06e84 --- /dev/null +++ b/src/ingest/rindexer/naming.ts @@ -0,0 +1,118 @@ +// rindexer names its Postgres objects by snake_casing the manifest, contract +// and event names it is given. Chainplot has to look those objects up, so it +// has to derive the very same names — lowercasing is not the same operation: +// `RelayERC20Deposit` lowercases to `relayerc20deposit` and snake_cases to +// `relay_erc_20_deposit`. Every current template uses `Transfer`, where the two +// coincide, which is how the difference stayed invisible. +// +// This is a port of rindexer's `camel_to_snake` (core/src/helpers/mod.rs, +// `camel_to_snake_advanced(s, false)`). Keep it byte-for-byte with upstream; +// the tests pin cases observed against a live rindexer database. + +/** Max identifier length Postgres accepts; rindexer compacts beyond it. */ +export const POSTGRES_IDENTIFIER_MAX = 63; + +export function camelToSnake(s: string): string { + let out = ""; + let previousWasUppercase = false; + let previousWasDigit = false; + let uppercaseRun = 0; + + const chars = [...s]; + for (let i = 0; i < chars.length; i++) { + const c = chars[i]!; + const isAlnum = /^[\p{L}\p{N}]$/u.test(c); + if (!isAlnum && c !== "_") continue; + + if (c !== c.toLowerCase() && c === c.toUpperCase()) { + // Uppercase letter. Split before it unless it continues an uppercase run + // — except when the run ends here because a lowercase letter follows + // (`ERC20Deposit` keeps `erc`, but `HTTPServer` splits to `http_server`). + const next = chars[i + 1]; + const nextIsLower = + next !== undefined && next !== next.toUpperCase() && next === next.toLowerCase(); + if (i > 0 && (!previousWasUppercase || nextIsLower)) { + out += "_"; + } + out += c.toLowerCase(); + previousWasUppercase = true; + previousWasDigit = false; + uppercaseRun += 1; + } else if (/^[0-9]$/.test(c)) { + // A digit run gets its own word, unless it directly follows a single + // capital (`V2` stays `v2`) or an underscore. + if (i > 0 && !previousWasDigit && !out.endsWith("_") && uppercaseRun !== 1) { + out += "_"; + } + out += c; + previousWasUppercase = false; + previousWasDigit = true; + uppercaseRun = 0; + } else { + out += c; + previousWasUppercase = false; + previousWasDigit = false; + uppercaseRun = 0; + } + } + return out; +} + +// rindexer derives table names from the manifest `name` (not the network): +// event table `{name}_{contract}.{event}`, cursor +// `rindexer_internal.{name}_{contract}_{event}`. renderConfig sets +// name = `chainplot_`. Each component is snake_cased upstream. +export function manifestName(networkName: string): string { + return camelToSnake(`chainplot_${networkName}`); +} + +export function schemaName(networkName: string, contractName: string): string { + return `${manifestName(networkName)}_${camelToSnake(contractName)}`; +} + +/** Bare (schema-less) cursor table name, before rindexer's length compaction. */ +export function cursorTableBareName( + networkName: string, + contractName: string, + event: string, +): string { + return `${schemaName(networkName, contractName)}_${camelToSnake(event)}`; +} + +export function cursorTableName( + networkName: string, + contractName: string, + event: string, +): string { + return `rindexer_internal.${cursorTableBareName(networkName, contractName, event)}`; +} + +export function eventTableName( + networkName: string, + contractName: string, + event: string, +): string { + return `${schemaName(networkName, contractName)}.${camelToSnake(event)}`; +} + +/** + * Past 63 characters rindexer rewrites the cursor table name with a keccak + * suffix, and Postgres silently truncates the schema. Neither can be looked + * up by the derivation above, so the combination is refused up front — at + * plan time, before a single block is indexed — with the fix spelled out. + */ +export function tableNameOverflow( + networkName: string, + contractName: string, + event: string, +): string | null { + const cursor = cursorTableBareName(networkName, contractName, event); + const schema = schemaName(networkName, contractName); + const longest = cursor.length >= schema.length ? cursor : schema; + if (longest.length <= POSTGRES_IDENTIFIER_MAX) return null; + return ( + `table name ${longest} is ${longest.length} characters; Postgres allows ` + + `${POSTGRES_IDENTIFIER_MAX}. Shorten the source id (${contractName}) — ` + + `it and the event name ${event} make up the excess` + ); +} diff --git a/src/project/validate.ts b/src/project/validate.ts index 7225137..7696436 100644 --- a/src/project/validate.ts +++ b/src/project/validate.ts @@ -5,6 +5,7 @@ import { Ajv2020, type ErrorObject } from "ajv/dist/2020.js"; import type { CommandError } from "../cli/envelope.js"; import { topoSortModels } from "./modelGraph.js"; import type { ProjectDocument } from "./types.js"; +import { tableNameOverflow } from "../ingest/rindexer/naming.js"; const SCHEMA_PATH = path.resolve( path.dirname(fileURLToPath(import.meta.url)), @@ -88,8 +89,27 @@ export function validateProject( (project.chain_sources ?? []).map((c) => [c.id, c] as const), ); for (const [index, source] of (project.event_sources ?? []).entries()) { - if (source.end.mode !== "follow_finalized") continue; const chain = chains.get(source.chain); + // rindexer snake_cases every name into a Postgres identifier, which has a + // 63-character ceiling; past it the table cannot be found by derivation. + // Refuse here, before a single block is indexed. + for (const event of source.events) { + const overflow = tableNameOverflow( + `chainplot_${chain?.chain_id ?? 0}`, + source.id.toLowerCase(), + event, + ); + if (overflow !== null) { + return { + ok: false, + error: error("validation", overflow, { + resource_id: source.id, + pointer: `/event_sources/${index}/events`, + }), + }; + } + } + if (source.end.mode !== "follow_finalized") continue; if (chain?.finality.policy === "confirmation_depth") { return { ok: false, diff --git a/tests/cli/validate.test.ts b/tests/cli/validate.test.ts index 785f383..06e568c 100644 --- a/tests/cli/validate.test.ts +++ b/tests/cli/validate.test.ts @@ -56,3 +56,29 @@ describe("validate", () => { expect(result.error?.message).toContain("missing model file"); }); }); + +// rindexer turns an event name into a Postgres identifier, and Postgres stops +// at 63 characters. A project past that limit indexes fine and then cannot +// find its own tables, so `validate` refuses it with the fix spelled out. +describe("validate refuses table names rindexer would compact", () => { + it("names the source and the event", async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "chainplot-longname-")); + fs.cpSync( + path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../../templates/ingest-transfers"), + dir, + { recursive: true }, + ); + const yamlPath = path.join(dir, "chainplot.yaml"); + fs.writeFileSync( + yamlPath, + fs + .readFileSync(yamlPath, "utf8") + .replace(" - Transfer", " - TransferWithAVeryLongEventNameThatOverflowsAPostgresIdentifier"), + ); + const result = await runCliJson(["validate", "--json"], dir); + expect(result.ok).toBe(false); + expect(result.error?.code).toBe("validation"); + expect(result.error?.message).toMatch(/63/); + expect(result.error?.pointer).toBe("/event_sources/0/events"); + }); +}); diff --git a/tests/ingest/live/e2e.live.test.ts b/tests/ingest/live/e2e.live.test.ts index 061b29f..f9f70a3 100644 --- a/tests/ingest/live/e2e.live.test.ts +++ b/tests/ingest/live/e2e.live.test.ts @@ -130,6 +130,64 @@ describe("live ingest end-to-end (M0 replay through the product)", () => { path.join(cwd, ".env"), `RPC_URL=${rpcUrl}\nDATABASE_URL=postgresql://chainplot:chainplot@postgres:5432/chainplot\n`, ); + + // A second source whose event name is more than one word. rindexer + // snake_cases it into `transfer_shares`; the template's `Transfer` is + // the one name where lowercasing gives the same answer, so only a + // multi-word event proves that coverage and export find the table. + // Lido stETH emits TransferShares with every transfer. + fs.writeFileSync( + path.join(cwd, "abis/stETH.json"), + JSON.stringify([ + { + type: "event", + name: "TransferShares", + anonymous: false, + inputs: [ + { indexed: true, name: "from", type: "address" }, + { indexed: true, name: "to", type: "address" }, + { indexed: false, name: "sharesValue", type: "uint256" }, + ], + }, + ]), + ); + const yamlPath = path.join(cwd, "chainplot.yaml"); + const yaml = fs.readFileSync(yamlPath, "utf8"); + expect(yaml).toContain("\ndatasets:\n"); + expect(yaml).toContain("\nqueries:\n"); + fs.writeFileSync( + yamlPath, + yaml + .replace( + "\ndatasets:\n", + [ + "", + " - id: steth", + " chain: mainnet", + " addresses:", + ' - "0xae7ab96520de3a18e5e111b5eaab095312d7fe84"', + " abi: abis/stETH.json", + " events:", + " - TransferShares", + " start_block: 18600000", + " end:", + " mode: pinned", + " block: 18600100", + "datasets:", + "", + ].join("\n"), + ) + .replace( + "\nqueries:\n", + [ + "", + " - id: steth", + " snapshot: .chainplot/snapshots/steth/steth_transfershares.parquet", + "queries:", + "", + ].join("\n"), + ), + ); await compose(cwd, ["up", "-d"]); const plan = await cli(cwd, ["plan", "--intent", "ingest"]); @@ -163,7 +221,16 @@ describe("live ingest end-to-end (M0 replay through the product)", () => { }[]; }; expect(coverage.chain_id).toBe(1); + expect(coverage.sources).toHaveLength(2); expect(coverage.sources[0].segments).toHaveLength(1); + // The multi-word event: rows were found in rindexer's snake_cased + // table, and the export wrote the parquet the dataset names. + const steth = coverage.sources[1].segments[0]!; + expect(steth).toMatchObject({ end_block: 18600100, status: "complete_with_rows" }); + expect(steth.row_count).toBeGreaterThan(0); + expect( + fs.existsSync(path.join(cwd, ".chainplot/snapshots/steth/steth_transfershares.parquet")), + ).toBe(true); const segment = coverage.sources[0].segments[0]!; expect(segment).toMatchObject({ start_block: 18600000, @@ -201,7 +268,8 @@ describe("live ingest end-to-end (M0 replay through the product)", () => { ), ) as { freshness: { kind: string; data_through: { block: number } } }; expect(release.freshness.kind).toBe("chain"); - expect(release.freshness.data_through.block).toBe(18600010); + // Data reaches as far as the furthest source: the stETH range ends later. + expect(release.freshness.data_through.block).toBe(18600100); // Truncation probe: drop coverage → build refuses (M2 gate). fs.rmSync(path.join(cwd, ".chainplot/coverage.json")); diff --git a/tests/ingest/naming.test.ts b/tests/ingest/naming.test.ts new file mode 100644 index 0000000..09ea01b --- /dev/null +++ b/tests/ingest/naming.test.ts @@ -0,0 +1,77 @@ +import { describe, expect, it } from "vitest"; +import { + camelToSnake, + cursorTableName, + eventTableName, + tableNameOverflow, +} from "../../src/ingest/rindexer/naming.js"; + +// rindexer creates the tables; chainplot only looks them up. Every template +// uses `Transfer`, where lowercase and snake_case coincide, so the difference +// was invisible until a project used a multi-word event: rows landed in +// `relay_erc_20_deposit` while apply asked for `relayerc20deposit`. +describe("camelToSnake follows rindexer's rule exactly", () => { + it.each([ + ["Transfer", "transfer"], + // Observed on a live rindexer database (the case that surfaced the bug). + ["RelayERC20Deposit", "relay_erc_20_deposit"], + ["UserOperationEvent", "user_operation_event"], + ["TransferShares", "transfer_shares"], + // An uppercase run ends where a lowercase letter follows. + ["HTTPServer", "http_server"], + ["ERC20Transfer", "erc_20_transfer"], + // A digit after a single capital attaches to it; after a run it does not. + ["SwapV2", "swap_v2"], + ["PoolV3Created", "pool_v3_created"], + // Source ids are lowercase already, but digits still split. + ["usdc", "usdc"], + ["usdc2", "usdc_2"], + ["erc20dep", "erc_20dep"], + // Underscores are kept and never doubled. + ["chainplot_chainplot_1", "chainplot_chainplot_1"], + ["already_snake", "already_snake"], + ])("%s → %s", (input, expected) => { + expect(camelToSnake(input)).toBe(expected); + }); + + it("drops characters rindexer drops", () => { + expect(camelToSnake("Weird-Name")).toBe("weird_name"); + }); +}); + +describe("table names", () => { + it("snake_cases the contract and the event, in both tables", () => { + expect(eventTableName("chainplot_1", "erc20dep", "RelayERC20Deposit")).toBe( + "chainplot_chainplot_1_erc_20dep.relay_erc_20_deposit", + ); + expect(cursorTableName("chainplot_1", "erc20dep", "RelayERC20Deposit")).toBe( + "rindexer_internal.chainplot_chainplot_1_erc_20dep_relay_erc_20_deposit", + ); + }); + + it("is unchanged for the single-word names every template uses", () => { + expect(eventTableName("chainplot_1", "usdc", "Transfer")).toBe( + "chainplot_chainplot_1_usdc.transfer", + ); + }); +}); + +// Past 63 characters rindexer compacts the cursor table name with a hash and +// Postgres truncates the schema; neither can be derived, so the combination is +// refused before anything is indexed. +describe("tableNameOverflow", () => { + it("accepts names that fit", () => { + expect(tableNameOverflow("chainplot_1", "usdc", "Transfer")).toBeNull(); + }); + + it("names the excess when the cursor table would not fit", () => { + const message = tableNameOverflow( + "chainplot_1", + "a_thirty_character_source_id_x", + "SomeVeryLongEventName", + ); + expect(message).toMatch(/63/); + expect(message).toMatch(/a_thirty_character_source_id_x/); + expect(message).toMatch(/SomeVeryLongEventName/); + }); +}); From 4d01094de0d2e61c7997f16bec076c06b360b6cf Mon Sep 17 00:00:00 2001 From: Eugene Aseev Date: Fri, 18 Sep 2026 10:46:32 +0800 Subject: [PATCH 03/19] Fail publish when the URL it returns does not serve MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit publish verified every uploaded byte against the bucket and then returned a dashboard_url it had never fetched. A public_base_url naming a different bucket, a bucket with public reads off, or an S3 endpoint that folded the bucket into every key all passed that verification and 404'd for every reader — while the command reported ok: true. After promoting latest.json, publish now GETs the dashboard URL and fails the run on anything but 200, spelling out the three causes. The upload has still happened, so the error is retryable and points back at publish. Co-Authored-By: Claude Fable 5.1 --- src/publish/publishRelease.ts | 44 +++++++++++++++++++---- tests/publish/publishCommand.test.ts | 53 ++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+), 7 deletions(-) diff --git a/src/publish/publishRelease.ts b/src/publish/publishRelease.ts index 0812227..83e0e0f 100644 --- a/src/publish/publishRelease.ts +++ b/src/publish/publishRelease.ts @@ -1,7 +1,7 @@ import fs from "node:fs"; import path from "node:path"; import { createHash } from "node:crypto"; -import { commandError } from "../plan/errors.js"; +import { commandError, errorMessage } from "../plan/errors.js"; import type { PublishTarget as PublishTargetDoc } from "../project/types.js"; import type { LatestPointer, @@ -153,17 +153,47 @@ export async function publishRelease( const pointer: LatestPointer = latestPointer(prefix, body); await target.promoteLatest(pointer); + const publicBase = targetDoc.public_base_url?.replace(/\/$/, "") ?? null; + const dashboardUrl = publicBase ? `${publicBase}/${prefix}/index.html` : null; + // verifyFiles proved the bytes are in the bucket. It says nothing about the + // URL handed back: a base URL naming a different bucket, a bucket with public + // access off, or an endpoint that folded the bucket into every key all pass + // that check and then 404 for every reader. One GET settles it. + if (dashboardUrl !== null) { + await assertPublicUrlServes(dashboardUrl, targetDoc.id); + } + return { target_id: targetDoc.id, release_prefix: prefix, - latest_url: targetDoc.public_base_url - ? `${targetDoc.public_base_url.replace(/\/$/, "")}/${base}latest.json` - : null, - dashboard_url: targetDoc.public_base_url - ? `${targetDoc.public_base_url.replace(/\/$/, "")}/${prefix}/index.html` - : null, + latest_url: publicBase ? `${publicBase}/${base}latest.json` : null, + dashboard_url: dashboardUrl, files_uploaded: files.length + referenced.length, datasets_referenced: referenced.map((r) => r.path), promoted: true, }; } + +const PUBLIC_CHECK_TIMEOUT_MS = 15_000; + +async function assertPublicUrlServes(url: string, targetId: string): Promise { + let outcome: string; + try { + const response = await fetch(url, { + signal: AbortSignal.timeout(PUBLIC_CHECK_TIMEOUT_MS), + }); + await response.body?.cancel(); + if (response.status === 200) return; + outcome = `HTTP ${response.status}`; + } catch (err) { + outcome = errorMessage(err); + } + throw commandError( + "transient_dependency", + `uploaded and promoted, but ${url} answered ${outcome} rather than 200. ` + + `The files are in the bucket; the public URL does not serve them. Check that ` + + `public_base_url is this bucket's public origin, that the bucket allows ` + + `public reads, and that CHAINPLOT_S3_ENDPOINT carries no path.`, + { resource_id: targetId, retryable: true, suggested_next: "publish" }, + ); +} diff --git a/tests/publish/publishCommand.test.ts b/tests/publish/publishCommand.test.ts index 0e0b561..613e646 100644 --- a/tests/publish/publishCommand.test.ts +++ b/tests/publish/publishCommand.test.ts @@ -4,6 +4,7 @@ import os from "node:os"; import path from "node:path"; import { fileURLToPath } from "node:url"; import { runCliJson } from "../helpers/run.js"; +import { startServe } from "../../src/publish/serve.js"; const template = path.resolve( path.dirname(fileURLToPath(import.meta.url)), @@ -160,3 +161,55 @@ interface PublishEnvelope { reused: boolean; publish: { release_prefix: string }; } + +// publish used to return a dashboard_url it had never fetched. A base URL that +// serves a different bucket, a bucket with public access off, or an endpoint +// that folded the bucket into every key all passed verifyFiles and then 404'd +// for every reader — while the command reported ok: true. +describe("publish verifies the public URL it hands back", () => { + it("fails when public_base_url does not serve what was uploaded", async () => { + const dir = setupProject(); + expect((await runCliJson(["build", "--json"], dir)).ok).toBe(true); + // A "public origin" that serves an unrelated, empty directory. + const elsewhere = fs.mkdtempSync(path.join(os.tmpdir(), "chainplot-elsewhere-")); + const origin = startServe(elsewhere, 0); + await origin.ready; + try { + fs.appendFileSync( + path.join(dir, "chainplot.yaml"), + ` public_base_url: http://127.0.0.1:${origin.port}\n`, + ); + const result = await runCliJson(["publish", "--json"], dir); + expect(result.ok).toBe(false); + expect(result.error?.code).toBe("transient_dependency"); + expect(result.error?.message).toMatch(/HTTP 404/); + expect(result.error?.message).toMatch(/public_base_url/); + // The upload itself did happen; only the URL is wrong. + expect(fs.existsSync(path.join(dir, "published", "latest.json"))).toBe(true); + } finally { + origin.close(); + } + }, 30_000); + + it("succeeds, and the URL it returns answers 200", async () => { + const dir = setupProject(); + expect((await runCliJson(["build", "--json"], dir)).ok).toBe(true); + const published = path.join(dir, "published"); + fs.mkdirSync(published); + const origin = startServe(published, 0); + await origin.ready; + try { + fs.appendFileSync( + path.join(dir, "chainplot.yaml"), + ` public_base_url: http://127.0.0.1:${origin.port}\n`, + ); + const result = await runCliJson(["publish", "--json"], dir); + expect(result.ok).toBe(true); + const { dashboard_url } = (result.data as { publish: { dashboard_url: string } }).publish; + expect(dashboard_url).toMatch(new RegExp(`^http://127\\.0\\.0\\.1:${origin.port}/releases/`)); + expect((await fetch(dashboard_url)).status).toBe(200); + } finally { + origin.close(); + } + }, 30_000); +}); From 3f9a15c6d102080675c7364b5d46f38fff7b92d1 Mon Sep 17 00:00:00 2001 From: Eugene Aseev Date: Fri, 18 Sep 2026 10:46:32 +0800 Subject: [PATCH 04/19] Let serve bind beyond loopback, so the documented preview step works MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit serve hard-coded 127.0.0.1. Run inside the producer container, as every template and example README instructs, that is the container's own loopback, and the compose files published no ports: the server started, printed a URL, and nothing on the host could open it. serve takes --host (default unchanged: 127.0.0.1). The compose files publish 4173 to the host's loopback only, and the READMEs show the form that works — `serve --host 0.0.0.0 --port 4173` in the container, http://127.0.0.1:4173 on the host. The URL the command reports is that one, never 0.0.0.0. Co-Authored-By: Claude Fable 5.1 --- README.md | 2 +- docs/capabilities.md | 2 +- docs/compatibility.md | 2 +- examples/protocol-flows/README.md | 6 +++-- examples/protocol-flows/compose.yaml | 4 +++ examples/transfer-traffic/README.md | 5 ++-- examples/transfer-traffic/compose.yaml | 4 +++ examples/usdc-supply/compose.yaml | 4 +++ src/cli/commands/serve.ts | 7 +++-- src/cli/run.ts | 15 ++++++++--- src/publish/serve.ts | 12 +++++++-- templates/ingest-transfers/README.md | 7 ++++- templates/ingest-transfers/compose.yaml | 4 +++ tests/cli/serve.test.ts | 35 +++++++++++++++++++++++++ 14 files changed, 94 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 2a713d5..9852983 100644 --- a/README.md +++ b/README.md @@ -123,7 +123,7 @@ limit 10 | `apply --plan ` | Execute that plan, and only that plan | | `refresh` | `plan --intent refresh` + `apply` | | `build` | Write the full static release | -| `serve` | Preview a release on 127.0.0.1 | +| `serve` | Preview a release; binds 127.0.0.1 unless `--host` says otherwise | | `publish` | Push a release to a directory or S3-compatible target | | `runs list\|show\|cancel` | Run journal; cancel is cooperative | | `fork` | Import a published release as a new project | diff --git a/docs/capabilities.md b/docs/capabilities.md index 80fc5c0..2120c1f 100644 --- a/docs/capabilities.md +++ b/docs/capabilities.md @@ -20,7 +20,7 @@ Machine-readable source of truth: `chainplot capabilities --json`. | `build` | shipped | full static release; `--mode results_only\|dataset_referenced` | | `refresh` | shipped | `--publish-target` publishes | | `query` / `dataset describe` / `test` | shipped | offline over snapshots | -| `serve` | shipped | 127.0.0.1 only | +| `serve` | shipped | binds 127.0.0.1 by default; `--host 0.0.0.0` inside a container | | `publish` | shipped | directory + S3-compatible (R2 verified) | | `runs list\|show\|cancel` | shipped | cooperative cancel | | `fork` | shipped | deny-by-default SSRF guard | diff --git a/docs/compatibility.md b/docs/compatibility.md index 5781e55..94cc2bc 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -51,7 +51,7 @@ Full static build on `main`: SELECT model graph (topo order, cycles refused at (full §16.1 layout minus `latest.json`: `index.html`, `assets/`, `dashboards/`, `results/`, `datasets//{manifest.json,tables/*.parquet}`, `source/` allowlist), viewer bundle (React + Vite + ECharts, committed under -`viewer/dist/`, no CDN), `serve` (127.0.0.1 only), `plan --intent build` +`viewer/dist/`, no CDN), `serve` (127.0.0.1 by default, `--host` to change), `plan --intent build` (no RPC). A2 + A9 groundwork pass offline. | Piece | Pin | diff --git a/examples/protocol-flows/README.md b/examples/protocol-flows/README.md index 3f10511..c18ac6b 100644 --- a/examples/protocol-flows/README.md +++ b/examples/protocol-flows/README.md @@ -9,8 +9,10 @@ activity of wrapped Ether. ## Run it Same runtime as example 1 (see its README): `docker compose up -d`, then -`plan --intent ingest` → `apply` → `test` → `build` → `serve` inside the -producer container. Requires `.env` with an archive-capable `RPC_URL`. +`plan --intent ingest` → `apply` → `test` → `build` → +`serve --host 0.0.0.0 --port 4173` inside the producer container, and open + on the host. Requires `.env` with an archive-capable +`RPC_URL`. ## What it shows diff --git a/examples/protocol-flows/compose.yaml b/examples/protocol-flows/compose.yaml index d9893ea..6dfba1e 100644 --- a/examples/protocol-flows/compose.yaml +++ b/examples/protocol-flows/compose.yaml @@ -28,6 +28,10 @@ services: volumes: - ./:/workspace working_dir: /workspace + # `serve --host 0.0.0.0 --port 4173` inside the container is reachable from + # a browser on this machine at http://127.0.0.1:4173, and from nowhere else. + ports: + - "127.0.0.1:4173:4173" entrypoint: ["sleep"] command: ["infinity"] diff --git a/examples/transfer-traffic/README.md b/examples/transfer-traffic/README.md index a6274a2..3deee14 100644 --- a/examples/transfer-traffic/README.md +++ b/examples/transfer-traffic/README.md @@ -20,8 +20,9 @@ docker compose exec producer chainplot plan --intent ingest --json docker compose exec producer chainplot apply --plan --json docker compose exec producer chainplot build --json -# 3. Preview the dashboard (from the host, or serve inside the container) -docker compose exec producer chainplot serve --port 4173 --json +# 3. Preview the dashboard: serve inside the container, open it from the host +docker compose exec producer chainplot serve --host 0.0.0.0 --port 4173 --json +# then open http://127.0.0.1:4173 in a browser on this machine ``` Coverage evidence comes from `rindexer_internal.*.last_synced_block` plus diff --git a/examples/transfer-traffic/compose.yaml b/examples/transfer-traffic/compose.yaml index 70858b6..6edb722 100644 --- a/examples/transfer-traffic/compose.yaml +++ b/examples/transfer-traffic/compose.yaml @@ -28,6 +28,10 @@ services: volumes: - ./:/workspace working_dir: /workspace + # `serve --host 0.0.0.0 --port 4173` inside the container is reachable from + # a browser on this machine at http://127.0.0.1:4173, and from nowhere else. + ports: + - "127.0.0.1:4173:4173" entrypoint: ["sleep"] command: ["infinity"] diff --git a/examples/usdc-supply/compose.yaml b/examples/usdc-supply/compose.yaml index 621bbbe..e8a2fe8 100644 --- a/examples/usdc-supply/compose.yaml +++ b/examples/usdc-supply/compose.yaml @@ -35,6 +35,10 @@ services: volumes: - ./:/workspace working_dir: /workspace + # `serve --host 0.0.0.0 --port 4173` inside the container is reachable from + # a browser on this machine at http://127.0.0.1:4173, and from nowhere else. + ports: + - "127.0.0.1:4173:4173" # The image entrypoint is the CLI itself, so a bare `command:` would be # read as CLI arguments. Hold the container open and drive it with # `docker compose exec producer chainplot --json`. diff --git a/src/cli/commands/serve.ts b/src/cli/commands/serve.ts index 20b7739..1900eb3 100644 --- a/src/cli/commands/serve.ts +++ b/src/cli/commands/serve.ts @@ -19,16 +19,19 @@ export async function serveCommand( cwd: string, dir?: string, port = 0, + host = "127.0.0.1", ): Promise { try { const target = path.resolve(cwd, dir ?? "dist/releases/local"); validateServeDir(target); - const handle = startServe(target, port); + const handle = startServe(target, port, host); activeServer = handle; // The assigned port is only knowable once listen() has called back, so // reporting it before that yields the literal 0 the caller passed in. await handle.ready; - const url = `http://127.0.0.1:${handle.port}`; + // An unspecified bind address is not something a browser can open. + const shownHost = host === "0.0.0.0" || host === "::" ? "127.0.0.1" : host; + const url = `http://${shownHost}:${handle.port}`; // Diagnostics to stderr; stdout stays reserved for the result envelope. process.stderr.write(`serving ${target} at ${url} (Ctrl+C to stop)\n`); const shutdown = () => { diff --git a/src/cli/run.ts b/src/cli/run.ts index 1ea9704..887e31e 100644 --- a/src/cli/run.ts +++ b/src/cli/run.ts @@ -244,12 +244,21 @@ export async function runCli( program .command("serve") - .description("preview a built release over loopback HTTP") + .description("preview a built release over HTTP (127.0.0.1 by default)") .option("--dir ", "release directory (default: dist/releases/local)") .option("--port ", "port (default: random)", (v: string) => parseInt(v, 10)) - .action(async (options: { dir?: string; port?: number }) => { + .option( + "--host ", + "bind address (default: 127.0.0.1; 0.0.0.0 inside a container)", + ) + .action(async (options: { dir?: string; port?: number; host?: string }) => { commandName = "serve"; - result = await serveCommand(opts.cwd, options.dir, options.port ?? 0); + result = await serveCommand( + opts.cwd, + options.dir, + options.port ?? 0, + options.host ?? "127.0.0.1", + ); if (result.ok) { // serve keeps the process alive; the envelope is printed by main.ts // only when it exits, so surface the URL immediately on stderr. diff --git a/src/publish/serve.ts b/src/publish/serve.ts index 37d7adc..814ee09 100644 --- a/src/publish/serve.ts +++ b/src/publish/serve.ts @@ -21,7 +21,11 @@ export interface ServeHandle { close(): void; } -export function startServe(rootDir: string, port: number): ServeHandle { +export function startServe( + rootDir: string, + port: number, + host = "127.0.0.1", +): ServeHandle { const root = path.resolve(rootDir); let readyResolve: (() => void) | null = null; const ready = new Promise((resolve) => { @@ -47,7 +51,11 @@ export function startServe(rootDir: string, port: number): ServeHandle { res.writeHead(200, { "content-type": type }); fs.createReadStream(filePath).pipe(res); }); - server.listen(port, "127.0.0.1", () => readyResolve?.()); + // Loopback by default. Inside a container that loopback is the container's + // own, so a documented preview step needs `--host 0.0.0.0` plus a published + // port; the URL reported to the user stays the one that works from a host + // browser. + server.listen(port, host, () => readyResolve?.()); const handle: ServeHandle = { get port(): number { const address = server.address(); diff --git a/templates/ingest-transfers/README.md b/templates/ingest-transfers/README.md index 64b0334..bc6d9c6 100644 --- a/templates/ingest-transfers/README.md +++ b/templates/ingest-transfers/README.md @@ -27,9 +27,14 @@ docker compose up -d docker compose exec producer chainplot plan --intent ingest --json docker compose exec producer chainplot apply --plan --json docker compose exec producer chainplot build --json -docker compose exec producer chainplot serve --port 4173 --json +docker compose exec producer chainplot serve --host 0.0.0.0 --port 4173 --json ``` +The last command keeps running; open in a browser on +this machine. `compose.yaml` publishes that port to the host's loopback only, +and `--host 0.0.0.0` is what lets the container answer on it — the default bind +is the container's own loopback, which nothing outside can reach. + `plan` is read-only and writes a digest-bound plan; `apply` executes that plan and nothing else. A release is refused unless the whole pinned block range is proven complete. diff --git a/templates/ingest-transfers/compose.yaml b/templates/ingest-transfers/compose.yaml index 621bbbe..e8a2fe8 100644 --- a/templates/ingest-transfers/compose.yaml +++ b/templates/ingest-transfers/compose.yaml @@ -35,6 +35,10 @@ services: volumes: - ./:/workspace working_dir: /workspace + # `serve --host 0.0.0.0 --port 4173` inside the container is reachable from + # a browser on this machine at http://127.0.0.1:4173, and from nowhere else. + ports: + - "127.0.0.1:4173:4173" # The image entrypoint is the CLI itself, so a bare `command:` would be # read as CLI arguments. Hold the container open and drive it with # `docker compose exec producer chainplot --json`. diff --git a/tests/cli/serve.test.ts b/tests/cli/serve.test.ts index 1178468..0423ea2 100644 --- a/tests/cli/serve.test.ts +++ b/tests/cli/serve.test.ts @@ -82,3 +82,38 @@ describe("cli gating", () => { expect(fs.existsSync(path.join(dir, "dist", "releases"))).toBe(false); }); }); + +// Inside the producer container the default loopback is the container's own, +// so the documented preview step printed a URL nothing on the host could +// reach. `--host 0.0.0.0` plus a published port is the fix; the reported URL +// must still be the one that works from a host browser. +describe("serve --host", () => { + it("binds the requested address and reports a loopback URL for it", async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "chainplot-serve-host-")); + fs.cpSync(template, dir, { recursive: true }); + await runCliJson(["build", "--json"], dir); + + const result = await serveCommand(dir, undefined, 0, "0.0.0.0"); + try { + expect(result.ok).toBe(true); + const { url } = result.data as { url: string }; + expect(new URL(url).hostname).toBe("127.0.0.1"); + expect((await fetch(`${url}/release.json`)).status).toBe(200); + } finally { + closeActiveServer(); + } + }); + + it("is accepted on the command line", async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "chainplot-serve-cli-")); + fs.cpSync(template, dir, { recursive: true }); + await runCliJson(["build", "--json"], dir); + const result = await runCliJson(["serve", "--host", "0.0.0.0", "--json"], dir); + try { + expect(result.ok).toBe(true); + expect((result.data as { url: string }).url).toMatch(/^http:\/\/127\.0\.0\.1:\d+$/); + } finally { + closeActiveServer(); + } + }); +}); From 7fb22ac494a0c71ea34d3e319897730218281377 Mon Sep 17 00:00:00 2001 From: Eugene Aseev Date: Fri, 18 Sep 2026 10:46:33 +0800 Subject: [PATCH 05/19] init: write a .gitignore and name the project after its directory MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A scaffolded project told the user to put their RPC endpoint in .env and shipped nothing that would keep that file out of git — nor the run journal or the built release. init now writes a .gitignore covering all three. The copied chainplot.yaml also kept the template's own id, so every project scaffolded from ingest-transfers was called ingest-transfers. The id is now derived from the output directory, reduced to the characters an id may carry, with the template id as the fallback for a name that leaves nothing. Co-Authored-By: Claude Fable 5.1 --- src/cli/commands/init.ts | 35 ++++++++++++++++++++++++++++++++++- tests/cli/init.test.ts | 21 +++++++++++++++++++++ tests/cli/initIngest.test.ts | 12 ++++++++++++ 3 files changed, 67 insertions(+), 1 deletion(-) diff --git a/src/cli/commands/init.ts b/src/cli/commands/init.ts index b707f71..cf44808 100644 --- a/src/cli/commands/init.ts +++ b/src/cli/commands/init.ts @@ -72,5 +72,38 @@ export function initTemplate(id: string, outputDir: string): CommandResult { return validation(message); } - return okResult("init", { template: id, output: dest }); + // A scaffold is a directory someone will `git init` in. The .env it asks + // them to create holds the RPC endpoint, and without this file nothing + // stops it being committed alongside the run journal and the built release. + fs.writeFileSync(path.join(dest, ".gitignore"), SCAFFOLD_GITIGNORE); + + // The template's own id is a placeholder. A project is named after the + // directory it was created in, the way `git init` or `npm init` would. + const projectId = projectIdFrom(path.basename(dest)) ?? id; + const yamlPath = path.join(dest, "chainplot.yaml"); + const yaml = fs.readFileSync(yamlPath, "utf8"); + const idLines = yaml.match(/^id: .*$/gm) ?? []; + if (idLines.length !== 1) { + return validation(`template ${id} has ${idLines.length} top-level id lines; expected 1`); + } + fs.writeFileSync(yamlPath, yaml.replace(/^id: .*$/m, `id: ${JSON.stringify(projectId)}`)); + + return okResult("init", { template: id, output: dest, id: projectId }); +} + +const SCAFFOLD_GITIGNORE = `# Written by chainplot init. +# The RPC endpoint lives here; never commit it. +.env +# Run journal, locks and exported snapshots; rebuilt by plan/apply. +.chainplot/ +# Built release; rebuilt by build. +dist/ +`; + +/** A directory name reduced to the characters a project id may carry. */ +export function projectIdFrom(dirName: string): string | null { + const cleaned = dirName + .replace(/[^A-Za-z0-9._-]+/g, "-") + .replace(/^[-.]+|[-.]+$/g, ""); + return cleaned.length > 0 ? cleaned : null; } diff --git a/tests/cli/init.test.ts b/tests/cli/init.test.ts index 7653cd4..3251e08 100644 --- a/tests/cli/init.test.ts +++ b/tests/cli/init.test.ts @@ -3,6 +3,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { runCliJson } from "../helpers/run.js"; +import { projectIdFrom } from "../../src/cli/commands/init.js"; describe("init", () => { it("lists fixture-transfers", async () => { @@ -28,3 +29,23 @@ describe("init", () => { expect(second.error?.code).toBe("validation"); }); }); + +describe("project id from the output directory", () => { + it("keeps what a project id may carry and folds the rest to a dash", () => { + expect(projectIdFrom("my-analytics")).toBe("my-analytics"); + expect(projectIdFrom("My Project!")).toBe("My-Project"); + expect(projectIdFrom("v1.2_data")).toBe("v1.2_data"); + expect(projectIdFrom("...")).toBeNull(); + }); + + it("falls back to the template id when the directory name is unusable", async () => { + const parent = fs.mkdtempSync(path.join(os.tmpdir(), "chainplot-init-id-")); + const out = path.join(parent, "!!!"); + const result = await runCliJson( + ["init", "--template", "fixture-transfers", "--output", out, "--json"], + parent, + ); + expect(result.ok).toBe(true); + expect((result.data as { id: string }).id).toBe("fixture-transfers"); + }); +}); diff --git a/tests/cli/initIngest.test.ts b/tests/cli/initIngest.test.ts index e581f40..468bedb 100644 --- a/tests/cli/initIngest.test.ts +++ b/tests/cli/initIngest.test.ts @@ -51,6 +51,18 @@ describe("ingest-transfers template", () => { expect(envExample).toContain("RPC_URL="); expect(envExample).not.toMatch(/https?:\/\/(?!127\.0\.0\.1|postgres)/); + // The scaffold tells the user to create .env with their RPC endpoint; + // nothing else would stop that file being committed with the project. + const gitignore = fs.readFileSync(path.join(out, ".gitignore"), "utf8"); + for (const entry of [".env", ".chainplot/", "dist/"]) { + expect(gitignore.split("\n")).toContain(entry); + } + // The project is named after its directory, not after the template. + expect((init.data as { id: string }).id).toBe("my-analytics"); + expect(fs.readFileSync(path.join(out, "chainplot.yaml"), "utf8")).toMatch( + /^id: "my-analytics"$/m, + ); + const validated = await runCliJson(["validate", "--json"], out); expect(validated.ok).toBe(true); }); From 1f3d55902b7cb3da8e853dc2226766008804b932 Mon Sep 17 00:00:00 2001 From: Eugene Aseev Date: Fri, 18 Sep 2026 10:46:33 +0800 Subject: [PATCH 06/19] fork: drop the author's publish targets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit publish_targets came across a fork verbatim. No credentials travel with a release, so nothing could be written there — but the forked project's first `publish` would aim at a stranger's bucket and prefix. fork now removes the block and says so in a warning, alongside the chain sources it already strips. Co-Authored-By: Claude Fable 5.1 --- src/fork/importRelease.ts | 13 ++++++++++++- tests/fork/importRelease.test.ts | 12 +++++++++--- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/src/fork/importRelease.ts b/src/fork/importRelease.ts index 82aaf25..5ba21d3 100644 --- a/src/fork/importRelease.ts +++ b/src/fork/importRelease.ts @@ -213,7 +213,9 @@ export async function importRelease( datasets?: { id: string; snapshot: string }[]; chain_sources?: unknown; event_sources?: unknown; + publish_targets?: unknown; }; + const warnings: string[] = []; for (const dataset of projectDoc.datasets ?? []) { const basename = path.basename(dataset.snapshot); dataset.snapshot = `datasets/${dataset.id}/tables/${basename}`; @@ -229,6 +231,16 @@ export async function importRelease( // forked project is dataset-only. Expanding history is a new ingest project. delete projectDoc.chain_sources; delete projectDoc.event_sources; + // The publish targets name the original author's bucket. No credentials + // come across, so nothing can be written there — but a fork that kept them + // would aim its first `publish` at a stranger's storage. + if (projectDoc.publish_targets !== undefined) { + delete projectDoc.publish_targets; + warnings.push( + "publish_targets dropped: they pointed at the original author's storage. " + + "Add your own publish_targets before running publish.", + ); + } fs.writeFileSync(path.join(outDir, "chainplot.yaml"), stringifyYaml(projectDoc)); // Recipe directories live at project root for the forked copy. @@ -243,7 +255,6 @@ export async function importRelease( // no dataset, so the fork is real and useful yet cannot rebuild until it is // pointed at a snapshot. Saying so here beats a bare "missing snapshot file" // from a `build` the forker has no reason to expect to fail. - const warnings: string[] = []; if (release.mode === "results_only") { warnings.push( `release mode is ${release.mode}: the dataset is not part of it, so the ` + diff --git a/tests/fork/importRelease.test.ts b/tests/fork/importRelease.test.ts index c7019b3..0eeafa4 100644 --- a/tests/fork/importRelease.test.ts +++ b/tests/fork/importRelease.test.ts @@ -63,6 +63,10 @@ describe("fork", () => { // rather than inheriting the producer's. expect(release.mode).toBe("results_only"); expect(release.queries).toContain("raw_amounts"); + // The author's bucket is theirs: a fork must not aim `publish` at it. + const forkedYaml = fs.readFileSync(path.join(out, "chainplot.yaml"), "utf8"); + expect(forkedYaml).not.toMatch(/publish_targets/); + expect(result.warnings).toEqual([expect.stringMatching(/publish_targets dropped/)]); const results = JSON.parse( fs.readFileSync( path.join(out, "dist/releases/local/results/raw_amounts.json"), @@ -144,7 +148,9 @@ describe("forking a release without its dataset", () => { parent, ); expect(result.ok).toBe(true); - expect(result.warnings).toEqual([]); + // The only warning is the one every fork of this fixture gets: its + // publish target was the producer's and did not come across. + expect(result.warnings).toEqual([expect.stringMatching(/publish_targets dropped/)]); }); }); @@ -186,8 +192,8 @@ describe("a referenced dataset round-trips", () => { expect((forked.data as { datasets_referenced: string[] }).datasets_referenced).toEqual([ "datasets/amounts/tables/amounts.parquet", ]); - // Nothing to warn about: the data did come across. - expect(forked.warnings).toEqual([]); + // The data did come across, so the only warning is the dropped target. + expect(forked.warnings).toEqual([expect.stringMatching(/publish_targets dropped/)]); const built = await runCliJson(["build", "--json"], out); expect(built.ok).toBe(true); From e53df349138968ed9c7d5ac99173d950d0c17556 Mon Sep 17 00:00:00 2001 From: Eugene Aseev Date: Fri, 18 Sep 2026 10:46:33 +0800 Subject: [PATCH 07/19] Docs: say how `chainplot` gets onto PATH MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The package is private and nothing links its bin, so `chainplot …` as written in the fork example was "command not found" on a fresh clone. The quickstart now gives the alias and the `pnpm link --global` alternative, and the fork example points at it. Co-Authored-By: Claude Fable 5.1 --- README.md | 10 ++++++++++ examples/fork/README.md | 3 +++ 2 files changed, 13 insertions(+) diff --git a/README.md b/README.md index 9852983..d7ff49c 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,16 @@ node dist/cli/main.js init --template fixture-transfers --output ./demo --json cd demo && node ../dist/cli/main.js build --json && node ../dist/cli/main.js serve --json ``` +Nothing puts a `chainplot` command on your PATH — the package is not published. +Where the docs write `chainplot …`, either alias it after `pnpm build`: + +```bash +alias chainplot="node $PWD/dist/cli/main.js" +``` + +or run `pnpm link --global` once from the checkout. Inside the producer +container the command is already on PATH. + --- ## Why this instead of a notebook diff --git a/examples/fork/README.md b/examples/fork/README.md index 4f3718b..1b1c94d 100644 --- a/examples/fork/README.md +++ b/examples/fork/README.md @@ -15,6 +15,9 @@ to a directory target or an S3-compatible target (see its `chainplot.yaml`). ## Fork +`chainplot` below is the CLI from a checkout of this repo: `pnpm build`, then +`alias chainplot="node $PWD/dist/cli/main.js"` (or `pnpm link --global`). + ```bash chainplot fork --from ./dist/releases/local --output ../forked-dashboard --json # or, against a publish root with latest.json: From 4f648cfd7ca568fc1556889145e0216a8028a836 Mon Sep 17 00:00:00 2001 From: Eugene Aseev Date: Fri, 18 Sep 2026 10:59:45 +0800 Subject: [PATCH 08/19] publish: replay a journaled success only while the release is still there MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The run journal reuses the outcome of a plan that already succeeded. For a publish that meant: empty the bucket, run publish again, and be told "N files uploaded" with nothing uploaded — for as long as the journal entry lived, and with deleting .chainplot/runs/ by hand as the only way out. Before replaying a publish, apply now asks the target whether release.json is still under the recorded prefix and latest.json still exists. If either is gone the plan runs again and the journal records why it was not reused. Co-Authored-By: Claude Fable 5.1 --- src/plan/apply.ts | 46 +++++++++++++++++++++------- src/publish/directory.ts | 4 +++ src/publish/publishRelease.ts | 17 ++++++++++ src/publish/s3.ts | 4 +++ src/publish/target.ts | 2 ++ tests/publish/publishCommand.test.ts | 30 ++++++++++++++++++ 6 files changed, 92 insertions(+), 11 deletions(-) diff --git a/src/plan/apply.ts b/src/plan/apply.ts index bf2a7d3..872d582 100644 --- a/src/plan/apply.ts +++ b/src/plan/apply.ts @@ -24,7 +24,11 @@ import { writeJournalStatus, } from "../runtime/journal.js"; import { acquireLocalLock } from "../runtime/locks.js"; -import { publishRelease, type PublishResult } from "../publish/publishRelease.js"; +import { + publishRelease, + publishedReleaseIntact, + type PublishResult, +} from "../publish/publishRelease.js"; import { commandError, errorMessage } from "./errors.js"; import { projectDigest, type PlanDocument } from "./generate.js"; @@ -105,16 +109,28 @@ export async function applyPlan(opts: ApplyOptions): Promise { } const existingStatus = readJournalStatus(opts.cwd, key); if (existingStatus?.status === "succeeded") { - return { - plan_id: plan.plan_id, - idempotency_key: key, - status: "succeeded", - reused: true, - release_dir: - (existingStatus.result as { release_dir?: string } | undefined)?.release_dir ?? null, - publish: - (existingStatus.result as { publish?: PublishResult | null } | undefined)?.publish ?? null, - }; + const previous = existingStatus.result as + | { release_dir?: string; publish?: PublishResult | null } + | undefined; + const published = previous?.publish ?? null; + // A journaled publish is worth replaying only while what it published is + // still there. Otherwise a bucket emptied since then gets "N files + // uploaded" with nothing uploaded, for as long as the journal lives. + if (published === null || (await publishStillThere(opts.cwd, published))) { + return { + plan_id: plan.plan_id, + idempotency_key: key, + status: "succeeded", + reused: true, + release_dir: previous?.release_dir ?? null, + publish: published, + }; + } + appendCheckpoint(opts.cwd, key, { + stage: "reuse_declined", + release_prefix: published.release_prefix, + message: "published release no longer present at the target", + }); } if (existingStatus?.status === "running") { throw commandError("policy_refused", `run ${key} is already running`, { @@ -222,6 +238,14 @@ export async function applyPlan(opts: ApplyOptions): Promise { } } +async function publishStillThere(cwd: string, published: PublishResult): Promise { + const targetDoc = (loadAndValidate(cwd).publish_targets ?? []).find( + (t) => t.id === published.target_id, + ); + if (!targetDoc) return false; + return publishedReleaseIntact(cwd, targetDoc, published.release_prefix); +} + function loadAndValidate(cwd: string): ProjectDocument { const doc = loadProject(cwd); const validated = validateProject(doc, cwd); diff --git a/src/publish/directory.ts b/src/publish/directory.ts index e88ddbd..9050253 100644 --- a/src/publish/directory.ts +++ b/src/publish/directory.ts @@ -58,6 +58,10 @@ export class DirectoryTarget implements PublishTarget { } } + async releaseExists(prefix: string): Promise { + return fs.existsSync(path.join(this.rootDir, prefix, "release.json")); + } + async readLatest(): Promise { const file = this.latestPath(); if (!fs.existsSync(file)) return null; diff --git a/src/publish/publishRelease.ts b/src/publish/publishRelease.ts index 83e0e0f..6f221cd 100644 --- a/src/publish/publishRelease.ts +++ b/src/publish/publishRelease.ts @@ -34,6 +34,23 @@ export function resolveTarget( return new S3Target(env, undefined, targetDoc.prefix ?? ""); } +/** + * Is a release that was published earlier still where it was put? + * + * The run journal replays a successful publish rather than repeating it. That + * is only right while the published files exist: a bucket emptied since then + * would otherwise be reported as "uploaded" for as long as the journal lives. + */ +export async function publishedReleaseIntact( + projectDir: string, + targetDoc: PublishTargetDoc, + prefix: string, +): Promise { + const target = resolveTarget(targetDoc, projectDir); + if (!(await target.releaseExists(prefix))) return false; + return (await target.readLatest()) !== null; +} + function walkFiles(dir: string, base = dir): string[] { const out: string[] = []; for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { diff --git a/src/publish/s3.ts b/src/publish/s3.ts index 2d64895..74e0150 100644 --- a/src/publish/s3.ts +++ b/src/publish/s3.ts @@ -244,6 +244,10 @@ export class S3Target implements PublishTarget { ifNoneMatch: "*", contentType: "application/json", }); + async releaseExists(prefix: string): Promise { + return (await this.ops.head(`${prefix}/release.json`)) !== null; + } + } } catch (err) { if ((err as { code?: string }).code === "policy_refused") throw err; diff --git a/src/publish/target.ts b/src/publish/target.ts index ff77883..0e219c9 100644 --- a/src/publish/target.ts +++ b/src/publish/target.ts @@ -36,4 +36,6 @@ export interface PublishTarget { ): Promise; readLatest(): Promise; promoteLatest(pointer: LatestPointer): Promise; + /** Whether a release published under `prefix` is still there. */ + releaseExists(prefix: string): Promise; } diff --git a/tests/publish/publishCommand.test.ts b/tests/publish/publishCommand.test.ts index 613e646..d4efc09 100644 --- a/tests/publish/publishCommand.test.ts +++ b/tests/publish/publishCommand.test.ts @@ -213,3 +213,33 @@ describe("publish verifies the public URL it hands back", () => { } }, 30_000); }); + +// The journal replayed a stored success for as long as it lived. Emptying the +// bucket and running publish again reported "N files uploaded" with nothing +// uploaded; recovery was deleting the journal entry by hand. +describe("publish reuse checks the target first", () => { + it("re-uploads when the published release has gone missing", async () => { + const dir = setupProject(); + expect((await runCliJson(["build", "--json"], dir)).ok).toBe(true); + const first = await runCliJson(["publish", "--json"], dir); + expect(first.ok).toBe(true); + expect((first.data as PublishEnvelope).reused).toBe(false); + + // Same release, target intact: replaying the journal is correct. + const replay = await runCliJson(["publish", "--json"], dir); + expect((replay.data as PublishEnvelope).reused).toBe(true); + + // Target emptied: the journal still says succeeded, but nothing is there. + fs.rmSync(path.join(dir, "published"), { recursive: true, force: true }); + const again = await runCliJson(["publish", "--json"], dir); + expect(again.ok).toBe(true); + expect((again.data as PublishEnvelope).reused).toBe(false); + expect(fs.existsSync(path.join(dir, "published", "latest.json"))).toBe(true); + expect( + fs.existsSync( + path.join(dir, "published", (again.data as PublishEnvelope).publish.release_prefix, "release.json"), + ), + ).toBe(true); + }, 30_000); +}); + From 6fbc12a12428caf6a6aa74dd7135dd4827093a1a Mon Sep 17 00:00:00 2001 From: Eugene Aseev Date: Fri, 18 Sep 2026 10:59:45 +0800 Subject: [PATCH 09/19] Refuse an S3 endpoint that carries a path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The client is path-style, so the bucket is the first path segment of every request. An endpoint that already has a path — R2's console shows `…/` beside the account URL — makes the real bucket a prefix on every key: uploads land one level too deep, verification passes, publish reports success, and the public URL serves nothing. CHAINPLOT_S3_ENDPOINT must now be a bare origin. Anything else is a validation error that names the origin to set instead. Co-Authored-By: Claude Fable 5.1 --- .env.example | 5 ++++- src/publish/s3.ts | 31 +++++++++++++++++++++----- tests/publish/s3unit.test.ts | 43 +++++++++++++++++++++++++++++++++++- 3 files changed, 72 insertions(+), 7 deletions(-) diff --git a/.env.example b/.env.example index b67fe6c..3bae342 100644 --- a/.env.example +++ b/.env.example @@ -14,7 +14,10 @@ RPC_URL= DATABASE_URL=postgresql://chainplot:chainplot@localhost:5432/chainplot # --- Publish to S3-compatible storage (M4) ------------------------------- -# Example: Cloudflare R2 endpoint (account-specific URL). +# The endpoint is an origin only — scheme and host, no path. For R2 that is +# https://.r2.cloudflarestorage.com; the bucket goes in +# CHAINPLOT_S3_BUCKET. A path here is sent as the bucket name, so every file +# lands one level too deep and the public URL serves nothing. CHAINPLOT_S3_ENDPOINT= CHAINPLOT_S3_BUCKET= CHAINPLOT_S3_REGION=auto diff --git a/src/publish/s3.ts b/src/publish/s3.ts index 74e0150..bca31ba 100644 --- a/src/publish/s3.ts +++ b/src/publish/s3.ts @@ -31,8 +31,29 @@ export function s3EnvFromProcess( ) { return null; } + // The client is path-style: the bucket becomes the first path segment of + // every request. An endpoint that already carries a path (R2's console + // shows `…/` next to the account URL) makes the real bucket a + // prefix on every key — uploads land one level too deep, publish reports + // success, and the public URL serves nothing. + let url: URL; + try { + url = new URL(env.CHAINPLOT_S3_ENDPOINT); + } catch { + throw commandError("validation", `CHAINPLOT_S3_ENDPOINT is not a URL: ${env.CHAINPLOT_S3_ENDPOINT}`, { + pointer: "/env/CHAINPLOT_S3_ENDPOINT", + }); + } + if (url.pathname !== "/" || url.search !== "" || url.hash !== "") { + throw commandError( + "validation", + `CHAINPLOT_S3_ENDPOINT must be an origin with no path: got ${env.CHAINPLOT_S3_ENDPOINT}. ` + + `The bucket goes in CHAINPLOT_S3_BUCKET; a path here would be sent as the bucket.`, + { pointer: "/env/CHAINPLOT_S3_ENDPOINT", suggested_next: `set CHAINPLOT_S3_ENDPOINT=${url.origin}` }, + ); + } return { - endpoint: env.CHAINPLOT_S3_ENDPOINT, + endpoint: url.origin, bucket: env.CHAINPLOT_S3_BUCKET, accessKeyId: env.AWS_ACCESS_KEY_ID, secretAccessKey: env.AWS_SECRET_ACCESS_KEY, @@ -223,6 +244,10 @@ export class S3Target implements PublishTarget { } } + async releaseExists(prefix: string): Promise { + return (await this.ops.head(`${prefix}/release.json`)) !== null; + } + async readLatest(): Promise { const obj = await this.ops.get(this.latestKey()); if (obj === null) return null; @@ -244,10 +269,6 @@ export class S3Target implements PublishTarget { ifNoneMatch: "*", contentType: "application/json", }); - async releaseExists(prefix: string): Promise { - return (await this.ops.head(`${prefix}/release.json`)) !== null; - } - } } catch (err) { if ((err as { code?: string }).code === "policy_refused") throw err; diff --git a/tests/publish/s3unit.test.ts b/tests/publish/s3unit.test.ts index 9f02370..082fc9b 100644 --- a/tests/publish/s3unit.test.ts +++ b/tests/publish/s3unit.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest"; import { createHash } from "node:crypto"; -import { S3Target, type S3Ops } from "../../src/publish/s3.js"; +import { S3Target, s3EnvFromProcess, type S3Ops } from "../../src/publish/s3.js"; interface Stored { body: string | Uint8Array; @@ -121,3 +121,44 @@ import { latestPointer } from "../../src/publish/latestPointer.js"; function latestPointerOf(prefix: string, body: string) { return latestPointer(prefix, body); } + +// R2's console shows the bucket next to the account URL, and a path pasted +// into CHAINPLOT_S3_ENDPOINT is sent as the bucket by a path-style client: +// every key lands one level too deep, publish reports success, the public +// URL serves nothing. +describe("s3EnvFromProcess", () => { + const base = { + CHAINPLOT_S3_BUCKET: "b", + AWS_ACCESS_KEY_ID: "a", + AWS_SECRET_ACCESS_KEY: "s", + }; + + it("is null while any variable is missing", () => { + expect(s3EnvFromProcess({ ...base })).toBeNull(); + }); + + it("accepts an origin, with or without the trailing slash", () => { + const endpoint = "https://acct.r2.cloudflarestorage.com"; + expect(s3EnvFromProcess({ ...base, CHAINPLOT_S3_ENDPOINT: endpoint })?.endpoint).toBe(endpoint); + expect(s3EnvFromProcess({ ...base, CHAINPLOT_S3_ENDPOINT: `${endpoint}/` })?.endpoint).toBe(endpoint); + }); + + it("refuses an endpoint that carries a path, naming the fix", () => { + expect(() => + s3EnvFromProcess({ ...base, CHAINPLOT_S3_ENDPOINT: "https://acct.r2.cloudflarestorage.com/b" }), + ).toThrow(/no path/); + try { + s3EnvFromProcess({ ...base, CHAINPLOT_S3_ENDPOINT: "https://acct.r2.cloudflarestorage.com/b" }); + } catch (err) { + expect((err as { suggested_next: string }).suggested_next).toBe( + "set CHAINPLOT_S3_ENDPOINT=https://acct.r2.cloudflarestorage.com", + ); + } + }); + + it("refuses something that is not a URL", () => { + expect(() => s3EnvFromProcess({ ...base, CHAINPLOT_S3_ENDPOINT: "acct.r2.dev" })).toThrow( + /not a URL/, + ); + }); +}); From 38e9bc2c94d2a7bc2b2eaa49fb95ba62066c49ba Mon Sep 17 00:00:00 2001 From: Eugene Aseev Date: Fri, 18 Sep 2026 10:59:45 +0800 Subject: [PATCH 10/19] CLI: register --jsonl, and answer --help without --json --jsonl was read straight from argv but never declared to the parser, so commander rejected it as an unknown option and progress streaming could not be switched on. It is an option on apply and refresh now. --help without --json returned a validation error, and with --json returned "help is not available in JSON mode". Asking what the commands are is not running one: help needs no --json, the text rides in the envelope when --json is given, and main prints it plain when it is not. The gate on real commands is unchanged. Co-Authored-By: Claude Fable 5.1 --- src/cli/main.ts | 6 +++++ src/cli/run.ts | 30 +++++++++++++---------- tests/cli/help.test.ts | 36 ++++++++++++++++++++++++++++ tests/publish/publishCommand.test.ts | 13 ++++++++++ 4 files changed, 73 insertions(+), 12 deletions(-) create mode 100644 tests/cli/help.test.ts diff --git a/src/cli/main.ts b/src/cli/main.ts index 9cfd364..5c65795 100644 --- a/src/cli/main.ts +++ b/src/cli/main.ts @@ -22,6 +22,12 @@ try { suggested_next: null, }); } +// `--help` without --json is a person asking; give them the text, not an +// envelope around it. +if (result.ok && result.command === "help" && !process.argv.includes("--json")) { + process.stdout.write((result.data as { text: string }).text); + process.exit(0); +} process.stdout.write(JSON.stringify(result) + "\n"); // `serve` is long-running: its result (with the URL) is printed once and the // process stays alive until SIGINT/SIGTERM (handled inside the command). diff --git a/src/cli/run.ts b/src/cli/run.ts index 887e31e..f1fafc3 100644 --- a/src/cli/run.ts +++ b/src/cli/run.ts @@ -1,6 +1,6 @@ import path from "node:path"; import { Command, CommanderError } from "commander"; -import { failResult, type CommandResult } from "./envelope.js"; +import { failResult, type CommandResult, okResult } from "./envelope.js"; import { capabilities } from "./commands/capabilities.js"; import { datasetDescribe } from "./commands/describe.js"; import { initTemplate } from "./commands/init.js"; @@ -46,12 +46,16 @@ export async function runCli( ): Promise { const json = argv.includes("--json"); const cmdArgv = argv.filter((arg) => arg !== "--json"); + // Asking what the commands are is not running one, so it needs no --json. + // The text comes back in the envelope either way; main.ts prints it plain + // when --json was not given. + const wantsHelp = argv.includes("--help") || argv.includes("-h"); // Gate before dispatch, not after. Checking this once the action has run // means a refused `build` still writes a release and a refused `publish` // still uploads — a caller that trusts `ok: false` and retries would then // double the side effects. - if (!json) { + if (!json && !wantsHelp) { return validationError( cmdArgv.find((arg) => !arg.startsWith("-")) ?? "", "--json is required", @@ -61,6 +65,7 @@ export async function runCli( let result: CommandResult | undefined; let commandName = ""; + let helpText = ""; const program = new Command(); program @@ -69,7 +74,9 @@ export async function runCli( .allowExcessArguments(false) .showHelpAfterError(false) .configureOutput({ - writeOut: () => {}, + writeOut: (text) => { + helpText += text; + }, writeErr: () => {}, }); @@ -194,13 +201,14 @@ export async function runCli( .description("execute a plan written by `plan`") .requiredOption("--plan ", "plan path or plan id") .option("--idempotency-key ", "idempotency key (default: plan digest)") - .action(async (options: { plan: string; idempotencyKey?: string }) => { + .option("--jsonl", "stream progress events as JSON lines before the result") + .action(async (options: { plan: string; idempotencyKey?: string; jsonl?: boolean }) => { commandName = "apply"; result = await applyCommand( opts.cwd, options.plan, options.idempotencyKey, - argv.includes("--jsonl"), + options.jsonl ?? false, ); }); @@ -208,12 +216,13 @@ export async function runCli( .command("refresh") .description("resume indexing, rebuild, optionally publish") .option("--publish-target ", "publish target id (must exist in chainplot.yaml)") - .action(async (options: { publishTarget?: string }) => { + .option("--jsonl", "stream progress events as JSON lines before the result") + .action(async (options: { publishTarget?: string; jsonl?: boolean }) => { commandName = "refresh"; result = await refreshCommand( opts.cwd, options.publishTarget, - argv.includes("--jsonl"), + options.jsonl ?? false, ); }); @@ -312,11 +321,8 @@ export async function runCli( err.code === "commander.helpDisplayed" || err.code === "commander.help" ) { - return validationError( - cmd, - cmd ? "help is not available in JSON mode" : "a command is required", - cmd ? null : "/command", - ); + if (wantsHelp) return okResult("help", { text: helpText }); + return validationError(cmd, "a command is required", "/command"); } if (err.code === "commander.unknownCommand") { return validationError(cmd, err.message, "/command"); diff --git a/tests/cli/help.test.ts b/tests/cli/help.test.ts new file mode 100644 index 0000000..53832be --- /dev/null +++ b/tests/cli/help.test.ts @@ -0,0 +1,36 @@ +import { describe, expect, it } from "vitest"; +import { runCli } from "../../src/cli/run.js"; + +// "Every command speaks JSON" was true of every command except asking for +// help: --help without --json came back as a validation error, and with +// --json as "help is not available in JSON mode". Help is not a command run, +// so it needs no --json; the text rides in the envelope either way. +describe("--help", () => { + it("is answered without --json", async () => { + const result = await runCli(["--help"], { cwd: process.cwd() }); + expect(result.ok).toBe(true); + expect(result.command).toBe("help"); + const { text } = result.data as { text: string }; + expect(text).toMatch(/Usage: chainplot/); + expect(text).toMatch(/\bplan\b/); + expect(text).toMatch(/\bpublish\b/); + }); + + it("describes a single command's options", async () => { + const result = await runCli(["plan", "--help"], { cwd: process.cwd() }); + expect(result.ok).toBe(true); + expect((result.data as { text: string }).text).toMatch(/--intent/); + }); + + it("still speaks JSON when asked to", async () => { + const result = await runCli(["serve", "--help", "--json"], { cwd: process.cwd() }); + expect(result.ok).toBe(true); + expect((result.data as { text: string }).text).toMatch(/--host/); + }); + + it("does not loosen the gate for real commands", async () => { + const result = await runCli(["validate"], { cwd: process.cwd() }); + expect(result.ok).toBe(false); + expect(result.error?.message).toContain("--json is required"); + }); +}); diff --git a/tests/publish/publishCommand.test.ts b/tests/publish/publishCommand.test.ts index d4efc09..7f3453a 100644 --- a/tests/publish/publishCommand.test.ts +++ b/tests/publish/publishCommand.test.ts @@ -243,3 +243,16 @@ describe("publish reuse checks the target first", () => { }, 30_000); }); +// --jsonl was read straight from argv and never registered, so commander +// rejected it as an unknown option: progress streaming was unreachable. +describe("apply --jsonl", () => { + it("is accepted and still returns the result envelope", async () => { + const dir = setupProject(); + expect((await runCliJson(["build", "--json"], dir)).ok).toBe(true); + const plan = await runCliJson(["plan", "--intent", "publish", "--json"], dir); + const planPath = (plan.data as { plan_path: string }).plan_path; + const applied = await runCliJson(["apply", "--plan", planPath, "--jsonl", "--json"], dir); + expect(applied.ok).toBe(true); + expect((applied.data as PublishEnvelope).publish.release_prefix).toMatch(/^releases\//); + }, 30_000); +}); From 6a9d3af007b3f4492ead66c05d2fe122f0f3dfa2 Mon Sep 17 00:00:00 2001 From: Eugene Aseev Date: Fri, 18 Sep 2026 10:59:45 +0800 Subject: [PATCH 11/19] Run the CLI in the container as the owner of the project MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The producer ran as root, so on Linux everything it wrote into the bind-mounted project — .chainplot/, dist/ — belonged to root, and the host user could neither read results nor delete the project without sudo. The `chainplot` shim now drops to the uid:gid that owns /workspace before starting the CLI; where the mount is root-owned it is a no-op. HOME moves to a world-readable directory so DuckDB finds its pre-installed postgres extension as any user. CI builds a project inside the container and asserts the host user owns, and can remove, what came out. The image also gains templates/, which `init` and `templates list` read at runtime and which had never been copied in. Co-Authored-By: Claude Fable 5.1 --- .github/workflows/ci.yml | 12 ++++++++++++ docker/producer.Dockerfile | 27 ++++++++++++++++++++++----- templates/ingest-transfers/README.md | 4 ++++ 3 files changed, 38 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7c3122e..2b17153 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -86,6 +86,18 @@ jobs: docker compose exec -T producer chainplot capabilities --json docker compose exec -T producer chainplot validate --json + # The container writes into the bind-mounted project as its owner, not as + # root, so a build run inside it leaves files the host user can read and + # delete without sudo. The fixture template builds offline. + - name: Files the container writes belong to the host user + working-directory: ${{ runner.temp }}/proj + run: | + docker compose exec -T producer chainplot init \ + --template fixture-transfers --output /workspace/fx --json + docker compose exec -T -w /workspace/fx producer chainplot build --json + test -O fx/dist/releases/local/release.json + rm -rf fx + - name: Tear down if: always() working-directory: ${{ runner.temp }}/proj diff --git a/docker/producer.Dockerfile b/docker/producer.Dockerfile index 0b5ade5..61ae19b 100644 --- a/docker/producer.Dockerfile +++ b/docker/producer.Dockerfile @@ -12,18 +12,35 @@ COPY package.json pnpm-lock.yaml ./ RUN pnpm install --frozen-lockfile COPY tsconfig.json ./ COPY schemas ./schemas +# `init` and `templates list` read these at runtime. +COPY templates ./templates COPY src ./src # The viewer bundle is a build artifact copied in, not rebuilt here, so the # producer image stays CLI + rindexer. Build it on the host first (`pnpm build`). COPY viewer/dist ./viewer/dist RUN pnpm build:cli COPY --from=rindexer /app/rindexer /usr/local/bin/rindexer +# The CLI runs as whichever user owns the mounted project (see the shim +# below), so HOME — where DuckDB looks for its extensions — has to be a +# directory any user can read and write, not /root. +ENV HOME=/home/chainplot +ENV DUCKDB_EXTENSION_DIRECTORY=/home/chainplot/.duckdb/extensions # Pre-install the DuckDB postgres extension so export works without network. -RUN node --input-type=module -e "import { DuckDBInstance } from '@duckdb/node-api'; const db = await DuckDBInstance.create(':memory:'); const c = await db.connect(); await c.run('INSTALL postgres; LOAD postgres;');" && mkdir -p /workspace/.duckdb -ENV DUCKDB_EXTENSION_DIRECTORY=/root/.duckdb/extensions -# A `chainplot` on PATH, so the documented +RUN mkdir -p /home/chainplot \ + && node --input-type=module -e "import { DuckDBInstance } from '@duckdb/node-api'; const db = await DuckDBInstance.create(':memory:'); const c = await db.connect(); await c.run('INSTALL postgres; LOAD postgres;');" \ + && chmod -R a+rwX /home/chainplot +# `chainplot` on PATH, so the documented # `docker compose exec producer chainplot --json` is the real command. -RUN printf '#!/bin/sh\nexec node /app/dist/cli/main.js "$@"\n' \ +# +# It drops to the uid:gid that owns /workspace before running the CLI. The +# files it writes into the bind-mounted project — .chainplot/, dist/ — are then +# the host user's to read and delete, not root's. Where the mount is owned by +# root this is a no-op. +RUN printf '%s\n' \ + '#!/bin/sh' \ + 'owner=$(stat -c %u:%g /workspace 2>/dev/null || echo 0:0)' \ + 'if [ "$owner" = "0:0" ]; then exec node /app/dist/cli/main.js "$@"; fi' \ + 'exec setpriv --reuid="${owner%%:*}" --regid="${owner##*:}" --clear-groups node /app/dist/cli/main.js "$@"' \ > /usr/local/bin/chainplot \ && chmod +x /usr/local/bin/chainplot -ENTRYPOINT ["node", "/app/dist/cli/main.js"] +ENTRYPOINT ["/usr/local/bin/chainplot"] diff --git a/templates/ingest-transfers/README.md b/templates/ingest-transfers/README.md index bc6d9c6..02c8982 100644 --- a/templates/ingest-transfers/README.md +++ b/templates/ingest-transfers/README.md @@ -39,6 +39,10 @@ is the container's own loopback, which nothing outside can reach. and nothing else. A release is refused unless the whole pinned block range is proven complete. +Everything the container writes into this directory — `.chainplot/`, `dist/` — +is owned by you, not by root: the CLI in the container runs as the owner of +the project directory. + ## Files | File | Purpose | From 4cca0836c12550435065db4dbe9f9079205f4b00 Mon Sep 17 00:00:00 2001 From: Eugene Aseev Date: Fri, 18 Sep 2026 10:59:46 +0800 Subject: [PATCH 12/19] Gate the live e2e on a mainnet RPC, not on any RPC The scaffolded project pins mainnet blocks, but the suite ran whenever RPC_URL was set. An endpoint for any other chain turned it red for a reason unrelated to the code. The gate now asks the endpoint which chain it serves and skips unless the answer is 1. Co-Authored-By: Claude Fable 5.1 --- tests/ingest/live/e2e.live.test.ts | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/tests/ingest/live/e2e.live.test.ts b/tests/ingest/live/e2e.live.test.ts index f9f70a3..9f92a34 100644 --- a/tests/ingest/live/e2e.live.test.ts +++ b/tests/ingest/live/e2e.live.test.ts @@ -39,8 +39,28 @@ async function dockerAvailable(): Promise { } } +// The scaffolded project pins mainnet blocks. An RPC for any other chain +// would fail the suite for a reason that has nothing to do with the code, so +// the gate asks the endpoint which chain it serves. +async function servesMainnet(url: string | undefined): Promise { + if (!url) return false; + try { + const response = await fetch(url, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ jsonrpc: "2.0", id: 1, method: "eth_chainId", params: [] }), + signal: AbortSignal.timeout(10_000), + }); + const body = (await response.json()) as { result?: string }; + return body.result === "0x1"; + } catch { + return false; + } +} + const hasDocker = await dockerAvailable(); -const d = rpcUrl && hasDocker ? it : it.skip; +const onMainnet = await servesMainnet(rpcUrl); +const d = onMainnet && hasDocker ? it : it.skip; async function compose(cwd: string, args: string[], timeout = 900_000) { return exec("docker", ["compose", ...args], { From 2d58b2d9c0e2c171a079ba54da9768bf53c12c17 Mon Sep 17 00:00:00 2001 From: Eugene Aseev Date: Fri, 18 Sep 2026 10:59:46 +0800 Subject: [PATCH 13/19] Say on the dashboard whether a release can be forked, and how "Anyone can fork it" was true of the format but told a reader nothing about this release, and nothing about needing the CLI. The footer now says either that the release is results-only and cannot be recomputed, or links to a README section that walks through cloning, building and forking. The README states plainly that forkability needs dataset_included or dataset_referenced. Co-Authored-By: Claude Fable 5.1 --- README.md | 32 ++++++++++++++++++++++++++++---- viewer/src/App.tsx | 11 +++++++++++ viewer/src/styles.css | 6 ++++++ 3 files changed, 45 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index d7ff49c..8cffda6 100644 --- a/README.md +++ b/README.md @@ -42,10 +42,12 @@ container the command is already on PATH. - **The output outlives the infrastructure.** A published release is static files. Your RPC provider, your Postgres, and this CLI can all be gone and the dashboard still renders. -- **Anyone can fork it.** A release always ships the recipe, and can ship the - dataset with it, so a second person imports it, writes a new query, and - rebuilds — no reindexing, no credentials, no access to your RPC. That costs - upload size, so it is opt-in: see [What a release weighs](#what-a-release-weighs). +- **Anyone can fork it.** A release always ships the recipe, and can ship or + reference the dataset, so a second person imports it, writes a new query, + and rebuilds — no reindexing, no credentials, no access to your RPC. The + default `results_only` release ships answers without the dataset, so a fork + of it cannot recompute; see [What a release weighs](#what-a-release-weighs) + and [Fork a published release](#fork-a-published-release). ## The pipeline @@ -192,6 +194,28 @@ page with `--mode results_only`; what you give up is the ability for someone forking it to recompute your numbers from source data, which is why the default keeps the data in. +## Fork a published release + +Every dashboard's footer says whether it can be recomputed. It can when the +release was built with `dataset_included` or `dataset_referenced`; a +`results_only` release carries the recipe and the answers but no data, so a +fork of it has nothing to rebuild from until it is pointed at a snapshot. + +Forking needs the CLI, which is this repository: + +```bash +git clone https://github.com/chainstacklabs/chainplot && cd chainplot +pnpm install && pnpm build +# --from is the publish root: the URL that has latest.json under it +node dist/cli/main.js fork --from https:/// --output ../my-fork --json +cd ../my-fork && node ../chainplot/dist/cli/main.js build --json +``` + +`fork` verifies every file against the release's checksums, pulls in a +referenced dataset and checks it against its manifest, and strips the chain +sources and publish targets — the new project is dataset-only and its first +`publish` goes wherever you say. No RPC, no Postgres, no credentials. + ## Security **Forking runs a stranger's SQL on your machine.** That is the whole point of a diff --git a/viewer/src/App.tsx b/viewer/src/App.tsx index 582a3cc..a236b4f 100644 --- a/viewer/src/App.tsx +++ b/viewer/src/App.tsx @@ -17,6 +17,9 @@ import { toChartNumber, } from "./format.js"; +/** Where a reader learns how to rebuild a release from its dataset. */ +const FORK_HOWTO_URL = "https://github.com/chainstacklabs/chainplot#fork-a-published-release"; + /** Columns the panel asked to compute but not show, e.g. an explicit sort key. */ function visibleColumns( columns: ColumnMeta[], @@ -527,6 +530,14 @@ export function App() {
Built by chainplot · {release.mode.replace(/_/g, " ")} ·{" "} + {" · "} + {release.mode === "results_only" ? ( + <>results only: the dataset is not published, so this release cannot be recomputed + ) : ( + + fork this release and recompute it + + )}
); diff --git a/viewer/src/styles.css b/viewer/src/styles.css index 914b2bd..400766f 100644 --- a/viewer/src/styles.css +++ b/viewer/src/styles.css @@ -421,6 +421,12 @@ table.data-table th.raw button::after { color: var(--text-secondary); } +.colophon a { + color: var(--text-secondary); + text-decoration: underline; + text-underline-offset: 2px; +} + @media (max-width: 760px) { .app { padding: 1.5rem 0.9rem 3rem; From 032dff28143d44a9be765e1a6218c5d5a4a6ebb5 Mon Sep 17 00:00:00 2001 From: Eugene Aseev Date: Fri, 18 Sep 2026 10:59:46 +0800 Subject: [PATCH 14/19] Explain block_budget in time as well as blocks The budget is a block count, and 100k blocks is two weeks on Ethereum, two days on Base and an afternoon on Arbitrum One. Someone who only knows one chain carries the default onto another and silently gets a different span. The template and the limits table now say so. Co-Authored-By: Claude Fable 5.1 --- docs/capabilities.md | 2 +- templates/ingest-transfers/chainplot.yaml | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/capabilities.md b/docs/capabilities.md index 2120c1f..885140e 100644 --- a/docs/capabilities.md +++ b/docs/capabilities.md @@ -60,7 +60,7 @@ visible in review rather than only in production. |---|---|---| | Chains per project | 1 | `schemas/project.schema.json` (`maxItems`) | | Contract addresses | 20 | `schemas/project.schema.json` (`maxItems`) | -| Blocks per approved run | 100_000, `policy.block_budget` to change | `src/plan/generate.ts` (`DEFAULT_BLOCK_BUDGET`) | +| Blocks per approved run | 100_000, `policy.block_budget` to change. A block count, not a duration: 100k blocks is about 14 days on Ethereum (12 s blocks), 2.3 days on Base (2 s), 7 hours on Arbitrum One (0.25 s) — set it for the chain you index | `src/plan/generate.ts` (`DEFAULT_BLOCK_BUDGET`) | | Query deadline | 60 s | `src/query/runQuery.ts` (`DEADLINE_MS`, SIGKILL) | | DuckDB memory | 1 GiB, spills to a temp dir | `src/query/workerMain.ts` (`MEMORY_LIMIT`) | | Returned rows | 10_000, `policy.row_limit` to change | `src/project/limits.ts`, enforced in `src/query/workerMain.ts` (the reader stops at the limit) | diff --git a/templates/ingest-transfers/chainplot.yaml b/templates/ingest-transfers/chainplot.yaml index 5b17a32..892c3df 100644 --- a/templates/ingest-transfers/chainplot.yaml +++ b/templates/ingest-transfers/chainplot.yaml @@ -1,6 +1,10 @@ format_version: 1 id: ingest-transfers policy: + # Blocks one approved run may cover; `plan` splits a longer range into runs + # of this size. It is a block count, so what it spans in time depends on the + # chain: 100000 blocks is about 14 days on Ethereum, 2.3 days on Base and + # 7 hours on Arbitrum One. Set it for the chain you index. block_budget: 100000 chain_sources: - id: mainnet From d5d93a465583db9b673b73e7f9090e1e31bf7e88 Mon Sep 17 00:00:00 2001 From: Eugene Aseev Date: Fri, 18 Sep 2026 10:59:46 +0800 Subject: [PATCH 15/19] Make the one-event export contract explicit apply fans a multi-event source out into one job per event before calling exportEventTable, which reads events[0]. Read on its own that looks like the second event is dropped; it is not, but a job carrying several would be exported as one without a word. Refuse it instead. Co-Authored-By: Claude Fable 5.1 --- src/ingest/exporter.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/ingest/exporter.ts b/src/ingest/exporter.ts index 35b128b..d313b04 100644 --- a/src/ingest/exporter.ts +++ b/src/ingest/exporter.ts @@ -70,6 +70,14 @@ export async function exportEventTable( job: BoundedJob, outDir: string, ): Promise { + // One parquet per event. apply fans a multi-event source out into one job + // per event before calling this; a job carrying several would silently + // export only the first, so it is refused rather than guessed at. + if (job.events.length !== 1) { + throw new Error( + `exportEventTable expects exactly one event per job, got ${job.events.length} for ${job.sourceId}`, + ); + } fs.mkdirSync(outDir, { recursive: true }); const req: ExportRequest = { databaseUrl: job.databaseUrl, From 23a5c16052f76d7524f419614ff526704fa15a96 Mon Sep 17 00:00:00 2001 From: Anton Sauchyk Date: Fri, 18 Sep 2026 22:19:08 +0200 Subject: [PATCH 16/19] Read DuckDB results by column, so the uniqueness gate can fire MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `getRowsJson()` returns positional arrays; the worker indexed them as objects, so `total` and `distinct_keys` were both `undefined` and were coerced to 0. Two consequences: every export reported `"rows": 0` to the progress stream and the run journal, and the duplicate-key gate compared 0 to 0, so it has never fired and could not. Read by column name instead, and make a missing count column an error rather than a default — the silent 0 is what hid this. The gate and the count move into exporter.ts where they can be tested; the worker keeps the sequencing. Tests cover both readers against real DuckDB output, and a table holding a genuinely duplicated physical key now trips the gate. Verified against the live database: the export that reported 0 reports 20618. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01CVa8K4piCcGRDA5f83RkYt --- src/ingest/exportWorkerMain.ts | 28 +++++---------- src/ingest/exporter.ts | 56 +++++++++++++++++++++++++++++ tests/ingest/exporter.test.ts | 66 ++++++++++++++++++++++++++++++++++ 3 files changed, 131 insertions(+), 19 deletions(-) diff --git a/src/ingest/exportWorkerMain.ts b/src/ingest/exportWorkerMain.ts index 0d907b9..839d4f8 100644 --- a/src/ingest/exportWorkerMain.ts +++ b/src/ingest/exportWorkerMain.ts @@ -1,6 +1,12 @@ import fs from "node:fs"; import { DuckDBInstance } from "@duckdb/node-api"; -import { buildExportSql, type ExportRequest } from "./exporter.js"; +import { + assertUniqueCounts, + buildExportSql, + readCounts, + readRowCount, + type ExportRequest, +} from "./exporter.js"; async function readRequest(): Promise { let buf = ""; @@ -29,25 +35,9 @@ async function execute(req: ExportRequest): Promise { for (let i = 0; i < statements.length; i++) { const sql = statements[i]; if (i === 3) { - const reader = await conn.runAndReadAll(sql); - const rows = reader.getRowsJson() as unknown as { - total: bigint | number; - distinct_keys: bigint | number; - }[]; - const total = Number(rows[0]?.total ?? 0); - const distinctKeys = Number(rows[0]?.distinct_keys ?? 0); - if (total !== distinctKeys) { - throw new Error( - JSON.stringify({ - code: "source_inconsistent", - message: `duplicate physical keys: total=${total} distinct=${distinctKeys}`, - }), - ); - } + assertUniqueCounts(readCounts(await conn.runAndReadAll(sql))); } else if (i === statements.length - 1) { - const reader = await conn.runAndReadAll(sql); - const rows = reader.getRowsJson() as unknown as { n: bigint | number }[]; - return Number(rows[0]?.n ?? 0); + return readRowCount(await conn.runAndReadAll(sql)); } else { await conn.run(sql); } diff --git a/src/ingest/exporter.ts b/src/ingest/exporter.ts index d313b04..5f70ba5 100644 --- a/src/ingest/exporter.ts +++ b/src/ingest/exporter.ts @@ -30,6 +30,62 @@ export function buildExportSql(req: ExportRequest): string { ].join("\n"); } +/** The shape `runAndReadAll` gives back; only the part we read from. */ +interface RowReader { + getRowObjectsJson(): unknown[]; +} + +export interface UniquenessCounts { + total: number; + distinctKeys: number; +} + +/** + * DuckDB has two readers and they are not interchangeable: `getRowsJson()` + * returns positional arrays, `getRowObjectsJson()` returns column-keyed + * objects. Reading the first as if it were the second yields `undefined` for + * every column. Coercing that to 0 is what left the gate below comparing 0 + * to 0, so a missing column is an error here rather than a default. + */ +function readCount(row: Record | undefined, column: string): number { + const value = row?.[column]; + if (value === undefined || value === null) { + throw new Error(`export: expected a ${column} count column, got none`); + } + return Number(value); +} + +function firstRow(reader: RowReader): Record | undefined { + return reader.getRowObjectsJson()[0] as Record | undefined; +} + +export function readCounts(reader: RowReader): UniquenessCounts { + const row = firstRow(reader); + return { + total: readCount(row, "total"), + distinctKeys: readCount(row, "distinct_keys"), + }; +} + +export function readRowCount(reader: RowReader): number { + return readCount(firstRow(reader), "n"); +} + +/** + * Refuses a source that holds the same physical key twice — what a replayed + * or reorged range leaves behind. Without it every aggregate downstream is + * silently inflated. + */ +export function assertUniqueCounts(counts: UniquenessCounts): void { + if (counts.total === counts.distinctKeys) return; + throw new Error( + JSON.stringify({ + code: "source_inconsistent", + message: `duplicate physical keys: total=${counts.total} distinct=${counts.distinctKeys}`, + }), + ); +} + export function buildUniquenessSql( networkName: string, contractName: string, diff --git a/tests/ingest/exporter.test.ts b/tests/ingest/exporter.test.ts index acc14e9..b4d02b7 100644 --- a/tests/ingest/exporter.test.ts +++ b/tests/ingest/exporter.test.ts @@ -1,7 +1,11 @@ import { describe, expect, it } from "vitest"; +import { DuckDBInstance } from "@duckdb/node-api"; import { + assertUniqueCounts, buildExportSql, buildUniquenessSql, + readCounts, + readRowCount, } from "../../src/ingest/exporter.js"; const req = { @@ -53,3 +57,65 @@ describe("exporter SQL", () => { expect(buildExportSql(req)).toBe(sql); }); }); + +describe("reading DuckDB results", () => { + // `getRowsJson()` returns positional arrays. Reading it as objects yields + // undefined for every column, which used to be coerced to 0 — leaving the + // uniqueness gate below comparing 0 to 0, so it could never fire. + async function query(sql: string) { + const instance = await DuckDBInstance.create(":memory:"); + const conn = await instance.connect(); + return { reader: await conn.runAndReadAll(sql), conn, instance }; + } + + it("reads the uniqueness counts by column name", async () => { + const { reader } = await query( + "SELECT 42::bigint AS total, 7::bigint AS distinct_keys", + ); + expect(readCounts(reader)).toEqual({ total: 42, distinctKeys: 7 }); + }); + + it("reads the exported row count by column name", async () => { + const { reader } = await query("SELECT 20618::bigint AS n"); + expect(readRowCount(reader)).toBe(20618); + }); + + it("refuses a missing column rather than calling it zero", async () => { + const { reader } = await query("SELECT 1 AS something_else"); + expect(() => readRowCount(reader)).toThrow(/count/i); + }); +}); + +describe("the uniqueness gate", () => { + it("passes when every physical key is distinct", () => { + expect(() => assertUniqueCounts({ total: 20618, distinctKeys: 20618 })).not.toThrow(); + }); + + it("fires when a physical key is duplicated", () => { + expect(() => assertUniqueCounts({ total: 3, distinctKeys: 2 })).toThrow( + /source_inconsistent/, + ); + }); + + it("fires on a table that actually holds a duplicated row", async () => { + const instance = await DuckDBInstance.create(":memory:"); + const conn = await instance.connect(); + await conn.run("ATTACH ':memory:' AS pg"); + await conn.run("CREATE SCHEMA pg.chainplot_chainplot_1_usdc"); + await conn.run( + "CREATE TABLE pg.chainplot_chainplot_1_usdc.transfer " + + "(contract_address VARCHAR, block_number BIGINT, tx_hash VARCHAR, log_index BIGINT)", + ); + // Same (address, block, tx, log_index) twice: what a replayed range leaves behind. + await conn.run( + "INSERT INTO pg.chainplot_chainplot_1_usdc.transfer VALUES " + + "('0xa', 1, '0xb', 0), ('0xa', 1, '0xb', 0), ('0xa', 2, '0xc', 0)", + ); + const reader = await conn.runAndReadAll( + buildUniquenessSql("chainplot_1", "usdc", "Transfer"), + ); + const counts = readCounts(reader); + expect(counts).toEqual({ total: 3, distinctKeys: 2 }); + expect(() => assertUniqueCounts(counts)).toThrow(/duplicate physical keys/); + }); +}); From 3e37d1239bf83257408b3198000893776797c0cc Mon Sep 17 00:00:00 2001 From: Anton Sauchyk Date: Fri, 18 Sep 2026 22:22:19 +0200 Subject: [PATCH 17/19] fork: accept a publish root remotely, as the local path already does MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `--from` takes a release directory or a publish root when the source is local, but remotely only ever fetched `${base}/release.json`. The README section added with the fork footer documents the publish root, so its own example returned `fork fetch: HTTP 404`. Remote now falls back to latest.json and verifies the checksum it names, mirroring the local branch. Two further things this exposed: the pointer's `release_prefix` is written relative to the *bucket*, not to `--from`, so appending it to a publish root that already ends in that prefix repeats it — the shared segments are dropped. And the remote file and dataset fetches ignored the prefix entirely, which nothing had noticed because no remote fork had ever followed a pointer. The publish root is the URL worth documenting: a release prefix changes on every publish. Verified against the live bucket — the README's own line now pulls 39 files and 3 referenced datasets and recomputes every figure. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01CVa8K4piCcGRDA5f83RkYt --- src/fork/importRelease.ts | 77 ++++++++++++++++++-- tests/fork/importRelease.test.ts | 117 +++++++++++++++++++++++++++++++ 2 files changed, 190 insertions(+), 4 deletions(-) diff --git a/src/fork/importRelease.ts b/src/fork/importRelease.ts index 5ba21d3..841e471 100644 --- a/src/fork/importRelease.ts +++ b/src/fork/importRelease.ts @@ -70,6 +70,66 @@ function readLocal(releaseRoot: string, rel: string, maxBytes: number): Buffer { return fs.readFileSync(resolved); } +/** guardedFetch only carries the status in its message, so match it there. */ +export function isReleaseNotFound(err: unknown): boolean { + return (err as { message?: string })?.message === "fork fetch: HTTP 404"; +} + +/** + * A remote `--from` may name a release directory or a publish root — the same + * two shapes the local branch already accepts. Try the release directly; if it + * is not there, follow latest.json and verify the checksum it names, so a + * pointer that has moved on cannot hand back a mismatched release. + * + * The publish root is the stable URL: the release prefix changes on every + * publish, so anything that documents one goes stale immediately. + */ +export async function resolveRemoteRelease( + base: string, + fetchOne: (url: string) => Promise, +): Promise<{ body: Buffer; prefix: string | null }> { + try { + return { body: await fetchOne(`${base}/release.json`), prefix: null }; + } catch (err) { + if (!isReleaseNotFound(err)) throw err; + } + const pointer = JSON.parse((await fetchOne(`${base}/latest.json`)).toString("utf8")) as { + release_prefix: string; + release_json_checksum: string; + }; + const releaseBase = pointerReleaseUrl(base, pointer.release_prefix); + const body = await fetchOne(`${releaseBase}/release.json`); + if (createHash("sha256").update(body).digest("hex") !== pointer.release_json_checksum) { + throw commandError( + "policy_refused", + "fork: release.json does not match the latest.json pointer checksum", + ); + } + return { body, prefix: relativePrefix(base, pointer.release_prefix) }; +} + +/** + * `release_prefix` in latest.json is written relative to the bucket, while + * `--from` is whatever URL the reader was given — usually the publish root, + * which already ends with the target's own prefix. Appending one to the other + * would repeat that prefix, so drop the segments they share. + */ +function relativePrefix(base: string, releasePrefix: string): string { + const basePath = new URL(base).pathname.split("/").filter(Boolean); + const parts = releasePrefix.split("/").filter(Boolean); + for (let n = Math.min(basePath.length, parts.length); n > 0; n--) { + if (basePath.slice(-n).join("/") === parts.slice(0, n).join("/")) { + return parts.slice(n).join("/"); + } + } + return parts.join("/"); +} + +function pointerReleaseUrl(base: string, releasePrefix: string): string { + const rest = relativePrefix(base, releasePrefix); + return rest ? `${base}/${rest}` : base; +} + export async function importRelease( from: string, outputDir: string, @@ -120,8 +180,12 @@ export async function importRelease( } } else { const base = source.location.replace(/\/$/, ""); - const res = await guardedFetch(`${base}/release.json`, guard); - releaseBody = res.body; + const resolved = await resolveRemoteRelease( + base, + async (url) => (await guardedFetch(url, guard)).body, + ); + releaseBody = resolved.body; + releasePrefix = resolved.prefix; } if (!validateRelease(JSON.parse(releaseBody.toString("utf8")))) { throw commandError("validation", "fork: release.json failed schema validation"); @@ -145,7 +209,7 @@ export async function importRelease( ); } else { const base = source.location.replace(/\/$/, ""); - const res = await guardedFetch(`${base}/${file.path}`, { + const res = await guardedFetch(`${base}/${remotePath(releasePrefix, file.path)}`, { ...guard, maxBytes: DEFAULT_LIMITS.totalBytes, }); @@ -319,13 +383,18 @@ async function readReferenced( ); } const base = source.location.replace(/\/$/, ""); - const res = await guardedFetch(`${base}/${dataset.path}`, { + const res = await guardedFetch(`${base}/${remotePath(releasePrefix, dataset.path)}`, { ...guard, maxBytes: DEFAULT_LIMITS.totalBytes, }); return res.body; } +/** Release-relative path, under the pointer's prefix when we followed one. */ +function remotePath(releasePrefix: string | null, filePath: string): string { + return releasePrefix ? `${releasePrefix}/${filePath}` : filePath; +} + function datasetPaths(doc: { datasets?: { id: string }[] }): string { const ids = (doc.datasets ?? []).map((d) => d.id); return ids.length ? `datasets (${ids.join(", ")})` : "the datasets"; diff --git a/tests/fork/importRelease.test.ts b/tests/fork/importRelease.test.ts index 0eeafa4..40e040e 100644 --- a/tests/fork/importRelease.test.ts +++ b/tests/fork/importRelease.test.ts @@ -1,3 +1,9 @@ +import { createHash } from "node:crypto"; +import { commandError } from "../../src/plan/errors.js"; +import { + isReleaseNotFound, + resolveRemoteRelease, +} from "../../src/fork/importRelease.js"; import { describe, expect, it } from "vitest"; import fs from "node:fs"; import os from "node:os"; @@ -229,3 +235,114 @@ describe("a referenced dataset round-trips", () => { expect(result.error?.message).toMatch(/checksum mismatch for referenced dataset/); }, 60_000); }); + +describe("resolving a remote --from", () => { + const releaseBody = Buffer.from(JSON.stringify({ schema_version: 1, mode: "results_only" })); + const checksum = createHash("sha256").update(releaseBody).digest("hex"); + const notFound = () => + commandError("transient_dependency", "fork fetch: HTTP 404", { retryable: true }); + + it("takes a release directory directly, with no prefix", async () => { + const seen: string[] = []; + const got = await resolveRemoteRelease("https://x/rel", async (url) => { + seen.push(url); + return releaseBody; + }); + expect(seen).toEqual(["https://x/rel/release.json"]); + expect(got).toEqual({ body: releaseBody, prefix: null }); + }); + + it("falls back to latest.json when the release is not there", async () => { + const seen: string[] = []; + const got = await resolveRemoteRelease("https://x/root", async (url) => { + seen.push(url); + if (url.endsWith("/root/release.json")) throw notFound(); + if (url.endsWith("/latest.json")) { + return Buffer.from( + JSON.stringify({ release_prefix: "releases/abc", release_json_checksum: checksum }), + ); + } + return releaseBody; + }); + expect(seen).toEqual([ + "https://x/root/release.json", + "https://x/root/latest.json", + "https://x/root/releases/abc/release.json", + ]); + expect(got.prefix).toBe("releases/abc"); + }); + + it("refuses a release that does not match the pointer checksum", async () => { + await expect( + resolveRemoteRelease("https://x/root", async (url) => { + if (url.endsWith("/root/release.json")) throw notFound(); + if (url.endsWith("/latest.json")) { + return Buffer.from( + JSON.stringify({ release_prefix: "releases/abc", release_json_checksum: "deadbeef" }), + ); + } + return releaseBody; + }), + ).rejects.toMatchObject({ code: "policy_refused" }); + }); + + it("does not fall back on an error that is not a 404", async () => { + await expect( + resolveRemoteRelease("https://x/root", async () => { + throw commandError("transient_dependency", "fork fetch: HTTP 500", { retryable: true }); + }), + ).rejects.toMatchObject({ message: "fork fetch: HTTP 500" }); + }); + + // The 404 is only distinguishable by the message guardedFetch builds, so + // pin that coupling here: change the wording there and this fails. + it("recognises the 404 that guardedFetch actually throws", () => { + expect(isReleaseNotFound(notFound())).toBe(true); + expect(isReleaseNotFound(new Error("something else"))).toBe(false); + }); +}); + +describe("latest.json prefixes are bucket-relative", () => { + const body = Buffer.from(JSON.stringify({ schema_version: 1 })); + const sum = createHash("sha256").update(body).digest("hex"); + const notFound = () => + commandError("transient_dependency", "fork fetch: HTTP 404", { retryable: true }); + + // What R2 actually serves: --from is the publish root, and the pointer + // repeats that prefix because it is written relative to the bucket. + it("does not repeat the prefix the publish root already carries", async () => { + const seen: string[] = []; + const got = await resolveRemoteRelease("https://h/fomo-rh", async (url) => { + seen.push(url); + if (url === "https://h/fomo-rh/release.json") throw notFound(); + if (url === "https://h/fomo-rh/latest.json") { + return Buffer.from( + JSON.stringify({ + release_prefix: "fomo-rh/releases/abc", + release_json_checksum: sum, + }), + ); + } + return body; + }); + expect(seen).toContain("https://h/fomo-rh/releases/abc/release.json"); + expect(seen).not.toContain("https://h/fomo-rh/fomo-rh/releases/abc/release.json"); + expect(got.prefix).toBe("releases/abc"); + }); + + it("still works from the bucket root, where nothing is shared", async () => { + const seen: string[] = []; + const got = await resolveRemoteRelease("https://h", async (url) => { + seen.push(url); + if (url === "https://h/release.json") throw notFound(); + if (url === "https://h/latest.json") { + return Buffer.from( + JSON.stringify({ release_prefix: "fomo-rh/releases/abc", release_json_checksum: sum }), + ); + } + return body; + }); + expect(seen).toContain("https://h/fomo-rh/releases/abc/release.json"); + expect(got.prefix).toBe("fomo-rh/releases/abc"); + }); +}); From d6b26e690a02d5ca8c88f292a02cba96558f436e Mon Sep 17 00:00:00 2001 From: Anton Sauchyk Date: Fri, 18 Sep 2026 22:24:03 +0200 Subject: [PATCH 18/19] Gate the live S3 suite on whether it can run, not on a strict read MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refusing an endpoint that carries a path was right, but the live suite's gate calls the same reader at module scope. A missing variable returned null and skipped; a malformed one now threw, so the file failed during collection as a bare "Unknown Error" with no test name on it. Both answers mean the same thing to a gate: these tests cannot run here. s3EnvIfUsable says so without losing the strict read that publish depends on. The incentive was backwards otherwise — no S3 config skipped cleanly, correct config ran, and slightly wrong config turned the suite red over a publish-time setting unrelated to the tests being run. CI could not catch it either, having no S3 environment to get wrong. Same shape as gating the live e2e on RPC_URL merely being set. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01CVa8K4piCcGRDA5f83RkYt --- src/publish/s3.ts | 17 +++++++++++++++ tests/publish/live/s3.live.test.ts | 4 ++-- tests/publish/s3unit.test.ts | 34 +++++++++++++++++++++++++++++- 3 files changed, 52 insertions(+), 3 deletions(-) diff --git a/src/publish/s3.ts b/src/publish/s3.ts index bca31ba..a65a41e 100644 --- a/src/publish/s3.ts +++ b/src/publish/s3.ts @@ -283,3 +283,20 @@ export class S3Target implements PublishTarget { } } } + +/** + * The gate a live-only suite should ask: can these tests run here? A missing + * variable and a malformed one are both no. `s3EnvFromProcess` distinguishes + * them — right for a publish, which should say exactly what is wrong — but a + * gate evaluated at module scope turns the second into a collection failure + * with no test name on it, and CI never sees it because CI has no S3 env. + */ +export function s3EnvIfUsable( + env: NodeJS.ProcessEnv = process.env, +): ReturnType { + try { + return s3EnvFromProcess(env); + } catch { + return null; + } +} diff --git a/tests/publish/live/s3.live.test.ts b/tests/publish/live/s3.live.test.ts index c2d1864..d56b76d 100644 --- a/tests/publish/live/s3.live.test.ts +++ b/tests/publish/live/s3.live.test.ts @@ -7,7 +7,7 @@ import { runCliJson } from "../../helpers/run.js"; import { S3Target, makeS3Ops, - s3EnvFromProcess, + s3EnvIfUsable, LATEST_KEY, } from "../../../src/publish/s3.js"; import { latestPointer } from "../../../src/publish/latestPointer.js"; @@ -15,7 +15,7 @@ import { latestPointer } from "../../../src/publish/latestPointer.js"; // Live-gated: needs CHAINPLOT_S3_ENDPOINT, CHAINPLOT_S3_BUCKET, // AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY (optionally CHAINPLOT_S3_REGION). // Skips when absent. Never commit endpoint URLs or keys. -const env = s3EnvFromProcess(); +const env = s3EnvIfUsable(); const d = env ? it : it.skip; describe("S3 conditional-write probe (M0 open question)", () => { diff --git a/tests/publish/s3unit.test.ts b/tests/publish/s3unit.test.ts index 082fc9b..1860707 100644 --- a/tests/publish/s3unit.test.ts +++ b/tests/publish/s3unit.test.ts @@ -1,6 +1,11 @@ import { describe, expect, it } from "vitest"; import { createHash } from "node:crypto"; -import { S3Target, s3EnvFromProcess, type S3Ops } from "../../src/publish/s3.js"; +import { + S3Target, + s3EnvFromProcess, + s3EnvIfUsable, + type S3Ops, +} from "../../src/publish/s3.js"; interface Stored { body: string | Uint8Array; @@ -162,3 +167,30 @@ describe("s3EnvFromProcess", () => { ); }); }); + +// A live suite's gate asks "can these tests run here?". A missing variable and +// a malformed one are both no — but one returned null and the other threw, and +// the throw happened at module scope, so the file failed collection with no +// test name attached. Same shape as gating the live e2e on RPC_URL being set. +describe("s3EnvIfUsable, the live-test gate", () => { + const base = { + CHAINPLOT_S3_BUCKET: "b", + AWS_ACCESS_KEY_ID: "a", + AWS_SECRET_ACCESS_KEY: "s", + }; + + it("is null when nothing is configured", () => { + expect(s3EnvIfUsable({ ...base })).toBeNull(); + }); + + it("is null when the endpoint is malformed, where the strict reader throws", () => { + const env = { ...base, CHAINPLOT_S3_ENDPOINT: "https://acct.r2.cloudflarestorage.com/b" }; + expect(() => s3EnvFromProcess(env)).toThrow(); + expect(s3EnvIfUsable(env)).toBeNull(); + }); + + it("still hands back a usable environment", () => { + const endpoint = "https://acct.r2.cloudflarestorage.com"; + expect(s3EnvIfUsable({ ...base, CHAINPLOT_S3_ENDPOINT: endpoint })?.endpoint).toBe(endpoint); + }); +}); From 79bb63b45f31e6d46cd8c6817dfcb94644712deb Mon Sep 17 00:00:00 2001 From: Eugene Aseev Date: Sat, 19 Sep 2026 12:53:11 +0800 Subject: [PATCH 19/19] Name snapshots the way rindexer names their tables The parquet a source/event exports to was `_`, while the table it came from is snake_cased: two conventions for one name, and the difference shows exactly when an event is more than one word. `snapshotFileName` now applies rindexer's rule to both parts, so `TransferShares` exports to `steth_transfer_shares.parquet` and a `snapshot:` path follows from the event name by one rule. Single-word events, which every template and example uses, are unchanged. The template and README say how the path is derived. Co-Authored-By: Claude Fable 5.1 --- README.md | 1 + src/ingest/exporter.ts | 17 +++++++++++++---- templates/ingest-transfers/chainplot.yaml | 3 +++ tests/ingest/exporter.test.ts | 17 +++++++++++++++++ tests/ingest/live/e2e.live.test.ts | 4 ++-- 5 files changed, 36 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 8cffda6..2775aa7 100644 --- a/README.md +++ b/README.md @@ -91,6 +91,7 @@ event_sources: datasets: - id: usdc + # apply writes _.parquet, snake_cased like rindexer's tables snapshot: .chainplot/snapshots/usdc/usdc_transfer.parquet queries: diff --git a/src/ingest/exporter.ts b/src/ingest/exporter.ts index 5f70ba5..3d5381e 100644 --- a/src/ingest/exporter.ts +++ b/src/ingest/exporter.ts @@ -3,6 +3,7 @@ import fs from "node:fs"; import path from "node:path"; import { fileURLToPath } from "node:url"; import { eventTableName } from "./rindexer/inspectCoverage.js"; +import { camelToSnake } from "./rindexer/naming.js"; import type { BoundedJob } from "./adapter.js"; export interface ExportRequest { @@ -122,6 +123,17 @@ export interface ExportResult { rowCount: number; } +/** + * The parquet a source/event pair exports to: `_.parquet`, + * both parts snake_cased exactly as rindexer names the table it came from — + * `RelayERC20Deposit` exports to `…_relay_erc_20_deposit.parquet`. One rule + * for the table and the file, so a `snapshot:` path can be derived from the + * event name without a second convention to remember. + */ +export function snapshotFileName(contractName: string, event: string): string { + return `${camelToSnake(contractName)}_${camelToSnake(event)}.parquet`; +} + export async function exportEventTable( job: BoundedJob, outDir: string, @@ -141,10 +153,7 @@ export async function exportEventTable( contractName: job.contractName, event: job.events[0], chainId: job.chainId, - // The file name is chainplot's own convention (documented in the - // templates' `snapshot:` paths); only the table it reads from follows - // rindexer's snake_case naming, via eventTableName. - outPath: path.join(outDir, `${job.contractName}_${job.events[0].toLowerCase()}.parquet`), + outPath: path.join(outDir, snapshotFileName(job.contractName, job.events[0])), }; const { modulePath, execArgv } = workerLaunch(); return await new Promise((resolve, reject) => { diff --git a/templates/ingest-transfers/chainplot.yaml b/templates/ingest-transfers/chainplot.yaml index 892c3df..5395355 100644 --- a/templates/ingest-transfers/chainplot.yaml +++ b/templates/ingest-transfers/chainplot.yaml @@ -26,6 +26,9 @@ event_sources: block: 18600010 datasets: - id: usdc + # Written by apply as .chainplot/snapshots//_.parquet, + # with both parts snake_cased the way rindexer names its tables: + # Transfer → transfer, RelayERC20Deposit → relay_erc_20_deposit. snapshot: .chainplot/snapshots/usdc/usdc_transfer.parquet queries: - id: transfer_count diff --git a/tests/ingest/exporter.test.ts b/tests/ingest/exporter.test.ts index b4d02b7..aec5b0b 100644 --- a/tests/ingest/exporter.test.ts +++ b/tests/ingest/exporter.test.ts @@ -6,6 +6,7 @@ import { buildUniquenessSql, readCounts, readRowCount, + snapshotFileName, } from "../../src/ingest/exporter.js"; const req = { @@ -119,3 +120,19 @@ describe("the uniqueness gate", () => { expect(() => assertUniqueCounts(counts)).toThrow(/duplicate physical keys/); }); }); + +// The snapshot is named the way rindexer names the table it came from, so a +// `snapshot:` path follows from the event name by one rule, not two. +describe("snapshotFileName", () => { + it("is unchanged for the single-word events every template uses", () => { + expect(snapshotFileName("usdc", "Transfer")).toBe("usdc_transfer.parquet"); + expect(snapshotFileName("weth", "Withdrawal")).toBe("weth_withdrawal.parquet"); + }); + + it("snake_cases a multi-word event exactly as the table is named", () => { + expect(snapshotFileName("steth", "TransferShares")).toBe("steth_transfer_shares.parquet"); + expect(snapshotFileName("erc20dep", "RelayERC20Deposit")).toBe( + "erc_20dep_relay_erc_20_deposit.parquet", + ); + }); +}); diff --git a/tests/ingest/live/e2e.live.test.ts b/tests/ingest/live/e2e.live.test.ts index 9f92a34..0e70831 100644 --- a/tests/ingest/live/e2e.live.test.ts +++ b/tests/ingest/live/e2e.live.test.ts @@ -202,7 +202,7 @@ describe("live ingest end-to-end (M0 replay through the product)", () => { [ "", " - id: steth", - " snapshot: .chainplot/snapshots/steth/steth_transfershares.parquet", + " snapshot: .chainplot/snapshots/steth/steth_transfer_shares.parquet", "queries:", "", ].join("\n"), @@ -249,7 +249,7 @@ describe("live ingest end-to-end (M0 replay through the product)", () => { expect(steth).toMatchObject({ end_block: 18600100, status: "complete_with_rows" }); expect(steth.row_count).toBeGreaterThan(0); expect( - fs.existsSync(path.join(cwd, ".chainplot/snapshots/steth/steth_transfershares.parquet")), + fs.existsSync(path.join(cwd, ".chainplot/snapshots/steth/steth_transfer_shares.parquet")), ).toBe(true); const segment = coverage.sources[0].segments[0]!; expect(segment).toMatchObject({