From c8fa0a15e57237499579f43c83ecd54fbc490812 Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Mon, 26 Oct 2026 10:28:18 +0100 Subject: [PATCH 1/4] test(testing): Run the monitor only when MONITOR_DIR is set The end-to-end harness starts the monitor from MONITOR_DIR and makes the monitor checks only then. Without it, runs print that they are without the monitor, skip the monitor-only checks and report them as skipped. --- testing/src/daemons.mjs | 20 ++++++++++++++++++-- testing/src/drills.mjs | 31 ++++++++++++++++++------------- testing/src/harness.mjs | 18 ++++++++++++++++-- testing/src/phase3-e2e.mjs | 26 ++++++++++++++++---------- testing/src/rehearsal.mjs | 19 +++++++++++-------- 5 files changed, 79 insertions(+), 35 deletions(-) diff --git a/testing/src/daemons.mjs b/testing/src/daemons.mjs index a0e3c3138..f05577c01 100644 --- a/testing/src/daemons.mjs +++ b/testing/src/daemons.mjs @@ -18,7 +18,7 @@ const LOG_DIR = path.join(ROOT, "testing", ".logs"); export function start(name, dir, env) { const proc = spawn("node", ["dist/main.js"], { - cwd: path.join(ROOT, dir), + cwd: path.resolve(ROOT, dir), env: { ...process.env, ...env }, stdio: ["ignore", "pipe", "pipe"], }); @@ -72,7 +72,23 @@ export function stopAll() { /** Remove a service's persisted state so a run starts clean. */ export function clearState(files) { - for (const f of files) { try { rmSync(path.join(ROOT, f)); } catch {} } + for (const f of files) { try { rmSync(path.resolve(ROOT, f)); } catch {} } +} + +/** The monitor is maintained separately. Set MONITOR_DIR to a built monitor + * checkout (absolute, or relative to the current directory) to include it in + * a run; unset, runs go ahead without it and skip the monitor checks. */ +export const MONITOR_DIR = process.env.MONITOR_DIR ? path.resolve(process.env.MONITOR_DIR) : undefined; + +/** Start the monitor from MONITOR_DIR with a clean state file, or say once + * that this run is without it. */ +export function startMonitor(env) { + if (!MONITOR_DIR) { + console.log(" MONITOR_DIR not set: running without the monitor, monitor checks are skipped"); + return; + } + clearState([path.join(MONITOR_DIR, "monitor-state.json")]); + start("monitor", MONITOR_DIR, { ...env, STATE_FILE: "./monitor-state.json" }); } export const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); diff --git a/testing/src/drills.mjs b/testing/src/drills.mjs index ea2a11335..6559f1578 100644 --- a/testing/src/drills.mjs +++ b/testing/src/drills.mjs @@ -29,8 +29,8 @@ import { Keyring } from "@polkadot/keyring"; import { cryptoWaitReady } from "@polkadot/util-crypto"; import { generatePrivateKey, privateKeyToAccount } from "viem/accounts"; import { erc20Abi, vaultAbi } from "./abi.mjs"; -import { alive, clearState, logs, start, stopAll, stopAndWait, waitFor } from "./daemons.mjs"; -import { assert, assertEq, check, section, summarise } from "./harness.mjs"; +import { alive, clearState, logs, MONITOR_DIR, start, startMonitor, stopAll, stopAndWait, waitFor } from "./daemons.mjs"; +import { assert, assertEq, check, section, skip, summarise } from "./harness.mjs"; import { assertTestnet, buildContext, CONVERSION_FACTOR, loadEnv, PEN_12, ROOT, send, TESTING, } from "./rehearsal-env.mjs"; @@ -38,6 +38,7 @@ import { deployToSepolia } from "./rehearsal-deploy.mjs"; import { discoverCollator, generateSpec, killMatching, sleep, spawnNetwork, teardown, waitForFinality } from "./zombienet.mjs"; const flags = new Set(process.argv.slice(2)); +const DAEMONS = ["attestor1", "attestor2", "attestor3", "attestor4", ...(MONITOR_DIR ? ["monitor"] : []), "releaser"]; const ATTACH = flags.has("--attach"); const KEEP = flags.has("--keep"); @@ -131,7 +132,7 @@ async function main() { section("Attestor fleet"); killMatching(["dist/main.js"]); clearState(["attestor/cp1.json", "attestor/cp2.json", "attestor/cp3.json", "attestor/cp4.json", - "monitor/monitor-state.json", "releaser/releaser-state.json"]); + "releaser/releaser-state.json"]); const pendulumWs = api._options?.provider?.endpoint ?? process.env.PENDULUM_WS; const baseEnv = { BASE_RPC_URL: env.BASE_SEPOLIA_RPC_URL, VAULT_ADDRESS: vault, @@ -151,7 +152,7 @@ async function main() { start(`attestor${i}`, "attestor", attestorEnv(i)); await sleep(3000); } - start("monitor", "monitor", { + startMonitor({ ...baseEnv, PENDULUM_WS: pendulumWs, GRACE_SECONDS: "300", @@ -159,7 +160,6 @@ async function main() { PENDULUM_START_BLOCK: (BigInt(pendulumHead) + 1n).toString(), PENDULUM_START_NONCE: monitorStartNonce, BASE_START_BLOCK: baseHead, - STATE_FILE: "./monitor-state.json", }); start("releaser", "releaser", { ...baseEnv, RELEASER_PRIVATE_KEY: env.RELEASER_PRIVATE_KEY, START_BLOCK: baseHead }); log(`fleet up (Pendulum from #${pendulumHead}, Base from #${baseHead})`); @@ -284,12 +284,17 @@ async function main() { }); section("RB-3 — the monitor tolerates a surplus (deficit is covered on Anvil)"); - await check("PEN sent into the vault raises no alarm and kills nothing", async () => { + const SURPLUS = 10n * 10n ** 18n; // RB-7's reconcile counts it, with or without the monitor + if (!MONITOR_DIR) { + await check("PEN sent into the vault lands as a surplus", () => + send(ctx, ctx.roles.admin, { ...P, functionName: "transfer", args: [vault, SURPLUS] })); + skip("PEN sent into the vault raises no alarm and kills nothing", "MONITOR_DIR not set"); + } else await check("PEN sent into the vault raises no alarm and kills nothing", async () => { // A deficit cannot be created on Sepolia — nothing but the vault itself // can move its tokens, which is the security property. Phase 3 covers // the deficit alarm via Anvil impersonation. Here: the benign inverse. const monitorLogBefore = logs("monitor").length; - await send(ctx, ctx.roles.admin, { ...P, functionName: "transfer", args: [vault, 10n * 10n ** 18n] }); + await send(ctx, ctx.roles.admin, { ...P, functionName: "transfer", args: [vault, SURPLUS] }); await sleep(40_000); // > two monitor poll cycles assert(alive("monitor"), `monitor died on a surplus:\n${logs("monitor")}`); const fresh = logs("monitor").slice(monitorLogBefore); @@ -326,11 +331,10 @@ async function main() { read(V, "totalSwept", []), read(P, "totalSupply", []), ]); // The RB-3 surplus sits in the vault's balance, ON TOP of conservation. - const surplus = 10n * 10n ** 18n; - assertEq(balance + rel + swept, supply + surplus, "balance + released + swept vs supply (+known surplus)"); + assertEq(balance + rel + swept, supply + SURPLUS, "balance + released + swept vs supply (+known surplus)"); }); - await check("the sweep executes and the monitor does not false-alarm", async () => { + await check(MONITOR_DIR ? "the sweep executes and the monitor does not false-alarm" : "the sweep executes", async () => { assert(Math.floor(Date.now() / 1000) >= sweepTs, "sweep floor not yet passed — raise DRILL_SWEEP_OFFSET"); assertEq(await read(V, "thresholdReducedAt", []), 0n, "thresholdReducedAt (settling gate must not be armed)"); const monitorLogBefore = logs("monitor").length; @@ -339,6 +343,7 @@ async function main() { await send(ctx, ctx.roles.admin, { ...V, functionName: "sweepRemainder", args: [recipient, sweepAmount] }); await eventually(async () => assertEq(await read(V, "totalSwept", []), sweptBefore + sweepAmount, "totalSwept"), "the sweep to settle"); + if (!MONITOR_DIR) return; await sleep(40_000); assert(alive("monitor"), `monitor died after the sweep:\n${logs("monitor")}`); const fresh = logs("monitor").slice(monitorLogBefore); @@ -346,7 +351,7 @@ async function main() { }); await check("every daemon survived every drill", () => { - for (const name of ["attestor1", "attestor2", "attestor3", "attestor4", "monitor", "releaser"]) { + for (const name of DAEMONS) { assert(alive(name), `${name} is dead:\n${logs(name)}`); } }); @@ -358,10 +363,10 @@ async function main() { startedAt: started.toISOString(), commit: execSync("git rev-parse HEAD", { cwd: ROOT, encoding: "utf8" }).trim(), vault, pen, sweepTimestamp: sweepTs, - covered: ["RB-1", "RB-3 (surplus half)", "RB-4", "RB-6", "RB-7"], + covered: ["RB-1", ...(MONITOR_DIR ? ["RB-3 (surplus half)"] : []), "RB-4", "RB-6", "RB-7"], coveredElsewhere: { "RB-2": "phase 5 rehearsal", "RB-3 deficit": "phase 3 (Anvil impersonation)", "RB-5": "drill-rb5-upgrade.mjs on Chopsticks" }, }, null, 2)); - for (const name of ["attestor1", "attestor2", "attestor3", "attestor4", "monitor", "releaser"]) { + for (const name of DAEMONS) { writeFileSync(path.join(runDir, `${name}.log`), logs(name)); } if (network) writeFileSync(path.join(runDir, "zombienet.log"), network.out.join("\n")); diff --git a/testing/src/harness.mjs b/testing/src/harness.mjs index 9e13d9a5f..9475265f0 100644 --- a/testing/src/harness.mjs +++ b/testing/src/harness.mjs @@ -20,13 +20,27 @@ export async function check(name, fn) { } } +/** Record a check that this run cannot perform, so the summary shows it as + * skipped rather than passed. */ +export function skip(name, reason) { + results.push({ name, skipped: true, reason }); + console.log(` SKIP ${name}`); + console.log(` ${reason}`); +} + export function section(title) { console.log(`\n${title}`); } export function summarise() { - const failed = results.filter((r) => !r.ok); - console.log(`\n${results.length - failed.length}/${results.length} checks passed`); + const failed = results.filter((r) => !r.ok && !r.skipped); + const skipped = results.filter((r) => r.skipped); + const run = results.length - skipped.length; + console.log(`\n${run - failed.length}/${run} checks passed${skipped.length ? `, ${skipped.length} skipped` : ""}`); + if (skipped.length > 0) { + console.log("\nSkipped:"); + for (const s of skipped) console.log(` - ${s.name}: ${s.reason}`); + } if (failed.length > 0) { console.log("\nFailed:"); for (const f of failed) console.log(` - ${f.name}: ${f.reason}`); diff --git a/testing/src/phase3-e2e.mjs b/testing/src/phase3-e2e.mjs index fc49a9cd8..94bc61b49 100644 --- a/testing/src/phase3-e2e.mjs +++ b/testing/src/phase3-e2e.mjs @@ -14,17 +14,20 @@ * remains a manual Zombienet exercise; see the README. * * Prerequisites: Chopsticks on :8000 (fresh), Anvil on :8545, and the services - * built (`npm run build` in attestor/, monitor/ and releaser/). + * built (`npm run build` in attestor/ and releaser/). The monitor runs, and its + * checks are made, only when MONITOR_DIR points at a built monitor checkout. */ import { ApiPromise, WsProvider } from "@polkadot/api"; import { Keyring } from "@polkadot/keyring"; import { cryptoWaitReady } from "@polkadot/util-crypto"; -import { assert, assertEq, check, section, summarise } from "./harness.mjs"; +import { assert, assertEq, check, section, skip, summarise } from "./harness.mjs"; import { accounts, attestors, keys, pub, releaser as releaserAcct, RPC, send, sendImpersonated } from "./anvil.mjs"; import { erc20Abi, vaultAbi } from "./abi.mjs"; import { deployStack, PARAMS } from "./deploy.mjs"; -import { alive, clearState, killStrays, logs, start, stopAll, stopAndWait, waitFor } from "./daemons.mjs"; +import { + alive, clearState, killStrays, logs, MONITOR_DIR, start, startMonitor, stopAll, stopAndWait, waitFor, +} from "./daemons.mjs"; const CHOPSTICKS = process.env.CHOPSTICKS_WS ?? "ws://127.0.0.1:8000"; const CF = PARAMS.CONVERSION_FACTOR; @@ -70,7 +73,7 @@ async function migrate(amount, baseAddress) { // --- start the fleet ------------------------------------------------------- killStrays(); clearState(["attestor/cp1.json", "attestor/cp2.json", "attestor/cp3.json", "attestor/cp4.json", - "monitor/monitor-state.json", "releaser/releaser-state.json"]); + "releaser/releaser-state.json"]); const baseEnv = { BASE_RPC_URL: RPC, VAULT_ADDRESS: vault, BASE_CHAIN_ID: "31337", POLL_INTERVAL_MS: "2000" }; const startBlock = String(await pub.getBlockNumber()); // Where the attestors begin scanning Pendulum. Must be the current head: a @@ -87,7 +90,7 @@ for (let i = 0; i < 4; i++) { ATTESTOR_PRIVATE_KEY: keys[i + 1], CHECKPOINT_FILE: `./cp${i + 1}.json`, START_BLOCK: pendulumHead, }); } -start("monitor", "monitor", { +startMonitor({ ...baseEnv, PENDULUM_WS: CHOPSTICKS, GRACE_SECONDS: "30", @@ -96,7 +99,6 @@ start("monitor", "monitor", { PENDULUM_START_BLOCK: monitorPendulumStart, PENDULUM_START_NONCE: monitorStartNonce, BASE_START_BLOCK: startBlock, - STATE_FILE: "./monitor-state.json", }); start("releaser", "releaser", { ...baseEnv, RELEASER_PRIVATE_KEY: keys[7], START_BLOCK: startBlock }); @@ -114,7 +116,8 @@ await check("a burn on Pendulum arrives on Base with no manual step", async () = } catch (e) { // A silent pipeline is the hardest thing to debug from the outside, so // surface what each daemon actually saw. - throw new Error(`${e.message}\n--- attestor1 ---\n${logs("attestor1")}\n--- monitor ---\n${logs("monitor")}`); + const monitorLog = MONITOR_DIR ? `\n--- monitor ---\n${logs("monitor")}` : ""; + throw new Error(`${e.message}\n--- attestor1 ---\n${logs("attestor1")}${monitorLog}`); } assertEq(await balanceOf(recipient), before + amount * CF, "released amount"); }); @@ -191,7 +194,10 @@ await check("a cap-deferred release is drained by the releaser, unattended", asy section("Invariants"); -await check("the monitor reports healthy and stays running", async () => { +// Checks that only the monitor can satisfy: skipped, not passed, without it. +const monitorCheck = MONITOR_DIR ? check : (name) => skip(name, "MONITOR_DIR not set"); + +await monitorCheck("the monitor reports healthy and stays running", async () => { assert(alive("monitor"), `monitor exited:\n${logs("monitor")}`); await waitFor(() => logs("monitor").includes("ok:"), { timeoutMs: 60_000, label: "a monitor ok line" }); const bad = logs("monitor").match(/ALERT: (CONSERVATION VIOLATION|VAULT BALANCE MISMATCH)/); @@ -205,7 +211,7 @@ await check("conservation holds across the whole run", async () => { assertEq(bal + released + swept, supply, "conservation identity"); }); -await check("a real vault deficit alerts and reaches a confirmed automatic pause", async () => { +await monitorCheck("a real vault deficit alerts and reaches a confirmed automatic pause", async () => { const deficit = 1n * 10n ** 18n; const before = logs("monitor").length; await sendImpersonated(vault, { ...P, functionName: "transfer", args: [admin.address, deficit] }); @@ -239,7 +245,7 @@ await check("a real vault deficit alerts and reaches a confirmed automatic pause assertEq(await read(V, "paused"), false, "vault remains unpaused after healthy checks"); }); -await check("a mismatched Base approval is caught before it can reach quorum", async () => { +await monitorCheck("a mismatched Base approval is caught before it can reach quorum", async () => { const before = logs("monitor").length; await send(attestors[0], { ...V, diff --git a/testing/src/rehearsal.mjs b/testing/src/rehearsal.mjs index 3e2847739..7acac0287 100644 --- a/testing/src/rehearsal.mjs +++ b/testing/src/rehearsal.mjs @@ -29,7 +29,7 @@ import { encodeAbiParameters, keccak256 } from "viem"; import { Keyring } from "@polkadot/keyring"; import { cryptoWaitReady } from "@polkadot/util-crypto"; import { erc20Abi, vaultAbi } from "./abi.mjs"; -import { alive, clearState, logs, start, stopAll, stopAndWait, waitFor } from "./daemons.mjs"; +import { alive, clearState, logs, MONITOR_DIR, start, startMonitor, stopAll, stopAndWait, waitFor } from "./daemons.mjs"; import { assert, assertEq, check, section, summarise } from "./harness.mjs"; import { assertTestnet, buildContext, CONVERSION_FACTOR, loadEnv, PEN_12, PEN_18, ROOT, send, TESTING, @@ -131,9 +131,13 @@ async function preflight(ctx) { section("Preflight"); await check("services are built", () => { - for (const svc of ["attestor", "monitor", "releaser"]) { + for (const svc of ["attestor", "releaser"]) { assert(existsSync(path.join(ROOT, svc, "dist/main.js")), `${svc}/dist/main.js missing — run \`npm run build\` in ${svc}/`); } + if (MONITOR_DIR) { + assert(existsSync(path.join(MONITOR_DIR, "dist/main.js")), + `${MONITOR_DIR}/dist/main.js missing — run \`npm run build\` there, or unset MONITOR_DIR`); + } }); await check("contract artifacts exist", () => { @@ -292,7 +296,7 @@ async function main() { section("Attestor fleet"); killMatching(["dist/main.js"]); clearState(["attestor/cp1.json", "attestor/cp2.json", "attestor/cp3.json", "attestor/cp4.json", - "monitor/monitor-state.json", "releaser/releaser-state.json"]); + "releaser/releaser-state.json"]); const pendulumWs = api._options?.provider?.endpoint ?? process.env.PENDULUM_WS; const baseEnv = { BASE_RPC_URL: env.BASE_SEPOLIA_RPC_URL, @@ -318,7 +322,7 @@ async function main() { }); await sleep(3000); } - start("monitor", "monitor", { + startMonitor({ ...baseEnv, PENDULUM_WS: pendulumWs, GRACE_SECONDS: "300", @@ -326,7 +330,6 @@ async function main() { PENDULUM_START_BLOCK: (BigInt(pendulumHead) + 1n).toString(), PENDULUM_START_NONCE: monitorStartNonce, BASE_START_BLOCK: baseHead, - STATE_FILE: "./monitor-state.json", }); start("releaser", "releaser", { ...baseEnv, RELEASER_PRIVATE_KEY: env.RELEASER_PRIVATE_KEY, START_BLOCK: baseHead }); log(`fleet started (Pendulum from #${pendulumHead}, Base from #${baseHead})`); @@ -415,13 +418,13 @@ async function main() { assert(await read(V, "totalReleased", []) >= releasedBefore, "totalReleased went backwards"); }); - await check("the monitor holds its conservation invariant", async () => { + await check(MONITOR_DIR ? "the monitor holds its conservation invariant" : "conservation holds", async () => { const [balance, released, swept, supply] = await Promise.all([ read(P, "balanceOf", [vault]), read(V, "totalReleased", []), read(V, "totalSwept", []), read(P, "totalSupply", []), ]); assertEq(balance + released + swept, supply, "balance + released + swept == totalSupply"); - assert(alive("monitor"), `monitor exited:\n${logs("monitor")}`); + if (MONITOR_DIR) assert(alive("monitor"), `monitor exited:\n${logs("monitor")}`); }); await check("the guardian can pause but cannot unpause; the admin can", async () => { @@ -485,7 +488,7 @@ async function main() { }, }; writeFileSync(path.join(runDir, "manifest.json"), JSON.stringify(manifest, null, 2)); - for (const name of ["attestor1", "attestor2", "attestor3", "attestor4", "monitor", "releaser"]) { + for (const name of ["attestor1", "attestor2", "attestor3", "attestor4", ...(MONITOR_DIR ? ["monitor"] : []), "releaser"]) { writeFileSync(path.join(runDir, `${name}.log`), logs(name)); } if (network) writeFileSync(path.join(runDir, "zombienet.log"), network.out.join("\n")); From 4c129a0fc12cd31d2744e91a87c4eadc148a969f Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Mon, 26 Oct 2026 10:29:07 +0100 Subject: [PATCH 2/4] chore: Remove the monitor from this repository The monitor is maintained separately. Drop it from the migration-tests workflow and point the testing docs at MONITOR_DIR for end-to-end runs. --- .github/workflows/migration-tests.yml | 5 +- docs/pen-migration-local-test-plan.md | 22 +- monitor/.gitignore | 4 - monitor/README.md | 181 -- monitor/package-lock.json | 1123 ------------ monitor/package.json | 21 - monitor/src/checks.test.ts | 212 --- monitor/src/checks.ts | 317 ---- monitor/src/main.test.ts | 2310 ------------------------- monitor/src/main.ts | 2110 ---------------------- monitor/src/state.test.ts | 105 -- monitor/src/state.ts | 172 -- monitor/tsconfig.json | 14 - releaser/src/checks.ts | 3 +- testing/README.md | 17 +- 15 files changed, 30 insertions(+), 6586 deletions(-) delete mode 100644 monitor/.gitignore delete mode 100644 monitor/README.md delete mode 100644 monitor/package-lock.json delete mode 100644 monitor/package.json delete mode 100644 monitor/src/checks.test.ts delete mode 100644 monitor/src/checks.ts delete mode 100644 monitor/src/main.test.ts delete mode 100644 monitor/src/main.ts delete mode 100644 monitor/src/state.test.ts delete mode 100644 monitor/src/state.ts delete mode 100644 monitor/tsconfig.json diff --git a/.github/workflows/migration-tests.yml b/.github/workflows/migration-tests.yml index cca04c952..746224a51 100644 --- a/.github/workflows/migration-tests.yml +++ b/.github/workflows/migration-tests.yml @@ -1,7 +1,7 @@ name: Migration Tests # The PEN -> Base migration stack lives outside the Rust workspace: the -# Foundry contracts and the three TypeScript daemons. `test-code.yml` covers +# Foundry contracts and the TypeScript daemons. `test-code.yml` covers # the pallet through `cargo test`; this job covers everything else, so a # green check on the PR means the contract and daemon suites actually ran. @@ -10,7 +10,6 @@ on: paths: - "contracts/**" - "attestor/**" - - "monitor/**" - "releaser/**" - ".github/workflows/migration-tests.yml" push: @@ -48,7 +47,7 @@ jobs: strategy: fail-fast: false matrix: - package: [attestor, monitor, releaser] + package: [attestor, releaser] defaults: run: working-directory: ${{ matrix.package }} diff --git a/docs/pen-migration-local-test-plan.md b/docs/pen-migration-local-test-plan.md index a38988210..a1b07240d 100644 --- a/docs/pen-migration-local-test-plan.md +++ b/docs/pen-migration-local-test-plan.md @@ -32,16 +32,20 @@ cargo build --release -p pendulum-runtime # Contracts + services cd contracts && forge build cd ../attestor && npm install && npm run build -cd ../monitor && npm install && npm run build +cd ../releaser && npm install && npm run build ``` +The monitor is maintained separately. To include it in the end-to-end phases, +build it in its own checkout (`npm install && npm run build`) and set +`MONITOR_DIR` to that checkout when running the harness. + Confirm the baseline is green before starting: ```bash cargo test -p token-migration --features runtime-benchmarks cd contracts && forge test cd ../attestor && npm test -cd ../monitor && npm test +cd ../releaser && npm test ``` **Pass:** all suites green. @@ -157,8 +161,9 @@ keeps a long run alive when the upstream public RPC drops the connection. Genuine relay-chain finality is covered separately in phase 4. -The script starts the four attestors, the monitor and the releaser itself, -each with its own key and checkpoint file. To run them by hand instead: +The script starts the four attestors and the releaser itself, plus the +monitor when `MONITOR_DIR` points at a built monitor checkout, each with its +own key and checkpoint file. To run them by hand instead: ```bash PENDULUM_WS=ws://127.0.0.1:8000 BASE_RPC_URL=http://localhost:8545 BASE_CHAIN_ID=31337 \ @@ -176,7 +181,8 @@ RELEASER_PRIVATE_KEY=0x… START_BLOCK=0 npm start The monitor's first-run boundaries are mandatory: use the first Pendulum block that can contain a migration (plus the nonce expected there) and the vault's -Base deployment block. Afterwards its atomic state file owns both cursors: +Base deployment block. Afterwards its atomic state file owns both cursors. +From the monitor checkout: ```bash PENDULUM_WS=ws://127.0.0.1:8000 BASE_RPC_URL=http://localhost:8545 BASE_CHAIN_ID=31337 \ @@ -213,7 +219,8 @@ Checks: `VITE_MIGRATION_VAULT_ADDRESS` set to the Anvil vault; migrate through the UI and watch the status card go 0/3 → 3/3 → released. -**Pass criterion:** 9/9 from the script. The final two checks impersonate the vault on +**Pass criterion:** 9/9 from the script with `MONITOR_DIR` set (without it, +6/6 with the three monitor checks reported as skipped). The final two checks impersonate the vault on Anvil to inject a real deficit and confirm the guardian pause through Base finality/state verification, then submit a fabricated one-attestor approval to prove the monitor rejects a tuple mismatch before it can reach quorum. @@ -320,7 +327,8 @@ It brings up Zombienet, waits for genuine parachain finality, unpauses the pallet **through the technical-committee origin** (this chain has no sudo, so unlike phase 2 there is no storage poke — the rehearsal drives the same origin that will unpause mainnet), deploys the contracts to Base Sepolia with the real -`Deploy.s.sol`, starts four attestors plus the monitor and releaser, and runs +`Deploy.s.sol`, starts four attestors plus the releaser (and the monitor when +`MONITOR_DIR` is set), and runs the scenarios. Every run writes `testing/.rehearsal//` containing a manifest (addresses, ports, block heights, commit) and each daemon's log, so a failed run stays diagnosable after teardown. diff --git a/monitor/.gitignore b/monitor/.gitignore deleted file mode 100644 index cbb90b2e6..000000000 --- a/monitor/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -node_modules/ -dist/ -monitor-state.json -monitor-state.json.*.tmp diff --git a/monitor/README.md b/monitor/README.md deleted file mode 100644 index 993a7c54a..000000000 --- a/monitor/README.md +++ /dev/null @@ -1,181 +0,0 @@ -# PEN Migration Invariant Monitor - -Independent watchdog for the PEN migration (PRD §6.5). **Must run on -infrastructure separate from every attestor** — its whole value is being an -independent pair of eyes on both chains. - -Checks every poll: - -| Check | Meaning | Reaction | -|---|---|---| -| M2 tuple match | Every safe Base `Approved` and `Released` event must match one finalized Pendulum `MigrationInitiated` nonce, recipient, and pallet amount; `Released.tokenAmount` must equal `palletAmount × conversionFactor`. After a violation the Base cursor stays on any block range whose events the source view may still reveal, so a pause caused by a lagging source does not lose those releases. | Alert + auto-pause on any mismatching `Released`, and on approvals of one nonce that disagree with the finalized Pendulum view from two or more distinct attestors, also across polls and restarts (from one at a vault threshold of two or less, where one more approval releases). Otherwise a single attestor's disagreeing approval pages `CRITICAL: MISMATCHED APPROVAL` instead: one approval can never release. A mismatch against a *known* nonce counts immediately. An *unknown* nonce first pages (`UNVERIFIED BASE EVENT`, without delaying the pause decision), then counts as a mismatch as soon as it is provable — the finalized source view passed the event's own timestamp and the nonce still does not exist — or after `UNMATCHED_EVENT_GRACE_SECONDS` if the source view stays behind or cannot be read (a stalled or unreadable source also pages on its own, see below) | -| M2a aggregate | `totalReleased <= TotalMigrated × conversionFactor + ACKNOWLEDGED_RELEASE_EXCESS`. This remains independent defence-in-depth around the event reconciliation: it also runs while a Base log range is held, and waits only while a Base event's burn is not in the source view yet. After a real, quantified loss, operators set `ACKNOWLEDGED_RELEASE_EXCESS` to it (RB-3) so the vault can be unpaused; any further excess still fires. | Alert + auto-pause | -| M2b: `balanceOf(vault) + totalReleased + totalSwept >= totalSupply` | Vault-internal conservation. Only a **deficit** alerts: a surplus is a harmless inbound transfer (donation, or a migration whose recipient is the vault) and is ignored, so it cannot false-trigger a pause. It reads Base only and runs first in every poll, so a failing Pendulum source or Base log scan cannot skip it. While the rest of a poll takes longer than `POLL_INTERVAL_MS` (a slow source, a long catch-up), it also runs on its own, so two of its runs are about three poll intervals apart at most. | Alert + auto-pause on a deficit | -| M4: every old migration reaches active approval quorum — and then releases | Liveness of the attestor fleet (`below approval quorum`), and of everything after it (`quorum reached but not released`: a release starved of daily allowance, paused, or made releasable by a threshold cut without a `ReleasePending`, which the releaser cannot see). | One summary page per kind and poll (count and the first nonces; the log lists every nonce), each nonce at most once per `GRACE_SECONDS` | -| Source supply | `totalIssuance + TotalMigrated` on Pendulum must never grow past its first-seen anchor. | Alert (`SOURCE SUPPLY GREW`) | -| Unreleasable burns | A burn to the zero or vault address can never be approved or released; it is paged once and excluded from the pending count so it cannot page liveness forever or block RB-7. | Alert once | -| Treasury destination | With `EXPECTED_TREASURY_DESTINATION` set, every finalized treasury migration (a `MigrationInitiated` whose `who` is the treasury account, `PalletId(*b"py/trsry")`) must pay that Base address. It is checked when the burn is ingested, before the attestors can release it; while the pause is not confirmed, the monitor holds at that block, and the violation pages again every poll saying so. | Alert + auto-pause (`TREASURY DESTINATION VIOLATION`) | -| Pendulum source | The finalized Pendulum view must stay fresh and readable. While it cannot be read (an RPC error, a nonce discontinuity, an event-shape change), the Base-side checks — vault balance and its auto-pause, Base view, auto-pause readiness, vault admin events, governance queue — still run, and Base events are reconciled against the last view read in full: an unreadable source counts as a stalled one, so a mismatch on a known nonce counts at once and an unknown nonce after `UNMATCHED_EVENT_GRACE_SECONDS`. M2a and liveness wait for the source. | Alert, throttled: `PENDULUM SOURCE VIEW STALLED` once the view is older than `SOURCE_STALE_ALERT_SECONDS` (naming the second node or the relay RPCs when they hold the view back), `PENDULUM SOURCE UNREADABLE` on a failure | -| Pendulum chain | Every finalized block the monitor ingests is read at a hash it verified against the blocks it ingested before and, where configured, a second node and the relay chain; a second node must also agree on the migration totals and on the migration events of each ingested block (see [Pendulum chain verification](#pendulum-chain-verification)). | Alert (`CRITICAL: PENDULUM SOURCE CONTRADICTED`), throttled; nothing from the source is ingested, and it is treated as unreadable (see Pendulum source) until it verifies again. A second node or relay RPC that does not answer only warns, throttled (`PENDULUM CROSS-CHECK UNAVAILABLE`, `RELAY ANCHOR UNAVAILABLE`), and so does a relay RPC outvoted by another (`RELAY RPCS DISAGREE`); relay RPCs that answer without anchoring the source's newer blocks hold them back (`RELAY ANCHOR UNAVAILABLE`); `CRITICAL: RELAY ANCHOR STALE` once no head was anchored for `RELAY_ANCHOR_GRACE_SECONDS` | -| Base log completeness | In every scanned block range the `Released` amounts must add up to the growth of `totalReleased`, and the `RemainderSwept` amounts to that of `totalSwept`, between the block before the range and its last block. A range whose logs do not (a backend serving truncated logs) is read once more, then held: the Base cursor stays on it and liveness waits until a read accounts for the totals; M2a still runs. | Alert (`BASE LOG SCAN INCOMPLETE`), throttled; `CRITICAL: BASE LOG SCAN INCOMPLETE` once a range has been held for longer than `UNMATCHED_EVENT_GRACE_SECONDS`, since no later Base event is matched against its burn meanwhile | -| Base view | The safe/finalized Base block must not be older than `BASE_STALE_ALERT_SECONDS`: a view that stops advancing hides every later vault event while every check still passes at the old block. | Alert (`BASE VIEW STALLED`), throttled | -| Auto-pause readiness | At startup and every poll: the Base RPC serves `BASE_CHAIN_ID` and, with a guardian key configured, the key parses, is `vault.guardian()` and holds at least `GUARDIAN_MIN_BALANCE_WEI`. A read that fails pages as well and never stops the poll's other checks. | Alert (`AUTO-PAUSE NOT READY`), throttled; `AUTO-PAUSE DISABLED` once at startup without a key | -| Pendulum governance | Every pending public proposal, the council's external proposal, every ongoing referendum and every open council and technical-committee motion, with the call decoded. A call that mints, moves or unlocks balances outside normal transfers, or changes the runtime or the migration pallet, pages as critical: minted PEN burned through the migration drains the fixed-supply vault, and only Pendulum governance can stop the call before enactment. A call the monitor cannot read pages as critical too: one over 64 KiB is never fetched, and one that does not decode or nests more than 256 calls or 16 levels deep is unreadable; one whose preimage is not noted yet is re-checked every poll and paged again once it can be read. The watch runs after the poll's conservation checks, so it cannot stop or delay them, and it decodes within fixed bounds, so its cost to the next poll is bounded. | Alert once per item (`PENDULUM GOVERNANCE ITEM`, `CRITICAL: Pendulum governance item …`); a motion proposed again with the same call (a new motion index), or an external proposal made again after it left the queue or became a referendum, pages again; a restart re-pages the items still pending | -| Migration pallet admin events | Every finalized `MigrationPauseSet` and `TreasuryDestinationSet`, however fast the motion behind it passed. | Alert once per event (`MIGRATION PALLET ADMIN EVENT`); `CRITICAL: MIGRATION PALLET ADMIN EVENT` for a `TreasuryDestinationSet` to another address than `EXPECTED_TREASURY_DESTINATION` | -| Vault admin events | Every admin event of the vault (token, pause, unpause, attestors, threshold, caps, guardian, admin transfer, sweep, stale pending cleared). Before the governance handover the admin acts without a delay, so these are the only trace of such a change. This scan (shared with the governance queue) keeps its own persisted cursor: while it fails, every other check still runs, and it resumes where it stopped, also after a restart. | Alert once per event (`VAULT ADMIN EVENT`); the first run also pages the deployment's own events; `BASE ADMIN SCAN FAILED`, throttled, on a failure, and `CRITICAL: BASE ADMIN SCAN FAILED` once it has failed for longer than `UNMATCHED_EVENT_GRACE_SECONDS` | -| Governance queue | Every operation the governance Timelock schedules (`CallScheduled`), with its calls decoded and the time it becomes executable. Once the delay has passed anyone may execute a ready operation, so the canceller has exactly the delay to review and veto it. | Alert once per operation and scheduling block (`GOVERNANCE OPERATION SCHEDULED`), so an operation cancelled and scheduled again with the same id pages again | - -## Pendulum chain verification - -The monitor reads every new finalized Pendulum block at a hash it verified -itself: - -- **Parent-hash chain.** Each poll walks the parent hashes down from the - finalized head to the block ingested last, through headers fetched from the - node and hashed by the monitor. The walk must arrive at the hash of the block - ingested last, which the state file keeps, so every ingested block lies on - one chain. -- **Second node** (`PENDULUM_WS_CROSSCHECK`). The source's chain must have - the second node's hash at the height of the second node's finalized head - (at the source's head once the second node has finalized past it), and no - block above that height is ingested. The migration totals (`nextNonce`, - `totalMigrated`) must agree at the block the monitor reads them, and every - ingested block with migration pallet events must carry the same ones on the - second node. A second node whose finalized head is older than - `SOURCE_STALE_ALERT_SECONDS` counts as unavailable. -- **Relay-chain anchor** (`POLKADOT_WS`). Each poll reads the Pendulum head - that the Polkadot relay chain recorded at its finalized block - (`paras.heads(PENDULUM_PARA_ID)`), asking the relay RPCs in turn until a - head at or above the source's finalized head anchors it; the highest head - that anchors the source is used. A relay RPC whose finalized relay block is - older than `SOURCE_STALE_ALERT_SECONDS` counts as not answering. The - source's chain must have that hash at that height. A head above the - source's finalized head counts once the source's best chain holds it, and - its chain must then pass through the finalized head. The newest such head - is kept in the state file, and the chain must keep passing through it, - also while it lies above the finalized head. Blocks ingested while no relay - RPC answered are tied to the next head that arrives by the same walk; a - head more than 100 blocks below the block ingested last is not used. A - head that contradicts the source counts only when another relay RPC's head - contradicts it too, or when `POLKADOT_WS` lists one RPC. A relay RPC whose - head anchors the source at or above the contradicting head's height - outvotes it: `RELAY RPCS DISAGREE` pages and that head anchors the source. - Any other answer leaves the contradiction unconfirmed; a lower head still - anchors the source up to its own height. No block above the newest - anchored head is ingested. - -Every block the monitor ingests is confirmed by each checker that answers, -and the values the checks use (the view's timestamp, `nextNonce`, -`totalMigrated` and issuance) are read at the newest ingested block. Relay -RPCs that answer but anchor no newer head, or leave a contradiction -unconfirmed, hold the source's newer blocks back and page -`RELAY ANCHOR UNAVAILABLE`. A contradiction pages -`CRITICAL: PENDULUM SOURCE CONTRADICTED`, nothing from the source is -ingested, and the source counts as unreadable until it verifies again. A -second node or relay RPC that cannot be read, or whose finalized head (for a -relay RPC, its finalized relay block) is older than -`SOURCE_STALE_ALERT_SECONDS`, only warns and holds nothing back; with no -checker answering, the monitor goes on with the source alone up to its -finalized head. `CRITICAL: RELAY ANCHOR STALE` pages once no head was -anchored for `RELAY_ANCHOR_GRACE_SECONDS`, whatever the reason (relay RPCs -down, the source behind the relay chain, a relay head that does not advance). - -## Configuration (environment) - -| Variable | Meaning | -|---|---| -| `PENDULUM_WS` | WebSocket of the monitor's own Pendulum node, run with `--state-pruning archive` (see below) | -| `PENDULUM_WS_CROSSCHECK` | Optional: WebSocket of a second Pendulum node, not the one behind `PENDULUM_WS`; only blocks on its finalized chain are ingested, and it must agree with the source on the migration totals in every poll and on the migration events of every ingested block that has some. Run it with `--state-pruning archive` too, or those events cannot be compared after a monitor outage longer than its pruning horizon (`PENDULUM CROSS-CHECK UNAVAILABLE`) | -| `POLKADOT_WS` | Optional: comma-separated WebSocket RPCs of the Polkadot relay chain, run by third parties and tried in order (for example `wss://rpc.polkadot.io`); turns on the relay-chain anchor, and while a relay RPC answers, no Pendulum block above the newest anchored head is ingested. List at least two: a record that contradicts the source needs a second relay RPC's confirmation, unless only one is listed | -| `PENDULUM_PARA_ID` | Pendulum's parachain id on the relay chain (default 2094) | -| `RELAY_ANCHOR_GRACE_SECONDS` | How long the source may go without a relay-anchored head before `CRITICAL: RELAY ANCHOR STALE` pages (default 3600) | -| `BASE_RPC_URL` | Base JSON-RPC endpoint — a **single dedicated node**, and a different provider than the attestors use. Load-balanced pools can serve `eth_getLogs` from a lagging replica; each scan range is probed for its end block first and its logs must account for the vault's running totals, which turns a behind-node into a loud replay instead of a silent gap, but a dedicated node removes the hazard entirely | -| `VAULT_ADDRESS` | MigrationVault address on Base | -| `POLL_INTERVAL_MS` | Poll cadence (default 60s); while a poll takes longer, the vault balance check runs on its own at this cadence | -| `GRACE_SECONDS` | Liveness alert threshold (default 30 min) | -| `UNMATCHED_EVENT_GRACE_SECONDS` | Time a lagging or unreadable Pendulum source view gets to reveal the burn behind a safe Base event before a cannot-verify pause (default 10 min). The first sighting of an unmatched event pages immediately, so this is the window operators have to repair a stalled or failing node. Size it to your node-ops response time | -| `SOURCE_CLOCK_SKEW_MARGIN_SECONDS` | Clock-skew allowance for the fabrication proof (default 120s): once the finalized Pendulum view has passed the moment the monitor first observed an unmatched Base event by this margin and the nonce still does not exist, the event is provably fabricated and pauses without waiting out the grace. The anchor is the monitor's own clock, not the Base block timestamp (which trails real time after a sequencer outage); a future-dated source view proves nothing | -| `SOURCE_STALE_ALERT_SECONDS` | Age of the verified Pendulum view (vs. wall clock) past which the monitor pages that it is going blind (default 300s); a second node whose finalized head, or a relay RPC whose finalized relay block, is older counts as not answering | -| `BASE_STALE_ALERT_SECONDS` | Age of the safe/finalized Base block (vs. wall clock) past which `BASE VIEW STALLED` pages (default 600s for `safe`, 1800s for `finalized`; Base's safe head normally trails by about a minute, its finalized head by about 20) | -| `ISSUANCE_TOLERANCE` | Growth of Pendulum `totalIssuance + TotalMigrated` over its first-seen anchor (12-decimal pallet units) tolerated before `SOURCE SUPPLY GREW` pages (default 0) | -| `ACKNOWLEDGED_RELEASE_EXCESS` | Excess of released over migrated, in token units (18 decimals), that operators quantified and accepted after a real loss; M2a fires only beyond it (default 0). Set it at the RB-3 sign-off and restart; the startup log shows it | -| `EXPECTED_TREASURY_DESTINATION` | Optional: the Base address every treasury migration must pay, the one set with `tokenMigration.setTreasuryDestination`; any other pauses the vault, and setting another one pages critical | -| `TIMELOCK_ADDRESS` | Optional: the governance TimelockController. Set it as soon as governance is deployed; the queue is scanned from the monitor's Base cursor onwards, so operations scheduled before the monitor knew the address are not paged | -| `ALERT_WEBHOOK_URL` | Slack incoming webhook receiving the pages (see [Pages](#pages)). A page counts as delivered only on a 2xx answer within 10s: once-only pages (governance items, admin events, scheduled operations, first sightings) are retried every poll until delivered, and failed deliveries are logged. Without it the log line is the page | -| `HEARTBEAT_URL` | Optional: a dead-man's switch URL (Healthchecks.io style), requested with a `GET` (10s timeout) after every poll that ran all checks; a failed request is logged and never stops the monitor. Have the service page when the requests stop: the monitor is down, stuck, or cannot complete its checks (for example while the Pendulum source cannot be read) | -| `GUARDIAN_PRIVATE_KEY` | Optional: the key of `vault.guardian()`, enabling automatic `pause()` on conservation violations; its readiness is checked at startup and every poll, and any other key pages `AUTO-PAUSE NOT READY` (the vault admin may pause too, but its key does not belong on the monitor) | -| `GUARDIAN_MIN_BALANCE_WEI` | Guardian ETH balance below which `AUTO-PAUSE NOT READY` pages (default 10^15 wei = 0.001 ETH) | -| `BASE_CHAIN_ID` | Default 8453; the Base RPC must serve it | -| `PENDULUM_START_BLOCK` | **Required on first run:** first block to scan, normally the pallet activation block or the next finalized block before unpausing | -| `PENDULUM_START_NONCE` | Nonce expected at that block (default 0; set explicitly when starting after any migrations) | -| `BASE_START_BLOCK` | **Required on first run:** vault deployment block or an earlier block | -| `STATE_FILE` | Durable cursors (Pendulum with the hash of its last block, Base, Base admin scan), the newest relay-anchored Pendulum head, pending tuples, and liveness timestamps (default `./monitor-state.json`) | -| `BASE_FINALITY_TAG` | `safe` (default) or `finalized`; only events inside this boundary are trusted | -| `BASE_MAX_BLOCK_RANGE` | Maximum `eth_getLogs` range (default 10,000 blocks) | - -## Pages - -Each page is a Slack incoming-webhook message, a JSON `POST` of `{ "text": … }` -to `ALERT_WEBHOOK_URL`. The text is `[CRITICAL] pen-monitor: SUBJECT` or -`[WARNING] pen-monitor: SUBJECT`, then the detail on the next line. Slack's -control characters `&`, `<` and `>` are escaped, so no detail can mention -anyone or hide a link, and a detail longer than 3,000 characters is cut short -(the monitor's log line keeps the full text). `CRITICAL` marks the violations -that pause the vault, `AUTO-PAUSE FAILED`, `AUTO-PAUSE UNAVAILABLE` and every -subject that starts with `CRITICAL`; everything else is a `WARNING`. - -| Subjects | Meaning | -|---|---| -| `MIGRATION TUPLE VIOLATION`, `VAULT BALANCE DEFICIT`, `CONSERVATION VIOLATION`, `TREASURY DESTINATION VIOLATION` | A conservation or treasury check failed; the auto-pause follows with `vault auto-pause confirmed`, `vault pause confirmed` (already paused, also without a guardian key), `AUTO-PAUSE PENDING FINALITY`, `AUTO-PAUSE FAILED` or `AUTO-PAUSE UNAVAILABLE` (no guardian key, and the vault is not paused) | -| `UNVERIFIED BASE EVENT` | First sighting of a Base event whose nonce the source view does not know yet | -| `CRITICAL: MISMATCHED APPROVAL` | One attestor approved a tuple that disagrees with the finalized Pendulum view; check its key and its Pendulum node. A second attestor doing so for the same nonce pauses the vault | -| `LIVENESS: migration below approval quorum`, `LIVENESS: quorum reached but not released` | Stalled migrations, one summary per kind and poll | -| `PENDULUM SOURCE VIEW STALLED`, `PENDULUM SOURCE UNREADABLE`, `BASE VIEW STALLED`, `BASE ADMIN SCAN FAILED`, `BASE LOG SCAN INCOMPLETE` | The monitor is going blind on one chain | -| `CRITICAL: BASE LOG SCAN INCOMPLETE`, `CRITICAL: BASE ADMIN SCAN FAILED` | For longer than `UNMATCHED_EVENT_GRACE_SECONDS`, a Base block range has been held, so no later Base event is matched against its burn, or the admin scan has failed, so no vault admin event or scheduled governance operation is paged while its veto window runs; fix the Base RPC | -| `CRITICAL: PENDULUM SOURCE CONTRADICTED` | The Pendulum source contradicts the chain the monitor verified; stop trusting that node | -| `PENDULUM CROSS-CHECK UNAVAILABLE` | The second Pendulum node does not answer or its finalized head is stale, so the source is used without that check | -| `RELAY ANCHOR UNAVAILABLE` | No relay RPC answers from a finalized relay block younger than `SOURCE_STALE_ALERT_SECONDS`, so the source is used without the anchor; or none anchors the source's newer blocks, or a relay RPC records a head that contradicts the source and no other one confirms or outvotes it, so those blocks wait | -| `RELAY RPCS DISAGREE` | One relay RPC records a Pendulum head that contradicts the source, and another records one at or above that height on the source's chain, which anchors the source. One of them is wrong; check both | -| `CRITICAL: RELAY ANCHOR STALE` | No Pendulum head was checked against the relay chain for `RELAY_ANCHOR_GRACE_SECONDS` | -| `AUTO-PAUSE NOT READY`, `AUTO-PAUSE DISABLED` | The next violation would not pause the vault automatically | -| `SOURCE SUPPLY GREW`, `CRITICAL: unreleasable migration burned on Pendulum` | Supply problems at the source | -| `PENDULUM GOVERNANCE ITEM`, `CRITICAL: Pendulum governance item mints, moves or unlocks balances, or changes the runtime`, `CRITICAL: Pendulum governance item cannot be read`, `MIGRATION PALLET ADMIN EVENT`, `CRITICAL: MIGRATION PALLET ADMIN EVENT` | Pendulum governance | -| `GOVERNANCE OPERATION SCHEDULED`, `VAULT ADMIN EVENT` | Base governance and vault administration | -| `monitor check failed`, `pendulum governance watch failed`, `multicall unavailable, …`, `multicall batch failed, …`, `monitor startup failed` | The monitor itself | - -## Run - -```sh -npm install -npm run build -npm start -``` - -Run it under a process manager and treat "monitor down" itself as a paging -condition (`HEARTBEAT_URL`): an unwatched migration is the risk model failing -silently. The two-chain cursors, unresolved canonical tuples, their original -finalized block timestamps, and alert throttles are atomically persisted. A malformed or -Pendulum-genesis/Base-chain/vault-mismatched state file is fatal rather than -silently resetting the security history. - -**Node requirements.** The Pendulum node must run with `--state-pruning -archive` (or `archive-canonical`): the monitor catches up block by block -through historical state, and a default-pruned node (256 blocks ≈ 51 min) -cannot serve that after any monitor outage longer than the pruning horizon — -the monitor then pages `PENDULUM SOURCE UNREADABLE` on restart. Recovery is -pointing `PENDULUM_WS` at an archive node, never editing the state file. A -monitor whose *running* source view stalls pages `PENDULUM SOURCE VIEW -STALLED` well before the unmatched-event grace can force a cannot-verify -pause; treat that page as "restore the node now", or the second node or -relay RPCs it names as holding the view back. The Base-side checks keep -running while the source cannot be read, and an unreadable source counts as -a stalled one: once a Base event it cannot verify appears, the monitor pages -`UNVERIFIED BASE EVENT` and pauses when the grace runs out, so treat -`PENDULUM SOURCE UNREADABLE` the same way. The Base node must answer -`eth_call` at the block before each scanned range and at its last block (the -log completeness check); after a monitor outage those are old blocks, so a -node that keeps only recent state stops the scan until it is an archive node. - -An automatic pause is reported as successful only after the transaction is -canonical inside the configured Base confirmation boundary and `paused()` is -true there. A merely-mined pause is reported as pending finality. diff --git a/monitor/package-lock.json b/monitor/package-lock.json deleted file mode 100644 index a48e1d8d5..000000000 --- a/monitor/package-lock.json +++ /dev/null @@ -1,1123 +0,0 @@ -{ - "name": "pen-migration-monitor", - "version": "0.1.0", - "lockfileVersion": 3, - "requires": true, - "packages": { - "": { - "name": "pen-migration-monitor", - "version": "0.1.0", - "dependencies": { - "@polkadot/api": "^11.3.1", - "viem": "^2.21.0" - }, - "devDependencies": { - "@types/node": "^20.14.0", - "typescript": "^5.5.0" - } - }, - "node_modules/@adraffy/ens-normalize": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@adraffy/ens-normalize/-/ens-normalize-1.11.1.tgz", - "integrity": "sha512-nhCBV3quEgesuf7c7KYfperqSS14T8bYuvJ8PcLJp6znkZpFc0AuW4qBtr8eKVyPPe/8RSr7sglCWPU5eaxwKQ==", - "license": "MIT" - }, - "node_modules/@noble/ciphers": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/@noble/ciphers/-/ciphers-1.3.0.tgz", - "integrity": "sha512-2I0gnIVPtfnMw9ee9h1dJG7tp81+8Ob3OJb3Mv37rx5L40/b0i7djjCVvGOVqc9AEIQyvyu1i6ypKdFw8R8gQw==", - "license": "MIT", - "engines": { - "node": "^14.21.3 || >=16" - }, - "funding": { - "url": "https://paulmillr.com/funding/" - } - }, - "node_modules/@noble/curves": { - "version": "1.9.7", - "resolved": "https://registry.npmjs.org/@noble/curves/-/curves-1.9.7.tgz", - "integrity": "sha512-gbKGcRUYIjA3/zCCNaWDciTMFI0dCkvou3TL8Zmy5Nc7sJ47a0jtOeZoTaMxkuqRo9cRhjOdZJXegxYE5FN/xw==", - "license": "MIT", - "dependencies": { - "@noble/hashes": "1.8.0" - }, - "engines": { - "node": "^14.21.3 || >=16" - }, - "funding": { - "url": "https://paulmillr.com/funding/" - } - }, - "node_modules/@noble/hashes": { - "version": "1.8.0", - "resolved": "https://registry.npmjs.org/@noble/hashes/-/hashes-1.8.0.tgz", - "integrity": "sha512-jCs9ldd7NwzpgXDIf6P3+NrHh9/sD6CQdxHyjQI+h/6rDNo88ypBxxz45UDuZHz9r3tNz7N/VInSVoVdtXEI4A==", - "license": "MIT", - "engines": { - "node": "^14.21.3 || >=16" - }, - "funding": { - "url": "https://paulmillr.com/funding/" - } - }, - "node_modules/@polkadot-api/json-rpc-provider": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/@polkadot-api/json-rpc-provider/-/json-rpc-provider-0.0.1.tgz", - "integrity": "sha512-/SMC/l7foRjpykLTUTacIH05H3mr9ip8b5xxfwXlVezXrNVLp3Cv0GX6uItkKd+ZjzVPf3PFrDF2B2/HLSNESA==", - "license": "MIT", - "optional": true - }, - "node_modules/@polkadot-api/json-rpc-provider-proxy": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/@polkadot-api/json-rpc-provider-proxy/-/json-rpc-provider-proxy-0.0.1.tgz", - "integrity": "sha512-gmVDUP8LpCH0BXewbzqXF2sdHddq1H1q+XrAW2of+KZj4woQkIGBRGTJHeBEVHe30EB+UejR1N2dT4PO/RvDdg==", - "license": "MIT", - "optional": true - }, - "node_modules/@polkadot-api/metadata-builders": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/@polkadot-api/metadata-builders/-/metadata-builders-0.0.1.tgz", - "integrity": "sha512-GCI78BHDzXAF/L2pZD6Aod/yl82adqQ7ftNmKg51ixRL02JpWUA+SpUKTJE5MY1p8kiJJIo09P2um24SiJHxNA==", - "license": "MIT", - "optional": true, - "dependencies": { - "@polkadot-api/substrate-bindings": "0.0.1", - "@polkadot-api/utils": "0.0.1" - } - }, - "node_modules/@polkadot-api/observable-client": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/@polkadot-api/observable-client/-/observable-client-0.1.0.tgz", - "integrity": "sha512-GBCGDRztKorTLna/unjl/9SWZcRmvV58o9jwU2Y038VuPXZcr01jcw/1O3x+yeAuwyGzbucI/mLTDa1QoEml3A==", - "license": "MIT", - "optional": true, - "dependencies": { - "@polkadot-api/metadata-builders": "0.0.1", - "@polkadot-api/substrate-bindings": "0.0.1", - "@polkadot-api/substrate-client": "0.0.1", - "@polkadot-api/utils": "0.0.1" - }, - "peerDependencies": { - "rxjs": ">=7.8.0" - } - }, - "node_modules/@polkadot-api/substrate-bindings": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/@polkadot-api/substrate-bindings/-/substrate-bindings-0.0.1.tgz", - "integrity": "sha512-bAe7a5bOPnuFVmpv7y4BBMRpNTnMmE0jtTqRUw/+D8ZlEHNVEJQGr4wu3QQCl7k1GnSV1wfv3mzIbYjErEBocg==", - "license": "MIT", - "optional": true, - "dependencies": { - "@noble/hashes": "^1.3.1", - "@polkadot-api/utils": "0.0.1", - "@scure/base": "^1.1.1", - "scale-ts": "^1.6.0" - } - }, - "node_modules/@polkadot-api/substrate-client": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/@polkadot-api/substrate-client/-/substrate-client-0.0.1.tgz", - "integrity": "sha512-9Bg9SGc3AwE+wXONQoW8GC00N3v6lCZLW74HQzqB6ROdcm5VAHM4CB/xRzWSUF9CXL78ugiwtHx3wBcpx4H4Wg==", - "license": "MIT", - "optional": true - }, - "node_modules/@polkadot-api/utils": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/@polkadot-api/utils/-/utils-0.0.1.tgz", - "integrity": "sha512-3j+pRmlF9SgiYDabSdZsBSsN5XHbpXOAce1lWj56IEEaFZVjsiCaxDOA7C9nCcgfVXuvnbxqqEGQvnY+QfBAUw==", - "license": "MIT", - "optional": true - }, - "node_modules/@polkadot/api": { - "version": "11.3.1", - "resolved": "https://registry.npmjs.org/@polkadot/api/-/api-11.3.1.tgz", - "integrity": "sha512-q4kFIIHTLvKxM24b0Eo8hJevsPMme+aITJGrDML9BgdZYTRN14+cu5nXiCsQvaEamdyYj+uCXWe2OV9X7pPxsA==", - "license": "Apache-2.0", - "dependencies": { - "@polkadot/api-augment": "11.3.1", - "@polkadot/api-base": "11.3.1", - "@polkadot/api-derive": "11.3.1", - "@polkadot/keyring": "^12.6.2", - "@polkadot/rpc-augment": "11.3.1", - "@polkadot/rpc-core": "11.3.1", - "@polkadot/rpc-provider": "11.3.1", - "@polkadot/types": "11.3.1", - "@polkadot/types-augment": "11.3.1", - "@polkadot/types-codec": "11.3.1", - "@polkadot/types-create": "11.3.1", - "@polkadot/types-known": "11.3.1", - "@polkadot/util": "^12.6.2", - "@polkadot/util-crypto": "^12.6.2", - "eventemitter3": "^5.0.1", - "rxjs": "^7.8.1", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/@polkadot/api-augment": { - "version": "11.3.1", - "resolved": "https://registry.npmjs.org/@polkadot/api-augment/-/api-augment-11.3.1.tgz", - "integrity": "sha512-Yj+6rb6h0WwY3yJ+UGhjGW+tyMRFUMsKQuGw+eFsXdjiNU9UoXsAqA2dG7Q1F+oeX/g+y2gLGBezNoCwbl6HfA==", - "license": "Apache-2.0", - "dependencies": { - "@polkadot/api-base": "11.3.1", - "@polkadot/rpc-augment": "11.3.1", - "@polkadot/types": "11.3.1", - "@polkadot/types-augment": "11.3.1", - "@polkadot/types-codec": "11.3.1", - "@polkadot/util": "^12.6.2", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/@polkadot/api-base": { - "version": "11.3.1", - "resolved": "https://registry.npmjs.org/@polkadot/api-base/-/api-base-11.3.1.tgz", - "integrity": "sha512-b8UkNL00NN7+3QaLCwL5cKg+7YchHoKCAhwKusWHNBZkkO6Oo2BWilu0dZkPJOyqV9P389Kbd9+oH+SKs9u2VQ==", - "license": "Apache-2.0", - "dependencies": { - "@polkadot/rpc-core": "11.3.1", - "@polkadot/types": "11.3.1", - "@polkadot/util": "^12.6.2", - "rxjs": "^7.8.1", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/@polkadot/api-derive": { - "version": "11.3.1", - "resolved": "https://registry.npmjs.org/@polkadot/api-derive/-/api-derive-11.3.1.tgz", - "integrity": "sha512-9dopzrh4cRuft1nANmBvMY/hEhFDu0VICMTOGxQLOl8NMfcOFPTLAN0JhSBUoicGZhV+c4vpv01NBx/7/IL1HA==", - "license": "Apache-2.0", - "dependencies": { - "@polkadot/api": "11.3.1", - "@polkadot/api-augment": "11.3.1", - "@polkadot/api-base": "11.3.1", - "@polkadot/rpc-core": "11.3.1", - "@polkadot/types": "11.3.1", - "@polkadot/types-codec": "11.3.1", - "@polkadot/util": "^12.6.2", - "@polkadot/util-crypto": "^12.6.2", - "rxjs": "^7.8.1", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/@polkadot/keyring": { - "version": "12.6.2", - "resolved": "https://registry.npmjs.org/@polkadot/keyring/-/keyring-12.6.2.tgz", - "integrity": "sha512-O3Q7GVmRYm8q7HuB3S0+Yf/q/EB2egKRRU3fv9b3B7V+A52tKzA+vIwEmNVaD1g5FKW9oB97rmpggs0zaKFqHw==", - "license": "Apache-2.0", - "dependencies": { - "@polkadot/util": "12.6.2", - "@polkadot/util-crypto": "12.6.2", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "@polkadot/util": "12.6.2", - "@polkadot/util-crypto": "12.6.2" - } - }, - "node_modules/@polkadot/networks": { - "version": "12.6.2", - "resolved": "https://registry.npmjs.org/@polkadot/networks/-/networks-12.6.2.tgz", - "integrity": "sha512-1oWtZm1IvPWqvMrldVH6NI2gBoCndl5GEwx7lAuQWGr7eNL+6Bdc5K3Z9T0MzFvDGoi2/CBqjX9dRKo39pDC/w==", - "license": "Apache-2.0", - "dependencies": { - "@polkadot/util": "12.6.2", - "@substrate/ss58-registry": "^1.44.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/@polkadot/rpc-augment": { - "version": "11.3.1", - "resolved": "https://registry.npmjs.org/@polkadot/rpc-augment/-/rpc-augment-11.3.1.tgz", - "integrity": "sha512-2PaDcKNju4QYQpxwVkWbRU3M0t340nMX9cMo+8awgvgL1LliV/fUDZueMKLuSS910JJMTPQ7y2pK4eQgMt08gQ==", - "license": "Apache-2.0", - "dependencies": { - "@polkadot/rpc-core": "11.3.1", - "@polkadot/types": "11.3.1", - "@polkadot/types-codec": "11.3.1", - "@polkadot/util": "^12.6.2", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/@polkadot/rpc-core": { - "version": "11.3.1", - "resolved": "https://registry.npmjs.org/@polkadot/rpc-core/-/rpc-core-11.3.1.tgz", - "integrity": "sha512-KKNepsDd/mpmXcA6v/h14eFFPEzLGd7nrvx2UUXUxoZ0Fq2MH1hplP3s93k1oduNY/vOXJR2K9S4dKManA6GVQ==", - "license": "Apache-2.0", - "dependencies": { - "@polkadot/rpc-augment": "11.3.1", - "@polkadot/rpc-provider": "11.3.1", - "@polkadot/types": "11.3.1", - "@polkadot/util": "^12.6.2", - "rxjs": "^7.8.1", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/@polkadot/rpc-provider": { - "version": "11.3.1", - "resolved": "https://registry.npmjs.org/@polkadot/rpc-provider/-/rpc-provider-11.3.1.tgz", - "integrity": "sha512-pqERChoHo45hd3WAgW8UuzarRF+G/o/eXEbl0PXLubiayw4X4qCmIzmtntUcKYgxGNcYGZaG87ZU8OjN97m6UA==", - "license": "Apache-2.0", - "dependencies": { - "@polkadot/keyring": "^12.6.2", - "@polkadot/types": "11.3.1", - "@polkadot/types-support": "11.3.1", - "@polkadot/util": "^12.6.2", - "@polkadot/util-crypto": "^12.6.2", - "@polkadot/x-fetch": "^12.6.2", - "@polkadot/x-global": "^12.6.2", - "@polkadot/x-ws": "^12.6.2", - "eventemitter3": "^5.0.1", - "mock-socket": "^9.3.1", - "nock": "^13.5.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18" - }, - "optionalDependencies": { - "@substrate/connect": "0.8.10" - } - }, - "node_modules/@polkadot/types": { - "version": "11.3.1", - "resolved": "https://registry.npmjs.org/@polkadot/types/-/types-11.3.1.tgz", - "integrity": "sha512-5c7uRFXQTT11Awi6T0yFIdAfD6xGDAOz06Kp7M5S9OGNZY28wSPk5x6BYfNphWPaIBmHHewYJB5qmnrdYQAWKQ==", - "license": "Apache-2.0", - "dependencies": { - "@polkadot/keyring": "^12.6.2", - "@polkadot/types-augment": "11.3.1", - "@polkadot/types-codec": "11.3.1", - "@polkadot/types-create": "11.3.1", - "@polkadot/util": "^12.6.2", - "@polkadot/util-crypto": "^12.6.2", - "rxjs": "^7.8.1", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/@polkadot/types-augment": { - "version": "11.3.1", - "resolved": "https://registry.npmjs.org/@polkadot/types-augment/-/types-augment-11.3.1.tgz", - "integrity": "sha512-eR3HVpvUmB3v7q2jTWVmVfAVfb1/kuNn7ij94Zqadg/fuUq0pKqIOKwkUj3OxRM3A/5BnW3MbgparjKD3r+fyw==", - "license": "Apache-2.0", - "dependencies": { - "@polkadot/types": "11.3.1", - "@polkadot/types-codec": "11.3.1", - "@polkadot/util": "^12.6.2", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/@polkadot/types-codec": { - "version": "11.3.1", - "resolved": "https://registry.npmjs.org/@polkadot/types-codec/-/types-codec-11.3.1.tgz", - "integrity": "sha512-i7IiiuuL+Z/jFoKTA9xeh4wGQnhnNNjMT0+1ohvlOvnFsoKZKFQQOaDPPntGJVL1JDCV+KjkN2uQKZSeW8tguQ==", - "license": "Apache-2.0", - "dependencies": { - "@polkadot/util": "^12.6.2", - "@polkadot/x-bigint": "^12.6.2", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/@polkadot/types-create": { - "version": "11.3.1", - "resolved": "https://registry.npmjs.org/@polkadot/types-create/-/types-create-11.3.1.tgz", - "integrity": "sha512-pBXtpz5FehcRJ6j5MzFUIUN8ZWM7z6HbqK1GxBmYbJVRElcGcOg7a/rL2pQVphU0Rx1E8bSO4thzGf4wUxSX7w==", - "license": "Apache-2.0", - "dependencies": { - "@polkadot/types-codec": "11.3.1", - "@polkadot/util": "^12.6.2", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/@polkadot/types-known": { - "version": "11.3.1", - "resolved": "https://registry.npmjs.org/@polkadot/types-known/-/types-known-11.3.1.tgz", - "integrity": "sha512-3BIof7u6tn9bk3ZCIxA07iNoQ3uj4+vn3DTOjCKECozkRlt6V+kWRvqh16Hc0SHMg/QjcMb2fIu/WZhka1McUQ==", - "license": "Apache-2.0", - "dependencies": { - "@polkadot/networks": "^12.6.2", - "@polkadot/types": "11.3.1", - "@polkadot/types-codec": "11.3.1", - "@polkadot/types-create": "11.3.1", - "@polkadot/util": "^12.6.2", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/@polkadot/types-support": { - "version": "11.3.1", - "resolved": "https://registry.npmjs.org/@polkadot/types-support/-/types-support-11.3.1.tgz", - "integrity": "sha512-jTFz1GKyF7nI29yIOq4v0NiWTOf5yX4HahJNeFD8TcxoLhF+6tH/XXqrUXJEfbaTlSrRWiW1LZYlb+snctqKHA==", - "license": "Apache-2.0", - "dependencies": { - "@polkadot/util": "^12.6.2", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/@polkadot/util": { - "version": "12.6.2", - "resolved": "https://registry.npmjs.org/@polkadot/util/-/util-12.6.2.tgz", - "integrity": "sha512-l8TubR7CLEY47240uki0TQzFvtnxFIO7uI/0GoWzpYD/O62EIAMRsuY01N4DuwgKq2ZWD59WhzsLYmA5K6ksdw==", - "license": "Apache-2.0", - "peer": true, - "dependencies": { - "@polkadot/x-bigint": "12.6.2", - "@polkadot/x-global": "12.6.2", - "@polkadot/x-textdecoder": "12.6.2", - "@polkadot/x-textencoder": "12.6.2", - "@types/bn.js": "^5.1.5", - "bn.js": "^5.2.1", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/@polkadot/util-crypto": { - "version": "12.6.2", - "resolved": "https://registry.npmjs.org/@polkadot/util-crypto/-/util-crypto-12.6.2.tgz", - "integrity": "sha512-FEWI/dJ7wDMNN1WOzZAjQoIcCP/3vz3wvAp5QQm+lOrzOLj0iDmaIGIcBkz8HVm3ErfSe/uKP0KS4jgV/ib+Mg==", - "license": "Apache-2.0", - "dependencies": { - "@noble/curves": "^1.3.0", - "@noble/hashes": "^1.3.3", - "@polkadot/networks": "12.6.2", - "@polkadot/util": "12.6.2", - "@polkadot/wasm-crypto": "^7.3.2", - "@polkadot/wasm-util": "^7.3.2", - "@polkadot/x-bigint": "12.6.2", - "@polkadot/x-randomvalues": "12.6.2", - "@scure/base": "^1.1.5", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "@polkadot/util": "12.6.2" - } - }, - "node_modules/@polkadot/wasm-bridge": { - "version": "7.5.4", - "resolved": "https://registry.npmjs.org/@polkadot/wasm-bridge/-/wasm-bridge-7.5.4.tgz", - "integrity": "sha512-6xaJVvoZbnbgpQYXNw9OHVNWjXmtcoPcWh7hlwx3NpfiLkkjljj99YS+XGZQlq7ks2fVCg7FbfknkNb8PldDaA==", - "license": "Apache-2.0", - "dependencies": { - "@polkadot/wasm-util": "7.5.4", - "tslib": "^2.7.0" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "@polkadot/util": "*", - "@polkadot/x-randomvalues": "*" - } - }, - "node_modules/@polkadot/wasm-crypto": { - "version": "7.5.4", - "resolved": "https://registry.npmjs.org/@polkadot/wasm-crypto/-/wasm-crypto-7.5.4.tgz", - "integrity": "sha512-1seyClxa7Jd7kQjfnCzTTTfYhTa/KUTDUaD3DMHBk5Q4ZUN1D1unJgX+v1aUeXSPxmzocdZETPJJRZjhVOqg9g==", - "license": "Apache-2.0", - "dependencies": { - "@polkadot/wasm-bridge": "7.5.4", - "@polkadot/wasm-crypto-asmjs": "7.5.4", - "@polkadot/wasm-crypto-init": "7.5.4", - "@polkadot/wasm-crypto-wasm": "7.5.4", - "@polkadot/wasm-util": "7.5.4", - "tslib": "^2.7.0" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "@polkadot/util": "*", - "@polkadot/x-randomvalues": "*" - } - }, - "node_modules/@polkadot/wasm-crypto-asmjs": { - "version": "7.5.4", - "resolved": "https://registry.npmjs.org/@polkadot/wasm-crypto-asmjs/-/wasm-crypto-asmjs-7.5.4.tgz", - "integrity": "sha512-ZYwxQHAJ8pPt6kYk9XFmyuFuSS+yirJLonvP+DYbxOrARRUHfN4nzp4zcZNXUuaFhpbDobDSFn6gYzye6BUotA==", - "license": "Apache-2.0", - "dependencies": { - "tslib": "^2.7.0" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "@polkadot/util": "*" - } - }, - "node_modules/@polkadot/wasm-crypto-init": { - "version": "7.5.4", - "resolved": "https://registry.npmjs.org/@polkadot/wasm-crypto-init/-/wasm-crypto-init-7.5.4.tgz", - "integrity": "sha512-U6s4Eo2rHs2n1iR01vTz/sOQ7eOnRPjaCsGWhPV+ZC/20hkVzwPAhiizu/IqMEol4tO2yiSheD4D6bn0KxUJhg==", - "license": "Apache-2.0", - "dependencies": { - "@polkadot/wasm-bridge": "7.5.4", - "@polkadot/wasm-crypto-asmjs": "7.5.4", - "@polkadot/wasm-crypto-wasm": "7.5.4", - "@polkadot/wasm-util": "7.5.4", - "tslib": "^2.7.0" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "@polkadot/util": "*", - "@polkadot/x-randomvalues": "*" - } - }, - "node_modules/@polkadot/wasm-crypto-wasm": { - "version": "7.5.4", - "resolved": "https://registry.npmjs.org/@polkadot/wasm-crypto-wasm/-/wasm-crypto-wasm-7.5.4.tgz", - "integrity": "sha512-PsHgLsVTu43eprwSvUGnxybtOEuHPES6AbApcs7y5ZbM2PiDMzYbAjNul098xJK/CPtrxZ0ePDFnaQBmIJyTFw==", - "license": "Apache-2.0", - "dependencies": { - "@polkadot/wasm-util": "7.5.4", - "tslib": "^2.7.0" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "@polkadot/util": "*" - } - }, - "node_modules/@polkadot/wasm-util": { - "version": "7.5.4", - "resolved": "https://registry.npmjs.org/@polkadot/wasm-util/-/wasm-util-7.5.4.tgz", - "integrity": "sha512-hqPpfhCpRAqCIn/CYbBluhh0TXmwkJnDRjxrU9Bnqtw9nMNa97D8JuOjdd2pi0rxm+eeLQ/f1rQMp71RMM9t4w==", - "license": "Apache-2.0", - "dependencies": { - "tslib": "^2.7.0" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "@polkadot/util": "*" - } - }, - "node_modules/@polkadot/x-bigint": { - "version": "12.6.2", - "resolved": "https://registry.npmjs.org/@polkadot/x-bigint/-/x-bigint-12.6.2.tgz", - "integrity": "sha512-HSIk60uFPX4GOFZSnIF7VYJz7WZA7tpFJsne7SzxOooRwMTWEtw3fUpFy5cYYOeLh17/kHH1Y7SVcuxzVLc74Q==", - "license": "Apache-2.0", - "dependencies": { - "@polkadot/x-global": "12.6.2", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/@polkadot/x-fetch": { - "version": "12.6.2", - "resolved": "https://registry.npmjs.org/@polkadot/x-fetch/-/x-fetch-12.6.2.tgz", - "integrity": "sha512-8wM/Z9JJPWN1pzSpU7XxTI1ldj/AfC8hKioBlUahZ8gUiJaOF7K9XEFCrCDLis/A1BoOu7Ne6WMx/vsJJIbDWw==", - "license": "Apache-2.0", - "dependencies": { - "@polkadot/x-global": "12.6.2", - "node-fetch": "^3.3.2", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/@polkadot/x-global": { - "version": "12.6.2", - "resolved": "https://registry.npmjs.org/@polkadot/x-global/-/x-global-12.6.2.tgz", - "integrity": "sha512-a8d6m+PW98jmsYDtAWp88qS4dl8DyqUBsd0S+WgyfSMtpEXu6v9nXDgPZgwF5xdDvXhm+P0ZfVkVTnIGrScb5g==", - "license": "Apache-2.0", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/@polkadot/x-randomvalues": { - "version": "12.6.2", - "resolved": "https://registry.npmjs.org/@polkadot/x-randomvalues/-/x-randomvalues-12.6.2.tgz", - "integrity": "sha512-Vr8uG7rH2IcNJwtyf5ebdODMcr0XjoCpUbI91Zv6AlKVYOGKZlKLYJHIwpTaKKB+7KPWyQrk4Mlym/rS7v9feg==", - "license": "Apache-2.0", - "peer": true, - "dependencies": { - "@polkadot/x-global": "12.6.2", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "@polkadot/util": "12.6.2", - "@polkadot/wasm-util": "*" - } - }, - "node_modules/@polkadot/x-textdecoder": { - "version": "12.6.2", - "resolved": "https://registry.npmjs.org/@polkadot/x-textdecoder/-/x-textdecoder-12.6.2.tgz", - "integrity": "sha512-M1Bir7tYvNappfpFWXOJcnxUhBUFWkUFIdJSyH0zs5LmFtFdbKAeiDXxSp2Swp5ddOZdZgPac294/o2TnQKN1w==", - "license": "Apache-2.0", - "dependencies": { - "@polkadot/x-global": "12.6.2", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/@polkadot/x-textencoder": { - "version": "12.6.2", - "resolved": "https://registry.npmjs.org/@polkadot/x-textencoder/-/x-textencoder-12.6.2.tgz", - "integrity": "sha512-4N+3UVCpI489tUJ6cv3uf0PjOHvgGp9Dl+SZRLgFGt9mvxnvpW/7+XBADRMtlG4xi5gaRK7bgl5bmY6OMDsNdw==", - "license": "Apache-2.0", - "dependencies": { - "@polkadot/x-global": "12.6.2", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/@polkadot/x-ws": { - "version": "12.6.2", - "resolved": "https://registry.npmjs.org/@polkadot/x-ws/-/x-ws-12.6.2.tgz", - "integrity": "sha512-cGZWo7K5eRRQCRl2LrcyCYsrc3lRbTlixZh3AzgU8uX4wASVGRlNWi/Hf4TtHNe1ExCDmxabJzdIsABIfrr7xw==", - "license": "Apache-2.0", - "dependencies": { - "@polkadot/x-global": "12.6.2", - "tslib": "^2.6.2", - "ws": "^8.15.1" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/@scure/base": { - "version": "1.2.6", - "resolved": "https://registry.npmjs.org/@scure/base/-/base-1.2.6.tgz", - "integrity": "sha512-g/nm5FgUa//MCj1gV09zTJTaM6KBAHqLN907YVQqf7zC49+DcO4B1so4ZX07Ef10Twr6nuqYEH9GEggFXA4Fmg==", - "license": "MIT", - "funding": { - "url": "https://paulmillr.com/funding/" - } - }, - "node_modules/@scure/bip32": { - "version": "1.7.0", - "resolved": "https://registry.npmjs.org/@scure/bip32/-/bip32-1.7.0.tgz", - "integrity": "sha512-E4FFX/N3f4B80AKWp5dP6ow+flD1LQZo/w8UnLGYZO674jS6YnYeepycOOksv+vLPSpgN35wgKgy+ybfTb2SMw==", - "license": "MIT", - "dependencies": { - "@noble/curves": "~1.9.0", - "@noble/hashes": "~1.8.0", - "@scure/base": "~1.2.5" - }, - "funding": { - "url": "https://paulmillr.com/funding/" - } - }, - "node_modules/@scure/bip39": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/@scure/bip39/-/bip39-1.6.0.tgz", - "integrity": "sha512-+lF0BbLiJNwVlev4eKelw1WWLaiKXw7sSl8T6FvBlWkdX+94aGJ4o8XjUdlyhTCjd8c+B3KT3JfS8P0bLRNU6A==", - "license": "MIT", - "dependencies": { - "@noble/hashes": "~1.8.0", - "@scure/base": "~1.2.5" - }, - "funding": { - "url": "https://paulmillr.com/funding/" - } - }, - "node_modules/@substrate/connect": { - "version": "0.8.10", - "resolved": "https://registry.npmjs.org/@substrate/connect/-/connect-0.8.10.tgz", - "integrity": "sha512-DIyQ13DDlXqVFnLV+S6/JDgiGowVRRrh18kahieJxhgvzcWicw5eLc6jpfQ0moVVLBYkO7rctB5Wreldwpva8w==", - "deprecated": "versions below 1.x are no longer maintained", - "license": "GPL-3.0-only", - "optional": true, - "dependencies": { - "@substrate/connect-extension-protocol": "^2.0.0", - "@substrate/connect-known-chains": "^1.1.4", - "@substrate/light-client-extension-helpers": "^0.0.6", - "smoldot": "2.0.22" - } - }, - "node_modules/@substrate/connect-extension-protocol": { - "version": "2.2.2", - "resolved": "https://registry.npmjs.org/@substrate/connect-extension-protocol/-/connect-extension-protocol-2.2.2.tgz", - "integrity": "sha512-t66jwrXA0s5Goq82ZtjagLNd7DPGCNjHeehRlE/gcJmJ+G56C0W+2plqOMRicJ8XGR1/YFnUSEqUFiSNbjGrAA==", - "license": "GPL-3.0-only", - "optional": true - }, - "node_modules/@substrate/connect-known-chains": { - "version": "1.10.3", - "resolved": "https://registry.npmjs.org/@substrate/connect-known-chains/-/connect-known-chains-1.10.3.tgz", - "integrity": "sha512-OJEZO1Pagtb6bNE3wCikc2wrmvEU5x7GxFFLqqbz1AJYYxSlrPCGu4N2og5YTExo4IcloNMQYFRkBGue0BKZ4w==", - "license": "GPL-3.0-only", - "optional": true - }, - "node_modules/@substrate/light-client-extension-helpers": { - "version": "0.0.6", - "resolved": "https://registry.npmjs.org/@substrate/light-client-extension-helpers/-/light-client-extension-helpers-0.0.6.tgz", - "integrity": "sha512-girltEuxQ1BvkJWmc8JJlk4ZxnlGXc/wkLcNguhY+UoDEMBK0LsdtfzQKIfrIehi4QdeSBlFEFBoI4RqPmsZzA==", - "license": "MIT", - "optional": true, - "dependencies": { - "@polkadot-api/json-rpc-provider": "0.0.1", - "@polkadot-api/json-rpc-provider-proxy": "0.0.1", - "@polkadot-api/observable-client": "0.1.0", - "@polkadot-api/substrate-client": "0.0.1", - "@substrate/connect-extension-protocol": "^2.0.0", - "@substrate/connect-known-chains": "^1.1.4", - "rxjs": "^7.8.1" - }, - "peerDependencies": { - "smoldot": "2.x" - } - }, - "node_modules/@substrate/ss58-registry": { - "version": "1.51.0", - "resolved": "https://registry.npmjs.org/@substrate/ss58-registry/-/ss58-registry-1.51.0.tgz", - "integrity": "sha512-TWDurLiPxndFgKjVavCniytBIw+t4ViOi7TYp9h/D0NMmkEc9klFTo+827eyEJ0lELpqO207Ey7uGxUa+BS1jQ==", - "license": "Apache-2.0" - }, - "node_modules/@types/bn.js": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/@types/bn.js/-/bn.js-5.2.0.tgz", - "integrity": "sha512-DLbJ1BPqxvQhIGbeu8VbUC1DiAiahHtAYvA0ZEAa4P31F7IaArc8z3C3BRQdWX4mtLQuABG4yzp76ZrS02Ui1Q==", - "license": "MIT", - "dependencies": { - "@types/node": "*" - } - }, - "node_modules/@types/node": { - "version": "20.19.43", - "resolved": "https://registry.npmjs.org/@types/node/-/node-20.19.43.tgz", - "integrity": "sha512-6oYBAi5ikg4Pl+kGsoYtawUMBT2zZMCvPNF7pVLnHZfd1zf38DRiWn/gT01RYCdUqkv7Fhr+C9ot4/tb+2sVvA==", - "license": "MIT", - "dependencies": { - "undici-types": "~6.21.0" - } - }, - "node_modules/abitype": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/abitype/-/abitype-1.2.3.tgz", - "integrity": "sha512-Ofer5QUnuUdTFsBRwARMoWKOH1ND5ehwYhJ3OJ/BQO+StkwQjHw0XyVh4vDttzHB7QOFhPHa/o413PJ82gU/Tg==", - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/wevm" - }, - "peerDependencies": { - "typescript": ">=5.0.4", - "zod": "^3.22.0 || ^4.0.0" - }, - "peerDependenciesMeta": { - "typescript": { - "optional": true - }, - "zod": { - "optional": true - } - } - }, - "node_modules/bn.js": { - "version": "5.2.4", - "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-5.2.4.tgz", - "integrity": "sha512-QL7sb18rJ1PbdsKsqPA0guxL563vIMwRHgzNrW/uzQuRGN1Cjqd/wonUBAVqHox9KwzHA6vCbM0lXx3k4iQMow==", - "license": "MIT" - }, - "node_modules/data-uri-to-buffer": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", - "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==", - "license": "MIT", - "engines": { - "node": ">= 12" - } - }, - "node_modules/debug": { - "version": "4.4.3", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", - "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", - "license": "MIT", - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/eventemitter3": { - "version": "5.0.4", - "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-5.0.4.tgz", - "integrity": "sha512-mlsTRyGaPBjPedk6Bvw+aqbsXDtoAyAzm5MO7JgU+yVRyMQ5O8bD4Kcci7BS85f93veegeCPkL8R4GLClnjLFw==", - "license": "MIT" - }, - "node_modules/fetch-blob": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", - "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/jimmywarting" - }, - { - "type": "paypal", - "url": "https://paypal.me/jimmywarting" - } - ], - "license": "MIT", - "dependencies": { - "node-domexception": "^1.0.0", - "web-streams-polyfill": "^3.0.3" - }, - "engines": { - "node": "^12.20 || >= 14.13" - } - }, - "node_modules/formdata-polyfill": { - "version": "4.0.10", - "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", - "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", - "license": "MIT", - "dependencies": { - "fetch-blob": "^3.1.2" - }, - "engines": { - "node": ">=12.20.0" - } - }, - "node_modules/isows": { - "version": "1.0.7", - "resolved": "https://registry.npmjs.org/isows/-/isows-1.0.7.tgz", - "integrity": "sha512-I1fSfDCZL5P0v33sVqeTDSpcstAg/N+wF5HS033mogOVIp4B+oHC7oOCsA3axAbBSGTJ8QubbNmnIRN/h8U7hg==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/wevm" - } - ], - "license": "MIT", - "peerDependencies": { - "ws": "*" - } - }, - "node_modules/json-stringify-safe": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", - "integrity": "sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==", - "license": "ISC" - }, - "node_modules/mock-socket": { - "version": "9.3.1", - "resolved": "https://registry.npmjs.org/mock-socket/-/mock-socket-9.3.1.tgz", - "integrity": "sha512-qxBgB7Qa2sEQgHFjj0dSigq7fX4k6Saisd5Nelwp2q8mlbAFh5dHV9JTTlF8viYJLSSWgMCZFUom8PJcMNBoJw==", - "license": "MIT", - "engines": { - "node": ">= 8" - } - }, - "node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "license": "MIT" - }, - "node_modules/nock": { - "version": "13.5.6", - "resolved": "https://registry.npmjs.org/nock/-/nock-13.5.6.tgz", - "integrity": "sha512-o2zOYiCpzRqSzPj0Zt/dQ/DqZeYoaQ7TUonc/xUPjCGl9WeHpNbxgVvOquXYAaJzI0M9BXV3HTzG0p8IUAbBTQ==", - "license": "MIT", - "dependencies": { - "debug": "^4.1.0", - "json-stringify-safe": "^5.0.1", - "propagate": "^2.0.0" - }, - "engines": { - "node": ">= 10.13" - } - }, - "node_modules/node-domexception": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", - "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", - "deprecated": "Use your platform's native DOMException instead", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/jimmywarting" - }, - { - "type": "github", - "url": "https://paypal.me/jimmywarting" - } - ], - "license": "MIT", - "engines": { - "node": ">=10.5.0" - } - }, - "node_modules/node-fetch": { - "version": "3.3.2", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", - "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", - "license": "MIT", - "dependencies": { - "data-uri-to-buffer": "^4.0.0", - "fetch-blob": "^3.1.4", - "formdata-polyfill": "^4.0.10" - }, - "engines": { - "node": "^12.20.0 || ^14.13.1 || >=16.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/node-fetch" - } - }, - "node_modules/ox": { - "version": "0.14.30", - "resolved": "https://registry.npmjs.org/ox/-/ox-0.14.30.tgz", - "integrity": "sha512-LI11uu+8iiM1B3CLckgd++YF1a0A2k5wDoM9ZeQMiL21BOzQs6L//BLS6hb1HSEKCyycdDIQLsVQx9MjpcC0hA==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/wevm" - } - ], - "license": "MIT", - "dependencies": { - "@adraffy/ens-normalize": "^1.11.0", - "@noble/ciphers": "^1.3.0", - "@noble/curves": "1.9.1", - "@noble/hashes": "^1.8.0", - "@scure/bip32": "^1.7.0", - "@scure/bip39": "^1.6.0", - "abitype": "^1.2.3", - "eventemitter3": "5.0.1" - }, - "peerDependencies": { - "typescript": ">=5.4.0" - }, - "peerDependenciesMeta": { - "typescript": { - "optional": true - } - } - }, - "node_modules/ox/node_modules/@noble/curves": { - "version": "1.9.1", - "resolved": "https://registry.npmjs.org/@noble/curves/-/curves-1.9.1.tgz", - "integrity": "sha512-k11yZxZg+t+gWvBbIswW0yoJlu8cHOC7dhunwOzoWH/mXGBiYyR4YY6hAEK/3EUs4UpB8la1RfdRpeGsFHkWsA==", - "license": "MIT", - "dependencies": { - "@noble/hashes": "1.8.0" - }, - "engines": { - "node": "^14.21.3 || >=16" - }, - "funding": { - "url": "https://paulmillr.com/funding/" - } - }, - "node_modules/ox/node_modules/eventemitter3": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-5.0.1.tgz", - "integrity": "sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA==", - "license": "MIT" - }, - "node_modules/propagate": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/propagate/-/propagate-2.0.1.tgz", - "integrity": "sha512-vGrhOavPSTz4QVNuBNdcNXePNdNMaO1xj9yBeH1ScQPjk/rhg9sSlCXPhMkFuaNNW/syTvYqsnbIJxMBfRbbag==", - "license": "MIT", - "engines": { - "node": ">= 8" - } - }, - "node_modules/rxjs": { - "version": "7.8.2", - "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-7.8.2.tgz", - "integrity": "sha512-dhKf903U/PQZY6boNNtAGdWbG85WAbjT/1xYoZIC7FAY0yWapOBQVsVrDl58W86//e1VpMNBtRV4MaXfdMySFA==", - "license": "Apache-2.0", - "peer": true, - "dependencies": { - "tslib": "^2.1.0" - } - }, - "node_modules/scale-ts": { - "version": "1.6.1", - "resolved": "https://registry.npmjs.org/scale-ts/-/scale-ts-1.6.1.tgz", - "integrity": "sha512-PBMc2AWc6wSEqJYBDPcyCLUj9/tMKnLX70jLOSndMtcUoLQucP/DM0vnQo1wJAYjTrQiq8iG9rD0q6wFzgjH7g==", - "license": "MIT", - "optional": true - }, - "node_modules/smoldot": { - "version": "2.0.22", - "resolved": "https://registry.npmjs.org/smoldot/-/smoldot-2.0.22.tgz", - "integrity": "sha512-B50vRgTY6v3baYH6uCgL15tfaag5tcS2o/P5q1OiXcKGv1axZDfz2dzzMuIkVpyMR2ug11F6EAtQlmYBQd292g==", - "license": "GPL-3.0-or-later WITH Classpath-exception-2.0", - "optional": true, - "peer": true, - "dependencies": { - "ws": "^8.8.1" - } - }, - "node_modules/tslib": { - "version": "2.8.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", - "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", - "license": "0BSD" - }, - "node_modules/typescript": { - "version": "5.9.3", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", - "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", - "devOptional": true, - "license": "Apache-2.0", - "peer": true, - "bin": { - "tsc": "bin/tsc", - "tsserver": "bin/tsserver" - }, - "engines": { - "node": ">=14.17" - } - }, - "node_modules/undici-types": { - "version": "6.21.0", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", - "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", - "license": "MIT" - }, - "node_modules/viem": { - "version": "2.54.6", - "resolved": "https://registry.npmjs.org/viem/-/viem-2.54.6.tgz", - "integrity": "sha512-OfybECKJYVmhiNqz+SHhed+O2h6niQ+0Wjg9J0b4bV+/QrvLgjxhfKO7hZqsuK1YtZ/0BErBKy708Zp+cU5T0Q==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/wevm" - } - ], - "license": "MIT", - "dependencies": { - "@noble/curves": "1.9.1", - "@noble/hashes": "1.8.0", - "@scure/bip32": "1.7.0", - "@scure/bip39": "1.6.0", - "abitype": "1.2.3", - "isows": "1.0.7", - "ox": "0.14.30", - "ws": "8.21.0" - }, - "peerDependencies": { - "typescript": ">=5.0.4" - }, - "peerDependenciesMeta": { - "typescript": { - "optional": true - } - } - }, - "node_modules/viem/node_modules/@noble/curves": { - "version": "1.9.1", - "resolved": "https://registry.npmjs.org/@noble/curves/-/curves-1.9.1.tgz", - "integrity": "sha512-k11yZxZg+t+gWvBbIswW0yoJlu8cHOC7dhunwOzoWH/mXGBiYyR4YY6hAEK/3EUs4UpB8la1RfdRpeGsFHkWsA==", - "license": "MIT", - "dependencies": { - "@noble/hashes": "1.8.0" - }, - "engines": { - "node": "^14.21.3 || >=16" - }, - "funding": { - "url": "https://paulmillr.com/funding/" - } - }, - "node_modules/web-streams-polyfill": { - "version": "3.3.3", - "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz", - "integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==", - "license": "MIT", - "engines": { - "node": ">= 8" - } - }, - "node_modules/ws": { - "version": "8.21.0", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.0.tgz", - "integrity": "sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g==", - "license": "MIT", - "peer": true, - "engines": { - "node": ">=10.0.0" - }, - "peerDependencies": { - "bufferutil": "^4.0.1", - "utf-8-validate": ">=5.0.2" - }, - "peerDependenciesMeta": { - "bufferutil": { - "optional": true - }, - "utf-8-validate": { - "optional": true - } - } - } - } -} diff --git a/monitor/package.json b/monitor/package.json deleted file mode 100644 index c9575d5be..000000000 --- a/monitor/package.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "name": "pen-migration-monitor", - "version": "0.1.0", - "private": true, - "description": "Independent invariant monitor for the PEN migration: verifies conservation between Pendulum burns and Base releases, watches liveness, and can auto-pause the vault", - "type": "module", - "scripts": { - "build": "tsc", - "start": "node dist/main.js", - "typecheck": "tsc --noEmit", - "test": "tsc && node --test dist/*.test.js" - }, - "dependencies": { - "@polkadot/api": "^11.3.1", - "viem": "^2.21.0" - }, - "devDependencies": { - "@types/node": "^20.14.0", - "typescript": "^5.5.0" - } -} diff --git a/monitor/src/checks.test.ts b/monitor/src/checks.test.ts deleted file mode 100644 index 24a33fbd5..000000000 --- a/monitor/src/checks.test.ts +++ /dev/null @@ -1,212 +0,0 @@ -/** - * Unit tests for the monitor's conservation/liveness predicates. - * - * Run with `npm test` (compiles, then `node --test`). Among others, these lock - * in that the M2b check ignores a token surplus (donation / vault recipient) - * and only fires on a real deficit. - */ - -import assert from "node:assert/strict"; -import { test } from "node:test"; -import { - approvalMismatch, - blockRanges, - chunks, - describeCall, - describeTimelockOperation, - disagreeingAttestors, - isStale, - provablyUnsourced, - releaseMismatch, - releasedExceedsMigrated, - riskyCalls, - shouldAwaitSource, - slackText, - toCallNode, - vaultConservationDeficit, -} from "./checks.js"; - -const CF = 1_000_000n; // 12 -> 18 decimals -const SUPPLY = 150_000_000n * 10n ** 18n; - -test("M2a: released within migrated does not fire", () => { - assert.equal(releasedExceedsMigrated(5n * CF, 5n, CF), false); - assert.equal(releasedExceedsMigrated(5n * CF, 10n, CF), false); // migrated lags releases via finality: fine -}); - -test("M2a: released exceeding migrated fires", () => { - assert.equal(releasedExceedsMigrated(6n * CF, 5n, CF), true); -}); - -test("M2a: an acknowledged loss is tolerated, and nothing beyond it", () => { - assert.equal(releasedExceedsMigrated(6n * CF, 5n, CF, CF), false); - assert.equal(releasedExceedsMigrated(6n * CF + 1n, 5n, CF, CF), true); -}); - -test("M2b: exact conservation is not a deficit", () => { - // balance + released + swept == totalSupply - assert.equal(vaultConservationDeficit(SUPPLY - 30n, 20n, 10n, SUPPLY), false); -}); - -test("M2b: a surplus (donation / vault-recipient self-transfer) must NOT fire", () => { - // Someone transferred dust into the vault: balance is higher than the - // identity predicts, so the sum EXCEEDS totalSupply. This is harmless and - // must never trip the check (previously a strict `!=` paused the vault here). - const donation = 5n * 10n ** 18n; - assert.equal(vaultConservationDeficit(SUPPLY - 30n + donation, 20n, 10n, SUPPLY), false); - // Even a 1-wei donation must not fire. - assert.equal(vaultConservationDeficit(SUPPLY + 1n, 0n, 0n, SUPPLY), false); -}); - -test("M2b: a real deficit (tokens vanished without a counter update) fires", () => { - // balance too low for the recorded released+swept: genuine loss. - assert.equal(vaultConservationDeficit(SUPPLY - 100n, 20n, 10n, SUPPLY), true); -}); - -test("M4: staleness respects the grace period", () => { - const grace = 1800; // seconds - const now = 10_000_000; - assert.equal(isStale(now - 1000, now, grace), false); // 1s old - assert.equal(isStale(now - 1800 * 1000, now, grace), false); // exactly at grace: not yet stale - assert.equal(isStale(now - 1801 * 1000, now, grace), true); // just past grace -}); - -test("an unseen future source nonce gets bounded RPC-lag grace, never an old nonce", () => { - const now = 10_000_000; - assert.equal(shouldAwaitSource(8n, 8n, now, now, 600), true); - assert.equal(shouldAwaitSource(9n, 8n, now - 599_000, now, 600), true); - assert.equal(shouldAwaitSource(9n, 8n, now - 601_000, now, 600), false); - assert.equal(shouldAwaitSource(7n, 8n, now, now, 600), false); -}); - -test("a fabricated event is proven unsourced once the source view passes its first observation", () => { - const firstSeen = 10_000_000; - const margin = 120_000; - const now = firstSeen + 600_000; - // Source finalized view has moved past the moment the event was first - // observed (plus skew margin): the burn would already have been ingested — - // fabricated, pause without waiting out the grace. - assert.equal(provablyUnsourced(firstSeen, firstSeen + margin, now, margin), true); - assert.equal(provablyUnsourced(firstSeen, firstSeen + margin + 1, now, margin), true); - // A lagging source view trails the observation: NOT proof, so the RPC-lag - // grace applies and node lag can never fast-path a false pause. - assert.equal(provablyUnsourced(firstSeen, firstSeen + margin - 1, now, margin), false); - assert.equal(provablyUnsourced(firstSeen, firstSeen - 300_000, now, margin), false); -}); - -test("a future-dated source view proves nothing", () => { - // A collator clock ahead of real time must not let the view "pass" an - // observation it has not genuinely caught up with. - const firstSeen = 10_000_000; - const margin = 120_000; - const now = firstSeen + 10_000; - assert.equal(provablyUnsourced(firstSeen, now + margin + 1, now, margin), false); - assert.equal(provablyUnsourced(firstSeen, now + margin, now, margin), true); -}); - -test("the proof anchor is the observation time, not a Base block timestamp that may trail real time", () => { - // An event observed now whose Base block timestamp trails real time by a - // sequencer catch-up: the view being past that OLD timestamp is not proof - // (the predicate never sees it), only being past the observation is. - const firstSeen = 10_000_000; - const margin = 120_000; - assert.equal(provablyUnsourced(firstSeen, firstSeen + 60_000, firstSeen + 60_000, margin), false); -}); - -test("M2 tuple matching rejects missing, wrong-recipient and wrong-amount approvals", () => { - const expected = { nonce: 7n, recipient: "0x1111111111111111111111111111111111111111", palletAmount: 100n }; - assert.match(approvalMismatch(undefined, expected)!, /no finalized/); - assert.match( - approvalMismatch(expected, { ...expected, recipient: "0x2222222222222222222222222222222222222222" })!, - /recipient/, - ); - assert.match(approvalMismatch(expected, { ...expected, palletAmount: 99n })!, /pallet amount/); - assert.equal(approvalMismatch(expected, { ...expected, recipient: expected.recipient.toUpperCase() }), undefined); -}); - -test("approvals against the burn count once per attestor and only while they disagree", () => { - const burn = { nonce: 7n, recipient: "0x1111111111111111111111111111111111111111", palletAmount: 100n }; - const [a, b] = ["0xA1a1a1a1a1A1a1a1a1a1a1A1a1a1a1a1a1a1A1a1", "0xa2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2"]; - const approvals = [ - { ...burn, palletAmount: 99n, attestor: a }, - { ...burn, palletAmount: 98n, attestor: a.toLowerCase() }, - { ...burn, attestor: b }, // agrees - { ...burn, nonce: 8n, palletAmount: 99n, attestor: b }, // another nonce - ]; - assert.deepEqual(disagreeingAttestors(burn, 7n, approvals), [a.toLowerCase()]); - assert.deepEqual(disagreeingAttestors(undefined, 7n, approvals), [a.toLowerCase(), b]); // no burn known - assert.deepEqual(disagreeingAttestors({ ...burn, palletAmount: 99n }, 7n, approvals), [a.toLowerCase(), b]); -}); - -test("M2 release matching includes the one-and-only decimal conversion", () => { - const expected = { nonce: 7n, recipient: "0x1111111111111111111111111111111111111111", palletAmount: 100n }; - assert.equal(releaseMismatch(expected, { ...expected, tokenAmount: 100n * CF }, CF), undefined); - assert.match(releaseMismatch(expected, { ...expected, tokenAmount: 100n * CF + 1n }, CF)!, /token amount/); -}); - -test("RPC work is split into inclusive block ranges and bounded batches", () => { - assert.deepEqual(blockRanges(1n, 10n, 4n), [[1n, 4n], [5n, 8n], [9n, 10n]]); - assert.deepEqual(blockRanges(11n, 10n, 4n), []); - assert.deepEqual(chunks([1, 2, 3, 4, 5], 2), [[1, 2], [3, 4], [5]]); -}); - -test("a scheduled governance operation is described with names, targets and the veto deadline", () => { - const vault = "0x2222222222222222222222222222222222222222"; - const labels = { [vault]: "vault" }; - const functions = { "0x0e18b681": "acceptAdmin()", "0x212bf316": "setCaps(uint256,uint256)" }; - const calls = [ - { index: 1n, target: vault.toUpperCase().replace("0X", "0x"), value: 0n, data: "0x212bf316" + "00".repeat(64) }, - { index: 0n, target: vault, value: 0n, data: "0x0e18b681" }, - ]; - const now = Date.UTC(2026, 9, 21, 10, 0, 0); - const text = describeTimelockOperation("0xabc", calls, BigInt((now + (48 * 60 + 5) * 60_000) / 1000), now, labels, functions); - assert.match(text, /^operation 0xabc: #0 vault\.acceptAdmin\(\) data=0x0e18b681; #1 vault\.setCaps\(uint256,uint256\)/); - assert.match(text, /executable by anyone from 2026-10-23T10:05:00\.000Z \(in 48h 5m\)/); - assert.match(text, /canceller must cancel it before then$/); -}); - -test("a scheduled operation with unknown parts or a settled timestamp still pages readably", () => { - const other = "0x3333333333333333333333333333333333333333"; - const call = { index: 0n, target: other, value: 5n, data: "0xdeadbeef" }; - assert.match(describeTimelockOperation("0x1", [call], 0n, 0, {}, {}), new RegExp(`#0 ${other}\\.unknown function 0xdeadbeef value=5 .*cancelled\\)$`)); - assert.match(describeTimelockOperation("0x1", [call], 1n, 0, {}, {}), /already executed$/); - assert.match(describeTimelockOperation("0x1", [call], 100n, 200_000, {}, {}), /\(ready now\)/); -}); - -test("a scheduled operation pages whatever its delay or its event carries", () => { - const call = { index: 0n, target: "0x3333333333333333333333333333333333333333", value: 0n, data: "0x8456cb59" }; - assert.match(describeTimelockOperation("0x1", [call], 10n ** 20n, 0, {}, {}), /from timestamp 100000000000000000000 \(in /); - const undecoded = { index: undefined, target: undefined, value: undefined, data: undefined } as unknown as typeof call; - assert.match(describeTimelockOperation("0x1", [undecoded, call], 5n, 0, {}, {}), /^operation 0x1: #undefined undefined\.unknown function undefined/); -}); - -test("a page is Slack text with its control characters escaped and a long detail cut short", () => { - assert.equal( - slackText("pen-monitor", "critical", "SUBJECT", "to & "), - "[CRITICAL] pen-monitor: SUBJECT\nto <!channel> & <https://x.example|click>", - ); - const text = slackText("pen-monitor", "warning", "SUBJECT", "a".repeat(10_000)); - assert.ok(text.startsWith(`[WARNING] pen-monitor: SUBJECT\n${"a".repeat(3_000)}… (cut short`), text.slice(0, 80)); - assert.ok(text.length < 3_200, String(text.length)); -}); - -test("a governance call tree is walked through batches and minting calls are flagged", () => { - const setBalance = { section: "tokens", method: "setBalance", args: ["who", "Native", 4_000_000n] }; - const batch = toCallNode({ section: "utility", method: "batch", args: [[setBalance, setBalance]] }); - assert.equal(describeCall(batch), "utility.batch(tokens.setBalance, tokens.setBalance)"); - assert.deepEqual(riskyCalls(batch), ["tokens.setBalance"]); - const nested = toCallNode({ section: "utility", method: "dispatchAs", args: ["Root", { section: "tokenMigration", method: "setPaused", args: [false] }] }); - assert.deepEqual(riskyCalls(nested), ["tokenMigration.setPaused"]); - assert.deepEqual(riskyCalls(toCallNode({ section: "system", method: "remark", args: ["0x00"] })), []); -}); - -test("the call-tree walk is bounded in size and depth", () => { - const remark = { section: "system", method: "remark", args: [] }; - const batch = (calls: unknown[]) => ({ section: "utility", method: "batch", args: [calls] }); - assert.equal(toCallNode(batch(Array(255).fill(remark))).calls.length, 255); - assert.throws(() => toCallNode(batch(Array(256).fill(remark))), /more than 256 calls/); - let nested: { section: string; method: string; args: unknown[] } = remark; - for (let depth = 1; depth < 16; depth++) nested = batch([nested]); - assert.equal(describeCall(toCallNode(nested)).split("(").length, 16); - assert.throws(() => toCallNode(batch([nested])), /more than 16 levels/); -}); diff --git a/monitor/src/checks.ts b/monitor/src/checks.ts deleted file mode 100644 index 17c01fa95..000000000 --- a/monitor/src/checks.ts +++ /dev/null @@ -1,317 +0,0 @@ -/** - * Pure conservation/liveness predicates for the invariant monitor. - * - * These are isolated from the chain-client plumbing in `main.ts` so they can be - * unit-tested exhaustively (see checks.test.ts). - */ - -/** - * (M2a) Nothing may leave the vault that was not burned on Pendulum. - * - * `totalReleased` (Base) must never exceed the burned total converted to token - * units. It lags `totalMigrated` only via finality delay, never the other way, - * because a release requires attestations of an already-finalized burn — so any - * excess is the strongest possible signal of attestor compromise. - * - * `acknowledgedExcess` is a loss operators have quantified and accepted: it - * stays in the cumulative totals for good, and without it the check would - * re-pause the vault after every recovery unpause. - */ -export function releasedExceedsMigrated( - totalReleased: bigint, - totalMigrated: bigint, - conversionFactor: bigint, - acknowledgedExcess = 0n, -): boolean { - return totalReleased > totalMigrated * conversionFactor + acknowledgedExcess; -} - -/** - * (M2b) Vault-internal conservation. - * - * Under all legitimate contract logic `balance + released + swept` is *exactly* - * `totalSupply`: every path that removes tokens bumps a counter in lockstep - * (`_release`→`totalReleased`, `sweepRemainder`→`totalSwept`). Real loss can - * therefore only ever show up as a DEFICIT. - * - * A SURPLUS (sum > totalSupply) is harmless and must NOT trip the check: it can - * be produced permissionlessly by anyone transferring PEN into the vault (a - * plain donation, or a migration whose recipient is the vault address). Treating - * a surplus as a violation would let a single dust transfer pause the vault on - * every poll — unrecoverable until the window-close sweep, and it would train - * on-call to ignore the highest-severity alert. Hence the strict `<`. - */ -export function vaultConservationDeficit( - vaultBalance: bigint, - totalReleased: bigint, - totalSwept: bigint, - totalSupply: bigint, -): boolean { - return vaultBalance + totalReleased + totalSwept < totalSupply; -} - -/** - * (M4) Liveness: a migration the monitor has known about for longer than the - * grace period, and which is still not consumed on Base, is stalled. - */ -export function isStale(firstSeenMs: number, nowMs: number, graceSeconds: number): boolean { - return nowMs - firstSeenMs > graceSeconds * 1000; -} - -/** - * A Base event whose nonce is at or beyond the source node's next nonce may be - * legitimate data observed through a temporarily faster RPC. Give that source - * view a bounded window to catch up. Older missing nonces cannot be explained - * by node lag and must fail immediately. - */ -export function shouldAwaitSource( - eventNonce: bigint, - nextExpectedNonce: bigint, - firstSeenMs: number, - nowMs: number, - graceSeconds: number, -): boolean { - return eventNonce >= nextExpectedNonce && !isStale(firstSeenMs, nowMs, graceSeconds); -} - -/** - * A Base event is PROVABLY without a Pendulum source once the monitor's - * finalized source view has passed the wall-clock moment the monitor FIRST - * OBSERVED the event (plus a clock-skew margin) and the nonce still does not - * exist. - * - * Why this is sound: a legitimate release's burn is relay-FINALIZED strictly - * before any attestor submits an approval, so the burn block's timestamp - * precedes the real time at which the monitor could first observe that - * approval on Base. Substrate timestamps are strictly monotone, so a finalized - * head whose timestamp is past that observation time by more than any collator - * clock drift already contains every block the burn could live in. If the - * nonce is still unknown then, no amount of further waiting can reveal it — - * the event is fabricated, and the auto-pause must not sit out the RPC-lag - * grace period. - * - * The anchor is the monitor's own clock, deliberately NOT the Base block - * timestamp: OP-stack L2 block timestamps trail real time by the length of any - * sequencer outage while it catches up, and an artificially old event - * timestamp would let a merely-lagging source "prove" a legitimate event - * fabricated. A lagging source never satisfies this predicate - * (its head timestamp trails the observation), and a future-dated source view - * (a collator clock ahead of real time) proves nothing either and is excluded, - * so neither can turn node lag into a false pause. - */ -export function provablyUnsourced( - firstSeenMs: number, - sourceFinalizedTsMs: number, - nowMs: number, - skewMarginMs: number, -): boolean { - if (sourceFinalizedTsMs > nowMs + skewMarginMs) return false; - return sourceFinalizedTsMs >= firstSeenMs + skewMarginMs; -} - -export interface MigrationTuple { - nonce: bigint; - recipient: string; - palletAmount: bigint; -} - -function sameAddress(left: string, right: string): boolean { - return left.toLowerCase() === right.toLowerCase(); -} - -export function approvalMismatch(expected: MigrationTuple | undefined, actual: MigrationTuple): string | undefined { - if (!expected) return `nonce ${actual.nonce} has no finalized Pendulum migration`; - if (!sameAddress(expected.recipient, actual.recipient)) { - return `nonce ${actual.nonce} recipient ${actual.recipient} != finalized ${expected.recipient}`; - } - if (expected.palletAmount !== actual.palletAmount) { - return `nonce ${actual.nonce} pallet amount ${actual.palletAmount} != finalized ${expected.palletAmount}`; - } - return undefined; -} - -/** - * The distinct attestors whose recorded approvals of `nonce` disagree with its - * finalized burn, or with its absence. Approvals that the source view has - * since confirmed no longer count. - */ -export function disagreeingAttestors( - expected: MigrationTuple | undefined, - nonce: bigint, - approvals: Array, -): string[] { - const attestors = approvals - .filter((approval) => approval.nonce === nonce && approvalMismatch(expected, approval)) - .map((approval) => approval.attestor.toLowerCase()); - return [...new Set(attestors)]; -} - -export function releaseMismatch( - expected: MigrationTuple | undefined, - actual: MigrationTuple & { tokenAmount: bigint }, - conversionFactor: bigint, -): string | undefined { - const tupleMismatch = approvalMismatch(expected, actual); - if (tupleMismatch) return tupleMismatch; - const expectedTokenAmount = actual.palletAmount * conversionFactor; - if (actual.tokenAmount !== expectedTokenAmount) { - return `nonce ${actual.nonce} token amount ${actual.tokenAmount} != converted ${expectedTokenAmount}`; - } - return undefined; -} - -export function blockRanges(fromBlock: bigint, toBlock: bigint, maxRange: bigint): Array<[bigint, bigint]> { - if (maxRange <= 0n) throw new Error(`invalid maxRange ${maxRange}`); - const result: Array<[bigint, bigint]> = []; - for (let start = fromBlock; start <= toBlock; start += maxRange) { - const end = start + maxRange - 1n; - result.push([start, end > toBlock ? toBlock : end]); - } - return result; -} - -export function chunks(items: T[], size: number): T[][] { - if (!Number.isSafeInteger(size) || size <= 0) throw new Error(`invalid chunk size ${size}`); - const result: T[][] = []; - for (let start = 0; start < items.length; start += size) result.push(items.slice(start, start + size)); - return result; -} - -/** One call of a governance Timelock operation (one `CallScheduled` event). */ -export interface ScheduledCall { - index: bigint; - target: string; - value: bigint; - data: string; -} - -/** - * The page for one operation the governance Timelock scheduled. Every such - * operation pages: once its delay has passed, anyone may execute it, so the - * canceller has exactly the delay to review and veto it. - * - * `readyAtSeconds` is the Timelock's `getTimestamp(id)`, whose sentinels are - * 0 (unknown or cancelled) and 1 (executed). `labels` maps lowercased - * addresses and `functions` maps selectors to readable names. Never throws, - * whatever the event carries: one operation must not hide the next. - */ -export function describeTimelockOperation( - id: string, - calls: ScheduledCall[], - readyAtSeconds: bigint, - nowMs: number, - labels: Record, - functions: Record, -): string { - const callText = [...calls] - .sort((a, b) => (a.index < b.index ? -1 : a.index > b.index ? 1 : 0)) - .map((call) => { - const target = labels[String(call.target).toLowerCase()] ?? call.target; - const selector = String(call.data).slice(0, 10).toLowerCase(); - const name = functions[selector] ?? `unknown function ${selector}`; - return `#${call.index} ${target}.${name}${call.value ? ` value=${call.value}` : ""} data=${call.data}`; - }) - .join("; "); - let status: string; - if (readyAtSeconds === 0n) status = "no longer pending (cancelled)"; - else if (readyAtSeconds === 1n) status = "already executed"; - else { - const readyMs = Number(readyAtSeconds) * 1000; - const leftMinutes = Math.floor((readyMs - nowMs) / 60_000); - // A Date holds at most 8.64e15 ms; a later time is shown in seconds. - const readyText = readyMs <= 8.64e15 ? new Date(readyMs).toISOString() : `timestamp ${readyAtSeconds}`; - status = - `executable by anyone from ${readyText}` + - (leftMinutes > 0 ? ` (in ${Math.floor(leftMinutes / 60)}h ${leftMinutes % 60}m)` : " (ready now)") + - "; if it is not intended, the canceller must cancel it before then"; - } - return `operation ${id}: ${callText} — ${status}`; -} - -export type Severity = "critical" | "warning"; - -/** Longest page detail sent; the monitor's log line keeps the full text. */ -const MAX_PAGE_DETAIL = 3_000; - -/** - * A page as the `text` of a Slack incoming-webhook message: severity, subject - * and detail, with Slack's control characters `&`, `<` and `>` escaped, so - * that no detail can mention anyone or hide a link. - */ -export function slackText(service: string, severity: Severity, subject: string, detail: string): string { - const body = detail.length > MAX_PAGE_DETAIL - ? `${detail.slice(0, MAX_PAGE_DETAIL)}… (cut short; the monitor log has the full text)` - : detail; - const escapes: Record = { "&": "&", "<": "<", ">": ">" }; - return `[${severity.toUpperCase()}] ${service}: ${subject}\n${body}`.replace(/[&<>]/g, (char) => escapes[char]); -} - -/** A decoded Pendulum call: `section.method` plus any calls nested in its arguments. */ -export interface CallNode { - name: string; - calls: CallNode[]; -} - -/** The duck type of a polkadot.js `Call`, enough to walk nested calls. */ -interface CallLike { - section: string; - method: string; - args: unknown[]; -} - -function isCallLike(value: unknown): value is CallLike { - const call = value as CallLike | null; - return typeof call?.section === "string" && typeof call.method === "string" && Array.isArray(call.args); -} - -/** Bounds on a walked call tree. A governance call is untrusted data; - * legitimate ones stay far below both, and a bigger tree is unreadable. */ -const MAX_CALL_NODES = 256; -const MAX_CALL_DEPTH = 16; - -/** Walks a decoded call, descending into arguments that are calls or lists of - * calls (batch, dispatch_as, ...). Throws past the node or depth bound. */ -export function toCallNode(call: CallLike): CallNode { - let nodes = 0; - const walk = (node: CallLike, depth: number): CallNode => { - if (++nodes > MAX_CALL_NODES) throw new Error(`call tree has more than ${MAX_CALL_NODES} calls`); - if (depth > MAX_CALL_DEPTH) throw new Error(`call tree is nested more than ${MAX_CALL_DEPTH} levels deep`); - const calls: CallNode[] = []; - for (const arg of node.args) { - if (isCallLike(arg)) calls.push(walk(arg, depth + 1)); - else if (Array.isArray(arg)) for (const item of arg) if (isCallLike(item)) calls.push(walk(item, depth + 1)); - } - return { name: `${node.section}.${node.method}`, calls }; - }; - return walk(call, 1); -} - -/** Calls that mint, move or unlock balances outside normal transfers, or change - * the runtime or the migration pallet. Minted PEN that is burned through the - * migration drains the fixed-supply vault on Base, and only Pendulum governance - * can stop such a call before it is enacted. */ -const RISKY_CALLS = new Set([ - "balances.forceSetBalance", "balances.setBalance", "balances.setBalanceDeprecated", "balances.forceTransfer", - "balances.forceUnreserve", "balances.forceAdjustTotalIssuance", "tokens.setBalance", "tokens.forceTransfer", - "currencies.updateBalance", "vesting.forceVestedTransfer", "vestingManager.removeVestingSchedule", - "system.setCode", "system.setCodeWithoutChecks", "system.setStorage", "system.killStorage", "system.killPrefix", - "system.authorizeUpgrade", "system.authorizeUpgradeWithoutChecks", "parachainSystem.authorizeUpgrade", - "parachainSystem.enactAuthorizedUpgrade", -]); -const RISKY_SECTIONS = new Set(["sudo", "tokenMigration"]); - -/** The risky calls anywhere in a call tree, deduplicated. */ -export function riskyCalls(call: CallNode): string[] { - const found = new Set(); - const walk = (node: CallNode) => { - if (RISKY_CALLS.has(node.name) || RISKY_SECTIONS.has(node.name.split(".")[0])) found.add(node.name); - node.calls.forEach(walk); - }; - walk(call); - return [...found]; -} - -/** Compact one-line form: `utility.batch(tokens.setBalance, tokens.setBalance)`. */ -export function describeCall(call: CallNode): string { - return call.calls.length ? `${call.name}(${call.calls.map(describeCall).join(", ")})` : call.name; -} diff --git a/monitor/src/main.test.ts b/monitor/src/main.test.ts deleted file mode 100644 index 2ef2ed682..000000000 --- a/monitor/src/main.test.ts +++ /dev/null @@ -1,2310 +0,0 @@ -/** - * Call-site tests for the poll loop in main.ts. Each test loads a fresh copy - * of the module (it reads its configuration at import) against stand-in - * chains: a JSON-RPC server for Base and a plain object shaped like the - * polkadot.js API for Pendulum. The same server receives every page. - */ - -import assert from "node:assert/strict"; -import { mkdtempSync, readFileSync } from "node:fs"; -import { createServer, type IncomingMessage } from "node:http"; -import type { AddressInfo } from "node:net"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; -import { test } from "node:test"; -import type { ApiPromise } from "@polkadot/api"; -import { - type AbiEvent, - type AbiFunction, - decodeFunctionData, - encodeAbiParameters, - encodeEventTopics, - type Hex, - keccak256, - parseAbi, - toEventSelector, - toFunctionSelector, - toHex, -} from "viem"; -import { privateKeyToAccount } from "viem/accounts"; - -(globalThis as { penMonitorUnderTest?: boolean }).penMonitorUnderTest = true; - -const VAULT = "0x1111111111111111111111111111111111111111"; -const TOKEN = "0x2222222222222222222222222222222222222222"; -const TIMELOCK = "0x3333333333333333333333333333333333333333"; -const RECIPIENT = "0x4444444444444444444444444444444444444444"; -const ATTESTORS = ["0xa1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1", "0xa2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2", "0xa3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3"]; -const CF = 1_000_000n; -const SUPPLY = 150_000_000n * 10n ** 18n; -const PEN = 10n ** 12n; -const GUARDIAN_KEY = `0x${"11".repeat(32)}` as const; -const GUARDIAN = privateKeyToAccount(GUARDIAN_KEY).address; - -const abi = parseAbi([ - "function totalReleased() view returns (uint256)", - "function totalSwept() view returns (uint256)", - "function conversionFactor() view returns (uint256)", - "function token() view returns (address)", - "function paused() view returns (bool)", - "function pause()", - "function threshold() view returns (uint256)", - "function guardian() view returns (address)", - "function activeApprovals(bytes32 payload) view returns (uint256)", - "function totalSupply() view returns (uint256)", - "function balanceOf(address owner) view returns (uint256)", - "function getTimestamp(bytes32 id) view returns (uint256)", - "event Approved(uint64 indexed nonce, address indexed recipient, uint256 palletAmount, address indexed attestor)", - "event Released(uint64 indexed nonce, address indexed recipient, uint256 palletAmount, uint256 tokenAmount)", - "event RemainderSwept(address indexed to, uint256 amount)", - "event GuardianUpdated(address indexed guardian)", - "event CapsUpdated(uint256 perReleaseCap, uint256 dailyCap)", - "event CallScheduled(bytes32 indexed id, uint256 indexed index, address target, uint256 value, bytes data, bytes32 predecessor, uint256 delay)", -]); - -interface Log { - address: string; - eventName: string; - args: Record; - blockNumber: bigint; -} - -/** The Base side: the vault, its token and the governance timelock. */ -function baseChain() { - return { - chainId: 8453, - safe: 1_010n, - /** How far the confirmed head trails the wall clock. */ - safeAgeSeconds: 2, - paused: false, - /** Where set, the vault is paused from this block on, and at the latest block. */ - pausedFrom: undefined as bigint | undefined, - threshold: 3n, - guardian: GUARDIAN as string, - totalReleased: 0n, - totalSwept: 0n, - vaultBalance: SUPPLY, - approvals: new Map(), - readyAt: new Map(), - /** Ether balances by lowercased address. */ - balances: new Map([[GUARDIAN.toLowerCase(), 10n ** 18n]]), - logs: [] as Log[], - /** Where set, log queries answer as a backend whose head is this block. */ - logsHead: undefined as bigint | undefined, - /** Where set and it returns a block, the log query for these topics - * answers as a backend whose head is that block. */ - logsHeadFor: undefined as ((topics: string[]) => bigint | undefined) | undefined, - /** Where set, the vault has no code before this block. */ - deployedAt: undefined as bigint | undefined, - /** Requests to answer with an RPC error. */ - failing: undefined as ((method: string, params: unknown[]) => boolean) | undefined, - /** Where set, the guardian's pause transaction is accepted and mined in the - * confirmed block; it pauses the vault there only where `true`. */ - pauseTakesEffect: undefined as boolean | undefined, - /** Hashes of the transactions sent. */ - sent: [] as string[], - }; -} - -/** A vault total at `block`: its current value less what later events added - * (`sign` -1 for the balance, which those events lowered). */ -function totalAt(base: Base, total: bigint, eventName: string, field: string, block: bigint | undefined, sign = 1n): bigint { - const later = base.logs.filter((log) => log.eventName === eventName && block !== undefined && log.blockNumber > block); - return later.reduce((sum, log) => sum - sign * (log.args[field] as bigint), total); -} -type Base = ReturnType; - -// eslint-disable-next-line @typescript-eslint/no-explicit-any -function answer(base: Base, method: string, params: any[]): unknown { - if (base.failing?.(method, params)) throw new Error("rate limited"); - switch (method) { - case "eth_getTransactionCount": - return "0x0"; - case "eth_gasPrice": - return "0x1"; - case "eth_estimateGas": - return "0x5208"; - case "eth_blockNumber": - return toHex(base.safe); - case "eth_sendRawTransaction": { - if (base.pauseTakesEffect === undefined) break; - base.sent.push(keccak256(params[0])); - if (base.pauseTakesEffect) base.pausedFrom ??= base.safe; - return keccak256(params[0]); - } - case "eth_getTransactionReceipt": { - if (!base.sent.includes(params[0])) return null; - return { - transactionHash: params[0], - transactionIndex: "0x0", - blockHash: toHex(base.safe, { size: 32 }), - blockNumber: toHex(base.safe), - from: GUARDIAN, - to: VAULT, - cumulativeGasUsed: "0x5208", - gasUsed: "0x5208", - effectiveGasPrice: "0x1", - contractAddress: null, - logs: [], - logsBloom: `0x${"00".repeat(256)}`, - status: "0x1", - type: "0x0", - }; - } - case "eth_chainId": - return toHex(base.chainId); - case "eth_getCode": { - const block = String(params[1]).startsWith("0x") ? BigInt(params[1]) : undefined; - const deployed = base.deployedAt !== undefined && (block === undefined || block >= base.deployedAt); - return String(params[0]).toLowerCase() === VAULT && deployed ? "0x6001" : "0x"; - } - case "eth_getBalance": - return toHex(base.balances.get(String(params[0]).toLowerCase()) ?? 0n); - case "eth_getBlockByNumber": { - const number = String(params[0]).startsWith("0x") ? BigInt(params[0]) : base.safe; - const timestamp = Math.floor(Date.now() / 1000) - base.safeAgeSeconds; - return { number: toHex(number), hash: toHex(number, { size: 32 }), timestamp: toHex(timestamp), transactions: [] }; - } - case "eth_call": { - const { functionName, args } = decodeFunctionData({ abi, data: params[0].data }); - if (functionName === "pause") { - if (base.pauseTakesEffect === undefined) throw new Error("unmocked pause"); - return "0x"; - } - const key = (args as readonly unknown[] | undefined)?.[0] as string; - const block = String(params[1]).startsWith("0x") ? BigInt(params[1]) : undefined; - const undeployed = base.deployedAt !== undefined && block !== undefined && block < base.deployedAt; - if (params[0].to.toLowerCase() === VAULT && undeployed) return "0x"; - const result = { - totalReleased: totalAt(base, base.totalReleased, "Released", "tokenAmount", block), - totalSwept: totalAt(base, base.totalSwept, "RemainderSwept", "amount", block), - conversionFactor: CF, - token: TOKEN, - paused: base.paused || (base.pausedFrom !== undefined && (block === undefined || block >= base.pausedFrom)), - threshold: base.threshold, - guardian: base.guardian, - activeApprovals: base.approvals.get(key) ?? 0n, - totalSupply: SUPPLY, - balanceOf: totalAt( - base, - totalAt(base, base.vaultBalance, "Released", "tokenAmount", block, -1n), - "RemainderSwept", - "amount", - block, - -1n, - ), - getTimestamp: base.readyAt.get(key) ?? 0n, - }[functionName]; - const { outputs } = abi.find((item) => item.type === "function" && item.name === functionName) as AbiFunction; - return encodeAbiParameters(outputs, [result]); - } - case "eth_getLogs": { - const [{ address, topics, fromBlock, toBlock }] = params; - const wanted = [topics?.[0] ?? []].flat() as string[]; - const head = base.logsHeadFor?.(wanted) ?? base.logsHead; - return base.logs.flatMap((log, index) => { - if (log.address.toLowerCase() !== String(address).toLowerCase()) return []; - if (log.blockNumber < BigInt(fromBlock) || log.blockNumber > BigInt(toBlock)) return []; - if (head !== undefined && log.blockNumber > head) return []; - const logTopics = encodeEventTopics({ abi, eventName: log.eventName, args: log.args } as never) as Hex[]; - if (wanted.length && !wanted.includes(logTopics[0])) return []; - const { inputs } = abi.find((item) => item.type === "event" && item.name === log.eventName) as AbiEvent; - const data = inputs.filter((input) => !input.indexed); - return [{ - address: log.address, - topics: logTopics, - data: encodeAbiParameters(data, data.map((input) => log.args[input.name!])), - blockNumber: toHex(log.blockNumber), - blockHash: toHex(log.blockNumber, { size: 32 }), - transactionHash: toHex(index + 1, { size: 32 }), - transactionIndex: "0x0", - logIndex: toHex(index), - removed: false, - }]; - }); - } - } - throw new Error(`unmocked ${method}`); -} - -/** A migration approved by three attestors and released on Base. */ -function release(base: Base, nonce: bigint, palletAmount: bigint, blockNumber = 1_005n): void { - for (const attestor of ATTESTORS) { - base.logs.push({ address: VAULT, eventName: "Approved", args: { nonce, recipient: RECIPIENT, palletAmount, attestor }, blockNumber }); - } - const tokenAmount = palletAmount * CF; - base.logs.push({ address: VAULT, eventName: "Released", args: { nonce, recipient: RECIPIENT, palletAmount, tokenAmount }, blockNumber }); - base.totalReleased += tokenAmount; - base.vaultBalance -= tokenAmount; -} - -interface Call { - section: string; - method: string; - args: unknown[]; -} - -/** Calls the stand-in registry decodes; a call's bytes are its name. */ -const CALLS: Record = { - remark: { section: "system", method: "remark", args: ["0x"] }, - mint: { section: "utility", method: "batch", args: [[{ section: "tokens", method: "setBalance", args: [] }]] }, - setTreasuryDestination: { section: "tokenMigration", method: "setTreasuryDestination", args: ["0x55"] }, -}; -const encodeCall = (name: string) => new TextEncoder().encode(name); -function decodeCall(bytes: Uint8Array): Call { - const call = CALLS[new TextDecoder().decode(bytes)]; - if (!call) throw new Error("Unable to find Call"); - return call; -} -const codec = (value: number | bigint) => ({ - toString: () => String(value), - toNumber: () => Number(value), - toBigInt: () => BigInt(value), -}); -const hash = (hex: string) => ({ toHex: () => hex, toString: () => hex, isEmpty: /^0x0*$/.test(hex) }); -const inline = (name: string) => ({ isInline: true, asInline: encodeCall(name), isLookup: false, isLegacy: false, toString: () => `inline ${name}` }); -const lookup = (hex: string, len: number) => ({ - isInline: false, - isLookup: true, - asLookup: { hash_: hash(hex), len: codec(len) }, - isLegacy: false, - toString: () => `lookup ${hex}`, -}); -const legacy = (hex: string) => ({ - isInline: false, - isLookup: false, - isLegacy: true, - asLegacy: { hash_: hash(hex) }, - toString: () => `legacy ${hex}`, -}); -type Bounded = ReturnType | ReturnType | ReturnType; - -interface PendulumEvent { - section: string; - method: string; - data: unknown[]; -} - -/** The Pendulum side: finalized blocks with their events, the migration - * pallet's totals and the governance queues. A block hash encodes the block - * number, and whether it is on the fork the node serves from `forkFrom` on. */ -function pendulumState() { - return { - /** Where set, the node serves another chain from this block on. */ - forkFrom: undefined as number | undefined, - /** Blocks whose header the node alters, so it no longer hashes to its hash. */ - alteredHeaders: new Set(), - /** Blocks whose header names the next block number. */ - renumberedHeaders: new Set(), - /** Whether the node names block hashes by number from the chain without the fork. */ - canonicalBlockHashes: false, - finalized: 100, - /** The newest block the node imported, where it is past the finalized one. */ - best: undefined as number | undefined, - /** Age of the finalized head; earlier blocks are 12 s apart. */ - headAgeMs: 0, - failure: undefined as Error | undefined, - /** Holds every finalized-head read until it settles, as a slow node would. */ - gate: undefined as Promise | undefined, - /** Finalized-head reads so far. */ - finalizedReads: 0, - events: new Map(), - totalMigrated: 0n, - nextNonce: 0n, - publicProps: [] as Array<[number, Bounded]>, - external: undefined as Bounded | undefined, - referenda: new Map(), - /** Noted preimages by `${hash}:${len}`, and their lengths by hash (old: - * noted before the pallet's status migration). */ - preimages: new Map(), - notedLengths: new Map(), - oldNotedLengths: new Map(), - preimageReads: 0, - /** Preimage reads still to fail, as an RPC error would. */ - preimageFailures: 0, - /** Open motions by hash, their index where it is not 0, the stored size - * where it is not the call's, and the motions whose call was fetched. */ - motions: { council: new Map(), technicalCommittee: new Map() }, - motionIndices: new Map(), - motionSizes: new Map(), - motionReads: new Set(), - }; -} -type Pendulum = ReturnType; - -/** A node serving `chain`, as much of the polkadot.js API as the monitor uses. */ -function pendulumChain(chain: Pendulum = pendulumState()) { - const collective = (motions: Map) => ({ - proposals: async () => [...motions.keys()].map(hash), - voting: async (motion: { toHex(): string }) => ({ - toJSON: () => (motions.has(motion.toHex()) ? { index: chain.motionIndices.get(motion.toHex()) ?? 0 } : null), - }), - proposalOf: Object.assign( - async (motion: { toHex(): string }) => { - chain.motionReads.add(motion.toHex()); - const name = motions.get(motion.toHex()); - return optional(name === undefined ? undefined : decodeCall(encodeCall(name))); - }, - { - size: async (motion: { toHex(): string }) => - codec(chain.motionSizes.get(motion.toHex()) ?? encodeCall(motions.get(motion.toHex()) ?? "").length), - }, - ), - }); - const optional = (value: T | undefined) => (value === undefined ? { isSome: false } : { isSome: true, unwrap: () => value }); - /** The migration totals at `block`: the current ones less the burns of later blocks. */ - const totalsAt = (block: number) => { - let { nextNonce, totalMigrated } = chain; - for (const [later, events] of chain.events) { - for (const { method, data } of later > block ? events : []) { - if (method !== "MigrationInitiated") continue; - nextNonce--; - totalMigrated -= (data[3] as { toBigInt(): bigint }).toBigInt(); - } - } - return { nextNonce, totalMigrated }; - }; - const at = (block: number) => ({ - registry: { createType: (_type: string, bytes: Uint8Array) => decodeCall(bytes) }, - consts: { democracy: { launchPeriod: codec(36_000) } }, - query: { - timestamp: { now: async () => codec(Date.now() - chain.headAgeMs - (chain.finalized - block) * 12_000) }, - system: { events: async () => (chain.events.get(block) ?? []).map((event) => ({ event })) }, - tokenMigration: { - totalMigrated: async () => codec(totalsAt(block).totalMigrated), - nextNonce: async () => codec(totalsAt(block).nextNonce), - }, - balances: { totalIssuance: async () => codec(100_000_000n * PEN - totalsAt(block).totalMigrated) }, - democracy: { - publicProps: async () => chain.publicProps.map(([index, bounded]) => [codec(index), bounded, "5Proposer"]), - nextExternal: async () => optional(chain.external && [chain.external, "SimpleMajority"]), - referendumCount: async () => codec(chain.referenda.size ? Math.max(...chain.referenda.keys()) + 1 : 0), - lowestUnbaked: async () => codec(chain.referenda.size ? Math.min(...chain.referenda.keys()) : 0), - referendumInfoOf: async (index: number) => { - const proposal = chain.referenda.get(index); - return optional(proposal && { isOngoing: true, asOngoing: { proposal, end: codec(9_000_000) } }); - }, - }, - preimage: { - preimageFor: async ([preimage, len]: [{ toHex(): string }, unknown]) => { - chain.preimageReads++; - if (chain.preimageFailures > 0 && chain.preimageFailures--) throw new Error("request timed out"); - const name = chain.preimages.get(`${preimage.toHex()}:${len}`); - return optional(name === undefined ? undefined : encodeCall(name)); - }, - requestStatusFor: async (preimage: { toHex(): string }) => { - const len = chain.notedLengths.get(preimage.toHex()); - return { toJSON: () => (len === undefined ? null : { unrequested: { ticket: ["5Noter", 1], len } }) }; - }, - statusFor: async (preimage: { toHex(): string }) => { - const len = chain.oldNotedLengths.get(preimage.toHex()); - return { toJSON: () => (len === undefined ? null : { unrequested: { deposit: ["5Noter", 1], len } }) }; - }, - }, - council: collective(chain.motions.council), - technicalCommittee: collective(chain.motions.technicalCommittee), - }, - }); - const api = { - genesisHash: hash(`0x${"5d".repeat(32)}`), - rpc: { - chain: { - getFinalizedHead: async () => { - chain.finalizedReads++; - await chain.gate; - if (chain.failure) throw chain.failure; - return hash(pendulumHash(chain, chain.finalized)); - }, - getHeader: async (value: string | { toHex(): string }) => { - const requested = typeof value === "string" ? value : value.toHex(); - const block = blockOf(requested); - const onFork = BigInt(requested) >> 64n !== 0n; - return { - number: codec(chain.renumberedHeaders.has(block) ? block + 1 : block), - parentHash: hash(pendulumHash(onFork ? chain : {}, block - 1)), - hash: hash(chain.alteredHeaders.has(block) ? pendulumHash(chain, block + 1_000_000) : requested), - }; - }, - getBlockHash: async (block: number) => - hash( - block > (chain.best ?? chain.finalized) - ? toHex(0, { size: 32 }) - : pendulumHash(chain.canonicalBlockHashes ? {} : chain, block), - ), - }, - }, - /** Decodes the head data of the stand-in relay chain. */ - createType: (_type: string, bytes: Uint8Array) => { - const header = JSON.parse(new TextDecoder().decode(bytes)) as { number: number; hash: string }; - return { number: codec(header.number), hash: hash(header.hash) }; - }, - at: async (value: string | { toHex(): string }) => at(blockOf(typeof value === "string" ? value : value.toHex())), - }; - return { chain, api: api as unknown as ApiPromise }; -} - -/** A relay chain whose finalized block, `ageMs()` old, records `head()` as the - * Pendulum head; every read fails while `failing()` holds. */ -function relayChain(head: () => { number: number; hash: string }, failing = () => false, ageMs = () => 0): ApiPromise { - const check = () => { - if (failing()) throw new Error("relay RPC unavailable"); - }; - const relay = { - rpc: { chain: { getFinalizedHead: async () => (check(), hash(`0x${"77".repeat(32)}`)) } }, - at: async () => ({ - query: { - timestamp: { now: async () => codec(Date.now() - ageMs()) }, - paras: { - heads: async (paraId: number) => { - check(); - const data = new TextEncoder().encode(JSON.stringify(head())); - return paraId === 2094 ? { isSome: true, unwrap: () => ({ toU8a: () => data }) } : { isSome: false }; - }, - }, - }, - }), - }; - return relay as unknown as ApiPromise; -} - -/** The hash of `block` on the chain the node serves. */ -function pendulumHash(chain: { forkFrom?: number }, block: number): string { - const fork = chain.forkFrom !== undefined && block >= chain.forkFrom ? 1n << 64n : 0n; - return toHex(fork + BigInt(block), { size: 32 }); -} -const blockOf = (blockHash: string) => Number(BigInt(blockHash) & 0xffff_ffff_ffff_ffffn); - -const MIGRATOR = `0x${"ab".repeat(32)}`; -const TREASURY_ACCOUNT = `0x${Buffer.from("modlpy/trsry").toString("hex")}${"00".repeat(20)}`; - -/** A finalized burn on Pendulum by `who` for `recipient`, in block 101 unless stated. */ -function burn( - pendulum: Pendulum, - nonce: bigint, - palletAmount: bigint, - block = 101, - { who = MIGRATOR, recipient = RECIPIENT } = {}, -): void { - pendulum.finalized = Math.max(pendulum.finalized, block); - const events = pendulum.events.get(block) ?? []; - events.push({ - section: "tokenMigration", - method: "MigrationInitiated", - data: [codec(nonce), hash(who), hash(recipient), codec(palletAmount)], - }); - pendulum.events.set(block, events); - pendulum.totalMigrated += palletAmount; - pendulum.nextNonce = nonce + 1n; -} - -interface Page { - subject: string; - detail: string; - severity: string; - /** The Slack message as sent. */ - message: Record; - status?: number; -} - -/** A page from the Slack message the monitor sent, with Slack's escapes undone. */ -function readPage(message: Record): Page { - const [, severity, subject, detail] = /^\[(CRITICAL|WARNING)\] pen-monitor: (.*)\n([\s\S]*)$/.exec(String(message.text)) ?? []; - assert.ok(subject, `not a monitor page: ${JSON.stringify(message)}`); - const unescaped = detail.replaceAll("<", "<").replaceAll(">", ">").replaceAll("&", "&"); - return { subject, detail: unescaped, severity, message }; -} - -async function body(request: IncomingMessage): Promise { - let text = ""; - for await (const chunk of request) text += chunk; - return text; -} - -let instances = 0; -let monitorEnv: string[] = []; - -/** A started monitor against fresh stand-ins, which `prepare` can change - * before startup. `webhook` answers each page with an HTTP status; `pages` - * lists them in arrival order from after startup, `startup` the ones before. - * `heartbeats` lists the heartbeat requests, which `heartbeat` answers. */ -async function setup( - t: { after(fn: () => void): void }, - { - env = {}, - webhook = () => 200, - heartbeat = () => 200, - prepare = () => {}, - connect, - }: { - env?: Record; - webhook?: (page: Page) => number | Promise; - heartbeat?: () => number; - prepare?: (base: Base, pendulum: Pendulum) => void; - /** Reaches the other nodes: the second Pendulum node and relay RPCs. */ - connect?: (url: string) => Promise; - } = {}, -) { - const base = baseChain(); - const { chain: pendulum, api } = pendulumChain(); - prepare(base, pendulum); - const pages: Page[] = []; - const heartbeats: string[] = []; - const server = createServer(async (request, response) => { - const text = await body(request); - if (request.url === "/heartbeat") { - heartbeats.push(request.method ?? ""); - response.writeHead(heartbeat()).end(); - return; - } - if (request.url === "/alert") { - const page = readPage(JSON.parse(text)); - pages.push(page); - page.status = await webhook(page); - response.writeHead(page.status).end(); - return; - } - const { id, method, params } = JSON.parse(text); - let reply: object; - try { - reply = { result: answer(base, method, params) }; - } catch (error) { - reply = { error: { code: -32603, message: String(error) } }; - } - response.setHeader("content-type", "application/json"); - response.end(JSON.stringify({ jsonrpc: "2.0", id, ...reply })); - }); - await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); - t.after(() => { - server.closeAllConnections(); - server.close(); - }); - const url = `http://127.0.0.1:${(server.address() as AddressInfo).port}`; - const stateFile = join(mkdtempSync(join(tmpdir(), "pen-monitor-test-")), "state.json"); - const variables: Record = { - PENDULUM_WS: "ws://127.0.0.1:1", - BASE_RPC_URL: url, - VAULT_ADDRESS: VAULT, - BASE_START_BLOCK: "1000", - PENDULUM_START_BLOCK: "101", - STATE_FILE: stateFile, - ALERT_WEBHOOK_URL: `${url}/alert`, - HEARTBEAT_URL: `${url}/heartbeat`, - ...env, - }; - for (const name of monitorEnv) delete process.env[name]; - monitorEnv = Object.keys(variables); - Object.assign(process.env, variables); - const launch = async () => { - const monitor: typeof import("./main.js") = await import(`./main.js?instance=${++instances}`); - await monitor.start(api, connect ?? (async (url) => assert.fail(`unexpected connection to ${url}`))); - return monitor; - }; - const monitor = await launch(); - const startup = pages.splice(0); - return { - base, - pendulum, - api, - pages, - startup, - heartbeats, - monitor, - /** A new monitor process on the same state file and stand-ins. */ - restart: launch, - subjects: () => pages.map((page) => page.subject), - persisted: () => JSON.parse(readFileSync(stateFile, "utf8")) as { - lastPendulumBlock: number; - lastPendulumHash: string; - relayAnchor?: { number: number; hash: string }; - baseFromBlock: string; - baseAdminFromBlock: string; - migrations: Array<{ nonce: string }>; - mismatchedApprovals: Array<{ nonce: string; attestor: string }>; - }, - }; -} - -test("a released migration reconciles and the poll pages nothing", async (t) => { - const { base, pendulum, api, monitor, subjects, persisted } = await setup(t); - burn(pendulum, 0n, 100n * PEN); - release(base, 0n, 100n * PEN); - await monitor.poll(api); - assert.deepEqual(subjects(), []); - assert.deepEqual(persisted().migrations, []); - assert.equal(persisted().baseFromBlock, "1011"); -}); - -/** Pages sent concurrently can arrive in either order. */ -const sorted = (subjects: string[]) => [...subjects].sort(); - -test("pages are Slack messages carrying severity, subject and detail in their text only", async (t) => { - const { base, api, monitor, pages } = await setup(t); - base.vaultBalance -= 1n; - base.safeAgeSeconds = 601; - await monitor.poll(api); - assert.ok(pages.length >= 3, JSON.stringify(pages)); - for (const page of pages) assert.deepEqual(Object.keys(page.message), ["text"]); - const severity = (subject: string) => pages.find((page) => page.subject === subject)?.severity; - assert.equal(severity("VAULT BALANCE DEFICIT"), "CRITICAL"); - assert.equal(severity("AUTO-PAUSE UNAVAILABLE"), "CRITICAL"); - assert.equal(severity("BASE VIEW STALLED"), "WARNING"); - assert.match(pages.find((page) => page.subject === "VAULT BALANCE DEFICIT")!.detail, /^balance \d+ \+ released 0 \+ swept 0 < supply/); -}); - -test("only a poll that runs every check requests the heartbeat", async (t) => { - const { base, pendulum, api, monitor, heartbeats } = await setup(t); - await monitor.poll(api); - assert.deepEqual(heartbeats, ["GET"]); - pendulum.failure = new Error("node unreachable"); - await monitor.poll(api); - pendulum.failure = undefined; - const guardianUpdated = toEventSelector("GuardianUpdated(address)"); // only the admin scan asks for it - base.failing = (method, params) => method === "eth_getLogs" && JSON.stringify(params).includes(guardianUpdated); - base.safe = 1_020n; // a new range for the admin scan - await monitor.poll(api); - assert.deepEqual(heartbeats, ["GET"]); - base.failing = undefined; - await monitor.poll(api); - assert.deepEqual(heartbeats, ["GET", "GET"]); -}); - -test("a failing heartbeat never stops the monitor", async (t) => { - const { api, monitor, heartbeats, subjects } = await setup(t, { heartbeat: () => 500 }); - await monitor.poll(api); - await monitor.poll(api); - assert.deepEqual(heartbeats, ["GET", "GET"]); - const unreachable = await setup(t, { env: { HEARTBEAT_URL: "http://127.0.0.1:1/heartbeat" } }); - await unreachable.monitor.poll(unreachable.api); - assert.deepEqual([...subjects(), ...unreachable.subjects()], []); -}); - -test("an auto-pause counts as confirmed only where the vault is paused at the confirmed block", async (t) => { - for (const pauseTakesEffect of [true, false]) { - const { base, api, monitor, subjects } = await setup(t, { - env: { GUARDIAN_PRIVATE_KEY: GUARDIAN_KEY, BASE_FINALITY_POLL_MS: "10" }, - prepare: (base) => (base.pauseTakesEffect = pauseTakesEffect), - }); - base.vaultBalance -= 1n; - await monitor.poll(api); - assert.equal(base.sent.length, 1); - const outcome = subjects().filter((subject) => subject.includes("PAUSE") || subject.includes("pause")); - assert.deepEqual(outcome, [pauseTakesEffect ? "vault auto-pause confirmed" : "AUTO-PAUSE FAILED"]); - } -}); - -test("a vault balance deficit pages and asks for the auto-pause", async (t) => { - const { base, api, monitor, subjects } = await setup(t); - base.vaultBalance -= 1n; - await monitor.poll(api); - assert.deepEqual(sorted(subjects()), ["AUTO-PAUSE UNAVAILABLE", "VAULT BALANCE DEFICIT"]); -}); - -test("a Pendulum failure pages on its own and the vault balance check still runs", async (t) => { - const { base, pendulum, api, monitor, subjects } = await setup(t); - pendulum.failure = new Error("node unreachable"); - base.vaultBalance -= 1n; - await monitor.poll(api); - assert.deepEqual(sorted(subjects()), [ - "AUTO-PAUSE UNAVAILABLE", - "PENDULUM SOURCE UNREADABLE", - "VAULT BALANCE DEFICIT", - "pendulum governance watch failed", - ]); -}); - -test("a failing Base log read does not skip the vault balance check", async (t) => { - const { base, pendulum, api, monitor, subjects } = await setup(t, { env: { GUARDIAN_PRIVATE_KEY: GUARDIAN_KEY } }); - pendulum.failure = new Error("node unreachable"); - const released = toEventSelector("Released(uint64,address,uint256,uint256)"); // only reconciliation asks for it - base.failing = (method, params) => method === "eth_getLogs" && JSON.stringify(params).includes(released); - base.vaultBalance -= 1n; - base.paused = true; // so the auto-pause only confirms - await monitor.poll(api); - assert.ok(subjects().includes("VAULT BALANCE DEFICIT"), JSON.stringify(subjects())); - assert.ok(subjects().includes("vault pause confirmed"), JSON.stringify(subjects())); -}); - -/** Waits until `predicate` holds, failing after `ms`. */ -async function until(predicate: () => boolean, label: string, ms = 2_000): Promise { - const deadline = Date.now() + ms; - while (!predicate()) { - if (Date.now() > deadline) assert.fail(`timed out waiting for ${label}`); - await new Promise((resolve) => setTimeout(resolve, 10)); - } -} - -test("a slow Pendulum source does not delay the vault balance check", async (t) => { - const { base, pendulum, api, monitor, subjects } = await setup(t); - let respond = () => {}; - pendulum.gate = new Promise((resolve) => (respond = resolve)); - base.vaultBalance -= 1n; - const polling = monitor.poll(api); - try { - await until(() => subjects().includes("VAULT BALANCE DEFICIT"), "the deficit page while Pendulum is still answering"); - } finally { - respond(); - await polling; - } -}); - -/** Holds every finalized-head read of `pendulum` until `respond` is called; - * `waiting` resolves once a read waits. */ -function holdSource(pendulum: Pendulum) { - let respond = () => {}; - pendulum.gate = new Promise((resolve) => (respond = resolve)); - const reads = pendulum.finalizedReads; - return { respond, waiting: () => until(() => pendulum.finalizedReads > reads, "a poll waiting on the source") }; -} - -test("while a poll waits on a slow source, the vault balance check runs on its own", async (t) => { - const { base, pendulum, api, monitor, subjects } = await setup(t, { env: { POLL_INTERVAL_MS: "20" } }); - const { respond, waiting } = holdSource(pendulum); - const polling = monitor.poll(api); // checks the balance, then waits on the source - try { - await waiting(); - await new Promise((resolve) => setTimeout(resolve, 30)); // a poll interval later - base.vaultBalance -= 1n; - await monitor.watchVaultBalance(); - assert.ok(subjects().includes("VAULT BALANCE DEFICIT"), JSON.stringify(subjects())); - } finally { - respond(); - await polling; - } -}); - -test("the vault balance check does not run on its own within a poll interval of a poll's, or between polls", async (t) => { - const { base, pendulum, api, monitor, subjects } = await setup(t, { env: { POLL_INTERVAL_MS: "500" } }); - const { respond, waiting } = holdSource(pendulum); - const polling = monitor.poll(api); - try { - await waiting(); - base.vaultBalance -= 1n; - await monitor.watchVaultBalance(); // the poll checked the balance less than an interval ago - } finally { - respond(); - await polling; - } - await new Promise((resolve) => setTimeout(resolve, 550)); - await monitor.watchVaultBalance(); // no poll is running: the next one checks it - assert.ok(!subjects().includes("VAULT BALANCE DEFICIT"), JSON.stringify(subjects())); -}); - -test("a pause attempt in flight is joined, not started again", async (t) => { - let answerFailedPause = () => {}; - const failedPauseAnswered = new Promise((resolve) => (answerFailedPause = resolve)); - const { base, pendulum, api, monitor, pages } = await setup(t, { - env: { GUARDIAN_PRIVATE_KEY: GUARDIAN_KEY, POLL_INTERVAL_MS: "20" }, - webhook: async ({ subject }) => { - if (subject === "AUTO-PAUSE FAILED") await failedPauseAnswered; // holds that attempt in flight - return 200; - }, - }); - burn(pendulum, 0n, 100n * PEN); - await monitor.poll(api); - const tokenAmount = 100n * PEN * CF; // a release of nonce 0 to another recipient - base.logs.push({ address: VAULT, eventName: "Released", args: { nonce: 0n, recipient: TOKEN, palletAmount: 100n * PEN, tokenAmount }, blockNumber: 1_015n }); - base.totalReleased += tokenAmount; - base.vaultBalance -= tokenAmount; - base.safe = 1_020n; - const { respond, waiting } = holdSource(pendulum); - const polling = monitor.poll(api); - const failedPauses = () => pages.filter((page) => page.subject === "AUTO-PAUSE FAILED").length; - try { - await waiting(); - await new Promise((resolve) => setTimeout(resolve, 30)); // a poll interval later - base.vaultBalance -= 1n; - const watching = monitor.watchVaultBalance(); // a deficit: the first pause attempt - await until(() => failedPauses() === 1, "the first failed pause page"); - respond(); // the poll goes on to the release, a second violation - await until(() => pages.some((page) => page.subject === "MIGRATION TUPLE VIOLATION"), "the second violation"); - await new Promise((resolve) => setTimeout(resolve, 50)); - answerFailedPause(); - await watching; - } finally { - respond(); - answerFailedPause(); - await polling; - } - assert.equal(failedPauses(), 1, JSON.stringify(pages.map((page) => page.subject))); -}); - -test("a source whose chain leaves the blocks ingested so far pages critical and ingests nothing", async (t) => { - const { pendulum, api, monitor, restart, pages, subjects, persisted } = await setup(t); - burn(pendulum, 0n, 100n * PEN); - await monitor.poll(api); - const restarted = await restart(); - pendulum.forkFrom = 101; // the node now serves another chain from block 101 on - burn(pendulum, 1n, 100n * PEN, 102); - await restarted.poll(api); - const page = pages.find((page) => page.subject === "CRITICAL: PENDULUM SOURCE CONTRADICTED"); - assert.equal(page?.severity, "CRITICAL", JSON.stringify(pages)); - assert.match(page.detail, /chain has 0x0+10+65 at Pendulum block 101, where the monitor ingested 0x0+65;/); - assert.ok(!subjects().includes("PENDULUM SOURCE UNREADABLE"), JSON.stringify(subjects())); - assert.equal(persisted().lastPendulumBlock, 101); - assert.deepEqual(persisted().migrations.map((migration) => migration.nonce), ["0"]); -}); - -test("a source whose finalized head falls behind the block ingested last is read at that block", async (t) => { - const { pendulum, api, monitor, pages, persisted } = await setup(t); - burn(pendulum, 0n, 100n * PEN); - await monitor.poll(api); - pendulum.finalized = 100; // a node that has not finalized block 101 yet - await monitor.poll(api); - assert.equal(pages.length, 0, JSON.stringify(pages)); - assert.equal(persisted().lastPendulumBlock, 101); -}); - -test("a source header that does not hash to its block hash pages critical", async (t) => { - const { pendulum, api, monitor, pages } = await setup(t); - burn(pendulum, 0n, 100n * PEN, 102); - pendulum.alteredHeaders.add(102); - await monitor.poll(api); - assert.match( - pages.find((page) => page.subject === "CRITICAL: PENDULUM SOURCE CONTRADICTED")?.detail ?? "", - /header for Pendulum block 102 does not hash to 0x0+66;/, - ); -}); - -test("a source header that names another block number pages critical", async (t) => { - const { pendulum, api, monitor, pages, persisted } = await setup(t); - burn(pendulum, 0n, 100n * PEN, 102); - pendulum.renumberedHeaders.add(101); - await monitor.poll(api); - assert.match( - pages.find((page) => page.subject === "CRITICAL: PENDULUM SOURCE CONTRADICTED")?.detail ?? "", - /header for Pendulum block 101 does not hash to 0x0+65;/, - ); - assert.equal(persisted().lastPendulumBlock, 100); -}); - -const SECOND_NODE = "ws://second-node.test:9944?apikey=secret"; - -/** A monitor whose second Pendulum node serves the source's chain, except for - * the values in `own`, which the test may change later; with `relays`, also - * against these relay RPCs, in this order. */ -async function crossCheckSetup(t: { after(fn: () => void): void }, own: Partial = {}, relays: ApiPromise[] = []) { - let source: Pendulum | undefined; - const chain = new Proxy({} as Pendulum, { - get: (_target, key) => (key in own ? own[key as keyof Pendulum] : source![key as keyof Pendulum]), - }); - const second = pendulumChain(chain); - const urls = relays.map((_relay, index) => `wss://relay-${index + 1}.test`); - const monitor = await setup(t, { - env: { PENDULUM_WS_CROSSCHECK: SECOND_NODE, ...(urls.length ? { POLKADOT_WS: urls.join(",") } : {}) }, - prepare: (_base, pendulum) => (source = pendulum), - connect: async (url) => (url === SECOND_NODE ? second.api : relays[urls.indexOf(url)] ?? assert.fail(url)), - }); - return { ...monitor, own }; -} - -test("a second node that disagrees on the newest block pages critical and nothing is ingested", async (t) => { - const { pendulum, api, monitor, pages, persisted, own } = await crossCheckSetup(t, { finalized: 200 }); - burn(pendulum, 0n, 100n * PEN); - await monitor.poll(api); - assert.equal(pages.length, 0, JSON.stringify(pages)); - assert.equal(persisted().lastPendulumBlock, 101); - own.forkFrom = 102; - burn(pendulum, 1n, 100n * PEN, 102); - await monitor.poll(api); - assert.match( - pages.find((page) => page.subject === "CRITICAL: PENDULUM SOURCE CONTRADICTED")?.detail ?? "", - /second Pendulum node has 0x0+10+66 at block 102, the source 0x0+66;/, - ); - assert.equal(persisted().lastPendulumBlock, 101); -}); - -test("a second node that disagrees on the block ingested last, its finalized head, pages critical", async (t) => { - const { pendulum, api, monitor, pages, persisted, own } = await crossCheckSetup(t, { finalized: 101 }); - burn(pendulum, 0n, 100n * PEN); - await monitor.poll(api); - assert.equal(persisted().lastPendulumBlock, 101); - own.forkFrom = 101; // the second node now has another block 101 - burn(pendulum, 1n, 100n * PEN, 102); - await monitor.poll(api); - assert.match( - pages.find((page) => page.subject === "CRITICAL: PENDULUM SOURCE CONTRADICTED")?.detail ?? "", - /second Pendulum node has 0x0+10+65 at block 101, the source 0x0+65;/, - ); - assert.equal(persisted().lastPendulumBlock, 101); -}); - -test("a second node that cannot answer only warns", async (t) => { - let connections = 0; - const { pendulum, api, monitor, pages, persisted, heartbeats } = await setup(t, { - env: { PENDULUM_WS_CROSSCHECK: SECOND_NODE }, - connect: async (url) => { - connections++; - throw new Error(`cannot connect to ${url}`); - }, - }); - burn(pendulum, 0n, 100n * PEN); - await monitor.poll(api); - assert.deepEqual(pages.map((page) => [page.subject, page.severity]), [["PENDULUM CROSS-CHECK UNAVAILABLE", "WARNING"]]); - assert.match(pages[0].detail, /cannot be read: Error: cannot connect to /); - assert.equal(persisted().lastPendulumBlock, 101); - assert.equal(connections, 1); // not asked again for the ingested block's migration events - assert.deepEqual(heartbeats, ["GET"]); - await monitor.poll(api); - assert.equal(pages.length, 1); // throttled -}); - -test("the URLs of the checking nodes are validated at startup", async (t) => { - await assert.rejects(setup(t, { env: { POLKADOT_WS: "wss://relay.test, https://relay.test" } }), /invalid POLKADOT_WS/); - await assert.rejects( - setup(t, { env: { PENDULUM_WS_CROSSCHECK: "wss://a.test,wss://b.test" } }), - /invalid PENDULUM_WS_CROSSCHECK: expected one ws/, - ); -}); - -test("a node connection that fails with an event says why", async (t) => { - const { pendulum, api, monitor, pages } = await setup(t, { - env: { PENDULUM_WS_CROSSCHECK: SECOND_NODE }, - connect: () => Promise.reject({ type: "error", message: "getaddrinfo ENOTFOUND second-node.test" }), - }); - burn(pendulum, 0n, 100n * PEN); - await monitor.poll(api); - assert.match(pages[0]?.detail ?? "", /cannot be read: Error: getaddrinfo ENOTFOUND second-node\.test;/); -}); - -test("a block the second node has not finalized yet waits for it, without a page", async (t) => { - const { pendulum, api, monitor, pages, persisted, own } = await crossCheckSetup(t, { finalized: 100 }); - burn(pendulum, 0n, 100n * PEN); - await monitor.poll(api); - assert.equal(pages.length, 0, JSON.stringify(pages)); - assert.equal(persisted().lastPendulumBlock, 100); - own.finalized = 101; - await monitor.poll(api); - assert.equal(pages.length, 0, JSON.stringify(pages)); - assert.equal(persisted().lastPendulumBlock, 101); - assert.deepEqual(persisted().migrations.map((migration) => migration.nonce), ["0"]); -}); - -test("a second node whose best chain holds another newest block, not yet finalized, holds that block back", async (t) => { - const { pendulum, api, monitor, pages, persisted } = await crossCheckSetup(t, { finalized: 100, best: 101, forkFrom: 101 }); - burn(pendulum, 0n, 100n * PEN); - await monitor.poll(api); - assert.equal(pages.length, 0, JSON.stringify(pages)); - assert.equal(persisted().lastPendulumBlock, 100); -}); - -test("a source on another chain one block past the second node's head ingests nothing from that chain", async (t) => { - const { base, pendulum, api, monitor, pages, persisted, own } = await crossCheckSetup(t, { finalized: 101, forkFrom: undefined }); - burn(pendulum, 0n, 100n * PEN); - pendulum.forkFrom = 102; // the source serves another chain from block 102 on... - burn(pendulum, 1n, 999n * PEN, 102); // ...with a fabricated burn, one block past the second node's head - release(base, 1n, 999n * PEN); - await monitor.poll(api); - assert.equal(persisted().lastPendulumBlock, 101); - assert.deepEqual(persisted().migrations.map((migration) => migration.nonce), ["0"]); - assert.equal(persisted().baseFromBlock, "1000"); // the release of the fabricated nonce awaits its burn - assert.deepEqual([...new Set(pages.map((page) => page.subject))], ["UNVERIFIED BASE EVENT"]); - own.finalized = 102; // the second node finalizes its own block 102... - burn(pendulum, 2n, 999n * PEN, 103); // ...and the source stays one block ahead - await monitor.poll(api); - assert.match( - pages.find((page) => page.subject === "CRITICAL: PENDULUM SOURCE CONTRADICTED")?.detail ?? "", - /second Pendulum node has 0x0+66 at block 102, the source 0x0+10+66;/, - ); - assert.equal(persisted().lastPendulumBlock, 101); - assert.deepEqual(persisted().migrations.map((migration) => migration.nonce), ["0"]); -}); - -test("with the second node and the relay chain, a block past both their heads waits, and a fork there is caught", async (t) => { - let recorded = 101; - const { pendulum, api, monitor, pages, persisted, own } = await crossCheckSetup(t, { finalized: 101, forkFrom: undefined }, [ - relayChain(() => ({ number: recorded, hash: pendulumHash({}, recorded) })), - ]); - burn(pendulum, 0n, 100n * PEN); - pendulum.forkFrom = 102; // the source finalizes another block 102... - burn(pendulum, 1n, 999n * PEN, 102); // ...with a fabricated burn - await monitor.poll(api); - assert.equal(pages.length, 0, JSON.stringify(pages)); - assert.equal(persisted().lastPendulumBlock, 101); - assert.deepEqual(persisted().migrations.map((migration) => migration.nonce), ["0"]); - own.finalized = 102; // both checkers reach the real block 102 - recorded = 102; - await monitor.poll(api); - assert.ok(pages.some((page) => page.subject === "CRITICAL: PENDULUM SOURCE CONTRADICTED"), JSON.stringify(pages)); - assert.equal(persisted().lastPendulumBlock, 101); -}); - -test("a second node behind the block ingested last holds newer blocks back, without a contradiction", async (t) => { - const { pendulum, api, monitor, pages, persisted, own } = await crossCheckSetup(t, { finalized: 100, headAgeMs: 400_000 }); - burn(pendulum, 0n, 100n * PEN); - await monitor.poll(api); // the stale second node is left out: block 101 is ingested - assert.equal(persisted().lastPendulumBlock, 101); - own.headAgeMs = 0; // it answers again, still at block 100 - burn(pendulum, 1n, 100n * PEN, 102); - await monitor.poll(api); - assert.deepEqual(pages.map((page) => page.subject), ["PENDULUM CROSS-CHECK UNAVAILABLE"]); - assert.equal(persisted().lastPendulumBlock, 101); - own.finalized = 102; - await monitor.poll(api); - assert.equal(persisted().lastPendulumBlock, 102); -}); - -test("a second node whose finalized head is stale counts as unavailable", async (t) => { - const { pendulum, api, monitor, pages, persisted } = await crossCheckSetup(t, { finalized: 100, headAgeMs: 400_000 }); - burn(pendulum, 0n, 100n * PEN); - await monitor.poll(api); - assert.deepEqual(pages.map((page) => [page.subject, page.severity]), [["PENDULUM CROSS-CHECK UNAVAILABLE", "WARNING"]]); - assert.match(pages[0].detail, /has a finalized head 40\ds old;/); - assert.equal(persisted().lastPendulumBlock, 101); -}); - -test("a second node with another nonce or total at the finalized head pages critical and nothing is ingested", async (t) => { - for (const own of [{ nextNonce: 2n }, { totalMigrated: 7n * PEN }]) { - const { pendulum, api, monitor, pages, persisted } = await crossCheckSetup(t, own); - burn(pendulum, 0n, 100n * PEN); - await monitor.poll(api); - assert.match( - pages.find((page) => page.subject === "CRITICAL: PENDULUM SOURCE CONTRADICTED")?.detail ?? "", - /at Pendulum block 101 the second node has nextNonce (2|1) and totalMigrated (100|7)000000000000, the source 1 and 100000000000000;/, - ); - assert.equal(persisted().lastPendulumBlock, 100); - } -}); - -test("a second node with other migration events in a block pages critical and nothing is ingested", async (t) => { - const { pendulum, api, monitor, pages, persisted, own } = await crossCheckSetup(t); - burn(pendulum, 0n, 100n * PEN); - const events = new Map(pendulum.events); - own.events = events; - burn(pendulum, 1n, 100n * PEN, 102); - events.set(102, [{ ...pendulum.events.get(102)![0], data: [codec(1n), hash(MIGRATOR), hash(TOKEN), codec(100n * PEN)] }]); - await monitor.poll(api); - assert.match( - pages.find((page) => page.subject === "CRITICAL: PENDULUM SOURCE CONTRADICTED")?.detail ?? "", - new RegExp(`in Pendulum block 102 the second node reports the migration events MigrationInitiated\\(1, ${MIGRATOR}, ${TOKEN}, `, "i"), - ); - assert.equal(persisted().lastPendulumBlock, 100); - assert.deepEqual(persisted().migrations, []); -}); - -test("a second node that cannot read a block's events only warns", async (t) => { - const failing = new (class extends Map { - get(): never { - throw new Error("state discarded"); - } - })(); - const { pendulum, api, monitor, pages, persisted } = await crossCheckSetup(t, { events: failing }); - burn(pendulum, 0n, 100n * PEN); - await monitor.poll(api); - assert.deepEqual(pages.map((page) => page.subject), ["PENDULUM CROSS-CHECK UNAVAILABLE"]); - assert.match(pages[0].detail, /cannot be read: .*state discarded/); - assert.equal(persisted().lastPendulumBlock, 101); -}); - -const RELAY = "wss://relay.test"; -type Head = { number: number; hash: string }; - -/** A monitor with the relay-chain anchor, against a relay chain recording - * `head` (by default the source's finalized head, on the chain without a fork). */ -async function relaySetup( - t: { after(fn: () => void): void }, - { head, failing, env = {} }: { head?: (pendulum: Pendulum) => Head; failing?: () => boolean; env?: Record } = {}, -) { - let chain: Pendulum | undefined; - const honest = (pendulum: Pendulum) => ({ number: pendulum.finalized, hash: pendulumHash({}, pendulum.finalized) }); - const relay = relayChain(() => (head ?? honest)(chain!), failing); - return setup(t, { - env: { POLKADOT_WS: RELAY, ...env }, - prepare: (_base, pendulum) => (chain = pendulum), - connect: async (url) => (url === RELAY ? relay : assert.fail(url)), - }); -} - -test("a relay-chain record of the source's head anchors it and is kept", async (t) => { - const { pendulum, api, monitor, pages, persisted } = await relaySetup(t); - burn(pendulum, 0n, 100n * PEN); - await monitor.poll(api); - assert.equal(pages.length, 0, JSON.stringify(pages)); - assert.deepEqual(persisted().relayAnchor, { number: 101, hash: pendulumHash({}, 101) }); - assert.equal(persisted().lastPendulumBlock, 101); - await monitor.poll(api); // neither the record nor the source moved: nothing waits, nothing pages - assert.equal(pages.length, 0, JSON.stringify(pages)); -}); - -test("the conservation check compares with the total of the view the relay chain confirms", async (t) => { - const { base, pendulum, api, monitor, subjects } = await relaySetup(t, { head: () => ({ number: 101, hash: pendulumHash({}, 101) }) }); - burn(pendulum, 0n, 100n * PEN); - burn(pendulum, 1n, 999n * PEN, 102); // past the relay chain's record - release(base, 0n, 100n * PEN); - base.totalReleased += 50n * PEN * CF; // released without a burn behind it - base.vaultBalance -= 50n * PEN * CF; - await monitor.poll(api); - assert.ok(subjects().includes("CONSERVATION VIOLATION"), JSON.stringify(subjects())); -}); - -test("a source that names another hash than the relay chain's record pages critical", async (t) => { - const { pendulum, api, monitor, pages, persisted } = await relaySetup(t, { - head: (pendulum) => ({ number: pendulum.finalized, hash: pendulumHash({ forkFrom: 0 }, pendulum.finalized) }), - }); - burn(pendulum, 0n, 100n * PEN); - await monitor.poll(api); - assert.match( - pages.find((page) => page.subject === "CRITICAL: PENDULUM SOURCE CONTRADICTED")?.detail ?? "", - /relay chain records 0x0+10+65 as Pendulum block 101, the source's chain has 0x0+65;/, - ); - assert.equal(persisted().lastPendulumBlock, 100); -}); - -test("a source whose chain avoids the relay chain's record pages critical, whatever hash it names", async (t) => { - const { pendulum, api, monitor, pages } = await relaySetup(t, { - head: () => ({ number: 101, hash: pendulumHash({}, 101) }), - }); - pendulum.forkFrom = 101; - pendulum.canonicalBlockHashes = true; // it names the relay chain's hash for block 101 - burn(pendulum, 0n, 100n * PEN, 102); - await monitor.poll(api); - assert.match( - pages.find((page) => page.subject === "CRITICAL: PENDULUM SOURCE CONTRADICTED")?.detail ?? "", - /relay chain records 0x0+65 as Pendulum block 101, the source's chain has 0x0+10+65;/, - ); -}); - -test("a relay record past the source's finalized head anchors it, and its chain must keep passing through it", async (t) => { - let relayDown = false; - const { pendulum, api, monitor, pages, persisted } = await relaySetup(t, { - head: (pendulum) => ({ number: 102, hash: pendulumHash(pendulum, 102) }), - failing: () => relayDown, - }); - burn(pendulum, 0n, 100n * PEN); - pendulum.best = 102; // imported, not yet finalized by the source - await monitor.poll(api); - assert.equal(pages.length, 0, JSON.stringify(pages)); - assert.deepEqual(persisted().relayAnchor, { number: 102, hash: pendulumHash({}, 102) }); - assert.equal(persisted().lastPendulumBlock, 101); - relayDown = true; - pendulum.forkFrom = 102; // the source moves to another chain above its last ingested block - burn(pendulum, 1n, 100n * PEN, 103); - await monitor.poll(api); - assert.match( - pages.find((page) => page.subject === "CRITICAL: PENDULUM SOURCE CONTRADICTED")?.detail ?? "", - /chain has 0x0+10+66 at Pendulum block 102, where the relay chain recorded 0x0+66;/, - ); -}); - -test("a kept relay record past the source's finalized head binds the source while no newer one arrives", async (t) => { - let relayDown = false; - const { pendulum, api, monitor, pages, persisted } = await relaySetup(t, { - head: () => ({ number: 103, hash: pendulumHash({}, 103) }), - failing: () => relayDown, - }); - burn(pendulum, 0n, 100n * PEN); - pendulum.best = 103; // imported, not yet finalized by the source - await monitor.poll(api); - assert.equal(pages.length, 0, JSON.stringify(pages)); - assert.deepEqual(persisted().relayAnchor, { number: 103, hash: pendulumHash({}, 103) }); - assert.equal(persisted().lastPendulumBlock, 101); - relayDown = true; - pendulum.forkFrom = 102; // the source finalizes another chain below the kept record... - burn(pendulum, 1n, 999n * PEN, 102); // ...with a burn on it - await monitor.poll(api); - assert.match( - pages.find((page) => page.subject === "CRITICAL: PENDULUM SOURCE CONTRADICTED")?.detail ?? "", - /chain has 0x0+66 at Pendulum block 102, where the source's finalized head is 0x0+10+66;/, - ); - assert.equal(persisted().lastPendulumBlock, 101); - assert.deepEqual(persisted().migrations.map((migration) => migration.nonce), ["0"]); -}); - -test("a kept relay record past the source's finalized head confirms the blocks below it while no newer one arrives", async (t) => { - const { pendulum, api, monitor, pages, persisted } = await relaySetup(t, { - head: () => ({ number: 103, hash: pendulumHash({}, 103) }), - }); - burn(pendulum, 0n, 100n * PEN); - pendulum.best = 103; // imported, not yet finalized by the source - await monitor.poll(api); - assert.equal(persisted().lastPendulumBlock, 101); - burn(pendulum, 1n, 100n * PEN, 102); // the source finalizes block 102; the record stays at 103 - await monitor.poll(api); - assert.equal(pages.length, 0, JSON.stringify(pages)); - assert.equal(persisted().lastPendulumBlock, 102); - assert.deepEqual(persisted().relayAnchor, { number: 103, hash: pendulumHash({}, 103) }); -}); - -test("a relay record past the source's finalized head must descend from that head", async (t) => { - const { pendulum, api, monitor, pages, persisted } = await relaySetup(t, { - head: () => ({ number: 102, hash: pendulumHash({}, 102) }), - }); - burn(pendulum, 0n, 100n * PEN); - pendulum.forkFrom = 101; // its finalized block 101 is on another chain... - pendulum.canonicalBlockHashes = true; // ...though it names the relay chain's hash for block 102 - pendulum.best = 102; - await monitor.poll(api); - assert.match( - pages.find((page) => page.subject === "CRITICAL: PENDULUM SOURCE CONTRADICTED")?.detail ?? "", - /records 0x0+66 as Pendulum block 102, whose chain has 0x0+65 at block 101, where the source has 0x0+10+65;/, - ); - assert.equal(persisted().lastPendulumBlock, 100); -}); - -test("a relay record past the source's finalized head that its best chain does not hold yet holds the source back", async (t) => { - const { pendulum, api, monitor, pages, persisted } = await relaySetup(t, { - head: () => ({ number: 102, hash: pendulumHash({}, 102) }), - }); - burn(pendulum, 0n, 100n * PEN); - pendulum.forkFrom = 102; // its best chain holds another block 102 for now - pendulum.best = 102; - await monitor.poll(api); - assert.deepEqual(pages.map((page) => [page.subject, page.severity]), [["RELAY ANCHOR UNAVAILABLE", "WARNING"]]); - assert.match( - pages[0].detail, - /^relay RPC 1: the source has not imported block 102, the relay chain's Pendulum head, yet; no Pendulum block after 100 is ingested until a relay RPC anchors a newer head,/, - ); - assert.equal(persisted().relayAnchor, undefined); - assert.equal(persisted().lastPendulumBlock, 100); -}); - -test("a block past the relay chain's record waits for it, and a fork there is caught once it is recorded", async (t) => { - let recorded = 101; - const { pendulum, api, monitor, pages, persisted } = await relaySetup(t, { - head: () => ({ number: recorded, hash: pendulumHash({}, recorded) }), - }); - burn(pendulum, 0n, 100n * PEN); - pendulum.forkFrom = 102; // the source finalizes another block 102... - burn(pendulum, 1n, 999n * PEN, 102); // ...with a fabricated burn, past the relay chain's record - await monitor.poll(api); - assert.equal(pages.length, 0, JSON.stringify(pages)); - assert.equal(persisted().lastPendulumBlock, 101); - assert.deepEqual(persisted().migrations.map((migration) => migration.nonce), ["0"]); - recorded = 102; // the relay chain records the real block 102 - await monitor.poll(api); - assert.match( - pages.find((page) => page.subject === "CRITICAL: PENDULUM SOURCE CONTRADICTED")?.detail ?? "", - /relay chain records 0x0+66 as Pendulum block 102, the source's chain has 0x0+10+66;/, - ); - assert.equal(persisted().lastPendulumBlock, 101); -}); - -test("a source that finalizes below a relay record it does not hold ingests nothing past the kept record", async (t) => { - let recorded = 101; - const { pendulum, api, monitor, pages, persisted } = await relaySetup(t, { - head: () => ({ number: recorded, hash: pendulumHash({}, recorded) }), - }); - burn(pendulum, 0n, 100n * PEN); - await monitor.poll(api); // block 101 anchored - pendulum.forkFrom = 102; // the source serves another chain from block 102 on... - for (let block = 102; block <= 105; block++) burn(pendulum, BigInt(block - 101), 999n * PEN, block); // ...with fabricated burns - pendulum.best = 106; // ...and holds another block 106 than the relay chain records - recorded = 106; - await monitor.poll(api); - assert.deepEqual(pages.map((page) => page.subject), ["RELAY ANCHOR UNAVAILABLE"]); - assert.match(pages[0].detail, /has not imported block 106, .*; no Pendulum block after 101 is ingested until a relay RPC anchors/); - assert.equal(persisted().lastPendulumBlock, 101); - assert.deepEqual(persisted().migrations.map((migration) => migration.nonce), ["0"]); - assert.deepEqual(persisted().relayAnchor, { number: 101, hash: pendulumHash({}, 101) }); -}); - -test("a Pendulum view a checker holds back pages as stalled, naming that checker", async (t) => { - const relayed = await relaySetup(t, { head: () => ({ number: 101, hash: pendulumHash({}, 101) }) }); - const crossChecked = await crossCheckSetup(t, { finalized: 101 }); - for (const [{ pendulum, api, monitor, pages }, checker] of [ - [relayed, "the relay RPCs \\(POLKADOT_WS\\)"], - [crossChecked, "the second node \\(PENDULUM_WS_CROSSCHECK\\)"], - ] as const) { - burn(pendulum, 0n, 100n * PEN); - pendulum.finalized = 130; // 29 blocks of 12 s past the checker's head - await monitor.poll(api); - assert.match( - pages.find((page) => page.subject === "PENDULUM SOURCE VIEW STALLED")?.detail ?? "", - new RegExp(`^verified Pendulum block 101 is 34\\ds old \\(the source finalized 130, not confirmed yet by ${checker}\\);.* — check ${checker} before`), - ); - } -}); - -test("a Base event whose burn the relay chain has not confirmed yet is not proven fabricated by a newer source head", async (t) => { - const { base, pendulum, api, monitor, subjects } = await relaySetup(t, { - head: () => ({ number: 101, hash: pendulumHash({}, 101) }), - env: { SOURCE_CLOCK_SKEW_MARGIN_SECONDS: "0" }, - }); - burn(pendulum, 0n, 100n * PEN); - burn(pendulum, 1n, 100n * PEN, 125); - pendulum.finalized = 130; - release(base, 1n, 100n * PEN); - await monitor.poll(api); - await new Promise((resolve) => setTimeout(resolve, 5)); - await monitor.poll(api); // the source's own head has passed the first sighting, the confirmed view has not - assert.ok(subjects().includes("UNVERIFIED BASE EVENT"), JSON.stringify(subjects())); - assert.ok(!subjects().includes("MIGRATION TUPLE VIOLATION"), JSON.stringify(subjects())); -}); - -test("a relay chain that records another hash at a height it anchored pages critical", async (t) => { - let recorded = pendulumHash({}, 101); - const { pendulum, api, monitor, pages, persisted } = await relaySetup(t, { - head: () => ({ number: 101, hash: recorded }), - }); - burn(pendulum, 0n, 100n * PEN); - await monitor.poll(api); - assert.deepEqual(persisted().relayAnchor, { number: 101, hash: pendulumHash({}, 101) }); - recorded = pendulumHash({ forkFrom: 0 }, 101); - await monitor.poll(api); - assert.match( - pages.find((page) => page.subject === "CRITICAL: PENDULUM SOURCE CONTRADICTED")?.detail ?? "", - /relay chain now records 0x0+10+65 as Pendulum block 101, where it recorded 0x0+65;/, - ); -}); - -test("a relay record below the block ingested last is checked down the source's chain", async (t) => { - for (const recorded of [pendulumHash({}, 100), pendulumHash({ forkFrom: 0 }, 100)]) { - let relayDown = true; - const { pendulum, api, monitor, pages, persisted } = await relaySetup(t, { - head: () => ({ number: 100, hash: recorded }), - failing: () => relayDown, - }); - burn(pendulum, 0n, 100n * PEN); - await monitor.poll(api); // ingested up to block 101 without the anchor - relayDown = false; - burn(pendulum, 1n, 100n * PEN, 102); - await monitor.poll(api); - if (recorded === pendulumHash({}, 100)) { - assert.deepEqual(pages.map((page) => page.subject), ["RELAY ANCHOR UNAVAILABLE"]); - assert.deepEqual(persisted().relayAnchor, { number: 100, hash: recorded }); - assert.equal(persisted().lastPendulumBlock, 101); // block 102 waits for a newer record - } else { - assert.match( - pages.find((page) => page.subject === "CRITICAL: PENDULUM SOURCE CONTRADICTED")?.detail ?? "", - /relay chain records 0x0+10+64 as Pendulum block 100, the source's chain has 0x0+64;/, - ); - assert.equal(persisted().lastPendulumBlock, 101); - } - } -}); - -test("a relay record far below the block ingested last is not walked down to", async (t) => { - const { pendulum, api, monitor, pages, persisted } = await relaySetup(t, { - head: () => ({ number: 150, hash: pendulumHash({}, 150) }), - env: { PENDULUM_START_BLOCK: "300" }, - }); - burn(pendulum, 0n, 100n * PEN, 300); - await monitor.poll(api); - assert.deepEqual(pages.map((page) => page.subject), ["RELAY ANCHOR UNAVAILABLE"]); - assert.match(pages[0].detail, /^relay RPC 1: the relay chain's Pendulum head 150 lies more than 100 blocks below block 299;/); - assert.equal(persisted().relayAnchor, undefined); - assert.equal(persisted().lastPendulumBlock, 299); -}); - -test("an unreadable relay chain warns, the source is used, and it turns critical after the grace", async (t) => { - const { pendulum, api, monitor, pages, persisted } = await relaySetup(t, { failing: () => true }); - burn(pendulum, 0n, 100n * PEN); - await monitor.poll(api); - assert.deepEqual(pages.map((page) => [page.subject, page.severity]), [["RELAY ANCHOR UNAVAILABLE", "WARNING"]]); - assert.match(pages[0].detail, /relay RPC 1: Error: relay RPC unavailable;/); - assert.equal(persisted().lastPendulumBlock, 101); - const late = await relaySetup(t, { failing: () => true, env: { RELAY_ANCHOR_GRACE_SECONDS: "0" } }); - burn(late.pendulum, 0n, 100n * PEN); - await new Promise((resolve) => setTimeout(resolve, 5)); - await late.monitor.poll(late.api); - assert.ok(late.subjects().includes("CRITICAL: RELAY ANCHOR STALE"), JSON.stringify(late.subjects())); -}); - -test("a source behind the relay chain is not anchored, its blocks wait and it pages, critical after the grace", async (t) => { - const ahead = (pendulum: Pendulum) => ({ number: pendulum.finalized + 1, hash: pendulumHash(pendulum, pendulum.finalized + 1) }); - const { pendulum, api, monitor, pages, persisted } = await relaySetup(t, { head: ahead }); - burn(pendulum, 0n, 100n * PEN); - await monitor.poll(api); - assert.deepEqual(pages.map((page) => page.subject), ["RELAY ANCHOR UNAVAILABLE"]); - assert.equal(persisted().relayAnchor, undefined); - assert.equal(persisted().lastPendulumBlock, 100); - const late = await relaySetup(t, { head: ahead, env: { RELAY_ANCHOR_GRACE_SECONDS: "0" } }); - await new Promise((resolve) => setTimeout(resolve, 5)); - await late.monitor.poll(late.api); - assert.match( - late.pages.find((page) => page.subject === "CRITICAL: RELAY ANCHOR STALE")?.detail ?? "", - /the source has not imported block 101, the relay chain's Pendulum head, yet/, - ); -}); - -test("the relay RPCs are tried in order", async (t) => { - let chain: Pendulum | undefined; - const relay = relayChain(() => ({ number: chain!.finalized, hash: pendulumHash({}, chain!.finalized) })); - const { pendulum, api, monitor, pages, persisted } = await setup(t, { - env: { POLKADOT_WS: "wss://down.test, wss://up.test" }, - prepare: (_base, pendulum) => (chain = pendulum), - connect: async (url) => (url === "wss://up.test" ? relay : assert.fail(`cannot reach ${url}`)), - }); - burn(pendulum, 0n, 100n * PEN); - await monitor.poll(api); - assert.equal(pages.length, 0, JSON.stringify(pages)); - assert.equal(persisted().relayAnchor?.number, 101); -}); - -/** A monitor with relay RPCs, in this order; each records `head` or, without one, is down. */ -async function relayRpcsSetup(t: { after(fn: () => void): void }, heads: Array<((pendulum: Pendulum) => Head) | undefined>) { - let chain: Pendulum | undefined; - const urls = heads.map((_head, index) => `wss://relay-${index + 1}.test`); - const relays = heads.map((head) => relayChain(() => head!(chain!), () => !head)); - return setup(t, { - env: { POLKADOT_WS: urls.join(",") }, - prepare: (_base, pendulum) => (chain = pendulum), - connect: async (url) => relays[urls.indexOf(url)] ?? assert.fail(url), - }); -} -const honestHead = (pendulum: Pendulum) => ({ number: pendulum.finalized, hash: pendulumHash({}, pendulum.finalized) }); -const forkedHead = (pendulum: Pendulum) => ({ number: pendulum.finalized, hash: pendulumHash({ forkFrom: 0 }, pendulum.finalized) }); - -test("a relay contradiction the next relay RPC outvotes warns, and that RPC's record anchors the source", async (t) => { - const { pendulum, api, monitor, pages, persisted } = await relayRpcsSetup(t, [forkedHead, honestHead]); - burn(pendulum, 0n, 100n * PEN); - await monitor.poll(api); - assert.deepEqual(pages.map((page) => [page.subject, page.severity]), [["RELAY RPCS DISAGREE", "WARNING"]]); - assert.match( - pages[0].detail, - /^relay RPC 1 reports that the relay chain records 0x0+10+65 as Pendulum block 101, .*; relay RPC 2 records 0x0+65 as Pendulum block 101, on the source's chain, .* check both$/, - ); - assert.deepEqual(persisted().relayAnchor, { number: 101, hash: pendulumHash({}, 101) }); - assert.equal(persisted().lastPendulumBlock, 101); -}); - -test("a relay record below a contradicting one does not outvote it, and anchors the source up to its own height", async (t) => { - const below = (pendulum: Pendulum) => ({ number: pendulum.finalized - 1, hash: pendulumHash({}, pendulum.finalized - 1) }); - for (const [heads, contradicting, lower] of [[[forkedHead, below], 1, 2], [[below, forkedHead], 2, 1]] as const) { - const { pendulum, api, monitor, pages, persisted } = await relayRpcsSetup(t, [...heads]); - burn(pendulum, 0n, 100n * PEN, 102); - await monitor.poll(api); - assert.deepEqual(pages.map((page) => page.subject), ["RELAY ANCHOR UNAVAILABLE"]); - assert.match( - pages[0].detail, - new RegExp( - `^relay RPC ${contradicting} reports that .* no other relay RPC confirms or outvotes it \\(relay RPC ${lower}: ` + - "its Pendulum head 101 lies below the contradicting record\\); no Pendulum block after 101 is ingested", - ), - ); - assert.deepEqual(persisted().relayAnchor, { number: 101, hash: pendulumHash({}, 101) }); - assert.equal(persisted().lastPendulumBlock, 101); - } -}); - -/** A relay RPC whose record stays at the source's block 101. */ -const stuckHead = () => ({ number: 101, hash: pendulumHash({}, 101) }); - -test("a relay contradiction that only a stuck relay RPC answers stays unconfirmed", async (t) => { - const { pendulum, api, monitor, pages, persisted } = await relayRpcsSetup(t, [honestHead, stuckHead]); - burn(pendulum, 0n, 100n * PEN); - await monitor.poll(api); // relay RPC 1 anchors block 101 - pendulum.forkFrom = 102; // the source moves to another chain... - burn(pendulum, 1n, 999n * PEN, 102); // ...with a fabricated burn, which relay RPC 1 contradicts - await monitor.poll(api); - assert.deepEqual(pages.map((page) => page.subject), ["RELAY ANCHOR UNAVAILABLE"]); - assert.match( - pages[0].detail, - /^relay RPC 1 reports that the relay chain records 0x0+66 as Pendulum block 102, the source's chain has 0x0+10+66, and no other relay RPC confirms or outvotes it \(relay RPC 2: the relay chain's Pendulum head 101 is not newer\);/, - ); - assert.deepEqual(persisted().relayAnchor, { number: 101, hash: pendulumHash({}, 101) }); - assert.equal(persisted().lastPendulumBlock, 101); - assert.deepEqual(persisted().migrations.map((migration) => migration.nonce), ["0"]); -}); - -test("a relay RPC whose record does not anchor the source leaves it to the next one", async (t) => { - const { pendulum, api, monitor, pages, persisted } = await relayRpcsSetup(t, [stuckHead, honestHead]); - burn(pendulum, 0n, 100n * PEN); - await monitor.poll(api); // relay RPC 1 anchors block 101 - burn(pendulum, 1n, 100n * PEN, 102); - await monitor.poll(api); // relay RPC 1 stays there - assert.equal(pages.length, 0, JSON.stringify(pages)); - assert.deepEqual(persisted().relayAnchor, { number: 102, hash: pendulumHash({}, 102) }); - assert.equal(persisted().lastPendulumBlock, 102); -}); - -test("a relay RPC whose record lags the source's head leaves the newer blocks to the next one", async (t) => { - const lagging = (pendulum: Pendulum) => ({ number: pendulum.finalized - 70, hash: pendulumHash({}, pendulum.finalized - 70) }); - const { base, pendulum, api, monitor, pages, persisted } = await relayRpcsSetup(t, [lagging, honestHead]); - burn(pendulum, 0n, 100n * PEN, 190); - pendulum.finalized = 200; - release(base, 0n, 100n * PEN); - await monitor.poll(api); - assert.equal(pages.length, 0, JSON.stringify(pages)); - assert.deepEqual(persisted().relayAnchor, { number: 200, hash: pendulumHash({}, 200) }); - assert.equal(persisted().lastPendulumBlock, 200); - assert.deepEqual(persisted().migrations, []); - assert.equal(persisted().baseFromBlock, "1011"); -}); - -test("a relay contradiction the next relay RPC confirms pages critical", async (t) => { - const { pendulum, api, monitor, pages, persisted } = await relayRpcsSetup(t, [forkedHead, forkedHead]); - burn(pendulum, 0n, 100n * PEN); - await monitor.poll(api); - assert.ok(pages.some((page) => page.subject === "CRITICAL: PENDULUM SOURCE CONTRADICTED"), JSON.stringify(pages)); - assert.equal(persisted().lastPendulumBlock, 100); -}); - -test("a relay contradiction above the source's head is confirmed by a later relay RPC, though a record between anchors", async (t) => { - const above = () => ({ number: 102, hash: pendulumHash({}, 102) }); - const onSourceChain = (pendulum: Pendulum) => ({ number: 101, hash: pendulumHash(pendulum, 101) }); - const { pendulum, api, monitor, pages, persisted } = await relayRpcsSetup(t, [above, onSourceChain, above]); - burn(pendulum, 0n, 100n * PEN); - pendulum.forkFrom = 101; // its finalized block 101 is on another chain... - pendulum.canonicalBlockHashes = true; // ...though its best chain holds the relay chain's block 102 - pendulum.best = 102; - await monitor.poll(api); - assert.match( - pages.find((page) => page.subject === "CRITICAL: PENDULUM SOURCE CONTRADICTED")?.detail ?? "", - /records 0x0+66 as Pendulum block 102, whose chain has 0x0+65 at block 101, where the source has 0x0+10+65;/, - ); - assert.equal(persisted().lastPendulumBlock, 100); -}); - -test("a relay contradiction no other relay RPC can confirm warns, and the source's blocks wait", async (t) => { - const { pendulum, api, monitor, pages, persisted } = await relayRpcsSetup(t, [forkedHead, undefined]); - burn(pendulum, 0n, 100n * PEN); - await monitor.poll(api); - assert.deepEqual(pages.map((page) => page.subject), ["RELAY ANCHOR UNAVAILABLE"]); - assert.match(pages[0].detail, /and no other relay RPC confirms or outvotes it \(relay RPC 2: Error: relay RPC unavailable\)/); - assert.equal(persisted().lastPendulumBlock, 100); -}); - -test("a relay RPC whose finalized relay block is stale counts as not answering, and the second node still caps the source", async (t) => { - let relayAgeMs = 0; - const { base, pendulum, api, monitor, pages, persisted, own } = await crossCheckSetup(t, { finalized: 101 }, [ - relayChain(stuckHead, () => false, () => relayAgeMs), - relayChain(stuckHead, () => true), - ]); - burn(pendulum, 0n, 100n * PEN); - await monitor.poll(api); // relay RPC 1 anchors block 101 - assert.deepEqual(persisted().relayAnchor, { number: 101, hash: pendulumHash({}, 101) }); - relayAgeMs = 400_000; // relay RPC 1 stops following the relay chain at that record; relay RPC 2 is down - burn(pendulum, 1n, 100n * PEN, 103); - pendulum.finalized = 104; - own.finalized = 103; - release(base, 1n, 100n * PEN, 1_015n); - base.safe = 1_020n; - await monitor.poll(api); - assert.deepEqual(pages.map((page) => [page.subject, page.severity]), [["RELAY ANCHOR UNAVAILABLE", "WARNING"]]); - assert.match( - pages[0].detail, - /^the relay chain cannot be read: relay RPC 1: its finalized relay block is 400s old; relay RPC 2: Error: relay RPC unavailable; the Pendulum source is used without the relay-chain anchor/, - ); - assert.equal(persisted().lastPendulumBlock, 103); - assert.deepEqual(persisted().migrations.map((migration) => migration.nonce), ["0"]); - assert.equal(persisted().baseFromBlock, "1021"); -}); - -test("a Base event whose nonce the source view passed without pauses before the grace, as fabricated", async (t) => { - const { base, pendulum, api, monitor, pages, subjects } = await setup(t, { env: { SOURCE_CLOCK_SKEW_MARGIN_SECONDS: "0" } }); - burn(pendulum, 0n, 100n * PEN); - pendulum.headAgeMs = 1_000; // the view trails the first sighting - release(base, 7n, 5_000n * PEN); // no burn behind it - await monitor.poll(api); - assert.ok(subjects().includes("UNVERIFIED BASE EVENT"), JSON.stringify(subjects())); - assert.ok(!subjects().includes("MIGRATION TUPLE VIOLATION"), JSON.stringify(subjects())); - pendulum.headAgeMs = 0; - await new Promise((resolve) => setTimeout(resolve, 5)); - await monitor.poll(api); // the view has passed the first sighting, still without nonce 7 - assert.match( - pages.find((page) => page.subject === "MIGRATION TUPLE VIOLATION")?.detail ?? "", - /nonce 7 .*source view has passed the event's timestamp — fabricated/, - ); -}); - -test("while Pendulum cannot be read, a Base event with an unknown nonce pauses after the grace", async (t) => { - const { base, pendulum, api, monitor, subjects } = await setup(t, { env: { UNMATCHED_EVENT_GRACE_SECONDS: "0" } }); - pendulum.failure = new Error("node unreachable"); - release(base, 7n, 5_000n * PEN); // no burn behind it - await monitor.poll(api); - assert.ok(subjects().includes("UNVERIFIED BASE EVENT"), JSON.stringify(subjects())); - assert.ok(!subjects().includes("MIGRATION TUPLE VIOLATION"), JSON.stringify(subjects())); - await new Promise((resolve) => setTimeout(resolve, 5)); - await monitor.poll(api); // the grace has run out - assert.ok(subjects().includes("MIGRATION TUPLE VIOLATION"), JSON.stringify(subjects())); -}); - -test("while Pendulum cannot be read, a mismatching release of a known nonce pauses at once", async (t) => { - const { base, pendulum, api, monitor, pages } = await setup(t); - burn(pendulum, 0n, 100n * PEN); - await monitor.poll(api); - pendulum.failure = new Error("node unreachable"); - const tokenAmount = 100n * PEN * CF; - const released = { nonce: 0n, recipient: TOKEN, palletAmount: 100n * PEN, tokenAmount }; - base.logs.push({ address: VAULT, eventName: "Released", args: released, blockNumber: 1_012n }); - base.totalReleased += tokenAmount; - base.vaultBalance -= tokenAmount; - base.safe = 1_020n; - await monitor.poll(api); - assert.match(pages.find((page) => page.subject === "MIGRATION TUPLE VIOLATION")?.detail ?? "", /recipient/); - assert.ok(pages.some((page) => page.subject === "AUTO-PAUSE UNAVAILABLE"), JSON.stringify(pages)); // the pause was asked for -}); - -test("a tuple violation whose pause is not confirmed is found again in the next poll", async (t) => { - const { base, pendulum, api, monitor, pages, persisted } = await setup(t); - burn(pendulum, 0n, 100n * PEN); - await monitor.poll(api); - const tokenAmount = 100n * PEN * CF; // a release of nonce 0 to another recipient - base.logs.push({ address: VAULT, eventName: "Released", args: { nonce: 0n, recipient: TOKEN, palletAmount: 100n * PEN, tokenAmount }, blockNumber: 1_015n }); - base.totalReleased += tokenAmount; - base.vaultBalance -= tokenAmount; - base.safe = 1_020n; - await monitor.poll(api); - await monitor.poll(api); - assert.equal(pages.filter((page) => page.subject === "MIGRATION TUPLE VIOLATION").length, 2, JSON.stringify(pages)); - assert.equal(persisted().baseFromBlock, "1011"); -}); - -test("a release the source never shows pauses after the grace, with no approval in view", async (t) => { - const { base, pendulum, api, monitor, pages } = await setup(t, { env: { UNMATCHED_EVENT_GRACE_SECONDS: "0" } }); - pendulum.failure = new Error("node unreachable"); - const tokenAmount = 5n * PEN * CF; - base.logs.push({ address: VAULT, eventName: "Released", args: { nonce: 7n, recipient: RECIPIENT, palletAmount: 5n * PEN, tokenAmount }, blockNumber: 1_005n }); - base.totalReleased += tokenAmount; - base.vaultBalance -= tokenAmount; - await monitor.poll(api); - await new Promise((resolve) => setTimeout(resolve, 5)); - await monitor.poll(api); // the grace has run out - assert.match(pages.find((page) => page.subject === "MIGRATION TUPLE VIOLATION")?.detail ?? "", /^Released at Base block 1005: nonce 7 has no finalized/); -}); - -test("a second release of one nonce is a violation", async (t) => { - const { base, pendulum, api, monitor, pages } = await setup(t); - burn(pendulum, 0n, 100n * PEN); - release(base, 0n, 100n * PEN, 1_004n); - release(base, 0n, 100n * PEN, 1_006n); - await monitor.poll(api); - assert.match(pages.find((page) => page.subject === "MIGRATION TUPLE VIOLATION")?.detail ?? "", /duplicate Released event for nonce 0/); -}); - -test("vault events past the confirmed Base block do not trip the balance check", async (t) => { - const { base, pendulum, api, monitor, pages } = await setup(t); - burn(pendulum, 0n, 100n * PEN); - release(base, 0n, 100n * PEN, 1_015n); // after the safe block 1010 - await monitor.poll(api); - assert.equal(pages.length, 0, JSON.stringify(pages)); -}); - -test("logs that miss a release hold the Base cursor and page until a read accounts for the totals", async (t) => { - const { base, pendulum, api, monitor, pages, persisted } = await setup(t); - burn(pendulum, 0n, 100n * PEN); - release(base, 0n, 100n * PEN, 1_005n); - base.logsHead = 1_003n; // logs from a backend that has not reached the release - await monitor.poll(api); - await until(() => pages.some((page) => page.subject === "BASE LOG SCAN INCOMPLETE"), "the incomplete scan page"); - assert.match( - pages.find((page) => page.subject === "BASE LOG SCAN INCOMPLETE")!.detail, - /in Base blocks 1000-1010 the Released events add up to 0 .* totalReleased grew by 100000000000000000000/, - ); - assert.equal(persisted().baseFromBlock, "1000"); - assert.deepEqual(persisted().migrations.map((migration) => migration.nonce), ["0"]); - base.logsHead = undefined; - await monitor.poll(api); - assert.equal(persisted().baseFromBlock, "1011"); - assert.deepEqual(persisted().migrations, []); -}); - -test("logs that miss a sweep hold the Base cursor too", async (t) => { - const { base, api, monitor, pages, persisted } = await setup(t); - base.logs.push({ address: VAULT, eventName: "RemainderSwept", args: { to: TOKEN, amount: 5n }, blockNumber: 1_005n }); - base.totalSwept += 5n; - base.vaultBalance -= 5n; - base.logsHead = 1_003n; - await monitor.poll(api); - await until(() => pages.some((page) => page.subject === "BASE LOG SCAN INCOMPLETE"), "the incomplete scan page"); - assert.match(pages.find((page) => page.subject === "BASE LOG SCAN INCOMPLETE")!.detail, /RemainderSwept events to 0, .* totalSwept by 5/); - assert.equal(persisted().baseFromBlock, "1000"); - base.logsHead = undefined; - await monitor.poll(api); - assert.equal(persisted().baseFromBlock, "1011"); -}); - -test("a Base log range held past the grace pages critical", async (t) => { - const { base, pendulum, api, monitor, pages } = await setup(t, { env: { UNMATCHED_EVENT_GRACE_SECONDS: "0" } }); - burn(pendulum, 0n, 100n * PEN); - release(base, 0n, 100n * PEN, 1_005n); - base.logsHead = 1_003n; - await monitor.poll(api); - await new Promise((resolve) => setTimeout(resolve, 5)); - await monitor.poll(api); - const held = () => pages.filter((page) => page.subject.endsWith("BASE LOG SCAN INCOMPLETE")); - await until(() => held().length === 2, "two incomplete scan pages"); - assert.deepEqual(held().map((page) => [page.subject, page.severity]), [ - ["BASE LOG SCAN INCOMPLETE", "WARNING"], - ["CRITICAL: BASE LOG SCAN INCOMPLETE", "CRITICAL"], - ]); - assert.match(held()[1].detail, /stays at block 1000 \(for \d+s\): no later Base event is matched against its burn/); - base.logsHead = undefined; - await monitor.poll(api); // the range reads in full - burn(pendulum, 1n, 100n * PEN, 102); - release(base, 1n, 100n * PEN, 1_015n); - base.logsHead = 1_012n; - base.safe = 1_020n; - await monitor.poll(api); // a new hold starts as a warning - await until(() => held().length === 3, "the next incomplete scan page"); - assert.equal(held()[2].subject, "BASE LOG SCAN INCOMPLETE"); -}); - -test("the conservation check still runs while a Base log range is held", async (t) => { - const { base, pendulum, api, monitor, subjects } = await setup(t); - burn(pendulum, 0n, 100n * PEN); - release(base, 0n, 100n * PEN, 1_005n); - release(base, 0n, 100n * PEN, 1_007n); // a second release lifts totalReleased past TotalMigrated... - base.logsHead = 1_006n; // ...while the log backend drops its block - await monitor.poll(api); - assert.ok(subjects().includes("CONSERVATION VIOLATION"), JSON.stringify(subjects())); - assert.ok(subjects().includes("AUTO-PAUSE UNAVAILABLE"), JSON.stringify(subjects())); // the pause was asked for -}); - -test("the conservation check waits while a Base event's burn is not in the source view yet", async (t) => { - const { base, api, monitor, subjects } = await setup(t); - release(base, 0n, 100n * PEN); // the burn is not in the lagging source view yet - await monitor.poll(api); - assert.deepEqual([...new Set(subjects())], ["UNVERIFIED BASE EVENT"]); -}); - -test("logs that miss an event once and then account for the totals hold nothing and page nothing", async (t) => { - const { base, pendulum, api, monitor, subjects, persisted } = await setup(t); - burn(pendulum, 0n, 100n * PEN); - release(base, 0n, 100n * PEN, 1_005n); - const released = toEventSelector("Released(uint64,address,uint256,uint256)"); // only reconciliation asks for it - let reads = 0; - base.logsHeadFor = (topics) => (topics.includes(released) && reads++ === 0 ? 1_003n : undefined); - await monitor.poll(api); - assert.equal(reads, 2); // read once more - assert.equal(persisted().baseFromBlock, "1011"); - assert.deepEqual(persisted().migrations, []); - await new Promise((resolve) => setTimeout(resolve, 50)); // an incomplete scan page is sent without waiting - assert.deepEqual(subjects(), []); -}); - -test("a vault total that cannot be read where the vault has code fails the poll, not counting as zero", async (t) => { - const { base, pendulum, api, monitor, subjects } = await setup(t, { prepare: (base) => (base.deployedAt = 900n) }); - burn(pendulum, 0n, 100n * PEN); - release(base, 0n, 100n * PEN, 950n); // before the first Base block the monitor scans - const totalReleased = toFunctionSelector("totalReleased()"); - base.failing = (method, params) => - method === "eth_call" && (params[0] as { data: string }).data === totalReleased && params[1] === toHex(999n); - await monitor.poll(api); - await new Promise((resolve) => setTimeout(resolve, 50)); // an incomplete scan page is sent without waiting - assert.ok(subjects().includes("monitor check failed"), JSON.stringify(subjects())); - assert.ok(!subjects().includes("BASE LOG SCAN INCOMPLETE"), JSON.stringify(subjects())); -}); - -test("releases and sweeps in the first and the last block of a range account for the totals", async (t) => { - const { base, pendulum, api, monitor, subjects, persisted } = await setup(t); - burn(pendulum, 0n, 100n * PEN); - burn(pendulum, 1n, 100n * PEN); - release(base, 0n, 100n * PEN, 1_000n); - release(base, 1n, 100n * PEN, 1_010n); - for (const blockNumber of [1_000n, 1_010n]) { - base.logs.push({ address: VAULT, eventName: "RemainderSwept", args: { to: TOKEN, amount: 5n }, blockNumber }); - base.totalSwept += 5n; - base.vaultBalance -= 5n; - } - await monitor.poll(api); - assert.equal(persisted().baseFromBlock, "1011"); - assert.deepEqual(persisted().migrations, []); - await new Promise((resolve) => setTimeout(resolve, 50)); // an incomplete scan page is sent without waiting - assert.deepEqual([...new Set(subjects())], ["VAULT ADMIN EVENT"]); // the sweeps -}); - -test("the totals count as zero before the vault existed", async (t) => { - const { base, pendulum, api, monitor, subjects, persisted } = await setup(t, { prepare: (base) => (base.deployedAt = 1_000n) }); - burn(pendulum, 0n, 100n * PEN); - release(base, 0n, 100n * PEN, 1_005n); - await monitor.poll(api); - assert.deepEqual(subjects(), []); - assert.equal(persisted().baseFromBlock, "1011"); -}); - -/** An approval of `nonce` for `recipient` from attestor `attestor`. */ -function approve(base: Base, nonce: bigint, recipient: string, attestor: string, blockNumber: bigint): void { - base.logs.push({ address: VAULT, eventName: "Approved", args: { nonce, recipient, palletAmount: 100n * PEN, attestor }, blockNumber }); -} - -test("the release of a nonce forgets the approvals of it that disagreed", async (t) => { - const { base, pendulum, api, monitor, persisted } = await setup(t); - burn(pendulum, 0n, 100n * PEN); - approve(base, 0n, TOKEN, ATTESTORS[0], 1_004n); - await monitor.poll(api); - assert.equal(persisted().mismatchedApprovals.length, 1); - release(base, 0n, 100n * PEN, 1_012n); - base.safe = 1_020n; - await monitor.poll(api); - assert.deepEqual(persisted().mismatchedApprovals, []); -}); - -test("one attestor approving against the burn pages critical; a second one pauses, also polls later", async (t) => { - const { base, pendulum, api, monitor, restart, pages, subjects } = await setup(t); - burn(pendulum, 0n, 100n * PEN); - await monitor.poll(api); - approve(base, 0n, TOKEN, ATTESTORS[0], 1_012n); - base.safe = 1_020n; - await monitor.poll(api); - const page = pages.find((page) => page.subject === "CRITICAL: MISMATCHED APPROVAL"); - assert.equal(page?.severity, "CRITICAL", JSON.stringify(pages)); - assert.match(page.detail, new RegExp(`by attestor ${ATTESTORS[0]}: nonce 0 recipient ${TOKEN}`, "i")); - assert.ok(!subjects().includes("MIGRATION TUPLE VIOLATION"), JSON.stringify(subjects())); - approve(base, 0n, RECIPIENT, ATTESTORS[1], 1_022n); // agrees with the burn - approve(base, 0n, "0x5555555555555555555555555555555555555555", ATTESTORS[0], 1_023n); // the same attestor again - base.safe = 1_030n; - await monitor.poll(api); - assert.ok(!subjects().includes("MIGRATION TUPLE VIOLATION"), JSON.stringify(subjects())); - const restarted = await restart(); - approve(base, 0n, TOKEN, ATTESTORS[2], 1_032n); - base.safe = 1_040n; - await restarted.poll(api); - assert.match( - pages.find((page) => page.subject === "MIGRATION TUPLE VIOLATION")?.detail ?? "", - new RegExp(`attestors ${ATTESTORS[0]}, ${ATTESTORS[2]} approved tuples for nonce 0`, "i"), - ); -}); - -test("at a threshold of two, one attestor approving against the burn pauses", async (t) => { - const { base, pendulum, api, monitor, pages, subjects } = await setup(t, { prepare: (base) => (base.threshold = 2n) }); - burn(pendulum, 0n, 100n * PEN); - approve(base, 0n, TOKEN, ATTESTORS[0], 1_005n); - await monitor.poll(api); - assert.match( - pages.find((page) => page.subject === "MIGRATION TUPLE VIOLATION")?.detail ?? "", - new RegExp(`; attestor ${ATTESTORS[0]} approved tuples for nonce 0`, "i"), - ); - assert.ok(!subjects().includes("CRITICAL: MISMATCHED APPROVAL"), JSON.stringify(subjects())); -}); - -test("auto-pause readiness is checked at startup and each problem pages", async (t) => { - const { startup } = await setup(t, { - env: { GUARDIAN_PRIVATE_KEY: GUARDIAN_KEY }, - prepare: (base) => { - base.chainId = 84532; - base.guardian = "0x5555555555555555555555555555555555555555"; - base.balances.clear(); - }, - }); - const detail = startup.find((page) => page.subject === "AUTO-PAUSE NOT READY")?.detail ?? ""; - assert.match(detail, /serves chain 84532, not BASE_CHAIN_ID 8453/); - assert.match(detail, new RegExp(`${GUARDIAN} is not the vault guardian 0x5{40}`)); - assert.match(detail, /holds 0 wei/); -}); - -test("the guardian balance floor rejects a negative value and keeps its default when empty", async (t) => { - await assert.rejects(setup(t, { env: { GUARDIAN_MIN_BALANCE_WEI: "-1" } }), /invalid GUARDIAN_MIN_BALANCE_WEI: -1/); - const { startup } = await setup(t, { - env: { GUARDIAN_PRIVATE_KEY: GUARDIAN_KEY, GUARDIAN_MIN_BALANCE_WEI: "" }, - prepare: (base) => base.balances.clear(), - }); - const detail = startup.find((page) => page.subject === "AUTO-PAUSE NOT READY")?.detail ?? ""; - assert.match(detail, /holds 0 wei, less than GUARDIAN_MIN_BALANCE_WEI 1000000000000000/); -}); - -test("auto-pause readiness is re-checked every poll", async (t) => { - const { base, api, monitor, startup, subjects } = await setup(t, { env: { GUARDIAN_PRIVATE_KEY: GUARDIAN_KEY } }); - assert.ok(!startup.some((page) => page.subject.startsWith("AUTO-PAUSE")), JSON.stringify(startup)); - base.guardian = "0x5555555555555555555555555555555555555555"; - await monitor.poll(api); - assert.deepEqual(subjects(), ["AUTO-PAUSE NOT READY"]); -}); - -test("a failed readiness read pages and the vault balance check still runs", async (t) => { - const { base, api, monitor, pages, subjects } = await setup(t, { env: { GUARDIAN_PRIVATE_KEY: GUARDIAN_KEY } }); - base.failing = (method) => method === "eth_getBalance"; - base.vaultBalance -= 1n; - base.paused = true; // so the auto-pause only confirms - await monitor.poll(api); - assert.deepEqual(sorted(subjects()), ["AUTO-PAUSE NOT READY", "VAULT BALANCE DEFICIT", "vault pause confirmed"]); - assert.match(pages.find((page) => page.subject === "AUTO-PAUSE NOT READY")?.detail ?? "", /readiness reads failed.*rate limited/s); -}); - -test("vault admin events page once each from the Base scan", async (t) => { - const { base, api, monitor, pages } = await setup(t); - const guardian = "0x5555555555555555555555555555555555555555"; - base.logs.push({ address: VAULT, eventName: "GuardianUpdated", args: { guardian }, blockNumber: 1_004n }); - base.logs.push({ address: VAULT, eventName: "CapsUpdated", args: { perReleaseCap: 5n, dailyCap: 7n }, blockNumber: 1_012n }); - await monitor.poll(api); - base.safe = 1_020n; - await monitor.poll(api); - const adminPages = pages.filter((page) => page.subject === "VAULT ADMIN EVENT").map((page) => page.detail); - assert.equal(adminPages.length, 2, JSON.stringify(pages)); - assert.match(adminPages[0], /^GuardianUpdated\(guardian=0x5{40}\) at Base block 1004/i); - assert.match(adminPages[1], /^CapsUpdated\(perReleaseCap=5, dailyCap=7\) at Base block 1012/); -}); - -test("a failed admin scan pages, every other check still runs and the scan resumes where it stopped", async (t) => { - const { base, pendulum, api, monitor, restart, subjects, persisted } = await setup(t, { - env: { GUARDIAN_PRIVATE_KEY: GUARDIAN_KEY }, - }); - burn(pendulum, 0n, 100n * PEN); - release(base, 0n, 100n * PEN); - base.logs.push({ address: VAULT, eventName: "GuardianUpdated", args: { guardian: GUARDIAN }, blockNumber: 1_004n }); - const guardianUpdated = toEventSelector("GuardianUpdated(address)"); // only the admin scan asks for it - base.failing = (method, params) => method === "eth_getLogs" && JSON.stringify(params).includes(guardianUpdated); - base.vaultBalance -= 1n; - base.paused = true; // so the auto-pause only confirms and the poll persists its state - await monitor.poll(api); - assert.deepEqual(sorted(subjects()), ["BASE ADMIN SCAN FAILED", "VAULT BALANCE DEFICIT", "vault pause confirmed"]); - assert.equal(persisted().baseFromBlock, "1011"); // reconciled although the admin scan failed - assert.equal(persisted().baseAdminFromBlock, "1000"); - base.failing = undefined; - base.vaultBalance += 1n; - await (await restart()).poll(api); - assert.ok(subjects().includes("VAULT ADMIN EVENT"), JSON.stringify(subjects())); - assert.equal(persisted().baseAdminFromBlock, "1011"); -}); - -test("an admin scan that keeps failing past the grace pages critical", async (t) => { - const { base, api, monitor, pages } = await setup(t, { env: { UNMATCHED_EVENT_GRACE_SECONDS: "0" } }); - const guardianUpdated = toEventSelector("GuardianUpdated(address)"); // only the admin scan asks for it - const failing = (method: string, params: unknown[]) => method === "eth_getLogs" && JSON.stringify(params).includes(guardianUpdated); - base.failing = failing; - await monitor.poll(api); - await new Promise((resolve) => setTimeout(resolve, 5)); - await monitor.poll(api); - const failed = () => pages.filter((page) => page.subject.endsWith("BASE ADMIN SCAN FAILED")); - assert.deepEqual(failed().map((page) => [page.subject, page.severity]), [ - ["BASE ADMIN SCAN FAILED", "WARNING"], - ["CRITICAL: BASE ADMIN SCAN FAILED", "CRITICAL"], - ]); - assert.match(failed()[1].detail, /\(failing for \d+s\)/); - base.failing = undefined; - await monitor.poll(api); // the scan reads again - base.failing = failing; - base.safe = 1_020n; // a new range for the admin scan - await monitor.poll(api); - assert.equal(failed()[2]?.subject, "BASE ADMIN SCAN FAILED"); // a new failure starts as a warning -}); - -test("a monitor without a guardian key says so at startup", async (t) => { - const { startup } = await setup(t); - assert.ok(startup.some((page) => page.subject === "AUTO-PAUSE DISABLED"), JSON.stringify(startup)); -}); - -test("a Base view that stops advancing pages", async (t) => { - const { base, api, monitor, pages } = await setup(t); - base.safeAgeSeconds = 601; - await monitor.poll(api); - assert.match(pages.find((page) => page.subject === "BASE VIEW STALLED")?.detail ?? "", /^safe block 1010 is 60\ds old/); -}); - -const UNREADABLE = "CRITICAL: Pendulum governance item cannot be read"; -const RISKY = "CRITICAL: Pendulum governance item mints, moves or unlocks balances, or changes the runtime"; -const pageFor = (pages: Page[], subject: string, label: string) => - pages.find((page) => page.subject === subject && page.detail.startsWith(label)); - -test("an undecodable governance item pages as unreadable and hides nothing after it", async (t) => { - const { pendulum, api, monitor, pages } = await setup(t); - pendulum.publicProps.push([5, inline("garbage")]); - pendulum.referenda.set(9, inline("mint")); - await monitor.poll(api); - assert.ok(pageFor(pages, UNREADABLE, "public proposal #5"), JSON.stringify(pages)); - assert.ok(pageFor(pages, RISKY, "referendum #9"), JSON.stringify(pages)); -}); - -test("an oversized governance call pages as unreadable without being fetched", async (t) => { - const { pendulum, api, monitor, pages } = await setup(t); - pendulum.publicProps.push([5, lookup(`0x${"ab".repeat(32)}`, 3_900_000)]); - pendulum.preimages.set(`0x${"ab".repeat(32)}:3900000`, "mint"); - await monitor.poll(api); - assert.match(pageFor(pages, UNREADABLE, "public proposal #5")?.detail ?? "", /3900000 bytes/); - assert.equal(pendulum.preimageReads, 0); -}); - -test("a governance call that cannot be read yet is re-checked and pages critical once readable", async (t) => { - const { pendulum, api, monitor, pages } = await setup(t); - const preimage = `0x${"cd".repeat(32)}`; - pendulum.referenda.set(9, lookup(preimage, 4)); - await monitor.poll(api); - assert.match(pageFor(pages, UNREADABLE, "referendum #9")?.detail ?? "", /not noted yet/); - pendulum.preimages.set(`${preimage}:4`, "mint"); - await monitor.poll(api); - assert.ok(pageFor(pages, RISKY, "referendum #9"), JSON.stringify(pages)); -}); - -test("a governance item whose read fails is retried and hides nothing after it", async (t) => { - const { pendulum, api, monitor, pages } = await setup(t); - const preimage = `0x${"aa".repeat(32)}`; - pendulum.publicProps.push([5, lookup(preimage, 4)]); - pendulum.preimages.set(`${preimage}:4`, "mint"); - pendulum.referenda.set(9, inline("mint")); - pendulum.preimageFailures = 1; - await monitor.poll(api); - assert.match(pageFor(pages, UNREADABLE, "public proposal #5")?.detail ?? "", /reading the call failed/); - assert.ok(pageFor(pages, RISKY, "referendum #9"), JSON.stringify(pages)); - await monitor.poll(api); - assert.ok(pageFor(pages, RISKY, "public proposal #5"), JSON.stringify(pages)); -}); - -test("a legacy call reference is read through the noted preimage's length", async (t) => { - const { pendulum, api, monitor, pages } = await setup(t); - const preimage = `0x${"ef".repeat(32)}`; - pendulum.referenda.set(9, legacy(preimage)); - pendulum.notedLengths.set(preimage, 4); - pendulum.preimages.set(`${preimage}:4`, "mint"); - await monitor.poll(api); - assert.ok(pageFor(pages, RISKY, "referendum #9"), JSON.stringify(pages)); -}); - -test("a legacy call reference to a preimage noted before the status migration is read too", async (t) => { - const { pendulum, api, monitor, pages } = await setup(t); - const preimage = `0x${"fe".repeat(32)}`; - pendulum.referenda.set(9, legacy(preimage)); - pendulum.oldNotedLengths.set(preimage, 4); - pendulum.preimages.set(`${preimage}:4`, "mint"); - await monitor.poll(api); - assert.ok(pageFor(pages, RISKY, "referendum #9"), JSON.stringify(pages)); -}); - -test("council and technical-committee motions are decoded with the same risk rules", async (t) => { - const { pendulum, api, monitor, pages } = await setup(t); - const [anchor, remark, huge] = ["0x01", "0x02", "0x03"].map((prefix) => prefix + "00".repeat(31)); - pendulum.motions.council.set(anchor, "setTreasuryDestination"); - pendulum.motions.technicalCommittee.set(remark, "remark"); - pendulum.motions.council.set(huge, "mint"); - pendulum.motionSizes.set(huge, 100_000); - await monitor.poll(api); - assert.ok(pageFor(pages, RISKY, `council motion ${anchor}`), JSON.stringify(pages)); - assert.ok(pageFor(pages, "PENDULUM GOVERNANCE ITEM", `technical committee motion ${remark}`), JSON.stringify(pages)); - assert.match(pageFor(pages, UNREADABLE, `council motion ${huge}`)?.detail ?? "", /100000 bytes/); - assert.ok(!pendulum.motionReads.has(huge)); -}); - -test("a motion or external proposal made again with the same call pages again", async (t) => { - const { pendulum, api, monitor, pages } = await setup(t); - const motion = `0x04${"00".repeat(31)}`; - pendulum.motions.council.set(motion, "remark"); - pendulum.motionIndices.set(motion, 3); - pendulum.external = inline("remark"); - await monitor.poll(api); - pendulum.motionIndices.set(motion, 4); // closed and proposed again between two polls - pendulum.referenda.set(9, inline("remark")); // the external proposal tabled, and proposed again between two polls - await monitor.poll(api); - pendulum.external = undefined; // tabled or vetoed - await monitor.poll(api); - pendulum.external = inline("remark"); // proposed again - await monitor.poll(api); - assert.equal(pages.filter((page) => page.detail.startsWith(`council motion ${motion}`)).length, 2, JSON.stringify(pages)); - assert.equal(pages.filter((page) => page.detail.startsWith("the council's external proposal")).length, 3, JSON.stringify(pages)); -}); - -const TREASURY_DESTINATION = "0x5555555555555555555555555555555555555555"; - -test("a treasury migration to any but the expected address pages and pauses", async (t) => { - const { base, pendulum, api, monitor, pages, persisted } = await setup(t, { - env: { EXPECTED_TREASURY_DESTINATION: TREASURY_DESTINATION, GUARDIAN_PRIVATE_KEY: GUARDIAN_KEY }, - }); - base.paused = true; // so the auto-pause only confirms - burn(pendulum, 0n, 100n * PEN, 101, { who: TREASURY_ACCOUNT, recipient: TREASURY_DESTINATION }); - burn(pendulum, 1n, 100n * PEN, 101, { recipient: TOKEN }); // anyone else may migrate anywhere - await monitor.poll(api); - assert.equal(pages.length, 0, JSON.stringify(pages)); - burn(pendulum, 2n, 7n * PEN, 102, { who: TREASURY_ACCOUNT, recipient: TOKEN }); - await monitor.poll(api); - const page = pages.find((page) => page.subject === "TREASURY DESTINATION VIOLATION"); - assert.equal(page?.severity, "CRITICAL", JSON.stringify(pages)); - assert.match(page.detail, new RegExp(`nonce 2 in finalized Pendulum block 102 pays 7000000000000 pallet units to ${TOKEN}`, "i")); - assert.ok(pages.some((page) => page.subject === "vault pause confirmed"), JSON.stringify(pages)); - assert.deepEqual(persisted().migrations.map((migration) => migration.nonce), ["0", "1", "2"]); -}); - -test("while the pause after a treasury violation is not confirmed, the monitor holds at its block", async (t) => { - const { pendulum, api, monitor, pages, subjects, persisted } = await setup(t, { - env: { EXPECTED_TREASURY_DESTINATION: TREASURY_DESTINATION }, - }); - burn(pendulum, 0n, 100n * PEN, 101, { who: TREASURY_ACCOUNT, recipient: TOKEN }); - await monitor.poll(api); - await monitor.poll(api); - const violations = pages.filter((page) => page.subject === "TREASURY DESTINATION VIOLATION"); - assert.equal(violations.length, 2); - assert.match(violations[1].detail, /; the monitor holds at that block until the vault is paused$/); - assert.equal(persisted().lastPendulumBlock, 100); - assert.ok(!subjects().includes("PENDULUM SOURCE UNREADABLE"), JSON.stringify(subjects())); // the source reads fine -}); - -test("without a guardian key, a vault paused by hand after a treasury violation lets the monitor move on", async (t) => { - const { base, pendulum, api, monitor, pages, persisted } = await setup(t, { - env: { EXPECTED_TREASURY_DESTINATION: TREASURY_DESTINATION }, - }); - burn(pendulum, 0n, 100n * PEN, 101, { who: TREASURY_ACCOUNT, recipient: TOKEN }); - await monitor.poll(api); - assert.equal(persisted().lastPendulumBlock, 100); - const pollSubjects = async () => { - const before = pages.length; - await monitor.poll(api); - return pages.slice(before).map((page) => page.subject); - }; - base.pausedFrom = 1_015n; // operators pause the vault by hand, after the confirmed Base block - let subjects = await pollSubjects(); - assert.ok(subjects.includes("AUTO-PAUSE PENDING FINALITY"), JSON.stringify(subjects)); - assert.ok(!subjects.includes("AUTO-PAUSE UNAVAILABLE"), JSON.stringify(subjects)); - assert.equal(persisted().lastPendulumBlock, 100); - base.safe = 1_020n; // the pause is confirmed - subjects = await pollSubjects(); - assert.ok(subjects.includes("vault pause confirmed"), JSON.stringify(subjects)); - assert.ok(!subjects.includes("AUTO-PAUSE UNAVAILABLE"), JSON.stringify(subjects)); - assert.equal(persisted().lastPendulumBlock, 101); - assert.deepEqual(persisted().migrations.map((migration) => migration.nonce), ["0"]); -}); - -test("admin events of the migration pallet page from the ingest", async (t) => { - const { pendulum, api, monitor, pages } = await setup(t); - pendulum.finalized = 101; - pendulum.events.set(101, [ - { section: "tokenMigration", method: "TreasuryDestinationSet", data: ["0x5555555555555555555555555555555555555555"] }, - { section: "tokenMigration", method: "MigrationPauseSet", data: [false] }, - ]); - await monitor.poll(api); - const adminPages = pages.filter((page) => page.subject === "MIGRATION PALLET ADMIN EVENT").map((page) => page.detail); - assert.equal(adminPages.length, 2, JSON.stringify(pages)); - assert.match(adminPages[0], /TreasuryDestinationSet\(0x5{40}\) in finalized Pendulum block 101/); - assert.match(adminPages[1], /MigrationPauseSet\(false\)/); -}); - -test("a treasury destination set to any but the expected address pages critical", async (t) => { - const { pendulum, api, monitor, pages } = await setup(t, { env: { EXPECTED_TREASURY_DESTINATION: TREASURY_DESTINATION } }); - pendulum.finalized = 102; - pendulum.events.set(101, [{ section: "tokenMigration", method: "TreasuryDestinationSet", data: [TREASURY_DESTINATION] }]); - pendulum.events.set(102, [{ section: "tokenMigration", method: "TreasuryDestinationSet", data: [TOKEN] }]); - await monitor.poll(api); - const set = pages.filter((page) => page.subject.endsWith("MIGRATION PALLET ADMIN EVENT")); - assert.deepEqual(set.map((page) => [page.subject, page.severity]), [ - ["MIGRATION PALLET ADMIN EVENT", "WARNING"], - ["CRITICAL: MIGRATION PALLET ADMIN EVENT", "CRITICAL"], - ]); - assert.match( - set[1].detail, - new RegExp(`^tokenMigration.TreasuryDestinationSet\\(${TOKEN}\\) in finalized Pendulum block 102: .* not to EXPECTED_TREASURY_DESTINATION ${TREASURY_DESTINATION}, `, "i"), - ); -}); - -test("the governance watch runs after the conservation checks", async (t) => { - const { base, pendulum, api, monitor, subjects } = await setup(t); - base.vaultBalance -= 1n; - pendulum.publicProps.push([5, inline("remark")]); - await monitor.poll(api); - assert.deepEqual(subjects().slice(2), ["PENDULUM GOVERNANCE ITEM"]); - assert.deepEqual(sorted(subjects().slice(0, 2)), ["AUTO-PAUSE UNAVAILABLE", "VAULT BALANCE DEFICIT"]); -}); - -test("stale migrations page one liveness summary per kind and poll", async (t) => { - const { base, pendulum, api, monitor, pages } = await setup(t, { env: { GRACE_SECONDS: "30" } }); - pendulum.headAgeMs = 60_000; - for (let nonce = 0n; nonce < 27n; nonce++) burn(pendulum, nonce, 100n * PEN); - for (const nonce of [25n, 26n]) { - const payload = keccak256(encodeAbiParameters( - [{ type: "uint64" }, { type: "address" }, { type: "uint256" }], - [nonce, RECIPIENT, 100n * PEN], - )); - base.approvals.set(payload, 3n); - } - await monitor.poll(api); - await monitor.poll(api); - const liveness = pages.filter((page) => page.subject.startsWith("LIVENESS")); - assert.equal(liveness.length, 2, JSON.stringify(liveness)); - assert.match(pageFor(liveness, "LIVENESS: migration below approval quorum", "25 ")!.detail, /nonce 0, 1, .*, 9 and 15 more\)/); - assert.match(pageFor(liveness, "LIVENESS: quorum reached but not released", "2 ")!.detail, /\(nonce 25, 26\)/); -}); - -test("first sightings of unmatched events do not delay the pause", async (t) => { - const events: string[] = []; - let violationArrived = () => {}; - const arrived = new Promise((resolve) => (violationArrived = resolve)); - const { base, pendulum, api, monitor } = await setup(t, { - webhook: async ({ subject }) => { - if (subject === "MIGRATION TUPLE VIOLATION") { - events.push("violation paged"); - violationArrived(); - } - if (subject === "UNVERIFIED BASE EVENT") { - events.push("first sighting paged"); - await Promise.race([arrived, new Promise((resolve) => setTimeout(resolve, 3_000).unref())]); - events.push("first sighting answered"); - } - return 200; - }, - }); - burn(pendulum, 0n, 100n * PEN); - base.logs.push({ - address: VAULT, - eventName: "Approved", - args: { nonce: 5n, recipient: RECIPIENT, palletAmount: 100n * PEN, attestor: ATTESTORS[0] }, - blockNumber: 1_005n, - }); - base.logs.push({ - address: VAULT, - eventName: "Released", - args: { nonce: 0n, recipient: RECIPIENT, palletAmount: 100n * PEN, tokenAmount: 1n }, - blockNumber: 1_006n, - }); - base.totalReleased += 1n; - base.vaultBalance -= 1n; - await monitor.poll(api); - await until(() => events.includes("first sighting answered"), "the first sighting's answer", 5_000); - assert.deepEqual(events, ["first sighting paged", "violation paged", "first sighting answered"]); -}); - -test("after a violation the Base cursor stays on events still awaiting their source", async (t) => { - const { base, pendulum, api, monitor, subjects, persisted } = await setup(t, { - env: { UNMATCHED_EVENT_GRACE_SECONDS: "0", GUARDIAN_PRIVATE_KEY: GUARDIAN_KEY }, - }); - base.paused = true; // so the pause is confirmed and the poll persists its state - release(base, 0n, 100n * PEN); // the burn is not in the lagging source view yet - await monitor.poll(api); - await new Promise((resolve) => setTimeout(resolve, 5)); - await monitor.poll(api); // the grace has run out: a cannot-verify violation - assert.ok(subjects().includes("MIGRATION TUPLE VIOLATION"), JSON.stringify(subjects())); - assert.equal(persisted().baseFromBlock, "1000"); - burn(pendulum, 0n, 100n * PEN); // the source view catches up - await monitor.poll(api); - assert.deepEqual(persisted().migrations, []); - assert.equal(persisted().baseFromBlock, "1011"); -}); - -test("the conservation check fires only beyond an acknowledged loss", async (t) => { - const { base, api, monitor, subjects } = await setup(t, { env: { ACKNOWLEDGED_RELEASE_EXCESS: "1000" } }); - base.totalReleased = 1_000n; // released without a burn, quantified and accepted - base.vaultBalance = SUPPLY - 1_000n; - await monitor.poll(api); - assert.deepEqual(subjects(), []); - base.totalReleased += 1n; - base.vaultBalance -= 1n; - await monitor.poll(api); - assert.ok(subjects().includes("CONSERVATION VIOLATION"), JSON.stringify(subjects())); -}); - -/** A timelock operation scheduling the vault's `pause()`, ready in 48 h. */ -function schedule(base: Base, id: Hex, blockNumber = 1_005n): void { - const call = { target: VAULT, value: 0n, data: "0x8456cb59", predecessor: toHex(0, { size: 32 }), delay: 172_800n }; - base.logs.push({ address: TIMELOCK, eventName: "CallScheduled", args: { id, index: 0n, ...call }, blockNumber }); - base.readyAt.set(id, BigInt(Math.floor(Date.now() / 1000) + 172_800)); -} - -test("a scheduled operation with an extreme delay pages and hides nothing after it", async (t) => { - const { base, api, monitor, pages } = await setup(t, { env: { TIMELOCK_ADDRESS: TIMELOCK } }); - const [extreme, ordinary] = [toHex(1, { size: 32 }), toHex(2, { size: 32 })]; - schedule(base, extreme, 1_004n); - base.readyAt.set(extreme, 10n ** 20n); - schedule(base, ordinary, 1_006n); - await monitor.poll(api); - const scheduled = pages.filter((page) => page.subject === "GOVERNANCE OPERATION SCHEDULED").map((page) => page.detail); - assert.equal(scheduled.length, 2, JSON.stringify(pages)); - assert.ok(scheduled.some((detail) => detail.startsWith(`operation ${ordinary}`)), JSON.stringify(scheduled)); - assert.ok(!pages.some((page) => page.subject === "BASE ADMIN SCAN FAILED"), JSON.stringify(pages)); -}); - -test("an operation cancelled and scheduled again with the same id pages again", async (t) => { - const { base, api, monitor, pages } = await setup(t, { env: { TIMELOCK_ADDRESS: TIMELOCK } }); - const id = toHex(7, { size: 32 }); - schedule(base, id, 1_005n); - await monitor.poll(api); - base.readyAt.set(id, 0n); // cancelled - schedule(base, id, 1_015n); - base.safe = 1_020n; - await monitor.poll(api); - const scheduled = pages.filter((page) => page.subject === "GOVERNANCE OPERATION SCHEDULED"); - assert.equal(scheduled.length, 2, JSON.stringify(pages)); -}); - -test("once-only pages the webhook rejects are sent again on the next poll", async (t) => { - const failedOnce = new Set(); - const { base, pendulum, api, monitor, pages } = await setup(t, { - env: { TIMELOCK_ADDRESS: TIMELOCK }, - webhook: ({ subject }) => { - if (failedOnce.has(subject)) return 200; - failedOnce.add(subject); - return 500; - }, - }); - schedule(base, toHex(1, { size: 32 })); - pendulum.publicProps.push([7, inline("remark")]); - await monitor.poll(api); - await monitor.poll(api); - const delivered = pages.filter((page) => page.status === 200).map((page) => page.subject); - assert.ok(delivered.includes("GOVERNANCE OPERATION SCHEDULED"), `delivered: ${delivered}`); - assert.ok(delivered.includes("PENDULUM GOVERNANCE ITEM"), `delivered: ${delivered}`); - assert.ok(delivered.includes("AUTO-PAUSE DISABLED"), `delivered: ${delivered}`); // first sent at startup -}); diff --git a/monitor/src/main.ts b/monitor/src/main.ts deleted file mode 100644 index 30c336c04..000000000 --- a/monitor/src/main.ts +++ /dev/null @@ -1,2110 +0,0 @@ -/** - * Independent PEN migration invariant monitor (PRD §6.5). - * - * The monitor owns durable cursors on both chains. Finalized Pendulum - * MigrationInitiated events are the canonical nonce/recipient/amount records; - * every safe Base Approved and Released event must match one of those records - * exactly. Aggregate conservation remains a second, independent defence. - */ - -import { ApiPromise, WsProvider } from "@polkadot/api"; -import { - createPublicClient, - createWalletClient, - defineChain, - encodeAbiParameters, - http, - keccak256, - parseAbi, - toFunctionSelector, -} from "viem"; -import { privateKeyToAccount } from "viem/accounts"; -import { - approvalMismatch, - blockRanges, - type CallNode, - chunks, - describeCall, - describeTimelockOperation, - disagreeingAttestors, - isStale, - provablyUnsourced, - releaseMismatch, - releasedExceedsMigrated, - riskyCalls, - type ScheduledCall, - type Severity, - shouldAwaitSource, - slackText, - toCallNode, - type MigrationTuple, - vaultConservationDeficit, -} from "./checks.js"; -import { - assertMonitorStateIdentity, - loadMonitorState, - saveMonitorState, - type PersistedMonitorState, -} from "./state.js"; - -const MULTICALL3_ADDRESS = "0xcA11bde05977b3631167028862bE2a173976CA11" as const; - -const vaultAbi = [ - { type: "function", name: "totalReleased", stateMutability: "view", inputs: [], outputs: [{ type: "uint256" }] }, - { type: "function", name: "totalSwept", stateMutability: "view", inputs: [], outputs: [{ type: "uint256" }] }, - { type: "function", name: "conversionFactor", stateMutability: "view", inputs: [], outputs: [{ type: "uint256" }] }, - { type: "function", name: "token", stateMutability: "view", inputs: [], outputs: [{ type: "address" }] }, - { type: "function", name: "paused", stateMutability: "view", inputs: [], outputs: [{ type: "bool" }] }, - { type: "function", name: "threshold", stateMutability: "view", inputs: [], outputs: [{ type: "uint256" }] }, - { type: "function", name: "guardian", stateMutability: "view", inputs: [], outputs: [{ type: "address" }] }, - { - type: "function", - name: "activeApprovals", - stateMutability: "view", - inputs: [{ name: "payload", type: "bytes32" }], - outputs: [{ type: "uint256" }], - }, - { type: "function", name: "pause", stateMutability: "nonpayable", inputs: [], outputs: [] }, - { - type: "event", - name: "Approved", - inputs: [ - { name: "nonce", type: "uint64", indexed: true }, - { name: "recipient", type: "address", indexed: true }, - { name: "palletAmount", type: "uint256", indexed: false }, - { name: "attestor", type: "address", indexed: true }, - ], - }, - { - type: "event", - name: "Released", - inputs: [ - { name: "nonce", type: "uint64", indexed: true }, - { name: "recipient", type: "address", indexed: true }, - { name: "palletAmount", type: "uint256", indexed: false }, - { name: "tokenAmount", type: "uint256", indexed: false }, - ], - }, - { - type: "event", - name: "RemainderSwept", - inputs: [ - { name: "to", type: "address", indexed: true }, - { name: "amount", type: "uint256", indexed: false }, - ], - }, -] as const; - -/** The vault events of its admin (and of the guardian's pause). */ -const vaultAdminEventsAbi = parseAbi([ - "event TokenSet(address indexed token)", - "event Paused(address indexed by)", - "event Unpaused(address indexed by)", - "event AttestorAdded(address indexed attestor)", - "event AttestorRemoved(address indexed attestor)", - "event ThresholdUpdated(uint256 threshold)", - "event CapsUpdated(uint256 perReleaseCap, uint256 dailyCap)", - "event GuardianUpdated(address indexed guardian)", - "event AdminTransferStarted(address indexed pendingAdmin)", - "event AdminTransferred(address indexed newAdmin)", - "event RemainderSwept(address indexed to, uint256 amount)", - "event StalePendingCleared(bytes32 indexed payload, uint256 tokenAmount)", -]); - -const erc20Abi = [ - { type: "function", name: "totalSupply", stateMutability: "view", inputs: [], outputs: [{ type: "uint256" }] }, - { - type: "function", - name: "balanceOf", - stateMutability: "view", - inputs: [{ name: "owner", type: "address" }], - outputs: [{ type: "uint256" }], - }, -] as const; - -const timelockAbi = [ - { - type: "function", - name: "getTimestamp", - stateMutability: "view", - inputs: [{ name: "id", type: "bytes32" }], - outputs: [{ type: "uint256" }], - }, - { - type: "event", - name: "CallScheduled", - inputs: [ - { name: "id", type: "bytes32", indexed: true }, - { name: "index", type: "uint256", indexed: true }, - { name: "target", type: "address", indexed: false }, - { name: "value", type: "uint256", indexed: false }, - { name: "data", type: "bytes", indexed: false }, - { name: "predecessor", type: "bytes32", indexed: false }, - { name: "delay", type: "uint256", indexed: false }, - ], - }, -] as const; - -/** Calls governance can be expected to schedule, named in the page. Anything - * else still pages, with its raw selector. */ -const KNOWN_FUNCTIONS = Object.fromEntries([ - // MigrationVault - "acceptAdmin()", "transferAdmin(address)", "setCaps(uint256,uint256)", "setGuardian(address)", - "setThreshold(uint256)", "addAttestor(address)", "removeAttestor(address)", "pause()", "unpause()", - "sweepRemainder(address,uint256)", "clearStalePending(uint64,address,uint256)", "setToken(address)", - // TimelockController and Governor self-administration - "updateDelay(uint256)", "grantRole(bytes32,address)", "revokeRole(bytes32,address)", - "setVotingDelay(uint48)", "setVotingPeriod(uint32)", "setProposalThreshold(uint256)", - "updateQuorumNumerator(uint256)", "updateTimelock(address)", "relay(address,uint256,bytes)", - // Treasury spends - "transfer(address,uint256)", "approve(address,uint256)", -].map((signature) => [toFunctionSelector(signature), signature])); - -function required(name: string): string { - const value = process.env[name]; - if (!value) throw new Error(`Missing required environment variable ${name}`); - return value; -} - -/** A numeric environment value, validated at startup: a typo must fail loudly - * here, not surface later as NaN-driven behavior (a NaN grace, for instance, - * would silently disable the very checks this daemon exists for). */ -function envNumber(name: string, fallback: number | undefined): number { - const raw = process.env[name]; - if (raw === undefined || raw === "") { - if (fallback === undefined) throw new Error(`Missing required environment variable ${name}`); - return fallback; - } - const value = Number(raw); - if (!Number.isFinite(value) || value < 0) throw new Error(`invalid ${name}: ${raw}`); - return value; -} - -/** A non-negative integer amount from the environment, validated at startup. */ -function envAmount(name: string, fallback = "0"): bigint { - const raw = process.env[name] || fallback; - if (!/^\d+$/.test(raw)) throw new Error(`invalid ${name}: ${raw}`); - return BigInt(raw); -} - -function optionalAddress(name: string): `0x${string}` | undefined { - const value = process.env[name]; - if (!value) return undefined; - if (!/^0x[0-9a-fA-F]{40}$/.test(value)) throw new Error(`invalid ${name}: ${value}`); - return value as `0x${string}`; -} - -/** Comma-separated WebSocket URLs from the environment, at most `most`, - * validated at startup. */ -function wsUrls(name: string, most = Infinity): string[] { - const urls = (process.env[name] ?? "").split(",").map((url) => url.trim()).filter(Boolean); - if (urls.length > most || urls.some((url) => !/^wss?:\/\//i.test(url))) { - throw new Error(`invalid ${name}: expected ${most === 1 ? "one ws:// or wss:// URL" : "ws:// or wss:// URLs"}`); - } - return urls; -} - -function finalityTag(): "safe" | "finalized" { - const value = process.env.BASE_FINALITY_TAG ?? "safe"; - if (value !== "safe" && value !== "finalized") throw new Error(`invalid BASE_FINALITY_TAG ${value}`); - return value; -} - -const baseFinalityTag = finalityTag(); - -const config = { - pendulumWs: required("PENDULUM_WS"), - /** Optional: a second Pendulum node whose block hashes must agree. */ - pendulumWsCrossCheck: wsUrls("PENDULUM_WS_CROSSCHECK", 1)[0] as string | undefined, - /** Optional: third-party Polkadot RPCs, tried in order, whose record of - * the Pendulum head anchors the source. */ - polkadotWs: wsUrls("POLKADOT_WS"), - pendulumParaId: envNumber("PENDULUM_PARA_ID", 2094), - /** How long the source may go without a relay-anchored head before that pages critical. */ - relayAnchorGraceMs: envNumber("RELAY_ANCHOR_GRACE_SECONDS", 3600) * 1000, - baseRpcUrl: required("BASE_RPC_URL"), - vaultAddress: required("VAULT_ADDRESS") as `0x${string}`, - pollIntervalMs: envNumber("POLL_INTERVAL_MS", 60_000), - graceSeconds: envNumber("GRACE_SECONDS", 1800), - unmatchedEventGraceSeconds: envNumber("UNMATCHED_EVENT_GRACE_SECONDS", 600), - alertWebhookUrl: process.env.ALERT_WEBHOOK_URL, - /** Optional: requested after every poll that ran all checks. */ - heartbeatUrl: process.env.HEARTBEAT_URL, - guardianPrivateKey: process.env.GUARDIAN_PRIVATE_KEY as `0x${string}` | undefined, - baseChainId: envNumber("BASE_CHAIN_ID", 8453), - baseFinalityTag, - baseFinalityPollMs: envNumber("BASE_FINALITY_POLL_MS", 2000), - /** Bound on confirming the auto-pause inside the finality boundary; - * `finalized` trails `safe` by L1 finality, hence the longer default. */ - baseFinalityTimeoutMs: envNumber( - "BASE_FINALITY_TIMEOUT_MS", - baseFinalityTag === "finalized" ? 2_700_000 : 900_000, - ), - pendulumStartBlock: envNumber("PENDULUM_START_BLOCK", undefined), - pendulumStartNonce: BigInt(process.env.PENDULUM_START_NONCE ?? "0"), - baseStartBlock: BigInt(required("BASE_START_BLOCK")), - baseMaxBlockRange: BigInt(process.env.BASE_MAX_BLOCK_RANGE ?? "10000"), - readBatchSize: envNumber("READ_BATCH_SIZE", 100), - stateFile: process.env.STATE_FILE ?? "./monitor-state.json", - /** Cross-chain clock-skew allowance for `provablyUnsourced`: how far the - * finalized Pendulum view must pass a Base event's timestamp before a - * missing nonce counts as proof of fabrication rather than lag. */ - sourceClockSkewMarginMs: envNumber("SOURCE_CLOCK_SKEW_MARGIN_SECONDS", 120) * 1000, - /** Age of the finalized Pendulum view (vs. wall clock) past which the - * monitor pages that it is going blind — BEFORE any unmatched event can - * reach the grace deadline, so operators get the whole grace window to - * restore the node ahead of a cannot-verify auto-pause. */ - sourceStaleAlertMs: envNumber("SOURCE_STALE_ALERT_SECONDS", 300) * 1000, - /** Growth of (Pendulum totalIssuance + TotalMigrated) over its first-seen - * anchor, in 12-decimal pallet units, tolerated before paging. */ - issuanceToleranceRaw: BigInt(process.env.ISSUANCE_TOLERANCE ?? "0"), - /** Optional: the governance TimelockController; every operation it - * schedules pages, because the canceller must veto inside the delay. */ - timelockAddress: optionalAddress("TIMELOCK_ADDRESS"), - /** Optional: the only Base address a treasury migration may pay. */ - expectedTreasuryDestination: optionalAddress("EXPECTED_TREASURY_DESTINATION"), - /** Age of the confirmed Base block (vs. wall clock) past which the monitor - * pages that its Base view stopped advancing. `safe` normally trails by - * about a minute, `finalized` by about 20 minutes. */ - baseStaleAlertMs: envNumber("BASE_STALE_ALERT_SECONDS", baseFinalityTag === "finalized" ? 1800 : 600) * 1000, - /** Guardian balance (wei) below which the auto-pause counts as not ready. */ - guardianMinBalanceWei: envAmount("GUARDIAN_MIN_BALANCE_WEI", "1000000000000000"), - /** Excess of released over migrated, in token units, that operators have - * quantified and accepted after a real loss (RB-3); M2a fires beyond it. */ - acknowledgedReleaseExcess: envAmount("ACKNOWLEDGED_RELEASE_EXCESS"), -}; - -const ZERO_ADDRESS = "0x0000000000000000000000000000000000000000"; -/** The runtime's treasury account, `PalletId(*b"py/trsry")` as an account - * ("modl" + id, zero-padded): the `who` of every treasury migration. */ -const PENDULUM_TREASURY_ACCOUNT = `0x6d6f646c70792f7472737279${"00".repeat(20)}`; - -const baseChain = defineChain({ - id: config.baseChainId, - name: "base", - nativeCurrency: { name: "Ether", symbol: "ETH", decimals: 18 }, - rpcUrls: { default: { http: [config.baseRpcUrl] } }, - contracts: { multicall3: { address: MULTICALL3_ADDRESS } }, -}); -const publicClient = createPublicClient({ chain: baseChain, transport: http(config.baseRpcUrl) }); - -interface MigrationRecord extends MigrationTuple { - firstSeenMs: number; - sourceBlock: number; -} - -interface RuntimeState { - baseChainId: number; - vaultAddress: `0x${string}`; - pendulumGenesisHash: `0x${string}` | undefined; - lastPendulumBlock: number; - /** The hash of `lastPendulumBlock`: the next blocks must descend from it. */ - lastPendulumHash: string | undefined; - /** The newest Pendulum head the relay chain recorded that the source was - * verified against; the source's chain must keep passing through it. */ - relayAnchor: { number: number; hash: string } | undefined; - nextExpectedNonce: bigint; - baseFromBlock: bigint; - /** The admin scan's own cursor: it only pages, so nothing waits for it. */ - baseAdminFromBlock: bigint; - migrations: Map; - livenessAlertedAt: Map; - unmatchedBaseFirstSeenAt: Map; - /** Approvals that disagreed with the finalized burn, or found none, kept - * until their nonce is released (see `disagreeingAttestors`). */ - mismatchedApprovals: Array; - /** Pendulum totalIssuance + TotalMigrated at first observation. Under every - * legitimate flow this sum can only fall (teleport-out) and later recover - * to at most its original value (teleport-in); growth means PEN was minted - * at the source, which honest attestation would then release from the - * fixed-supply vault. Alert-only: the response is a human decision. */ - issuanceAnchor: bigint | undefined; -} - -function hydrateState(): RuntimeState { - const persisted = loadMonitorState(config.stateFile); - if (!persisted) { - return { - baseChainId: config.baseChainId, - vaultAddress: config.vaultAddress, - pendulumGenesisHash: undefined, - lastPendulumBlock: config.pendulumStartBlock - 1, - lastPendulumHash: undefined, - relayAnchor: undefined, - nextExpectedNonce: config.pendulumStartNonce, - baseFromBlock: config.baseStartBlock, - baseAdminFromBlock: config.baseStartBlock, - migrations: new Map(), - livenessAlertedAt: new Map(), - unmatchedBaseFirstSeenAt: new Map(), - mismatchedApprovals: [], - issuanceAnchor: undefined, - }; - } - return { - baseChainId: persisted.baseChainId, - vaultAddress: persisted.vaultAddress, - pendulumGenesisHash: persisted.pendulumGenesisHash, - lastPendulumBlock: persisted.lastPendulumBlock, - lastPendulumHash: persisted.lastPendulumHash, - relayAnchor: persisted.relayAnchor, - nextExpectedNonce: BigInt(persisted.nextExpectedNonce), - baseFromBlock: BigInt(persisted.baseFromBlock), - baseAdminFromBlock: BigInt(persisted.baseAdminFromBlock ?? persisted.baseFromBlock), - migrations: new Map(persisted.migrations.map((migration) => [ - BigInt(migration.nonce), - { - nonce: BigInt(migration.nonce), - recipient: migration.recipient, - palletAmount: BigInt(migration.palletAmount), - firstSeenMs: migration.firstSeenMs, - sourceBlock: migration.sourceBlock, - }, - ])), - livenessAlertedAt: new Map(persisted.livenessAlertedAt.map(([nonce, at]) => [BigInt(nonce), at])), - unmatchedBaseFirstSeenAt: new Map(persisted.unmatchedBaseFirstSeenAt), - mismatchedApprovals: (persisted.mismatchedApprovals ?? []).map((approval) => ({ - ...approval, - nonce: BigInt(approval.nonce), - palletAmount: BigInt(approval.palletAmount), - })), - issuanceAnchor: persisted.issuanceAnchor === undefined ? undefined : BigInt(persisted.issuanceAnchor), - }; -} - -function cloneState(source: RuntimeState): RuntimeState { - return { - baseChainId: source.baseChainId, - vaultAddress: source.vaultAddress, - pendulumGenesisHash: source.pendulumGenesisHash, - lastPendulumBlock: source.lastPendulumBlock, - lastPendulumHash: source.lastPendulumHash, - relayAnchor: source.relayAnchor, - nextExpectedNonce: source.nextExpectedNonce, - baseFromBlock: source.baseFromBlock, - baseAdminFromBlock: source.baseAdminFromBlock, - migrations: new Map(source.migrations), - livenessAlertedAt: new Map(source.livenessAlertedAt), - unmatchedBaseFirstSeenAt: new Map(source.unmatchedBaseFirstSeenAt), - mismatchedApprovals: [...source.mismatchedApprovals], - issuanceAnchor: source.issuanceAnchor, - }; -} - -function persistState(state: RuntimeState): void { - if (!state.pendulumGenesisHash) throw new Error("cannot persist monitor state before Pendulum identity is known"); - const persisted: PersistedMonitorState = { - version: 1, - baseChainId: state.baseChainId, - vaultAddress: state.vaultAddress, - pendulumGenesisHash: state.pendulumGenesisHash, - lastPendulumBlock: state.lastPendulumBlock, - lastPendulumHash: state.lastPendulumHash as `0x${string}` | undefined, - relayAnchor: state.relayAnchor as { number: number; hash: `0x${string}` } | undefined, - nextExpectedNonce: state.nextExpectedNonce.toString(), - baseFromBlock: state.baseFromBlock.toString(), - baseAdminFromBlock: state.baseAdminFromBlock.toString(), - migrations: [...state.migrations.values()].map((migration) => ({ - nonce: migration.nonce.toString(), - recipient: migration.recipient as `0x${string}`, - palletAmount: migration.palletAmount.toString(), - firstSeenMs: migration.firstSeenMs, - sourceBlock: migration.sourceBlock, - })), - livenessAlertedAt: [...state.livenessAlertedAt].map(([nonce, at]) => [nonce.toString(), at]), - unmatchedBaseFirstSeenAt: [...state.unmatchedBaseFirstSeenAt], - mismatchedApprovals: state.mismatchedApprovals.map((approval) => ({ - nonce: approval.nonce.toString(), - recipient: approval.recipient as `0x${string}`, - palletAmount: approval.palletAmount.toString(), - attestor: approval.attestor as `0x${string}`, - })), - issuanceAnchor: state.issuanceAnchor?.toString(), - }; - saveMonitorState(config.stateFile, persisted); -} - -let state: RuntimeState; -let multicallUnavailable = false; -let lastSourceStallAlertMs = 0; -let lastSourceFailureAlertMs = 0; -let lastSourceIntegrityAlertMs = 0; -let lastCrossCheckAlertMs = 0; -let lastRelayAlertMs = 0; -let lastRelayDisagreementAlertMs = 0; -let lastRelayStaleAlertMs = 0; -/** When the source last verified against a new relay-chain record (or the start). */ -let lastRelayAnchorMs = Date.now(); -let lastBaseStallAlertMs = 0; -let lastReadinessAlertMs = 0; -let lastAdminScanFailureAlertMs = 0; -/** Since when the Base admin scan fails, while it does. */ -let adminScanFailingSinceMs: number | undefined; -let lastLogScanAlertMs = 0; -/** Since when the Base log scan holds a range, while it does. */ -let logScanHeldSinceMs: number | undefined; -let lastIssuanceAlertMs = 0; -/** Once-only pages by key, each held until the webhook accepts it. */ -// ponytail: in memory; a restart re-pages what it finds again (Base admin events after the persisted cursor, pending governance items) and drops undelivered pages it does not find again. -const onceOnlyPages = new Map }>(); -let lastGovernanceWatchFailureMs = 0; - -function log(message: string): void { - console.log(`${new Date().toISOString()} ${message}`); -} - -/** Strip URLs before anything leaves the process: RPC endpoints commonly embed - * API keys, and viem and polkadot.js error texts quote the endpoint verbatim. */ -function redact(text: string): string { - return text.replace(/(?:https?|wss?):\/\/[^\s"')]+/gi, ""); -} - -/** Sends one page, as a Slack incoming-webhook message, and resolves to - * whether it was delivered: the webhook must answer 2xx within the timeout. - * Without a webhook the log line is the page. */ -async function alert( - subject: string, - detail: string, - severity: Severity = subject.startsWith("CRITICAL") ? "critical" : "warning", -): Promise { - console.error(`${new Date().toISOString()} ALERT: ${subject} — ${detail}`); - if (!config.alertWebhookUrl) return true; - try { - const response = await fetch(config.alertWebhookUrl, { - method: "POST", - headers: { "content-type": "application/json" }, - body: JSON.stringify({ text: slackText("pen-monitor", severity, subject, redact(detail)) }), - // The webhook is an external dependency; it must never stall the - // check loop (or the pause that follows a violation) indefinitely. - signal: AbortSignal.timeout(10_000), - }); - if (response.ok) return true; - console.error(`alert webhook answered ${response.status} for: ${subject}`); - } catch (webhookError) { - console.error(`alert webhook failed for: ${subject}: ${redact(`${webhookError}`)}`); - } - return false; -} - -/** Pages `key` once. A page the webhook does not accept stays queued, and - * every poll retries it until it is delivered. */ -function pageOnce(key: string, subject: string, detail: string): Promise { - const queued = onceOnlyPages.get(key); - if (queued) return queued.delivery; - const delivery = alert(subject, detail); - onceOnlyPages.set(key, { subject, detail, delivery }); - return delivery; -} - -async function retryUndeliveredPages(): Promise { - await Promise.all([...onceOnlyPages.values()].map(async (page) => { - if (await page.delivery) return; - page.delivery = alert(page.subject, page.detail); - await page.delivery; - })); -} - -async function waitForBaseFinality( - hash: `0x${string}`, - receiptBlock: bigint, - receiptBlockHash: `0x${string}`, -): Promise { - const deadline = Date.now() + config.baseFinalityTimeoutMs; - for (;;) { - const confirmed = await publicClient.getBlock({ blockTag: config.baseFinalityTag }); - if (confirmed.number !== null && confirmed.number >= receiptBlock) { - const canonicalReceipt = await publicClient.getTransactionReceipt({ hash }); - if (canonicalReceipt.status !== "success" || canonicalReceipt.blockHash !== receiptBlockHash) { - throw new Error(`pause transaction ${hash} was reorged before becoming ${config.baseFinalityTag}`); - } - return confirmed.number; - } - if (Date.now() >= deadline) { - throw new Error(`timed out waiting for pause ${hash} to become ${config.baseFinalityTag}`); - } - await new Promise((resolve) => setTimeout(resolve, config.baseFinalityPollMs)); - } -} - -/** The pause attempt in flight, which a concurrent violation joins. */ -let pausing: Promise | undefined; - -function pauseVault(): Promise { - pausing ??= pauseOnce().finally(() => (pausing = undefined)); - return pausing; -} - -async function pauseOnce(): Promise { - try { - const confirmed = await publicClient.getBlock({ blockTag: config.baseFinalityTag }); - if (confirmed.number === null) throw new Error(`${config.baseFinalityTag} block has no number`); - const pausedConfirmed = await publicClient.readContract({ - address: config.vaultAddress, - abi: vaultAbi, - functionName: "paused", - blockNumber: confirmed.number, - }); - if (pausedConfirmed) { - await alert("vault pause confirmed", `already paused in Base ${config.baseFinalityTag} block ${confirmed.number}`); - return true; - } - const pausedLatest = await publicClient.readContract({ - address: config.vaultAddress, - abi: vaultAbi, - functionName: "paused", - }); - if (pausedLatest) { - await alert("AUTO-PAUSE PENDING FINALITY", `pause is mined but not yet ${config.baseFinalityTag}`); - return false; - } - if (!config.guardianPrivateKey) { - await alert("AUTO-PAUSE UNAVAILABLE", "no GUARDIAN_PRIVATE_KEY configured; pause manually NOW", "critical"); - return false; - } - - const guardian = privateKeyToAccount(config.guardianPrivateKey); - const walletClient = createWalletClient({ account: guardian, chain: baseChain, transport: http(config.baseRpcUrl) }); - const { request } = await publicClient.simulateContract({ - account: guardian, - address: config.vaultAddress, - abi: vaultAbi, - functionName: "pause", - }); - const txHash = await walletClient.writeContract(request); - const receipt = await publicClient.waitForTransactionReceipt({ hash: txHash }); - if (receipt.status !== "success") throw new Error(`pause transaction reverted: ${txHash}`); - const safeBlock = await waitForBaseFinality(txHash, receipt.blockNumber, receipt.blockHash); - const paused = await publicClient.readContract({ - address: config.vaultAddress, - abi: vaultAbi, - functionName: "paused", - blockNumber: safeBlock, - }); - if (!paused) throw new Error(`pause transaction ${txHash} is confirmed but paused() is false`); - await alert("vault auto-pause confirmed", `tx=${txHash} ${config.baseFinalityTag}Block=${safeBlock}`); - return true; - } catch (pauseError) { - await alert("AUTO-PAUSE FAILED", `${pauseError}; pause manually NOW`, "critical"); - return false; - } -} - -/** What would make an auto-pause fail now: an RPC on another chain, or a key - * that does not parse, is not the vault's guardian or cannot pay for the - * pause. Pages, throttled, so it surfaces before a violation needs it. A read - * that fails is a problem too: it must not stop the poll's other checks. */ -async function checkAutoPauseReadiness(): Promise { - const problems: string[] = []; - try { - const chainId = await publicClient.getChainId(); - if (chainId !== config.baseChainId) { - problems.push(`the Base RPC serves chain ${chainId}, not BASE_CHAIN_ID ${config.baseChainId}`); - } - if (config.guardianPrivateKey) { - let guardian: `0x${string}` | undefined; - try { - guardian = privateKeyToAccount(config.guardianPrivateKey).address; - } catch { - problems.push("GUARDIAN_PRIVATE_KEY is not a valid private key"); - } - if (guardian) { - const [vaultGuardian, balance] = await Promise.all([ - publicClient.readContract({ address: config.vaultAddress, abi: vaultAbi, functionName: "guardian" }), - publicClient.getBalance({ address: guardian }), - ]); - if (vaultGuardian.toLowerCase() !== guardian.toLowerCase()) { - problems.push(`the key's address ${guardian} is not the vault guardian ${vaultGuardian}`); - } - if (balance < config.guardianMinBalanceWei) { - problems.push(`guardian ${guardian} holds ${balance} wei, less than GUARDIAN_MIN_BALANCE_WEI ${config.guardianMinBalanceWei}`); - } - } - } - } catch (readError) { - problems.push(`the readiness reads failed: ${readError}`); - } - if (problems.length && Date.now() - lastReadinessAlertMs > config.graceSeconds * 1000) { - lastReadinessAlertMs = Date.now(); - await alert("AUTO-PAUSE NOT READY", `${problems.join("; ")} — fix it before a violation needs the pause`); - } - return problems; -} - -async function securityViolation(subject: string, detail: string): Promise { - // Pause FIRST. The alert webhook is an external dependency with its own - // latency; nothing may sit between detecting a violation and the pause. - const notified = alert(subject, detail, "critical"); - const paused = await pauseVault(); - await notified; - return paused; -} - -/** - * Decide what to do with a safe Base event whose nonce the finalized Pendulum - * view does not (yet) know. - * - * Three outcomes: - * - "proven": the source view has passed the moment this event was first - * observed and the nonce still does not exist — fabricated, - * pause immediately (the RPC-lag grace must not extend an - * attacker's dwell time); - * - "await": the source view still trails the event and the grace has not - * expired — legitimate node lag looks exactly like this; - * - "expired": the grace ran out with the source still behind — the monitor - * has been unable to verify releases for the whole window. - * - * The first observation of an unmatched event pages immediately (deduplicated - * via the persisted first-seen map), so operators get the entire grace window - * to repair a lagging source node before "expired" forces the pause. The page - * is not awaited: nothing may sit between detecting a violation and the pause. - */ -function unmatchedDisposition( - kind: "Approved" | "Released", - key: string, - nonce: bigint, - eventBlockNumber: bigint, - draft: RuntimeState, - observedAt: number, - sourceFinalizedTsMs: number, -): "proven" | "await" | "expired" { - const alreadyKnown = draft.unmatchedBaseFirstSeenAt.has(key); - const firstSeen = draft.unmatchedBaseFirstSeenAt.get(key) ?? observedAt; - draft.unmatchedBaseFirstSeenAt.set(key, firstSeen); - if (!alreadyKnown) { - void pageOnce( - `unverified:${key}`, - "UNVERIFIED BASE EVENT", - `${kind} nonce ${nonce} at Base block ${eventBlockNumber} has no finalized Pendulum source yet; ` + - `waiting up to ${config.unmatchedEventGraceSeconds}s for the source view to reveal its burn`, - ); - } - if (provablyUnsourced(firstSeen, sourceFinalizedTsMs, observedAt, config.sourceClockSkewMarginMs)) return "proven"; - return shouldAwaitSource(nonce, draft.nextExpectedNonce, firstSeen, observedAt, config.unmatchedEventGraceSeconds) - ? "await" - : "expired"; -} - -interface BlockRef { - number: number; - hash: string; -} - -/** The Pendulum source contradicts the chain it must be on: the one ingested - * so far, the second node's or the relay chain's. Handled as unreadable. */ -class SourceIntegrityError extends Error {} - -/** The source holds at a block whose violation must pause the vault first; - * the violation's own page says so. */ -class PauseHold extends Error {} - -/** Bound on connecting to, and on each read from, the nodes of the integrity checks. */ -const NODE_TIMEOUT_MS = 20_000; - -function withTimeout(promise: Promise, ms: number): Promise { - let timer: NodeJS.Timeout | undefined; - const timeout = new Promise((_, reject) => { - timer = setTimeout(() => reject(new Error(`no answer within ${ms / 1000}s`)), ms); - }); - return Promise.race([promise, timeout]).finally(() => clearTimeout(timer)); -} - -/** Connects to a node of the integrity checks; tests replace it. */ -let connectNode = async (url: string): Promise => { - const provider = new WsProvider(url, 2_500, {}, NODE_TIMEOUT_MS); - try { - return await withTimeout(ApiPromise.create({ provider, throwOnConnect: true, noInitWarn: true }), NODE_TIMEOUT_MS); - } catch (connectError) { - await provider.disconnect().catch(() => {}); - throw connectError; - } -}; -const nodeConnections = new Map>(); - -/** Runs `read` against the node at `url` over a kept connection, bounded in - * time; a failure drops the connection, so the next poll connects afresh. */ -async function onNode(url: string, read: (node: ApiPromise) => Promise): Promise { - let connection = nodeConnections.get(url); - if (!connection) nodeConnections.set(url, (connection = connectNode(url))); - try { - return await withTimeout(connection.then(read), NODE_TIMEOUT_MS); - } catch (nodeError) { - nodeConnections.delete(url); - connection.then((node) => node.disconnect()).catch(() => {}); - // A failed WebSocket connection rejects with an event, not an Error. - throw nodeError instanceof Error ? nodeError : new Error(String((nodeError as { message?: unknown })?.message ?? nodeError)); - } -} - -interface MigrationTotals { - nextNonce: bigint; - totalMigrated: bigint; -} - -async function migrationTotals(apiAt: ApiAt): Promise { - const [nextNonce, totalMigrated] = await Promise.all([ - apiAt.query.tokenMigration.nextNonce(), - apiAt.query.tokenMigration.totalMigrated(), - ]); - return { - nextNonce: BigInt(nextNonce.toString().replaceAll(",", "")), - totalMigrated: BigInt(totalMigrated.toString().replaceAll(",", "")), - }; -} - -/** The height up to which the second node confirms the source's chain - * `hashes`, walked down from `top` to `bottom`: the hashes must agree at its - * finalized head, or at `top` once it has finalized past it. Undefined while - * it cannot be read or its finalized head is older than - * SOURCE_STALE_ALERT_SECONDS, which only warns, throttled: it is a check, not - * a source. */ -async function crossCheckSource(hashes: Map, top: BlockRef, bottom: number): Promise { - const url = config.pendulumWsCrossCheck; - if (!url) return undefined; - let other: BlockRef & { ageMs: number }; - try { - other = await onNode(url, async (node) => { - const finalizedHash = await node.rpc.chain.getFinalizedHead(); - const finalized = (await node.rpc.chain.getHeader(finalizedHash)).number.toNumber(); - const timestamp = await (await node.at(finalizedHash)).query.timestamp.now(); - // Above its finalized head, a node's block at a height may still lose. - const number = Math.min(finalized, top.number); - const hash = number === finalized ? finalizedHash.toHex() : (await node.rpc.chain.getBlockHash(number)).toHex(); - return { number, hash, ageMs: Date.now() - Number(timestamp.toString().replaceAll(",", "")) }; - }); - } catch (crossCheckError) { - await crossCheckUnavailable(`cannot be read: ${crossCheckError}`); - return undefined; - } - if (other.ageMs > config.sourceStaleAlertMs) { - await crossCheckUnavailable(`has a finalized head ${Math.round(other.ageMs / 1000)}s old`); - return undefined; - } - if (other.number >= bottom && hashes.get(other.number) !== other.hash) { - throw new SourceIntegrityError( - `the second Pendulum node has ${other.hash} at block ${other.number}, the source ${hashes.get(other.number)}`, - ); - } - return other.number; -} - -/** The second node must have the source's migration totals at `blockHash`, a - * block it confirmed. One that cannot answer only warns, throttled. */ -async function crossCheckTotals(block: number, blockHash: string, totals: MigrationTotals): Promise { - let other: MigrationTotals; - try { - other = await onNode(config.pendulumWsCrossCheck as string, async (node) => migrationTotals(await node.at(blockHash))); - } catch (crossCheckError) { - return crossCheckUnavailable(`cannot be read: ${crossCheckError}`); - } - if (other.nextNonce !== totals.nextNonce || other.totalMigrated !== totals.totalMigrated) { - throw new SourceIntegrityError( - `at Pendulum block ${block} the second node has nextNonce ${other.nextNonce} and ` + - `totalMigrated ${other.totalMigrated}, the source ${totals.nextNonce} and ${totals.totalMigrated}`, - ); - } -} - -type EventRecords = Array<{ event: { section: string; method: string; data: unknown[] } }>; - -/** The migration pallet's events in a block, as one text; empty without any. */ -function migrationEventText(records: EventRecords): string { - return records - .filter(({ event }) => event.section === "tokenMigration") - .map(({ event }) => `${event.method}(${event.data.join(", ")})`) - .join("; "); -} - -/** The second node must report the same migration events in the block at - * `blockHash`. One that cannot answer only warns, throttled. */ -async function crossCheckEvents(block: number, blockHash: string, events: string): Promise { - const url = config.pendulumWsCrossCheck; - if (!url) return; - let other: string; - try { - other = await onNode(url, async (node) => - migrationEventText((await (await node.at(blockHash)).query.system.events()) as unknown as EventRecords) - ); - } catch (crossCheckError) { - return crossCheckUnavailable(`cannot be read: ${crossCheckError}`); - } - if (other !== events) { - throw new SourceIntegrityError( - `in Pendulum block ${block} the second node reports the migration events ${other || "none"}, the source ${events}`, - ); - } -} - -/** Pages, throttled, that the source cannot be compared with the second node. */ -async function crossCheckUnavailable(reason: string): Promise { - if (Date.now() - lastCrossCheckAlertMs <= config.graceSeconds * 1000) return; - lastCrossCheckAlertMs = Date.now(); - await alert( - "PENDULUM CROSS-CHECK UNAVAILABLE", - `the second Pendulum node (PENDULUM_WS_CROSSCHECK) ${reason}; the source's blocks are not compared ` + - "with it until it answers — restore it", - ); -} - -/** Walks parent hashes down from `top` to `bottom` through headers fetched - * from the node, recomputing each hash here, so that the result is one - * chain the node cannot fake: block hashes by height, `bottom` included. */ -async function walkHeaders(api: ApiPromise, top: BlockRef, bottom: number): Promise> { - // ponytail: one pass, held in memory and started over after a failed read; make it resumable if catch-ups grow to weeks. - const hashes = new Map([[top.number, top.hash]]); - let hash = top.hash; - for (let height = top.number; height > bottom; height--) { - const header = await api.rpc.chain.getHeader(hash); - if (header.hash.toHex() !== hash || header.number.toNumber() !== height) { - throw new SourceIntegrityError(`the source's header for Pendulum block ${height} does not hash to ${hash}`); - } - hash = header.parentHash.toHex(); - hashes.set(height - 1, hash); - } - return hashes; -} - -/** The Pendulum head that the relay chain behind the RPC at `url` recorded - * at its finalized block, and the age of that block. */ -function relayHead(api: ApiPromise, url: string): Promise { - return onNode(url, async (relay) => { - const relayAt = await relay.at(await relay.rpc.chain.getFinalizedHead()); - const timestamp = await relayAt.query.timestamp.now(); - const head = (await relayAt.query.paras.heads(config.pendulumParaId)) as unknown as { - isSome: boolean; - unwrap(): { toU8a(bare: boolean): Uint8Array }; - }; - if (!head.isSome) throw new Error(`no head recorded for para ${config.pendulumParaId}`); - // The head data is the Pendulum header; its hash is computed here. - const header = api.createType("Header", head.unwrap().toU8a(true)) as unknown as { - number: { toNumber(): number }; - hash: { toHex(): string }; - }; - return { - number: header.number.toNumber(), - hash: header.hash.toHex(), - ageMs: Date.now() - Number(timestamp.toString().replaceAll(",", "")), - }; - }); -} - -/** How far below the block ingested last the relay chain's record may lie and - * still be walked down to; a deeper one is not used. */ -const RELAY_HEAD_MAX_DEPTH = 100; - -/** Why the relay chain's record `head` does not anchor the source this poll, - * or undefined when it does. Throws when it contradicts the source's chain: - * `hashes`, walked down from `top` to `bottom`. */ -async function relayVerdict( - api: ApiPromise, - draft: RuntimeState, - hashes: Map, - top: BlockRef, - bottom: BlockRef, - head: BlockRef, -): Promise { - const known = draft.relayAnchor; - if (known && head.number <= known.number) { - if (head.number === known.number && head.hash !== known.hash) { - throw new SourceIntegrityError( - `the relay chain now records ${head.hash} as Pendulum block ${head.number}, where it recorded ${known.hash}`, - ); - } - return `the relay chain's Pendulum head ${head.number} is not newer`; - } - if (head.number > top.number) { - // Past the finalized head, the source's best chain may still hold another - // block at that height. The record anchors the source once it holds the - // record, whose chain must then pass through `top`. - if ((await api.rpc.chain.getBlockHash(head.number)).toHex() !== head.hash) { - return `the source has not imported block ${head.number}, the relay chain's Pendulum head, yet`; - } - const hash = (await walkHeaders(api, head, top.number)).get(top.number); - if (hash !== top.hash) { - throw new SourceIntegrityError( - `the relay chain records ${head.hash} as Pendulum block ${head.number}, whose chain has ${hash} at block ` + - `${top.number}, where the source has ${top.hash}`, - ); - } - return undefined; - } - if (head.number < bottom.number - RELAY_HEAD_MAX_DEPTH) { - return `the relay chain's Pendulum head ${head.number} lies more than ${RELAY_HEAD_MAX_DEPTH} blocks below block ${bottom.number}`; - } - const hash = head.number >= bottom.number - ? hashes.get(head.number) - : (await walkHeaders(api, bottom, head.number)).get(head.number); - if (hash !== head.hash) { - throw new SourceIntegrityError(`the relay chain records ${head.hash} as Pendulum block ${head.number}, the source's chain has ${hash}`); - } - return undefined; -} - -/** Anchors the source to the relay chain's record of the Pendulum head, kept - * as `draft.relayAnchor`: the highest record that anchors it, from the relay - * RPCs in POLKADOT_WS, asked in turn until a record anchors the source's - * head `top`. A relay RPC whose finalized relay block is older than - * SOURCE_STALE_ALERT_SECONDS counts as not answering. A record that - * contradicts the source counts only when another relay RPC's record - * contradicts it too, or when POLKADOT_WS lists one RPC. A record that - * anchors the source at or above the contradicting record's height outvotes - * it; otherwise it stays unconfirmed. Resolves to the height up to which the - * relay chain confirms the source's chain `hashes`, walked down from `top` to - * `bottom`: the newest anchored head, or no higher than `bottom` before the - * first; undefined while no relay RPC answers. Pages while no new head is - * anchored and the source's blocks wait for one, a contradiction stays - * unconfirmed or no relay RPC answers: a warning, critical once none was - * anchored for RELAY_ANCHOR_GRACE_SECONDS; and when relay RPCs disagree. All - * throttled. */ -async function anchorToRelay( - api: ApiPromise, - draft: RuntimeState, - hashes: Map, - top: BlockRef, - bottom: BlockRef, -): Promise { - if (!config.polkadotWs.length) return undefined; - const answers: string[] = []; - let answered = false; - let anchor: (BlockRef & { rpc: number }) | undefined; - let contradiction: { rpc: number; height: number; error: SourceIntegrityError } | undefined; - for (const [index, url] of config.polkadotWs.entries()) { - if (anchor && anchor.number >= Math.max(top.number, contradiction?.height ?? 0)) break; - let head: BlockRef & { ageMs: number }; - try { - head = await relayHead(api, url); - } catch (relayError) { - answers.push(`relay RPC ${index + 1}: ${relayError}`); - continue; - } - if (head.ageMs > config.sourceStaleAlertMs) { - answers.push(`relay RPC ${index + 1}: its finalized relay block is ${Math.round(head.ageMs / 1000)}s old`); - continue; - } - answered = true; - let verdict: string | undefined; - try { - verdict = await relayVerdict(api, draft, hashes, top, bottom, head); - } catch (verdictError) { - if (!(verdictError instanceof SourceIntegrityError)) throw verdictError; - if (contradiction || config.polkadotWs.length === 1) throw contradiction?.error ?? verdictError; - contradiction = { rpc: index + 1, height: head.number, error: verdictError }; - continue; - } - if (verdict !== undefined) answers.push(`relay RPC ${index + 1}: ${verdict}`); - else if (!anchor || head.number > anchor.number) anchor = { number: head.number, hash: head.hash, rpc: index + 1 }; - } - if (anchor) { - draft.relayAnchor = { number: anchor.number, hash: anchor.hash }; - lastRelayAnchorMs = Date.now(); - if (!contradiction) return anchor.number; - if (anchor.number >= contradiction.height) { - if (Date.now() - lastRelayDisagreementAlertMs > config.graceSeconds * 1000) { - lastRelayDisagreementAlertMs = Date.now(); - await alert( - "RELAY RPCS DISAGREE", - `relay RPC ${contradiction.rpc} reports that ${contradiction.error.message}; relay RPC ${anchor.rpc} records ` + - `${anchor.hash} as Pendulum block ${anchor.number}, on the source's chain, and the source is anchored to ` + - "that record — one of the two relay RPCs is wrong, check both", - ); - } - return anchor.number; - } - answers.push(`relay RPC ${anchor.rpc}: its Pendulum head ${anchor.number} lies below the contradicting record`); - } - const reason = contradiction - ? `relay RPC ${contradiction.rpc} reports that ${contradiction.error.message}, and no other relay RPC confirms ` + - `or outvotes it${answers.length ? ` (${answers.join("; ")})` : ""}` - : `${answered ? "" : "the relay chain cannot be read: "}${answers.join("; ")}`; - const confirmed = draft.relayAnchor?.number ?? bottom.number; - const consequence = answered - ? `no Pendulum block after ${Math.max(confirmed, draft.lastPendulumBlock)} is ingested until a relay RPC anchors a newer head` - : "the Pendulum source is used without the relay-chain anchor until a relay RPC answers"; - if ( - (contradiction || !answered || top.number > confirmed) && - Date.now() - lastRelayAlertMs > config.graceSeconds * 1000 - ) { - lastRelayAlertMs = Date.now(); - await alert( - "RELAY ANCHOR UNAVAILABLE", - `${reason}; ${consequence}, and this turns critical after ${config.relayAnchorGraceMs / 1000}s`, - ); - } - const unanchoredMs = Date.now() - lastRelayAnchorMs; - if (unanchoredMs > config.relayAnchorGraceMs && Date.now() - lastRelayStaleAlertMs > config.unmatchedEventGraceSeconds * 1000) { - lastRelayStaleAlertMs = Date.now(); - await alert( - "CRITICAL: RELAY ANCHOR STALE", - `no Pendulum head was checked against the relay chain for ${Math.round(unanchoredMs / 1000)}s (${reason}); ` + - `${consequence} — restore a relay RPC, or the source node if it lags`, - ); - } - return answered ? confirmed : undefined; -} - -/** The hashes of the finalized blocks to ingest, on one chain with every - * block ingested before and with the relay-chain record kept from an earlier - * poll, and the newest of them that every checker that answers confirms - * (`confirmed`, at most the source's finalized head): the second node up to - * `crossChecked`, and the relay chain up to `anchored`, its newest record - * that anchors the source. A checker that does not answer confirms nothing - * and holds nothing back. */ -async function sourceHashes( - api: ApiPromise, - draft: RuntimeState, - finalized: BlockRef, -): Promise<{ hashes: Map; confirmed: number; crossChecked: number | undefined; anchored: number | undefined }> { - // A kept record past the finalized head binds the source too: the walk starts there. - const known = draft.relayAnchor; - const top = known && known.number > finalized.number ? known : finalized; - const hashes = await walkHeaders(api, top, draft.lastPendulumBlock); - const ingested = draft.lastPendulumHash ? { number: draft.lastPendulumBlock, hash: draft.lastPendulumHash } : undefined; - const expected: Array<[BlockRef | undefined, string]> = [ - [finalized, "the source's finalized head is"], - [ingested, "the monitor ingested"], - [known, "the relay chain recorded"], - ]; - for (const [block, what] of expected) { - const hash = block && hashes.get(block.number); - if (block && hash !== undefined && hash !== block.hash) { - throw new SourceIntegrityError(`the source's chain has ${hash} at Pendulum block ${block.number}, where ${what} ${block.hash}`); - } - } - const lowest = Math.min(top.number, draft.lastPendulumBlock); - const crossChecked = await crossCheckSource(hashes, top, lowest); - const anchored = await anchorToRelay(api, draft, hashes, top, { number: lowest, hash: hashes.get(lowest) as string }); - return { hashes, confirmed: Math.min(finalized.number, crossChecked ?? Infinity, anchored ?? Infinity), crossChecked, anchored }; -} - -/** The migration pallet's admin events, each with what to check. */ -const PALLET_ADMIN_EVENTS = new Map([ - ["MigrationPauseSet", "migrations were paused or resumed; check that it was intended"], - [ - "TreasuryDestinationSet", - "treasury migrations are released to this Base address from now on; check that it is the governance timelock", - ], -]); - -/** Ingests the blocks after the cursor up to `lastBlock`, each read at its - * hash in `hashes`: the chain that `sourceHashes` verified. Their migration - * events are compared with the second node's while it answers. */ -async function ingestPendulumEvents( - api: ApiPromise, - draft: RuntimeState, - lastBlock: number, - hashes: Map, - crossChecked: boolean, -): Promise { - for (let block = draft.lastPendulumBlock + 1; block <= lastBlock; block++) { - const blockHash = hashes.get(block); - if (!blockHash) throw new Error(`no verified hash for Pendulum block ${block}`); - const apiAt = await api.at(blockHash); - const firstSeenMs = Number((await apiAt.query.timestamp.now()).toString().replaceAll(",", "")); - const records = (await apiAt.query.system.events()) as unknown as EventRecords; - const migrationEvents = migrationEventText(records); - if (migrationEvents && crossChecked) await crossCheckEvents(block, blockHash, migrationEvents); - for (const [index, record] of records.entries()) { - const { section, method, data } = record.event; - if (section !== "tokenMigration") continue; - const adminCheck = PALLET_ADMIN_EVENTS.get(method); - if (adminCheck) { - // Every admin change, however fast the motion behind it passed; a - // treasury destination other than the expected one is critical. - const expected = config.expectedTreasuryDestination; - const unexpected = method === "TreasuryDestinationSet" && expected !== undefined && - String(data[0]).toLowerCase() !== expected.toLowerCase(); - await pageOnce( - `pallet:${block}:${index}`, - `${unexpected ? "CRITICAL: " : ""}MIGRATION PALLET ADMIN EVENT`, - `tokenMigration.${method}(${data.join(", ")}) in finalized Pendulum block ${block}: ` + - (unexpected - ? `treasury migrations are released to this Base address from now on, not to ` + - `EXPECTED_TREASURY_DESTINATION ${expected}, and each one pauses the vault — check the motion behind it` - : adminCheck), - ); - continue; - } - if (method !== "MigrationInitiated") continue; - if (data.length !== 4) throw new Error(`MigrationInitiated in block ${block} has ${data.length} fields, expected 4`); - const [nonceValue, whoValue, recipientValue, amountValue] = data as [ - { toBigInt(): bigint }, - { toHex(): string }, - { toHex(): string }, - { toBigInt(): bigint }, - ]; - const nonce = nonceValue.toBigInt(); - const recipient = recipientValue.toHex() as `0x${string}`; - if (!/^0x[0-9a-fA-F]{40}$/.test(recipient)) { - throw new Error(`cannot decode base_address in block ${block}: ${recipient}`); - } - if (nonce !== draft.nextExpectedNonce) { - throw new Error(`migration nonce gap: finalized event ${nonce}, expected ${draft.nextExpectedNonce}`); - } - draft.nextExpectedNonce++; - const expectedTreasury = config.expectedTreasuryDestination; - if ( - expectedTreasury && - whoValue.toHex().toLowerCase() === PENDULUM_TREASURY_ACCOUNT && - recipient.toLowerCase() !== expectedTreasury.toLowerCase() - ) { - // The vault is paused before the attestors can release it; while - // the pause is not confirmed, the monitor does not move past it. - const paused = await securityViolation( - "TREASURY DESTINATION VIOLATION", - `treasury migration nonce ${nonce} in finalized Pendulum block ${block} pays ${amountValue.toBigInt()} ` + - `pallet units to ${recipient}, not EXPECTED_TREASURY_DESTINATION ${expectedTreasury}; the monitor ` + - "holds at that block until the vault is paused", - ); - if (!paused) throw new PauseHold(`holding at Pendulum block ${block} until the vault is paused`); - } - if (recipient === ZERO_ADDRESS || recipient.toLowerCase() === config.vaultAddress.toLowerCase()) { - // The vault deterministically rejects this tuple (ZeroAddress / - // RecipientIsVault) and the attestors skip it, so it can never be - // approved or released. Tracking it would keep the pending count - // non-zero forever and page liveness every grace period (and make - // RB-7's "zero outstanding nonces" unsatisfiable). Record it once - // and leave it out of reconciliation. - await pageOnce( - `unreleasable:${nonce}`, - "CRITICAL: unreleasable migration burned on Pendulum", - `nonce ${nonce} in block ${block} burned ${amountValue.toBigInt()} to ${recipient}; the vault can never release it`, - ); - continue; - } - draft.migrations.set(nonce, { - nonce, - recipient, - palletAmount: amountValue.toBigInt(), - firstSeenMs, - sourceBlock: block, - }); - } - draft.lastPendulumBlock = block; - draft.lastPendulumHash = blockHash; - // Checkpoint a long catch-up as it goes: the Pendulum-side cursor is - // self-consistent on its own (the Base cursor only ever trails it), so a - // transient RPC error hours into a replay must not discard the progress - // and restart the whole replay — against a flaky endpoint that never - // converges, leaving the monitor blind indefinitely. - if (block % 200 === 0) { - state = cloneState(draft); - persistState(state); - } - } -} - -/** A vault total at `blockNumber`: zero before the vault existed. */ -async function vaultTotalAt(functionName: "totalReleased" | "totalSwept", blockNumber: bigint): Promise { - if (blockNumber < 0n) return 0n; - try { - return await publicClient.readContract({ address: config.vaultAddress, abi: vaultAbi, functionName, blockNumber }); - } catch (readError) { - if (!(await publicClient.getBytecode({ address: config.vaultAddress, blockNumber }))) return 0n; - throw readError; - } -} - -/** The vault's events in one block range, and `gap` when they do not account - * for the change of its running totals over the range: the Released amounts - * must add up to the growth of totalReleased, the RemainderSwept amounts to - * that of totalSwept. A backend that serves a truncated log result fails it. */ -async function readVaultRange(start: bigint, end: bigint) { - const [events, releasedBefore, sweptBefore, releasedAfter, sweptAfter] = await Promise.all([ - publicClient.getContractEvents({ address: config.vaultAddress, abi: vaultAbi, fromBlock: start, toBlock: end }), - vaultTotalAt("totalReleased", start - 1n), - vaultTotalAt("totalSwept", start - 1n), - vaultTotalAt("totalReleased", end), - vaultTotalAt("totalSwept", end), - ]); - const approvals = events.filter((event) => event.eventName === "Approved"); - const releases = events.filter((event) => event.eventName === "Released"); - const released = releases.reduce((sum, event) => sum + (event.args as { tokenAmount: bigint }).tokenAmount, 0n); - const swept = events - .filter((event) => event.eventName === "RemainderSwept") - .reduce((sum, event) => sum + (event.args as { amount: bigint }).amount, 0n); - const gap = released !== releasedAfter - releasedBefore || swept !== sweptAfter - sweptBefore - ? `in Base blocks ${start}-${end} the Released events add up to ${released} and the RemainderSwept events to ` + - `${swept}, but totalReleased grew by ${releasedAfter - releasedBefore} and totalSwept by ${sweptAfter - sweptBefore}` - : undefined; - return { approvals, releases, gap }; -} - -async function reconcileBaseEvents( - draft: RuntimeState, - toBlock: bigint, - conversionFactor: bigint, - sourceFinalizedTsMs: number, - threshold: bigint, -): Promise<{ violation?: string; awaitingSource?: string; incomplete?: string }> { - const releasedInScan = new Set(); - let firstViolation: string | undefined; - for (const [start, end] of blockRanges(draft.baseFromBlock, toBlock, config.baseMaxBlockRange)) { - // Probe the range's end block first. Against a load-balanced endpoint, - // `eth_getLogs` can be served by a node that has not reached `end` yet - // and some providers then silently truncate rather than error — which - // would advance the cursor past events this monitor never reconciled. A - // lagging node fails this probe loudly instead, and the cycle replays. - await publicClient.getBlock({ blockNumber: end }); - // The probe and the log query can still reach different backends, so the - // logs must also account for the vault's totals; a range whose logs do - // not is read once more, then held for the next poll. - let range = await readVaultRange(start, end); - if (range.gap) range = await readVaultRange(start, end); - if (range.gap) { - const heldMs = logScanHeldSinceMs === undefined ? 0 : Date.now() - logScanHeldSinceMs; - logScanHeldSinceMs ??= Date.now(); - if (Date.now() - lastLogScanAlertMs > config.unmatchedEventGraceSeconds * 1000) { - lastLogScanAlertMs = Date.now(); - // Held past the grace, no later Base event has been matched against its burn for that long. - const critical = heldMs > config.unmatchedEventGraceSeconds * 1000; - void alert( - `${critical ? "CRITICAL: " : ""}BASE LOG SCAN INCOMPLETE`, - `${range.gap}; the Base RPC served logs that miss events, so the range is read again every poll ` + - `and the Base cursor stays at block ${start}` + - (critical ? ` (for ${Math.round(heldMs / 1000)}s): no later Base event is matched against its burn` : "") + - " — check the Base RPC", - ); - } - return { violation: firstViolation, incomplete: range.gap }; - } - logScanHeldSinceMs = undefined; - const { approvals, releases } = range; - const observedAt = Date.now(); - let awaitingSource: string | undefined; - - // Preflight the complete range before deleting released migrations. If a - // source RPC is merely behind, the range must be replayed intact later. - for (const event of approvals) { - const actual = event.args as { nonce: bigint; recipient: `0x${string}`; palletAmount: bigint; attestor: string }; - const expected = draft.migrations.get(actual.nonce); - let mismatch = approvalMismatch(expected, actual); - const key = `Approved:${actual.nonce}`; - if (!mismatch) { - draft.unmatchedBaseFirstSeenAt.delete(key); - continue; - } - if (!expected) { - const disposition = unmatchedDisposition( - "Approved", - key, - actual.nonce, - event.blockNumber, - draft, - observedAt, - sourceFinalizedTsMs, - ); - if (disposition !== "proven") awaitingSource ??= `Approved nonce ${actual.nonce} at Base block ${event.blockNumber}`; - if (disposition === "await") continue; - mismatch += disposition === "proven" - ? " (source view has passed the event's timestamp — fabricated)" - : " (source view still behind after the full grace period)"; - } - // Approvals that disagree with the burn pause the vault once they come - // from two attestors, or from one at a threshold of two or less, where - // one more approval releases. One attestor's approval alone pages. - const { nonce, recipient, palletAmount, attestor } = actual; - const seen = draft.mismatchedApprovals.some((approval) => - approval.nonce === nonce && approval.attestor.toLowerCase() === attestor.toLowerCase() && - !approvalMismatch(approval, actual) - ); - if (!seen) draft.mismatchedApprovals.push({ nonce, recipient, palletAmount, attestor }); - const attestors = disagreeingAttestors(expected, nonce, draft.mismatchedApprovals); - if (attestors.length >= (threshold > 2n ? 2 : 1)) { - firstViolation ??= `Approved at Base block ${event.blockNumber}: ${mismatch}; attestor` + - `${attestors.length > 1 ? "s" : ""} ${attestors.join(", ")} approved tuples for nonce ${nonce} that ` + - "disagree with the finalized Pendulum view"; - continue; - } - void pageOnce( - `approval:${event.transactionHash}:${event.logIndex}`, - "CRITICAL: MISMATCHED APPROVAL", - `Approved at Base block ${event.blockNumber} by attestor ${attestor}: ${mismatch}; one attestor's approval ` + - "cannot release, so the vault is not paused — check that attestor's key and Pendulum node; a second " + - "attestor approving this nonce against its burn pauses the vault", - ); - } - for (const event of releases) { - const actual = event.args as { - nonce: bigint; - recipient: `0x${string}`; - palletAmount: bigint; - tokenAmount: bigint; - }; - const expected = draft.migrations.get(actual.nonce); - const mismatch = releaseMismatch(expected, actual, conversionFactor); - const key = `Released:${actual.nonce}`; - if (!mismatch) { - draft.unmatchedBaseFirstSeenAt.delete(key); - continue; - } - if (!expected) { - const disposition = unmatchedDisposition( - "Released", - key, - actual.nonce, - event.blockNumber, - draft, - observedAt, - sourceFinalizedTsMs, - ); - if (disposition !== "proven") awaitingSource ??= `Released nonce ${actual.nonce} at Base block ${event.blockNumber}`; - if (disposition === "await") continue; - firstViolation ??= `Released at Base block ${event.blockNumber}: ${mismatch} (` + - (disposition === "proven" - ? "source view has passed the event's timestamp — fabricated" - : "source view still behind after the full grace period") + ")"; - continue; - } - firstViolation ??= `Released at Base block ${event.blockNumber}: ${mismatch}`; - } - // A range holding an event the source may still reveal (awaiting it, or - // past the grace) is replayed later even after a violation: passing it - // would lose a legitimate release for good. - if (awaitingSource) return { violation: firstViolation, awaitingSource }; - - for (const event of releases) { - const actual = event.args as { - nonce: bigint; - recipient: `0x${string}`; - palletAmount: bigint; - tokenAmount: bigint; - }; - if (releasedInScan.has(actual.nonce)) { - firstViolation ??= `duplicate Released event for nonce ${actual.nonce}`; - continue; - } - const mismatch = releaseMismatch(draft.migrations.get(actual.nonce), actual, conversionFactor); - if (mismatch) { - firstViolation ??= `Released at Base block ${event.blockNumber}: ${mismatch}`; - continue; - } - releasedInScan.add(actual.nonce); - draft.migrations.delete(actual.nonce); - draft.livenessAlertedAt.delete(actual.nonce); - draft.mismatchedApprovals = draft.mismatchedApprovals.filter((approval) => approval.nonce !== actual.nonce); - } - draft.baseFromBlock = end + 1n; - } - return { violation: firstViolation }; -} - -/** Page every vault admin event, and every operation the governance Timelock - * scheduled, up to `toBlock`, the poll's `safe`/`finalized` Base block. - * Before the governance handover the vault admin acts without a delay, so - * its events are the only trace of a guardian, attestor or cap change. The - * scan keeps its own cursor in `state`, so a restart resumes where it stopped. */ -async function watchBaseAdmin(toBlock: bigint): Promise { - const timelock = config.timelockAddress; - /** Scheduled calls by operation id and block: a cancelled operation can be - * scheduled again with the same id, and then pages again. */ - const operations = new Map(); - const pages: Promise[] = []; - const fromBlock = state.baseAdminFromBlock; - for (const [start, end] of blockRanges(fromBlock, toBlock, config.baseMaxBlockRange)) { - await publicClient.getBlock({ blockNumber: end }); // same lagging-node probe as the vault scan - const adminEvents = await publicClient.getContractEvents({ - address: config.vaultAddress, - abi: vaultAdminEventsAbi, - fromBlock: start, - toBlock: end, - }); - for (const event of adminEvents) { - const args = Object.entries(event.args).map(([name, value]) => `${name}=${value}`).join(", "); - pages.push(pageOnce( - `vault:${event.transactionHash}:${event.logIndex}`, - "VAULT ADMIN EVENT", - `${event.eventName}(${args}) at Base block ${event.blockNumber}, tx ${event.transactionHash}`, - )); - } - if (!timelock) continue; - const scheduled = await publicClient.getContractEvents({ - address: timelock, - abi: timelockAbi, - eventName: "CallScheduled", - fromBlock: start, - toBlock: end, - }); - for (const event of scheduled) { - const { id, index, target, value, data } = event.args as { - id: `0x${string}`; - index: bigint; - target: `0x${string}`; - value: bigint; - data: `0x${string}`; - }; - const key = `timelock:${id}:${event.blockNumber}`; - const operation = operations.get(key) ?? { id, calls: [] }; - operation.calls.push({ index, target, value, data }); - operations.set(key, operation); - } - } - if (timelock && operations.size > 0) { - const token = await publicClient.readContract({ - address: config.vaultAddress, - abi: vaultAbi, - functionName: "token", - blockNumber: toBlock, - }); - const labels = { - [config.vaultAddress.toLowerCase()]: "vault", - [token.toLowerCase()]: "token", - [timelock.toLowerCase()]: "timelock", - }; - for (const [key, { id, calls }] of operations) { - if (onceOnlyPages.has(key)) continue; - const readyAt = await publicClient.readContract({ - address: timelock, - abi: timelockAbi, - functionName: "getTimestamp", - args: [id], - blockNumber: toBlock, - }); - pages.push(pageOnce( - key, - "GOVERNANCE OPERATION SCHEDULED", - describeTimelockOperation(id, calls, readyAt, Date.now(), labels, KNOWN_FUNCTIONS), - )); - } - } - await Promise.all(pages); - if (toBlock >= fromBlock) state.baseAdminFromBlock = toBlock + 1n; -} - -type ApiAt = Awaited>; - -/** A democracy `Bounded`, as much of it as the watcher reads. */ -interface BoundedCall { - isInline: boolean; - asInline: Uint8Array; - isLookup: boolean; - asLookup: { hash_: { toHex(): string }; len: { toNumber(): number } }; - asLegacy: { hash_: { toHex(): string } }; - toString(): string; -} - -/** A governance item's call: decoded, `pending` while it cannot be read yet - * (re-checked every poll), or neither when it is unreadable for good. */ -interface CallRead { - node?: CallNode; - text: string; - pending?: boolean; -} - -/** - * Largest governance call the watcher fetches and decodes. Anyone can note a - * preimage and propose it, and the decoding cost grows with its size; 64 KiB - * decodes in about 0.15 s even when packed with the smallest calls. - * Legitimate calls stay far below it; a bigger one is never fetched and pages - * as unreadable. - */ -const MAX_GOVERNANCE_CALL_BYTES = 64 * 1024; - -function readCall(decode: () => unknown): CallRead { - try { - const node = toCallNode(decode() as Parameters[0]); - return { node, text: describeCall(node) }; - } catch (decodeError) { - return { text: `the call cannot be decoded: ${String(decodeError).slice(0, 300)}` }; - } -} - -const oversizedCall = (len: number): CallRead => ({ - text: `the call is ${len} bytes, over the ${MAX_GOVERNANCE_CALL_BYTES} bytes the monitor decodes`, -}); - -/** The length of a noted preimage, which a legacy call reference leaves out. - * One noted before the pallet's status migration stays in the deprecated - * `statusFor` until something touches it. */ -async function notedLength(apiAt: ApiAt, hash: { toHex(): string }): Promise { - const status = (await apiAt.query.preimage.requestStatusFor(hash)).toJSON() as { - unrequested?: { len: number }; - requested?: { maybeLen: number | null }; - } | null; - if (status) return status.unrequested?.len ?? status.requested?.maybeLen ?? undefined; - const old = (await apiAt.query.preimage.statusFor(hash)).toJSON() as { - unrequested?: { len: number }; - requested?: { len: number | null }; - } | null; - return old?.unrequested?.len ?? old?.requested?.len ?? undefined; -} - -/** Reads a proposal's call. Its preimage may be noted at any time before - * enactment, so until then the call is pending, not unreadable. */ -async function boundedCall(apiAt: ApiAt, bounded: BoundedCall): Promise { - if (bounded.isInline) return readCall(() => apiAt.registry.createType("Call", bounded.asInline)); - const hash = bounded.isLookup ? bounded.asLookup.hash_ : bounded.asLegacy.hash_; - const len = bounded.isLookup ? bounded.asLookup.len.toNumber() : await notedLength(apiAt, hash); - if (len !== undefined && len > MAX_GOVERNANCE_CALL_BYTES) return oversizedCall(len); - const preimage = len === undefined - ? undefined - : (await apiAt.query.preimage.preimageFor([hash, len])) as unknown as { isSome: boolean; unwrap(): Uint8Array }; - if (!preimage?.isSome) return { text: `preimage ${hash.toHex()} is not noted yet`, pending: true }; - return readCall(() => apiAt.registry.createType("Call", preimage.unwrap())); -} - -/** Reads a council or technical-committee motion's call, sized before it is fetched. */ -async function motionCall(apiAt: ApiAt, collective: "council" | "technicalCommittee", hash: { toHex(): string }): Promise { - const proposalOf = apiAt.query[collective].proposalOf; - const len = Number((await proposalOf.size(hash)).toString()); - if (len > MAX_GOVERNANCE_CALL_BYTES) return oversizedCall(len); - const call = (await proposalOf(hash)) as unknown as { isSome: boolean; unwrap(): unknown }; - if (!call.isSome) return { text: "the motion's call is not stored", pending: true }; - return readCall(() => call.unwrap()); -} - -/** - * Page every pending Pendulum governance item once: public proposals, the - * council's external proposal, ongoing referenda and open council and - * technical-committee motions, with the call decoded. Calls that mint, move - * or unlock balances or change the runtime or the migration pallet page as - * CRITICAL: minted PEN burned through the migration drains the fixed-supply - * vault, and only Pendulum governance can stop the call before enactment. - * So does a call the watcher cannot read; until it can, it is re-checked - * every poll. Alert-only; it runs after the poll's conservation checks, so it - * cannot stop or delay them, and decodes within fixed bounds, so its cost to - * the next poll is bounded. - */ -async function watchPendulumGovernance(api: ApiPromise): Promise { - const finalizedHash = await api.rpc.chain.getFinalizedHead(); - const finalizedBlock = (await api.rpc.chain.getHeader(finalizedHash)).number.toNumber(); - const apiAt = await api.at(finalizedHash); - const democracy = apiAt.query.democracy; - const launchPeriod = Number(apiAt.consts.democracy.launchPeriod.toString()); - const nextLaunch = (Math.floor(finalizedBlock / launchPeriod) + 1) * launchPeriod; - const items: Array<{ key: string; label: string; hint: string; read(): Promise }> = []; - const proposals = (await democracy.publicProps()) as unknown as Array< - [{ toNumber(): number }, BoundedCall, { toString(): string }] - >; - for (const [index, bounded, proposer] of proposals) { - items.push({ - key: `proposal:${index.toNumber()}`, - label: `public proposal #${index.toNumber()} from ${proposer.toString()}`, - hint: `the best-backed public proposal becomes a referendum at block ${nextLaunch}; ` + - "a hostile one needs a council or technical-committee cancel before then, or nay votes after", - read: () => boundedCall(apiAt, bounded), - }); - } - const [count, lowest] = (await Promise.all([democracy.referendumCount(), democracy.lowestUnbaked()])).map((value) => - Number(value.toString()) - ); - /** The newest ongoing referendum of each call. */ - const referendumOf = new Map(); - for (let index = lowest; index < count; index++) { - const info = (await democracy.referendumInfoOf(index)) as unknown as { - isSome: boolean; - unwrap(): { isOngoing: boolean; asOngoing: { proposal: BoundedCall; end: { toNumber(): number } } }; - }; - if (!info.isSome || !info.unwrap().isOngoing) continue; - const ongoing = info.unwrap().asOngoing; - referendumOf.set(ongoing.proposal.toString(), index); - items.push({ - key: `referendum:${index}`, - label: `referendum #${index}`, - hint: `voting ends at block ${ongoing.end.toNumber()}; a hostile one needs nay votes or an emergency cancel`, - read: () => boundedCall(apiAt, ongoing.proposal), - }); - } - const external = (await democracy.nextExternal()) as unknown as { isSome: boolean; unwrap(): [BoundedCall, unknown] }; - if (external.isSome) { - const [bounded] = external.unwrap(); - // Only its call identifies an external proposal. A tabled one becomes a - // referendum with the same call, so that referendum keys the page of the - // same call proposed again. - items.push({ - key: `external:${referendumOf.get(bounded.toString()) ?? "-"}:${bounded.toString()}`, - label: "the council's external proposal", - hint: `it can become a referendum at block ${nextLaunch}`, - read: () => boundedCall(apiAt, bounded), - }); - } else { - // Once it has left the queue, the same call proposed again pages again. - for (const key of onceOnlyPages.keys()) if (key.startsWith("external:")) onceOnlyPages.delete(key); - } - for (const collective of ["council", "technicalCommittee"] as const) { - const motions = (await apiAt.query[collective].proposals()) as unknown as Array<{ toHex(): string }>; - for (const hash of motions) { - // A closed motion can be proposed again with the same call and hash, - // but never with the same index, so the index keys the page. - const votes = (await apiAt.query[collective].voting(hash)).toJSON() as { index: number } | null; - items.push({ - key: `${collective}:${votes?.index}:${hash.toHex()}`, - label: `${collective === "council" ? "council" : "technical committee"} motion ${hash.toHex()}`, - hint: "it executes as soon as enough members vote aye; a hostile one needs nay votes", - read: () => motionCall(apiAt, collective, hash), - }); - } - } - // Pages go out together, so a burst of new items costs one webhook round trip. - const pages: Promise[] = []; - for (const item of items) { - if (onceOnlyPages.has(item.key)) continue; - let call: CallRead; - try { - call = await item.read(); - } catch (readError) { - // One item must never hide the items after it; a failed read is retried. - call = { text: `reading the call failed: ${String(readError).slice(0, 300)}`, pending: true }; - } - if (!call.node) { - // A call nobody can review is critical. One that cannot be read yet - // is re-checked every poll and pages again once it can be read. - pages.push(pageOnce( - call.pending ? `${item.key}:unread` : item.key, - "CRITICAL: Pendulum governance item cannot be read", - `${item.label}: ${call.text}; ` + - `${call.pending ? "the monitor pages its call once it can read it" : "review it manually"}; ${item.hint}`, - )); - continue; - } - const risky = riskyCalls(call.node); - pages.push(pageOnce( - item.key, - risky.length - ? "CRITICAL: Pendulum governance item mints, moves or unlocks balances, or changes the runtime" - : "PENDULUM GOVERNANCE ITEM", - `${item.label}: ${call.text}${risky.length ? ` (flagged: ${risky.join(", ")})` : ""}; ${item.hint}`, - )); - } - await Promise.all(pages); -} - -function payloadHash(migration: MigrationRecord): `0x${string}` { - return keccak256(encodeAbiParameters( - [{ type: "uint64" }, { type: "address" }, { type: "uint256" }], - [migration.nonce, migration.recipient as `0x${string}`, migration.palletAmount], - )); -} - -async function activeApprovalCounts(migrations: MigrationRecord[], blockNumber: bigint): Promise { - const counts: bigint[] = []; - for (const batch of chunks(migrations, config.readBatchSize)) { - const single = (migration: MigrationRecord) => publicClient.readContract({ - address: config.vaultAddress, - abi: vaultAbi, - functionName: "activeApprovals", - args: [payloadHash(migration)], - blockNumber, - }); - if (!multicallUnavailable) { - try { - const result = await publicClient.multicall({ - allowFailure: false, - blockNumber, - contracts: batch.map((migration) => ({ - address: config.vaultAddress, - abi: vaultAbi, - functionName: "activeApprovals" as const, - args: [payloadHash(migration)] as const, - })), - }); - counts.push(...(result as bigint[])); - continue; - } catch (error) { - await alert("multicall batch failed, using bounded reads for this batch", `${error}`); - } - } - counts.push(...(await Promise.all(batch.map(single)))); - } - return counts; -} - -interface SourceView { - finalizedBlock: number; - timestampMs: number; - totalMigrated: bigint; -} - -/** Reads finalized Pendulum and ingests its new migrations into `draft`. - * Throws on any read, decode or continuity failure. */ -async function readSource(api: ApiPromise, draft: RuntimeState): Promise { - // Read Pendulum after choosing the Base snapshot. TotalMigrated can only - // grow, so this ordering cannot false-positive if a burn finalizes mid-poll. - const finalizedHash = await api.rpc.chain.getFinalizedHead(); - const finalizedHeader = await api.rpc.chain.getHeader(finalizedHash); - const finalized = { number: finalizedHeader.number.toNumber(), hash: finalizedHash.toHex() }; - const { hashes, confirmed, crossChecked, anchored } = await sourceHashes(api, draft, finalized); - // The view: the newest block the checkers confirm, or the block ingested - // last while they confirm none after it. Every value read from Pendulum - // below is read there. - const viewBlock = Math.max(confirmed, draft.lastPendulumBlock); - const viewHash = hashes.get(viewBlock) ?? draft.lastPendulumHash; - if (!viewHash) throw new Error(`no verified hash for Pendulum block ${viewBlock}`); - const apiAt = await api.at(viewHash); - const sourceFinalizedTsMs = Number((await apiAt.query.timestamp.now()).toString().replaceAll(",", "")); - // A stalling source view is this daemon going blind: it cannot verify any - // new Base event, and once the unmatched-event grace expires that forces a - // cannot-verify pause. Page as soon as the view goes stale, so operators - // get the entire grace window to restore the node, or the checker that - // holds the view back, before that happens. - if ( - Date.now() - sourceFinalizedTsMs > config.sourceStaleAlertMs && - Date.now() - lastSourceStallAlertMs > config.unmatchedEventGraceSeconds * 1000 - ) { - lastSourceStallAlertMs = Date.now(); - // The checkers that confirm less than the source finalized hold the view back. - const holding = finalized.number <= viewBlock ? "" : [ - crossChecked !== undefined && crossChecked < finalized.number && "the second node (PENDULUM_WS_CROSSCHECK)", - anchored !== undefined && anchored < finalized.number && "the relay RPCs (POLKADOT_WS)", - ].filter(Boolean).join(" and "); - await alert( - "PENDULUM SOURCE VIEW STALLED", - `verified Pendulum block ${viewBlock} is ${Math.round((Date.now() - sourceFinalizedTsMs) / 1000)}s old` + - (holding ? ` (the source finalized ${finalized.number}, not confirmed yet by ${holding})` : "") + - `; the monitor cannot verify new Base events against a stalled view — ` + - `${holding ? `check ${holding}` : "restore the node"} before the ${config.unmatchedEventGraceSeconds}s ` + - "unmatched-event grace forces a pause", - ); - } - const [totals, totalIssuanceValue] = await Promise.all([migrationTotals(apiAt), apiAt.query.balances.totalIssuance()]); - const { nextNonce: chainNextNonce, totalMigrated } = totals; - if (crossChecked !== undefined && crossChecked >= viewBlock) await crossCheckTotals(viewBlock, viewHash, totals); - await ingestPendulumEvents(api, draft, confirmed, hashes, crossChecked !== undefined); - if (chainNextNonce !== draft.nextExpectedNonce) { - throw new Error(`finalized nextNonce ${chainNextNonce} != monitor event cursor ${draft.nextExpectedNonce}`); - } - // Source-chain supply guard. Every burn of freshly MINTED Pendulum PEN (a - // passed setBalance referendum, an unexpected teleport-in) is a genuine - // finalized burn: attestors approve it honestly, every tuple matches, and - // M2a/M2b stay satisfied — while the fixed-supply vault drains ahead of - // honest late migrators. issuance + migrated cannot grow under any - // legitimate flow, so growth over the first-seen anchor is the one signal - // for this. Alert only: pausing is a human decision here. - const totalIssuance = BigInt(totalIssuanceValue.toString().replaceAll(",", "")); - const sourceSupply = totalIssuance + totalMigrated; - if (draft.issuanceAnchor === undefined) { - draft.issuanceAnchor = sourceSupply; - log(`anchored source supply: issuance ${totalIssuance} + migrated ${totalMigrated} = ${sourceSupply}`); - } else if ( - sourceSupply > draft.issuanceAnchor + config.issuanceToleranceRaw && - Date.now() - lastIssuanceAlertMs > config.graceSeconds * 1000 - ) { - lastIssuanceAlertMs = Date.now(); - await alert( - "SOURCE SUPPLY GREW", - `Pendulum issuance ${totalIssuance} + migrated ${totalMigrated} = ${sourceSupply} exceeds the anchor ` + - `${draft.issuanceAnchor} by ${sourceSupply - draft.issuanceAnchor} pallet units: PEN was minted at the ` + - `source and can be burned against the fixed-supply vault — investigate before it migrates (RB-3)`, - ); - } - return { finalizedBlock: viewBlock, timestampMs: sourceFinalizedTsMs, totalMigrated }; -} - -/** When the vault balance check last started, and whether a poll is running. */ -let lastBalanceCheckMs = 0; -let polling = false; - -/** The vault's totals at the confirmed Base block `blockNumber`, after the - * vault balance check (M2b) on them, which pauses the vault on a deficit. */ -async function checkVaultBalance(blockNumber: bigint) { - lastBalanceCheckMs = Date.now(); - const [totalReleased, totalSwept, conversionFactor, tokenAddress] = await Promise.all([ - publicClient.readContract({ address: config.vaultAddress, abi: vaultAbi, functionName: "totalReleased", blockNumber }), - publicClient.readContract({ address: config.vaultAddress, abi: vaultAbi, functionName: "totalSwept", blockNumber }), - publicClient.readContract({ address: config.vaultAddress, abi: vaultAbi, functionName: "conversionFactor", blockNumber }), - publicClient.readContract({ address: config.vaultAddress, abi: vaultAbi, functionName: "token", blockNumber }), - ]); - const [totalSupply, vaultBalance] = await Promise.all([ - publicClient.readContract({ address: tokenAddress, abi: erc20Abi, functionName: "totalSupply", blockNumber }), - publicClient.readContract({ - address: tokenAddress, - abi: erc20Abi, - functionName: "balanceOf", - args: [config.vaultAddress], - blockNumber, - }), - ]); - if (vaultConservationDeficit(vaultBalance, totalReleased, totalSwept, totalSupply)) { - await securityViolation( - "VAULT BALANCE DEFICIT", - `balance ${vaultBalance} + released ${totalReleased} + swept ${totalSwept} < supply ${totalSupply}`, - ); - } - return { totalReleased, totalSwept, conversionFactor, vaultBalance }; -} - -/** Runs the vault balance check on its own while a poll is still busy a poll - * interval after its last one started (a slow Pendulum source, a long - * catch-up). main() calls this every poll interval, so two checks are about - * three poll intervals apart at most. */ -export async function watchVaultBalance(): Promise { - if (!polling || Date.now() - lastBalanceCheckMs < config.pollIntervalMs) return; - lastBalanceCheckMs = Date.now(); - try { - const confirmed = await publicClient.getBlock({ blockTag: config.baseFinalityTag }); - if (confirmed.number === null) throw new Error(`${config.baseFinalityTag} Base block has no number`); - await checkVaultBalance(confirmed.number); - } catch (balanceError) { - await alert("monitor check failed", `the vault balance check failed: ${balanceError}`); - } -} - -/** One pass of the checks; resolves to whether every one of them ran. */ -async function check(api: ApiPromise): Promise { - const confirmedBase = await publicClient.getBlock({ blockTag: config.baseFinalityTag }); - if (confirmedBase.number === null) throw new Error(`${config.baseFinalityTag} Base block has no number`); - const baseBlock = confirmedBase.number; - // A Base view that stops advancing hides every vault event after it while - // all checks keep passing at the old block; page it like a stalled source. - const baseAgeMs = Date.now() - Number(confirmedBase.timestamp) * 1000; - if ( - baseAgeMs > config.baseStaleAlertMs && - Date.now() - lastBaseStallAlertMs > config.unmatchedEventGraceSeconds * 1000 - ) { - lastBaseStallAlertMs = Date.now(); - await alert( - "BASE VIEW STALLED", - `${config.baseFinalityTag} block ${baseBlock} is ${Math.round(baseAgeMs / 1000)}s old; vault events after it ` + - `are not being checked — restore the Base node, or check whether Base's ${config.baseFinalityTag} head stalled`, - ); - } - await checkAutoPauseReadiness(); - - // The vault balance check (M2b) reads Base only, so it runs before anything - // that waits on Pendulum or scans Base logs: a failing source or log scan - // cannot skip it, and while the rest of a poll takes longer than a poll - // interval, `watchVaultBalance` runs it on its own. Its pause does not end - // the poll. - const { totalReleased, totalSwept, conversionFactor, vaultBalance } = await checkVaultBalance(baseBlock); - - // The admin scan only pages and keeps its own cursor, so nothing else waits - // for it, and its failure skips no other check. - let adminScanned = true; - try { - await watchBaseAdmin(baseBlock); - adminScanFailingSinceMs = undefined; - } catch (adminError) { - adminScanned = false; - const failingMs = adminScanFailingSinceMs === undefined ? 0 : Date.now() - adminScanFailingSinceMs; - adminScanFailingSinceMs ??= Date.now(); - if (Date.now() - lastAdminScanFailureAlertMs > config.unmatchedEventGraceSeconds * 1000) { - lastAdminScanFailureAlertMs = Date.now(); - // Meanwhile the veto window of any operation scheduled since runs unseen. - const critical = failingMs > config.unmatchedEventGraceSeconds * 1000; - await alert( - `${critical ? "CRITICAL: " : ""}BASE ADMIN SCAN FAILED`, - `${adminError}; vault admin events and the governance queue are not paged until the scan reads again` + - (critical ? ` (failing for ${Math.round(failingMs / 1000)}s)` : "") + - ", and it then resumes where it stopped; every other check still runs", - ); - } - } - - // A Pendulum failure must not switch off the Base-side checks. Base events - // are then reconciled against the last view read in full, as against a - // lagging source but with no fabrication proof: a mismatch on a known nonce - // is judged at once, an unknown nonce after the grace. The vault balance - // check needs no source; M2a and liveness wait for it. - let draft = cloneState(state); - let source: SourceView | undefined; - try { - source = await readSource(api, draft); - } catch (sourceError) { - draft = cloneState(state); // drops a half-ingested block; the 200-block checkpoints already moved `state` - const contradicted = sourceError instanceof SourceIntegrityError; - const consequence = `Base events are checked against the last Pendulum view read in full: one whose nonce ` + - `that view lacks pauses the vault ${config.unmatchedEventGraceSeconds}s after its first sighting, and M2a ` + - "and liveness are skipped"; - if (sourceError instanceof PauseHold) log(sourceError.message); - else if (contradicted && Date.now() - lastSourceIntegrityAlertMs > config.unmatchedEventGraceSeconds * 1000) { - lastSourceIntegrityAlertMs = Date.now(); - await alert( - "CRITICAL: PENDULUM SOURCE CONTRADICTED", - `${sourceError}; nothing from the source is ingested until it verifies again, and until then ` + - `${consequence} — stop trusting the source node and compare it with other nodes and the relay chain`, - ); - } else if (!contradicted && Date.now() - lastSourceFailureAlertMs > config.unmatchedEventGraceSeconds * 1000) { - lastSourceFailureAlertMs = Date.now(); - await alert("PENDULUM SOURCE UNREADABLE", `${sourceError}; until the source reads again, ${consequence}`); - } - } - - const threshold = await publicClient.readContract({ - address: config.vaultAddress, - abi: vaultAbi, - functionName: "threshold", - blockNumber: baseBlock, - }); - const reconciliation = await reconcileBaseEvents(draft, baseBlock, conversionFactor, source?.timestampMs ?? 0, threshold); - if (reconciliation.violation) { - if (await securityViolation("MIGRATION TUPLE VIOLATION", reconciliation.violation)) { - state = draft; - persistState(state); - } - return false; - } - if (!source) { - state = draft; // reconciled against the last view read in full - persistState(state); - return false; - } - const { finalizedBlock, totalMigrated } = source; - // M2a needs only the totals, so a held log range does not stop it. A Base - // event whose burn the source view does not show yet would make it fire - // falsely, so it waits for that. - if ( - !reconciliation.awaitingSource && - releasedExceedsMigrated(totalReleased, totalMigrated, conversionFactor, config.acknowledgedReleaseExcess) - ) { - if (await securityViolation( - "CONSERVATION VIOLATION", - `released ${totalReleased} > migrated ${totalMigrated * conversionFactor} + acknowledged excess ` + - `${config.acknowledgedReleaseExcess} (token units)`, - )) { - state = draft; - persistState(state); - } - return false; - } - if (reconciliation.awaitingSource || reconciliation.incomplete) { - state = draft; - persistState(state); - log( - reconciliation.incomplete - ? `holding the Base cursor at ${draft.baseFromBlock}: ${reconciliation.incomplete}` - : `waiting up to ${config.unmatchedEventGraceSeconds}s for Pendulum RPC to catch up with ` + - reconciliation.awaitingSource, - ); - return false; - } - - const pending = [...draft.migrations.values()]; - const approvals = await activeApprovalCounts(pending, baseBlock); - const now = Date.now(); - // One page per kind and poll however long the backlog, with the detail - // per nonce in the log: a page per nonce floods the receiver and, awaited - // in turn, delays the next poll's checks. - const quorum: MigrationRecord[] = []; - const belowQuorum: MigrationRecord[] = []; - for (let index = 0; index < pending.length; index++) { - const migration = pending[index]; - if (!isStale(migration.firstSeenMs, now, config.graceSeconds)) continue; - const lastAlerted = draft.livenessAlertedAt.get(migration.nonce); - if (lastAlerted !== undefined && now - lastAlerted < config.graceSeconds * 1000) continue; - log(`liveness: nonce ${migration.nonce} has ${approvals[index]}/${threshold} active approvals and is unreleased`); - (approvals[index] >= threshold ? quorum : belowQuorum).push(migration); - } - const pageLiveness = async (migrations: MigrationRecord[], subject: string, detail: string) => { - if (!migrations.length) return; - const nonces = migrations.slice(0, 10).map((migration) => migration.nonce).join(", ") + - (migrations.length > 10 ? ` and ${migrations.length - 10} more` : ""); - if (await alert(subject, `${migrations.length} migration(s) (nonce ${nonces}) ${detail}`)) { - for (const migration of migrations) draft.livenessAlertedAt.set(migration.nonce, now); - } - }; - // Quorum is present but nothing released it: cap-deferred and starved of - // daily allowance, paused, or made releasable by a threshold cut without - // ever emitting ReleasePending (so the releaser cannot see it). - await pageLiveness( - quorum, - "LIVENESS: quorum reached but not released", - `have ${threshold} or more active approvals and are still unreleased after ${config.graceSeconds}s — check the ` + - `daily allowance / pause state, or call release() directly if one qualified through a threshold cut (RB-6)`, - ); - await pageLiveness( - belowQuorum, - "LIVENESS: migration below approval quorum", - `have fewer than ${threshold} active approvals after ${config.graceSeconds}s`, - ); - - state = draft; - persistState(state); - log( - `ok: migrated=${totalMigrated} released=${totalReleased} swept=${totalSwept} ` + - `pending=${state.migrations.size} vaultBalance=${vaultBalance} base=${baseBlock} pendulum=${finalizedBlock}`, - ); - return adminScanned; -} - -/** Tells an external dead-man's switch (Healthchecks.io style) that a poll - * ran every check; it pages when these requests stop. Never fatal. */ -async function heartbeat(): Promise { - if (!config.heartbeatUrl) return; - try { - const response = await fetch(config.heartbeatUrl, { signal: AbortSignal.timeout(10_000) }); - if (!response.ok) console.error(`heartbeat answered ${response.status}`); - } catch (heartbeatError) { - console.error(`heartbeat failed: ${redact(`${heartbeatError}`)}`); - } -} - -/** Startup once the Pendulum API is connected: durable state, the multicall - * probe, the state identity check and the auto-pause readiness. `connect` - * replaces how the nodes of the integrity checks are reached (tests). */ -export async function start(api: ApiPromise, connect?: (url: string) => Promise): Promise { - if (connect) connectNode = connect; - state = hydrateState(); - const multicallCode = await publicClient.getBytecode({ address: MULTICALL3_ADDRESS }); - if (!multicallCode || multicallCode === "0x") { - multicallUnavailable = true; - await alert("multicall unavailable, using bounded individual reads", `no contract code at ${MULTICALL3_ADDRESS}`); - } - const pendulumGenesisHash = api.genesisHash.toHex(); - if (state.pendulumGenesisHash) { - assertMonitorStateIdentity( - { - baseChainId: state.baseChainId, - vaultAddress: state.vaultAddress, - pendulumGenesisHash: state.pendulumGenesisHash, - }, - { ...config, pendulumGenesisHash }, - ); - } else { - state.pendulumGenesisHash = pendulumGenesisHash; - } - if (!config.guardianPrivateKey) { - await pageOnce( - "autopause-disabled", - "AUTO-PAUSE DISABLED", - "no GUARDIAN_PRIVATE_KEY configured: violations page, but nothing pauses the vault automatically", - ); - } - const readiness = await checkAutoPauseReadiness(); - log( - `monitor started; Pendulum after ${state.lastPendulumBlock}, Base from ${state.baseFromBlock}, ` + - `polling every ${config.pollIntervalMs}ms; governance queue ` + - (config.timelockAddress ? `watched at ${config.timelockAddress}` : "not watched (TIMELOCK_ADDRESS unset)") + - `; second Pendulum node ${config.pendulumWsCrossCheck ? "cross-checked" : "not set"}` + - `; auto-pause ${!config.guardianPrivateKey ? "disabled" : readiness.length ? "NOT READY" : "ready"}` + - `; acknowledged release excess ${config.acknowledgedReleaseExcess} token units`, - ); -} - -/** One poll; a failure pages and the next poll starts afresh. */ -export async function poll(api: ApiPromise): Promise { - polling = true; - let complete = false; - try { - complete = await check(api); - } catch (checkError) { - await alert("monitor check failed", `${checkError}`); - } - try { - await watchPendulumGovernance(api); - } catch (governanceError) { - complete = false; - if (Date.now() - lastGovernanceWatchFailureMs > config.graceSeconds * 1000) { - lastGovernanceWatchFailureMs = Date.now(); - await alert("pendulum governance watch failed", `${governanceError}`); - } - } - await retryUndeliveredPages(); - if (complete) await heartbeat(); - polling = false; -} - -async function main(): Promise { - const api = await ApiPromise.create({ provider: new WsProvider(config.pendulumWs) }); - await start(api); - setInterval(() => void watchVaultBalance(), config.pollIntervalMs); - for (;;) { - await poll(api); - await new Promise((resolve) => setTimeout(resolve, config.pollIntervalMs)); - } -} - -// main.test.ts sets this before importing the module, to drive single polls. -if (!(globalThis as { penMonitorUnderTest?: boolean }).penMonitorUnderTest) { - main().catch(async (error) => { - await alert("monitor startup failed", `${error}`); - process.exit(1); - }); -} diff --git a/monitor/src/state.test.ts b/monitor/src/state.test.ts deleted file mode 100644 index 017bb11fb..000000000 --- a/monitor/src/state.test.ts +++ /dev/null @@ -1,105 +0,0 @@ -import assert from "node:assert/strict"; -import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; -import { test } from "node:test"; -import { - assertMonitorStateIdentity, - loadMonitorState, - saveMonitorState, - type PersistedMonitorState, -} from "./state.js"; - -const state: PersistedMonitorState = { - version: 1, - baseChainId: 8453, - vaultAddress: "0x1111111111111111111111111111111111111111", - pendulumGenesisHash: `0x${"33".repeat(32)}`, - lastPendulumBlock: 100, - nextExpectedNonce: "3", - baseFromBlock: "200", - migrations: [{ - nonce: "2", - recipient: "0x2222222222222222222222222222222222222222", - palletAmount: "100", - firstSeenMs: 1_700_000_000_000, - sourceBlock: 100, - }], - livenessAlertedAt: [["2", 1_700_000_001_000]], - unmatchedBaseFirstSeenAt: [["Approved:3", 1_700_000_002_000]], -}; - -test("monitor state survives a durable replacement with its security history", () => { - const directory = mkdtempSync(join(tmpdir(), "pen-monitor-state-")); - const path = join(directory, "state.json"); - try { - assert.equal(loadMonitorState(path), undefined); - saveMonitorState(path, state); - const loaded = loadMonitorState(path); - assert.deepEqual(loaded, state); - assert.doesNotThrow(() => assertMonitorStateIdentity(loaded!, state)); - assert.throws( - () => assertMonitorStateIdentity(loaded!, { ...state, pendulumGenesisHash: `0x${"44".repeat(32)}` }), - /Pendulum genesis/, - ); - } finally { - rmSync(directory, { recursive: true, force: true }); - } -}); - -test("corrupt monitor state fails rather than restarting grace timers", () => { - const directory = mkdtempSync(join(tmpdir(), "pen-monitor-state-")); - const path = join(directory, "state.json"); - try { - writeFileSync(path, "{"); - assert.throws(() => loadMonitorState(path), /refusing to reset security history/); - } finally { - rmSync(directory, { recursive: true, force: true }); - } -}); - -test("missing unmatched-event history is rejected rather than resetting its grace period", () => { - const directory = mkdtempSync(join(tmpdir(), "pen-monitor-state-")); - const path = join(directory, "state.json"); - try { - const { unmatchedBaseFirstSeenAt: _, ...incomplete } = state; - writeFileSync(path, JSON.stringify(incomplete)); - assert.throws(() => loadMonitorState(path), /invalid or legacy schema/); - } finally { - rmSync(directory, { recursive: true, force: true }); - } -}); - -test("optional fields are checked when present", () => { - const directory = mkdtempSync(join(tmpdir(), "pen-monitor-state-")); - const path = join(directory, "state.json"); - const approval = { - nonce: "2", - recipient: "0x3333333333333333333333333333333333333333", - palletAmount: "100", - attestor: "0x4444444444444444444444444444444444444444", - } as const; - try { - const complete = { - ...state, - baseAdminFromBlock: "150", - mismatchedApprovals: [approval], - lastPendulumHash: `0x${"55".repeat(32)}` as const, - relayAnchor: { number: 100, hash: `0x${"66".repeat(32)}` as const }, - }; - saveMonitorState(path, complete); - assert.deepEqual(loadMonitorState(path), complete); - for (const bad of [ - { baseAdminFromBlock: "-1" }, - { mismatchedApprovals: [{ ...approval, attestor: "0x44" }] }, - { lastPendulumHash: "0x55" }, - { relayAnchor: { number: "100", hash: `0x${"66".repeat(32)}` } }, - { relayAnchor: { number: 100, hash: "0x66" } }, - ]) { - writeFileSync(path, JSON.stringify({ ...state, ...bad })); - assert.throws(() => loadMonitorState(path), /invalid or legacy schema/); - } - } finally { - rmSync(directory, { recursive: true, force: true }); - } -}); diff --git a/monitor/src/state.ts b/monitor/src/state.ts deleted file mode 100644 index 320ae7cff..000000000 --- a/monitor/src/state.ts +++ /dev/null @@ -1,172 +0,0 @@ -import { - closeSync, - fsyncSync, - openSync, - readFileSync, - renameSync, - unlinkSync, - writeFileSync, -} from "node:fs"; -import { dirname } from "node:path"; - -export interface PersistedMigration { - nonce: string; - recipient: `0x${string}`; - palletAmount: string; - firstSeenMs: number; - sourceBlock: number; -} - -export interface PersistedMonitorState { - version: 1; - baseChainId: number; - vaultAddress: `0x${string}`; - pendulumGenesisHash: `0x${string}`; - lastPendulumBlock: number; - nextExpectedNonce: string; - baseFromBlock: string; - migrations: PersistedMigration[]; - livenessAlertedAt: Array<[string, number]>; - unmatchedBaseFirstSeenAt: Array<[string, number]>; - /** Pendulum totalIssuance + TotalMigrated when first observed; absent in - * state written before it existed and re-anchored on load. */ - issuanceAnchor?: string; - /** Next Base block of the admin scan; absent in older state, which never - * persisted a Base cursor past it, and then `baseFromBlock`. */ - baseAdminFromBlock?: string; - /** Approvals that disagreed with the finalized burn, until their nonce is - * released; absent in older state. */ - mismatchedApprovals?: PersistedApproval[]; - /** The hash of `lastPendulumBlock`; absent in older state. */ - lastPendulumHash?: `0x${string}`; - /** The newest relay-chain record of the Pendulum head the source was - * verified against; absent without the relay-chain anchor. */ - relayAnchor?: { number: number; hash: `0x${string}` }; -} - -export interface PersistedApproval { - nonce: string; - recipient: `0x${string}`; - palletAmount: string; - attestor: `0x${string}`; -} - -const isAddress = (value: unknown) => typeof value === "string" && /^0x[0-9a-fA-F]{40}$/.test(value); -const isHash = (value: unknown) => typeof value === "string" && /^0x[0-9a-fA-F]{64}$/.test(value); - -function isMissing(error: unknown): boolean { - return (error as NodeJS.ErrnoException)?.code === "ENOENT"; -} - -function isUint(value: unknown): value is string { - return typeof value === "string" && /^\d+$/.test(value); -} - -export function loadMonitorState(path: string): PersistedMonitorState | undefined { - let raw: string; - try { - raw = readFileSync(path, "utf8"); - } catch (error) { - if (isMissing(error)) return undefined; - throw new Error(`cannot read monitor state ${path}`, { cause: error }); - } - let value: unknown; - try { - value = JSON.parse(raw); - } catch (error) { - throw new Error(`monitor state ${path} is not valid JSON; refusing to reset security history`, { cause: error }); - } - const state = value as Partial; - const badMigration = !Array.isArray(state.migrations) || state.migrations.some((migration) => - !isUint(migration?.nonce) || - typeof migration?.recipient !== "string" || - !/^0x[0-9a-fA-F]{40}$/.test(migration.recipient) || - !isUint(migration?.palletAmount) || - !Number.isSafeInteger(migration?.firstSeenMs) || - !Number.isSafeInteger(migration?.sourceBlock), - ); - const badAlerts = !Array.isArray(state.livenessAlertedAt) || state.livenessAlertedAt.some( - (entry) => !Array.isArray(entry) || entry.length !== 2 || !isUint(entry[0]) || !Number.isSafeInteger(entry[1]), - ); - const badApprovals = state.mismatchedApprovals !== undefined && ( - !Array.isArray(state.mismatchedApprovals) || state.mismatchedApprovals.some((approval) => - !isUint(approval?.nonce) || !isAddress(approval?.recipient) || !isUint(approval?.palletAmount) || - !isAddress(approval?.attestor) - ) - ); - const badUnmatchedEvents = !Array.isArray(state.unmatchedBaseFirstSeenAt) || state.unmatchedBaseFirstSeenAt.some( - (entry) => - !Array.isArray(entry) || - entry.length !== 2 || - typeof entry[0] !== "string" || - !/^(Approved|Released):\d+$/.test(entry[0]) || - !Number.isSafeInteger(entry[1]), - ); - if ( - state.version !== 1 || - !Number.isSafeInteger(state.baseChainId) || - typeof state.vaultAddress !== "string" || - !/^0x[0-9a-fA-F]{40}$/.test(state.vaultAddress) || - typeof state.pendulumGenesisHash !== "string" || - !/^0x[0-9a-fA-F]{64}$/.test(state.pendulumGenesisHash) || - !Number.isSafeInteger(state.lastPendulumBlock) || - !isUint(state.nextExpectedNonce) || - !isUint(state.baseFromBlock) || - (state.issuanceAnchor !== undefined && !isUint(state.issuanceAnchor)) || - (state.baseAdminFromBlock !== undefined && !isUint(state.baseAdminFromBlock)) || - (state.lastPendulumHash !== undefined && !isHash(state.lastPendulumHash)) || - (state.relayAnchor !== undefined && - (!Number.isSafeInteger(state.relayAnchor?.number) || !isHash(state.relayAnchor?.hash))) || - badMigration || - badAlerts || - badApprovals || - badUnmatchedEvents - ) { - throw new Error(`monitor state ${path} has an invalid or legacy schema; refusing to reset security history`); - } - return state as PersistedMonitorState; -} - -export function assertMonitorStateIdentity( - state: { baseChainId: number; vaultAddress: string; pendulumGenesisHash: string }, - expected: { baseChainId: number; vaultAddress: string; pendulumGenesisHash: string }, -): void { - if (state.baseChainId !== expected.baseChainId) { - throw new Error(`state Base chain ${state.baseChainId} does not match configured chain ${expected.baseChainId}`); - } - if (state.vaultAddress.toLowerCase() !== expected.vaultAddress.toLowerCase()) { - throw new Error(`state vault ${state.vaultAddress} does not match configured vault ${expected.vaultAddress}`); - } - if (state.pendulumGenesisHash.toLowerCase() !== expected.pendulumGenesisHash.toLowerCase()) { - throw new Error( - `state Pendulum genesis ${state.pendulumGenesisHash} does not match connected chain ${expected.pendulumGenesisHash}`, - ); - } -} - -export function saveMonitorState(path: string, state: PersistedMonitorState): void { - const temporary = `${path}.${process.pid}.tmp`; - let file: number | undefined; - try { - file = openSync(temporary, "w", 0o600); - writeFileSync(file, JSON.stringify(state)); - fsyncSync(file); - closeSync(file); - file = undefined; - renameSync(temporary, path); - const directory = openSync(dirname(path), "r"); - try { - fsyncSync(directory); - } finally { - closeSync(directory); - } - } catch (error) { - if (file !== undefined) closeSync(file); - try { - unlinkSync(temporary); - } catch (cleanupError) { - if (!isMissing(cleanupError)) console.error(`could not remove temporary state ${temporary}`, cleanupError); - } - throw new Error(`cannot durably write monitor state ${path}`, { cause: error }); - } -} diff --git a/monitor/tsconfig.json b/monitor/tsconfig.json deleted file mode 100644 index 59b02f6e5..000000000 --- a/monitor/tsconfig.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "compilerOptions": { - "target": "ES2022", - "module": "NodeNext", - "moduleResolution": "NodeNext", - "outDir": "dist", - "rootDir": "src", - "strict": true, - "esModuleInterop": true, - "skipLibCheck": true, - "forceConsistentCasingInFileNames": true - }, - "include": ["src"] -} diff --git a/releaser/src/checks.ts b/releaser/src/checks.ts index 8dab54b9a..d7d336f42 100644 --- a/releaser/src/checks.ts +++ b/releaser/src/checks.ts @@ -2,8 +2,7 @@ * Pure predicates for the releaser. * * Isolated from the chain-client plumbing in `main.ts` so they can be unit - * tested exhaustively (see checks.test.ts), mirroring `attestor/src/checks.ts` - * and `monitor/src/checks.ts`. + * tested exhaustively (see checks.test.ts), mirroring `attestor/src/checks.ts`. */ import { BaseError, ContractFunctionRevertedError } from "viem"; diff --git a/testing/README.md b/testing/README.md index 067d29390..ebb3f545c 100644 --- a/testing/README.md +++ b/testing/README.md @@ -63,17 +63,24 @@ warp afterwards, never the reverse. ## Phase 3 — the whole pipeline together -Runs four attestors, the monitor and the releaser against Chopsticks and Anvil, -and asserts that a burn on the Substrate side arrives on Base with no manual -step, that the fleet tolerates outages, and that a cap-deferred release drains -itself. +Runs four attestors and the releaser against Chopsticks and Anvil, and asserts +that a burn on the Substrate side arrives on Base with no manual step, that the +fleet tolerates outages, and that a cap-deferred release drains itself. + +The monitor is maintained separately. Set `MONITOR_DIR` to a built monitor +checkout (absolute, or relative to the current directory) to include it: the +harness then starts it and runs the monitor checks. Without `MONITOR_DIR` the +run says so on start-up and reports the monitor-only checks as skipped. The +phase 5 rehearsal and phase 6 drills honour the same variable. ```bash anvil --port 8545 npx @acala-network/chopsticks@latest --config testing/chopsticks-e2e.yml \ --wasm-override target/release/wbuild/pendulum-runtime/pendulum_runtime.compact.compressed.wasm -# build the services once: (cd attestor && npm run build), same for monitor/ and releaser/ +# build the services once: (cd attestor && npm run build), same for releaser/ node testing/src/phase3-e2e.mjs +# or, with the monitor (built with `npm run build` in its checkout): +MONITOR_DIR=/path/to/monitor node testing/src/phase3-e2e.mjs ``` Chopsticks stands in for Pendulum rather than Zombienet: it reports finalized From d5ff97b118277f818e2204f83633ef6ce14a3b58 Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Mon, 26 Oct 2026 10:52:08 +0100 Subject: [PATCH 3/4] docs(testing): Start Anvil with immediate finality for phase 3 --- testing/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/README.md b/testing/README.md index ebb3f545c..a6758e7cc 100644 --- a/testing/README.md +++ b/testing/README.md @@ -74,7 +74,7 @@ run says so on start-up and reports the monitor-only checks as skipped. The phase 5 rehearsal and phase 6 drills honour the same variable. ```bash -anvil --port 8545 +anvil --port 8545 --slots-in-an-epoch 0 # finalized blocks right away; the releaser reads at Base's finalized tag npx @acala-network/chopsticks@latest --config testing/chopsticks-e2e.yml \ --wasm-override target/release/wbuild/pendulum-runtime/pendulum_runtime.compact.compressed.wasm # build the services once: (cd attestor && npm run build), same for releaser/ From e1c90685973c60e0b2823a6ad9886330af141b3b Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Mon, 26 Oct 2026 11:02:34 +0100 Subject: [PATCH 4/4] test(testing): Cover how the monitor handles fabricated approvals --- testing/src/phase3-e2e.mjs | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/testing/src/phase3-e2e.mjs b/testing/src/phase3-e2e.mjs index 94bc61b49..56238e07d 100644 --- a/testing/src/phase3-e2e.mjs +++ b/testing/src/phase3-e2e.mjs @@ -245,20 +245,26 @@ await monitorCheck("a real vault deficit alerts and reaches a confirmed automati assertEq(await read(V, "paused"), false, "vault remains unpaused after healthy checks"); }); -await monitorCheck("a mismatched Base approval is caught before it can reach quorum", async () => { +await monitorCheck("fabricated approvals are reported and pause the vault before they can reach quorum", async () => { + // One attestor's approval cannot release on its own, so the monitor reports + // it; a second attestor approving the same tuple pauses the vault before a + // third approval could release it. + const fabricated = { ...V, functionName: "approve", args: [9_999_999n, admin.address, MIN] }; const before = logs("monitor").length; - await send(attestors[0], { - ...V, - functionName: "approve", - args: [9_999_999n, admin.address, MIN], + await send(attestors[0], fabricated); + await waitFor(() => logs("monitor").slice(before).includes("MISMATCHED APPROVAL"), { + timeoutMs: 30_000, + label: "the monitor to report a fabricated approval", }); + assertEq(await read(V, "paused"), false, "one fabricated approval leaves the vault running"); + await send(attestors[1], fabricated); await waitFor(() => logs("monitor").slice(before).includes("MIGRATION TUPLE VIOLATION"), { timeoutMs: 30_000, - label: "the monitor to reject a fabricated approval tuple", + label: "the monitor to reject the second fabricated approval", }); await waitFor(async () => (await read(V, "paused")) === true, { timeoutMs: 30_000, - label: "the mismatched approval to trigger an automatic pause", + label: "the second fabricated approval to trigger an automatic pause", }); });