From f84130ab4dac620d94883ebbc3a2c296a3349449 Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Wed, 21 Oct 2026 15:30:46 +0200 Subject: [PATCH 01/75] feat(monitor): Page every operation the governance timelock schedules Once a queued operation's delay has passed, anyone may execute it, so the canceller has exactly the delay to veto it. With TIMELOCK_ADDRESS set, the monitor scans CallScheduled events up to its safe Base block at the start of each poll and pages each operation once, with its calls decoded and the time it becomes executable. --- monitor/README.md | 2 + monitor/src/checks.test.ts | 24 ++++++++ monitor/src/checks.ts | 48 +++++++++++++++ monitor/src/main.ts | 119 ++++++++++++++++++++++++++++++++++++- 4 files changed, 192 insertions(+), 1 deletion(-) diff --git a/monitor/README.md b/monitor/README.md index 59e9c237b..5519f8cd5 100644 --- a/monitor/README.md +++ b/monitor/README.md @@ -14,6 +14,7 @@ Checks every poll: | M4: every old migration reaches active approval quorum — and then releases | Liveness of the attestor fleet (`below approval quorum`), and of everything after it (`quorum reached but not released`: a release starved of daily allowance, paused, or made releasable by a threshold cut without a `ReleasePending`, which the releaser cannot see). | Alert, throttled per nonce | | Source supply | `totalIssuance + TotalMigrated` on Pendulum must never grow past its first-seen anchor. | Alert (`SOURCE SUPPLY GREW`) | | Unreleasable burns | A burn to the zero or vault address can never be approved or released; it is paged once and excluded from the pending count so it cannot page liveness forever or block RB-7. | Alert once | +| Governance queue | Every operation the governance Timelock schedules (`CallScheduled`), with its calls decoded and the time it becomes executable. Once the delay has passed anyone may execute a ready operation, so the canceller has exactly the delay to review and veto it. | Alert once per operation (`GOVERNANCE OPERATION SCHEDULED`) | ## Configuration (environment) @@ -28,6 +29,7 @@ Checks every poll: | `SOURCE_CLOCK_SKEW_MARGIN_SECONDS` | Clock-skew allowance for the fabrication proof (default 120s): once the finalized Pendulum view has passed the moment the monitor first observed an unmatched Base event by this margin and the nonce still does not exist, the event is provably fabricated and pauses without waiting out the grace. The anchor is the monitor's own clock, not the Base block timestamp (which trails real time after a sequencer outage); a future-dated source view proves nothing | | `SOURCE_STALE_ALERT_SECONDS` | Age of the finalized Pendulum view (vs. wall clock) past which the monitor pages that it is going blind (default 300s) | | `ISSUANCE_TOLERANCE` | Growth of Pendulum `totalIssuance + TotalMigrated` over its first-seen anchor (12-decimal pallet units) tolerated before `SOURCE SUPPLY GREW` pages (default 0) | +| `TIMELOCK_ADDRESS` | Optional: the governance TimelockController. Set it as soon as governance is deployed; the queue is scanned from the monitor's Base cursor onwards, so operations scheduled before the monitor knew the address are not paged | | `ALERT_WEBHOOK_URL` | Webhook receiving JSON alerts — wire this to paging | | `GUARDIAN_PRIVATE_KEY` | Optional: a guardian key enabling automatic `pause()` on conservation violations | | `BASE_CHAIN_ID` | Default 8453 | diff --git a/monitor/src/checks.test.ts b/monitor/src/checks.test.ts index c4803013b..ecc6670e7 100644 --- a/monitor/src/checks.test.ts +++ b/monitor/src/checks.test.ts @@ -12,6 +12,7 @@ import { approvalMismatch, blockRanges, chunks, + describeTimelockOperation, isStale, provablyUnsourced, releaseMismatch, @@ -124,3 +125,26 @@ test("RPC work is split into inclusive block ranges and bounded batches", () => assert.deepEqual(blockRanges(11n, 10n, 4n), []); assert.deepEqual(chunks([1, 2, 3, 4, 5], 2), [[1, 2], [3, 4], [5]]); }); + +test("a scheduled governance operation is described with names, targets and the veto deadline", () => { + const vault = "0x2222222222222222222222222222222222222222"; + const labels = { [vault]: "vault" }; + const functions = { "0x0e18b681": "acceptAdmin()", "0x212bf316": "setCaps(uint256,uint256)" }; + const calls = [ + { index: 1n, target: vault.toUpperCase().replace("0X", "0x"), value: 0n, data: "0x212bf316" + "00".repeat(64) }, + { index: 0n, target: vault, value: 0n, data: "0x0e18b681" }, + ]; + const now = Date.UTC(2026, 9, 21, 10, 0, 0); + const text = describeTimelockOperation("0xabc", calls, BigInt((now + (48 * 60 + 5) * 60_000) / 1000), now, labels, functions); + assert.match(text, /^operation 0xabc: #0 vault\.acceptAdmin\(\) data=0x0e18b681; #1 vault\.setCaps\(uint256,uint256\)/); + assert.match(text, /executable by anyone from 2026-10-23T10:05:00\.000Z \(in 48h 5m\)/); + assert.match(text, /canceller must cancel it before then$/); +}); + +test("a scheduled operation with unknown parts or a settled timestamp still pages readably", () => { + const other = "0x3333333333333333333333333333333333333333"; + const call = { index: 0n, target: other, value: 5n, data: "0xdeadbeef" }; + assert.match(describeTimelockOperation("0x1", [call], 0n, 0, {}, {}), new RegExp(`#0 ${other}\\.unknown function 0xdeadbeef value=5 .*cancelled\\)$`)); + assert.match(describeTimelockOperation("0x1", [call], 1n, 0, {}, {}), /already executed$/); + assert.match(describeTimelockOperation("0x1", [call], 100n, 200_000, {}, {}), /\(ready now\)/); +}); diff --git a/monitor/src/checks.ts b/monitor/src/checks.ts index 92082992b..56901a0e1 100644 --- a/monitor/src/checks.ts +++ b/monitor/src/checks.ts @@ -156,3 +156,51 @@ export function chunks(items: T[], size: number): T[][] { for (let start = 0; start < items.length; start += size) result.push(items.slice(start, start + size)); return result; } + +/** One call of a governance Timelock operation (one `CallScheduled` event). */ +export interface ScheduledCall { + index: bigint; + target: string; + value: bigint; + data: string; +} + +/** + * The page for one operation the governance Timelock scheduled. Every such + * operation pages: once its delay has passed, anyone may execute it, so the + * canceller has exactly the delay to review and veto it. + * + * `readyAtSeconds` is the Timelock's `getTimestamp(id)`, whose sentinels are + * 0 (unknown or cancelled) and 1 (executed). `labels` maps lowercased + * addresses and `functions` maps selectors to readable names. + */ +export function describeTimelockOperation( + id: string, + calls: ScheduledCall[], + readyAtSeconds: bigint, + nowMs: number, + labels: Record, + functions: Record, +): string { + const callText = [...calls] + .sort((a, b) => (a.index < b.index ? -1 : a.index > b.index ? 1 : 0)) + .map((call) => { + const target = labels[call.target.toLowerCase()] ?? call.target; + const selector = call.data.slice(0, 10).toLowerCase(); + const name = functions[selector] ?? `unknown function ${selector}`; + return `#${call.index} ${target}.${name}${call.value ? ` value=${call.value}` : ""} data=${call.data}`; + }) + .join("; "); + let status: string; + if (readyAtSeconds === 0n) status = "no longer pending (cancelled)"; + else if (readyAtSeconds === 1n) status = "already executed"; + else { + const readyMs = Number(readyAtSeconds) * 1000; + const leftMinutes = Math.floor((readyMs - nowMs) / 60_000); + status = + `executable by anyone from ${new Date(readyMs).toISOString()}` + + (leftMinutes > 0 ? ` (in ${Math.floor(leftMinutes / 60)}h ${leftMinutes % 60}m)` : " (ready now)") + + "; if it is not intended, the canceller must cancel it before then"; + } + return `operation ${id}: ${callText} — ${status}`; +} diff --git a/monitor/src/main.ts b/monitor/src/main.ts index 533620efb..91464b6cb 100644 --- a/monitor/src/main.ts +++ b/monitor/src/main.ts @@ -15,16 +15,19 @@ import { encodeAbiParameters, http, keccak256, + toFunctionSelector, } from "viem"; import { privateKeyToAccount } from "viem/accounts"; import { approvalMismatch, blockRanges, chunks, + describeTimelockOperation, isStale, provablyUnsourced, releaseMismatch, releasedExceedsMigrated, + type ScheduledCall, shouldAwaitSource, type MigrationTuple, vaultConservationDeficit, @@ -86,6 +89,44 @@ const erc20Abi = [ }, ] as const; +const timelockAbi = [ + { + type: "function", + name: "getTimestamp", + stateMutability: "view", + inputs: [{ name: "id", type: "bytes32" }], + outputs: [{ type: "uint256" }], + }, + { + type: "event", + name: "CallScheduled", + inputs: [ + { name: "id", type: "bytes32", indexed: true }, + { name: "index", type: "uint256", indexed: true }, + { name: "target", type: "address", indexed: false }, + { name: "value", type: "uint256", indexed: false }, + { name: "data", type: "bytes", indexed: false }, + { name: "predecessor", type: "bytes32", indexed: false }, + { name: "delay", type: "uint256", indexed: false }, + ], + }, +] as const; + +/** Calls governance can be expected to schedule, named in the page. Anything + * else still pages, with its raw selector. */ +const KNOWN_FUNCTIONS = Object.fromEntries([ + // MigrationVault + "acceptAdmin()", "transferAdmin(address)", "setCaps(uint256,uint256)", "setGuardian(address)", + "setThreshold(uint256)", "addAttestor(address)", "removeAttestor(address)", "pause()", "unpause()", + "sweepRemainder(address,uint256)", "clearStalePending(uint64,address,uint256)", "setToken(address)", + // TimelockController and Governor self-administration + "updateDelay(uint256)", "grantRole(bytes32,address)", "revokeRole(bytes32,address)", + "setVotingDelay(uint48)", "setVotingPeriod(uint32)", "setProposalThreshold(uint256)", + "updateQuorumNumerator(uint256)", "updateTimelock(address)", "relay(address,uint256,bytes)", + // Treasury spends + "transfer(address,uint256)", "approve(address,uint256)", +].map((signature) => [toFunctionSelector(signature), signature])); + function required(name: string): string { const value = process.env[name]; if (!value) throw new Error(`Missing required environment variable ${name}`); @@ -106,6 +147,13 @@ function envNumber(name: string, fallback: number | undefined): number { return value; } +function optionalAddress(name: string): `0x${string}` | undefined { + const value = process.env[name]; + if (!value) return undefined; + if (!/^0x[0-9a-fA-F]{40}$/.test(value)) throw new Error(`invalid ${name}: ${value}`); + return value as `0x${string}`; +} + function finalityTag(): "safe" | "finalized" { const value = process.env.BASE_FINALITY_TAG ?? "safe"; if (value !== "safe" && value !== "finalized") throw new Error(`invalid BASE_FINALITY_TAG ${value}`); @@ -150,6 +198,9 @@ const config = { /** Growth of (Pendulum totalIssuance + TotalMigrated) over its first-seen * anchor, in 12-decimal pallet units, tolerated before paging. */ issuanceToleranceRaw: BigInt(process.env.ISSUANCE_TOLERANCE ?? "0"), + /** Optional: the governance TimelockController; every operation it + * schedules pages, because the canceller must veto inside the delay. */ + timelockAddress: optionalAddress("TIMELOCK_ADDRESS"), }; const ZERO_ADDRESS = "0x0000000000000000000000000000000000000000"; @@ -268,6 +319,12 @@ let state: RuntimeState; let multicallUnavailable = false; let lastSourceStallAlertMs = 0; let lastIssuanceAlertMs = 0; +/** The governance-queue scan runs ahead of everything else in a poll and + * throws on failure, so the persisted Base cursor never passes a range it has + * not scanned; a restart resumes from that cursor. */ +let timelockFromBlock: bigint | undefined; +// ponytail: in-memory dedupe; a restart can re-page operations between the persisted cursor and the last scan. +const pagedTimelockOperations = new Set(); function log(message: string): void { console.log(`${new Date().toISOString()} ${message}`); @@ -614,6 +671,64 @@ async function reconcileBaseEvents( return { violation: firstViolation }; } +/** Page every operation the governance Timelock scheduled up to `toBlock`, + * the poll's `safe`/`finalized` Base block. */ +async function watchTimelock(toBlock: bigint): Promise { + if (!config.timelockAddress) return; + const operations = new Map<`0x${string}`, ScheduledCall[]>(); + for (const [start, end] of blockRanges(timelockFromBlock ?? state.baseFromBlock, toBlock, config.baseMaxBlockRange)) { + await publicClient.getBlock({ blockNumber: end }); // same lagging-node probe as the vault scan + const scheduled = await publicClient.getContractEvents({ + address: config.timelockAddress, + abi: timelockAbi, + eventName: "CallScheduled", + fromBlock: start, + toBlock: end, + }); + for (const event of scheduled) { + const { id, index, target, value, data } = event.args as { + id: `0x${string}`; + index: bigint; + target: `0x${string}`; + value: bigint; + data: `0x${string}`; + }; + const calls = operations.get(id) ?? []; + calls.push({ index, target, value, data }); + operations.set(id, calls); + } + } + if (operations.size > 0) { + const token = await publicClient.readContract({ + address: config.vaultAddress, + abi: vaultAbi, + functionName: "token", + blockNumber: toBlock, + }); + const labels = { + [config.vaultAddress.toLowerCase()]: "vault", + [token.toLowerCase()]: "token", + [config.timelockAddress.toLowerCase()]: "timelock", + }; + for (const [id, calls] of operations) { + if (pagedTimelockOperations.has(id)) continue; + const readyAt = await publicClient.readContract({ + address: config.timelockAddress, + abi: timelockAbi, + functionName: "getTimestamp", + args: [id], + blockNumber: toBlock, + }); + await alert( + "GOVERNANCE OPERATION SCHEDULED", + describeTimelockOperation(id, calls, readyAt, Date.now(), labels, KNOWN_FUNCTIONS), + ); + pagedTimelockOperations.add(id); + } + } + timelockFromBlock = toBlock + 1n; +} + function payloadHash(migration: MigrationRecord): `0x${string}` { return keccak256(encodeAbiParameters( [{ type: "uint64" }, { type: "address" }, { type: "uint256" }], @@ -658,6 +773,7 @@ async function check(api: ApiPromise): Promise { const confirmedBase = await publicClient.getBlock({ blockTag: config.baseFinalityTag }); if (confirmedBase.number === null) throw new Error(`${config.baseFinalityTag} Base block has no number`); const baseBlock = confirmedBase.number; + await watchTimelock(baseBlock); // Read Pendulum after choosing the Base snapshot. TotalMigrated can only // grow, so this ordering cannot false-positive if a burn finalizes mid-poll. @@ -840,7 +956,8 @@ async function main(): Promise { } log( `monitor started; Pendulum after ${state.lastPendulumBlock}, Base from ${state.baseFromBlock}, ` + - `polling every ${config.pollIntervalMs}ms`, + `polling every ${config.pollIntervalMs}ms; governance queue ` + + (config.timelockAddress ? `watched at ${config.timelockAddress}` : "not watched (TIMELOCK_ADDRESS unset)"), ); for (;;) { try { From ab76af8ca5133b89beb8e1d045c7e35a63c2650a Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Wed, 21 Oct 2026 15:30:46 +0200 Subject: [PATCH 02/75] docs: Note that the monitor pages queued timelock operations --- docs/pen-migration-security.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/pen-migration-security.md b/docs/pen-migration-security.md index d8b5d630b..e4ab6b42c 100644 --- a/docs/pen-migration-security.md +++ b/docs/pen-migration-security.md @@ -37,7 +37,8 @@ team, and the safeguards below limit what a compromise can do. never unpause; unpausing requires the vault admin. - **Admin changes.** Admin transfers are two-step. After the governance handover every admin action passes a 48-hour timelock, during which a - designated canceller can veto a queued operation. + designated canceller can veto a queued operation. The monitor pages every + operation the timelock schedules. - **Sweep floor.** The unmigrated remainder cannot be moved before 2027-03-01 00:00 UTC (immutable), and only through a governance decision. - **Separation of duties.** The guardian and the monitor are operated by From 918aeb198704207fb2f48adaddb536d7cd1f6a1f Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Wed, 21 Oct 2026 16:57:58 +0200 Subject: [PATCH 03/75] feat(monitor): Page pending Pendulum governance proposals and referenda Every public proposal, the council's external proposal and every ongoing referendum pages once, with its call decoded through batches. Calls that mint, move or unlock balances, or change the runtime or the migration pallet, page as critical: minted PEN burned through the migration drains the fixed-supply vault, and only Pendulum governance can stop it before enactment. The watch is alert-only and cannot stop the conservation checks. --- monitor/README.md | 1 + monitor/src/checks.test.ts | 14 +++++ monitor/src/checks.ts | 58 +++++++++++++++++++ monitor/src/main.ts | 110 +++++++++++++++++++++++++++++++++++++ 4 files changed, 183 insertions(+) diff --git a/monitor/README.md b/monitor/README.md index 5519f8cd5..d48b328e5 100644 --- a/monitor/README.md +++ b/monitor/README.md @@ -14,6 +14,7 @@ Checks every poll: | M4: every old migration reaches active approval quorum — and then releases | Liveness of the attestor fleet (`below approval quorum`), and of everything after it (`quorum reached but not released`: a release starved of daily allowance, paused, or made releasable by a threshold cut without a `ReleasePending`, which the releaser cannot see). | Alert, throttled per nonce | | Source supply | `totalIssuance + TotalMigrated` on Pendulum must never grow past its first-seen anchor. | Alert (`SOURCE SUPPLY GREW`) | | Unreleasable burns | A burn to the zero or vault address can never be approved or released; it is paged once and excluded from the pending count so it cannot page liveness forever or block RB-7. | Alert once | +| Pendulum governance | Every pending public proposal, the council's external proposal and every ongoing referendum, with the call decoded. A call that mints, moves or unlocks balances outside normal transfers, or changes the runtime or the migration pallet, pages as critical: minted PEN burned through the migration drains the fixed-supply vault, and only Pendulum governance can stop the call before enactment. | Alert once per item (`PENDULUM GOVERNANCE ITEM`, or `CRITICAL: Pendulum governance item …`); a restart re-pages the items still pending | | Governance queue | Every operation the governance Timelock schedules (`CallScheduled`), with its calls decoded and the time it becomes executable. Once the delay has passed anyone may execute a ready operation, so the canceller has exactly the delay to review and veto it. | Alert once per operation (`GOVERNANCE OPERATION SCHEDULED`) | ## Configuration (environment) diff --git a/monitor/src/checks.test.ts b/monitor/src/checks.test.ts index ecc6670e7..d01eeacce 100644 --- a/monitor/src/checks.test.ts +++ b/monitor/src/checks.test.ts @@ -12,12 +12,15 @@ import { approvalMismatch, blockRanges, chunks, + describeCall, describeTimelockOperation, isStale, provablyUnsourced, releaseMismatch, releasedExceedsMigrated, + riskyCalls, shouldAwaitSource, + toCallNode, vaultConservationDeficit, } from "./checks.js"; @@ -148,3 +151,14 @@ test("a scheduled operation with unknown parts or a settled timestamp still page assert.match(describeTimelockOperation("0x1", [call], 1n, 0, {}, {}), /already executed$/); assert.match(describeTimelockOperation("0x1", [call], 100n, 200_000, {}, {}), /\(ready now\)/); }); + +test("a governance call tree is walked through batches and minting calls are flagged", () => { + const setBalance = { section: "tokens", method: "setBalance", args: ["who", "Native", 4_000_000n] }; + // The recurring attack proposal: utility.batch of tokens.setBalance to the proposer. + const attack = toCallNode({ section: "utility", method: "batch", args: [[setBalance, setBalance]] }); + assert.equal(describeCall(attack), "utility.batch(tokens.setBalance, tokens.setBalance)"); + assert.deepEqual(riskyCalls(attack), ["tokens.setBalance"]); + const nested = toCallNode({ section: "utility", method: "dispatchAs", args: ["Root", { section: "tokenMigration", method: "setPaused", args: [false] }] }); + assert.deepEqual(riskyCalls(nested), ["tokenMigration.setPaused"]); + assert.deepEqual(riskyCalls(toCallNode({ section: "system", method: "remark", args: ["0x00"] })), []); +}); diff --git a/monitor/src/checks.ts b/monitor/src/checks.ts index 56901a0e1..87f3b4527 100644 --- a/monitor/src/checks.ts +++ b/monitor/src/checks.ts @@ -204,3 +204,61 @@ export function describeTimelockOperation( } return `operation ${id}: ${callText} — ${status}`; } + +/** A decoded Pendulum call: `section.method` plus any calls nested in its arguments. */ +export interface CallNode { + name: string; + calls: CallNode[]; +} + +/** The duck type of a polkadot.js `Call`, enough to walk nested calls. */ +interface CallLike { + section: string; + method: string; + args: unknown[]; +} + +function isCallLike(value: unknown): value is CallLike { + const call = value as CallLike | null; + return typeof call?.section === "string" && typeof call.method === "string" && Array.isArray(call.args); +} + +/** Walks a decoded call, descending into arguments that are calls or lists of calls (batch, dispatch_as, ...). */ +export function toCallNode(call: CallLike): CallNode { + const calls: CallNode[] = []; + for (const arg of call.args) { + if (isCallLike(arg)) calls.push(toCallNode(arg)); + else if (Array.isArray(arg)) for (const item of arg) if (isCallLike(item)) calls.push(toCallNode(item)); + } + return { name: `${call.section}.${call.method}`, calls }; +} + +/** Calls that mint, move or unlock balances outside normal transfers, or change + * the runtime or the migration pallet. Minted PEN that is burned through the + * migration drains the fixed-supply vault on Base, and only Pendulum governance + * can stop such a call before it is enacted. */ +const RISKY_CALLS = new Set([ + "balances.forceSetBalance", "balances.setBalance", "balances.setBalanceDeprecated", "balances.forceTransfer", + "balances.forceUnreserve", "balances.forceAdjustTotalIssuance", "tokens.setBalance", "tokens.forceTransfer", + "currencies.updateBalance", "vesting.forceVestedTransfer", "vestingManager.removeVestingSchedule", + "system.setCode", "system.setCodeWithoutChecks", "system.setStorage", "system.killStorage", "system.killPrefix", + "system.authorizeUpgrade", "system.authorizeUpgradeWithoutChecks", "parachainSystem.authorizeUpgrade", + "parachainSystem.enactAuthorizedUpgrade", +]); +const RISKY_SECTIONS = new Set(["sudo", "tokenMigration"]); + +/** The risky calls anywhere in a call tree, deduplicated. */ +export function riskyCalls(call: CallNode): string[] { + const found = new Set(); + const walk = (node: CallNode) => { + if (RISKY_CALLS.has(node.name) || RISKY_SECTIONS.has(node.name.split(".")[0])) found.add(node.name); + node.calls.forEach(walk); + }; + walk(call); + return [...found]; +} + +/** Compact one-line form: `utility.batch(tokens.setBalance, tokens.setBalance)`. */ +export function describeCall(call: CallNode): string { + return call.calls.length ? `${call.name}(${call.calls.map(describeCall).join(", ")})` : call.name; +} diff --git a/monitor/src/main.ts b/monitor/src/main.ts index 91464b6cb..b1d4ded66 100644 --- a/monitor/src/main.ts +++ b/monitor/src/main.ts @@ -21,14 +21,18 @@ import { privateKeyToAccount } from "viem/accounts"; import { approvalMismatch, blockRanges, + type CallNode, chunks, + describeCall, describeTimelockOperation, isStale, provablyUnsourced, releaseMismatch, releasedExceedsMigrated, + riskyCalls, type ScheduledCall, shouldAwaitSource, + toCallNode, type MigrationTuple, vaultConservationDeficit, } from "./checks.js"; @@ -325,6 +329,9 @@ let lastIssuanceAlertMs = 0; let timelockFromBlock: bigint | undefined; // ponytail: in-memory dedupe; a restart can re-page operations between the persisted cursor and the last scan. const pagedTimelockOperations = new Set(); +// ponytail: in-memory dedupe; a restart re-pages the Pendulum governance items still pending. +const pagedGovernanceItems = new Set(); +let lastGovernanceWatchFailureMs = 0; function log(message: string): void { console.log(`${new Date().toISOString()} ${message}`); @@ -729,6 +736,100 @@ async function watchTimelock(toBlock: bigint): Promise { timelockFromBlock = toBlock + 1n; } +type ApiAt = Awaited>; + +/** A democracy `Bounded`, as much of it as the watcher reads. */ +interface BoundedCall { + isInline: boolean; + asInline: Uint8Array; + isLookup: boolean; + asLookup: { hash_: { toHex(): string }; len: { toNumber(): number } }; + toString(): string; +} + +/** Decodes a proposal's call; a preimage that is not noted yet cannot be read. */ +async function boundedCall(apiAt: ApiAt, bounded: BoundedCall): Promise<{ node?: CallNode; text: string }> { + let bytes: Uint8Array; + if (bounded.isInline) bytes = bounded.asInline; + else if (bounded.isLookup) { + const { hash_, len } = bounded.asLookup; + const preimage = (await apiAt.query.preimage.preimageFor([hash_, len])) as unknown as { + isSome: boolean; + unwrap(): Uint8Array; + }; + if (!preimage.isSome) return { text: `preimage ${hash_.toHex()} is not noted yet; review the call once it is` }; + bytes = preimage.unwrap(); + } else return { text: `legacy call reference ${bounded.toString()}; review it manually` }; + const node = toCallNode(apiAt.registry.createType("Call", bytes) as unknown as Parameters[0]); + return { node, text: describeCall(node) }; +} + +/** + * Page every pending Pendulum democracy item once: public proposals, the + * council's external proposal and ongoing referenda, with the call decoded. + * Calls that mint, move or unlock balances or change the runtime page as + * CRITICAL: minted PEN burned through the migration drains the fixed-supply + * vault, and only Pendulum governance can stop the call before enactment. + * Alert-only. + */ +async function watchPendulumGovernance(apiAt: ApiAt, finalizedBlock: number): Promise { + const democracy = apiAt.query.democracy; + const launchPeriod = Number(apiAt.consts.democracy.launchPeriod.toString()); + const nextLaunch = (Math.floor(finalizedBlock / launchPeriod) + 1) * launchPeriod; + const items: Array<{ key: string; label: string; hint: string; bounded: BoundedCall }> = []; + const proposals = (await democracy.publicProps()) as unknown as Array< + [{ toNumber(): number }, BoundedCall, { toString(): string }] + >; + for (const [index, bounded, proposer] of proposals) { + items.push({ + key: `proposal:${index.toNumber()}`, + label: `public proposal #${index.toNumber()} from ${proposer.toString()}`, + hint: `the best-backed public proposal becomes a referendum at block ${nextLaunch}; ` + + "a hostile one needs a council or technical-committee cancel before then, or nay votes after", + bounded, + }); + } + const external = (await democracy.nextExternal()) as unknown as { isSome: boolean; unwrap(): [BoundedCall, unknown] }; + if (external.isSome) { + const [bounded] = external.unwrap(); + items.push({ + key: `external:${bounded.toString()}`, + label: "the council's external proposal", + hint: `it can become a referendum at block ${nextLaunch}`, + bounded, + }); + } + const [count, lowest] = (await Promise.all([democracy.referendumCount(), democracy.lowestUnbaked()])).map((value) => + Number(value.toString()) + ); + for (let index = lowest; index < count; index++) { + const info = (await democracy.referendumInfoOf(index)) as unknown as { + isSome: boolean; + unwrap(): { isOngoing: boolean; asOngoing: { proposal: BoundedCall; end: { toNumber(): number } } }; + }; + if (!info.isSome || !info.unwrap().isOngoing) continue; + const ongoing = info.unwrap().asOngoing; + items.push({ + key: `referendum:${index}`, + label: `referendum #${index}`, + hint: `voting ends at block ${ongoing.end.toNumber()}; a hostile one needs nay votes or an emergency cancel`, + bounded: ongoing.proposal, + }); + } + for (const item of items) { + if (pagedGovernanceItems.has(item.key)) continue; + const call = await boundedCall(apiAt, item.bounded); + const risky = call.node ? riskyCalls(call.node) : []; + await alert( + risky.length + ? "CRITICAL: Pendulum governance item mints, moves or unlocks balances, or changes the runtime" + : "PENDULUM GOVERNANCE ITEM", + `${item.label}: ${call.text}${risky.length ? ` (flagged: ${risky.join(", ")})` : ""}; ${item.hint}`, + ); + pagedGovernanceItems.add(item.key); + } +} + function payloadHash(migration: MigrationRecord): `0x${string}` { return keccak256(encodeAbiParameters( [{ type: "uint64" }, { type: "address" }, { type: "uint256" }], @@ -798,6 +899,15 @@ async function check(api: ApiPromise): Promise { `the ${config.unmatchedEventGraceSeconds}s unmatched-event grace forces a pause`, ); } + try { + await watchPendulumGovernance(apiAt, finalizedBlock); + } catch (governanceError) { + // Alert-only: an RPC or decode failure here must not stop the conservation checks. + if (Date.now() - lastGovernanceWatchFailureMs > config.graceSeconds * 1000) { + lastGovernanceWatchFailureMs = Date.now(); + await alert("pendulum governance watch failed", `${governanceError}`); + } + } const draft = cloneState(state); await ingestPendulumEvents(api, draft, finalizedBlock); const [totalMigratedValue, chainNextNonceValue, totalIssuanceValue] = await Promise.all([ From a3dddce409affcb5fe8778748a30cb9a5d51e498 Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Thu, 22 Oct 2026 10:11:51 +0200 Subject: [PATCH 04/75] test(monitor): Drive single polls against stand-in chains Split main() into start() and poll() so a test can run one poll of the real check code against a JSON-RPC stand-in for Base and an object shaped like the polkadot.js API for Pendulum, with the webhook recording every page. --- monitor/src/main.test.ts | 363 +++++++++++++++++++++++++++++++++++++++ monitor/src/main.ts | 36 ++-- 2 files changed, 388 insertions(+), 11 deletions(-) create mode 100644 monitor/src/main.test.ts diff --git a/monitor/src/main.test.ts b/monitor/src/main.test.ts new file mode 100644 index 000000000..a62ee82ac --- /dev/null +++ b/monitor/src/main.test.ts @@ -0,0 +1,363 @@ +/** + * Call-site tests for the poll loop in main.ts. Each test loads a fresh copy + * of the module (it reads its configuration at import) against stand-in + * chains: a JSON-RPC server for Base and a plain object shaped like the + * polkadot.js API for Pendulum. The same server receives every page. + */ + +import assert from "node:assert/strict"; +import { mkdtempSync, readFileSync } from "node:fs"; +import { createServer, type IncomingMessage } from "node:http"; +import type { AddressInfo } from "node:net"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { test } from "node:test"; +import type { ApiPromise } from "@polkadot/api"; +import { + type AbiEvent, + type AbiFunction, + decodeFunctionData, + encodeAbiParameters, + encodeEventTopics, + type Hex, + parseAbi, + toHex, +} from "viem"; + +(globalThis as { penMonitorUnderTest?: boolean }).penMonitorUnderTest = true; + +const VAULT = "0x1111111111111111111111111111111111111111"; +const TOKEN = "0x2222222222222222222222222222222222222222"; +const RECIPIENT = "0x4444444444444444444444444444444444444444"; +const ATTESTORS = ["0xa1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1", "0xa2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2", "0xa3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3"]; +const CF = 1_000_000n; +const SUPPLY = 150_000_000n * 10n ** 18n; +const PEN = 10n ** 12n; + +const abi = parseAbi([ + "function totalReleased() view returns (uint256)", + "function totalSwept() view returns (uint256)", + "function conversionFactor() view returns (uint256)", + "function token() view returns (address)", + "function paused() view returns (bool)", + "function threshold() view returns (uint256)", + "function activeApprovals(bytes32 payload) view returns (uint256)", + "function totalSupply() view returns (uint256)", + "function balanceOf(address owner) view returns (uint256)", + "function getTimestamp(bytes32 id) view returns (uint256)", + "event Approved(uint64 indexed nonce, address indexed recipient, uint256 palletAmount, address indexed attestor)", + "event Released(uint64 indexed nonce, address indexed recipient, uint256 palletAmount, uint256 tokenAmount)", + "event CallScheduled(bytes32 indexed id, uint256 indexed index, address target, uint256 value, bytes data, bytes32 predecessor, uint256 delay)", +]); + +interface Log { + address: string; + eventName: string; + args: Record; + blockNumber: bigint; +} + +/** The Base side: the vault, its token and the governance timelock. */ +function baseChain() { + return { + chainId: 8453, + safe: 1_010n, + /** How far the confirmed head trails the wall clock. */ + safeAgeSeconds: 2, + paused: false, + threshold: 3n, + totalReleased: 0n, + totalSwept: 0n, + vaultBalance: SUPPLY, + approvals: new Map(), + readyAt: new Map(), + logs: [] as Log[], + }; +} +type Base = ReturnType; + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +function answer(base: Base, method: string, params: any[]): unknown { + switch (method) { + case "eth_chainId": + return toHex(base.chainId); + case "eth_getCode": + return "0x"; + case "eth_getBlockByNumber": { + const number = String(params[0]).startsWith("0x") ? BigInt(params[0]) : base.safe; + const timestamp = Math.floor(Date.now() / 1000) - base.safeAgeSeconds; + return { number: toHex(number), hash: toHex(number, { size: 32 }), timestamp: toHex(timestamp), transactions: [] }; + } + case "eth_call": { + const { functionName, args } = decodeFunctionData({ abi, data: params[0].data }); + const key = (args as readonly unknown[] | undefined)?.[0] as string; + const result = { + totalReleased: base.totalReleased, + totalSwept: base.totalSwept, + conversionFactor: CF, + token: TOKEN, + paused: base.paused, + threshold: base.threshold, + activeApprovals: base.approvals.get(key) ?? 0n, + totalSupply: SUPPLY, + balanceOf: base.vaultBalance, + getTimestamp: base.readyAt.get(key) ?? 0n, + }[functionName]; + const { outputs } = abi.find((item) => item.type === "function" && item.name === functionName) as AbiFunction; + return encodeAbiParameters(outputs, [result]); + } + case "eth_getLogs": { + const [{ address, topics, fromBlock, toBlock }] = params; + const wanted = [topics?.[0] ?? []].flat() as string[]; + return base.logs.flatMap((log, index) => { + if (log.address.toLowerCase() !== String(address).toLowerCase()) return []; + if (log.blockNumber < BigInt(fromBlock) || log.blockNumber > BigInt(toBlock)) return []; + const logTopics = encodeEventTopics({ abi, eventName: log.eventName, args: log.args } as never) as Hex[]; + if (wanted.length && !wanted.includes(logTopics[0])) return []; + const { inputs } = abi.find((item) => item.type === "event" && item.name === log.eventName) as AbiEvent; + const data = inputs.filter((input) => !input.indexed); + return [{ + address: log.address, + topics: logTopics, + data: encodeAbiParameters(data, data.map((input) => log.args[input.name!])), + blockNumber: toHex(log.blockNumber), + blockHash: toHex(log.blockNumber, { size: 32 }), + transactionHash: toHex(index + 1, { size: 32 }), + transactionIndex: "0x0", + logIndex: toHex(index), + removed: false, + }]; + }); + } + } + throw new Error(`unmocked ${method}`); +} + +/** A migration approved by three attestors and released on Base. */ +function release(base: Base, nonce: bigint, palletAmount: bigint, blockNumber = 1_005n): void { + for (const attestor of ATTESTORS) { + base.logs.push({ address: VAULT, eventName: "Approved", args: { nonce, recipient: RECIPIENT, palletAmount, attestor }, blockNumber }); + } + const tokenAmount = palletAmount * CF; + base.logs.push({ address: VAULT, eventName: "Released", args: { nonce, recipient: RECIPIENT, palletAmount, tokenAmount }, blockNumber }); + base.totalReleased += tokenAmount; + base.vaultBalance -= tokenAmount; +} + +interface Call { + section: string; + method: string; + args: unknown[]; +} + +/** Calls the stand-in registry decodes; a call's bytes are its name. */ +const CALLS: Record = { + remark: { section: "system", method: "remark", args: ["0x"] }, + mint: { section: "utility", method: "batch", args: [[{ section: "tokens", method: "setBalance", args: [] }]] }, + setTreasuryDestination: { section: "tokenMigration", method: "setTreasuryDestination", args: ["0x55"] }, +}; +const encodeCall = (name: string) => new TextEncoder().encode(name); +function decodeCall(bytes: Uint8Array): Call { + const call = CALLS[new TextDecoder().decode(bytes)]; + if (!call) throw new Error("Unable to find Call"); + return call; +} +const codec = (value: number | bigint) => ({ + toString: () => String(value), + toNumber: () => Number(value), + toBigInt: () => BigInt(value), +}); +const hash = (hex: string) => ({ toHex: () => hex }); +const inline = (name: string) => ({ isInline: true, asInline: encodeCall(name), isLookup: false, isLegacy: false, toString: () => `inline ${name}` }); +const lookup = (hex: string, len: number) => ({ + isInline: false, + isLookup: true, + asLookup: { hash_: hash(hex), len: codec(len) }, + isLegacy: false, + toString: () => `lookup ${hex}`, +}); +type Bounded = ReturnType | ReturnType; + +interface PendulumEvent { + section: string; + method: string; + data: unknown[]; +} + +/** The Pendulum side: finalized blocks with their events, the migration + * pallet's totals and the governance queues. Block numbers double as hashes. */ +function pendulumChain() { + const chain = { + finalized: 100, + /** Age of the finalized head; earlier blocks are 12 s apart. */ + headAgeMs: 0, + failure: undefined as Error | undefined, + events: new Map(), + totalMigrated: 0n, + nextNonce: 0n, + publicProps: [] as Array<[number, Bounded]>, + referenda: new Map(), + /** Noted preimages by `${hash}:${len}`. */ + preimages: new Map(), + preimageReads: 0, + }; + const optional = (value: T | undefined) => (value === undefined ? { isSome: false } : { isSome: true, unwrap: () => value }); + const at = (block: number) => ({ + registry: { createType: (_type: string, bytes: Uint8Array) => decodeCall(bytes) }, + consts: { democracy: { launchPeriod: codec(36_000) } }, + query: { + timestamp: { now: async () => codec(Date.now() - chain.headAgeMs - (chain.finalized - block) * 12_000) }, + system: { events: async () => (chain.events.get(block) ?? []).map((event) => ({ event })) }, + tokenMigration: { + totalMigrated: async () => codec(chain.totalMigrated), + nextNonce: async () => codec(chain.nextNonce), + }, + balances: { totalIssuance: async () => codec(100_000_000n * PEN - chain.totalMigrated) }, + democracy: { + publicProps: async () => chain.publicProps.map(([index, bounded]) => [codec(index), bounded, "5Proposer"]), + nextExternal: async () => optional(undefined), + referendumCount: async () => codec(chain.referenda.size ? Math.max(...chain.referenda.keys()) + 1 : 0), + lowestUnbaked: async () => codec(chain.referenda.size ? Math.min(...chain.referenda.keys()) : 0), + referendumInfoOf: async (index: number) => { + const proposal = chain.referenda.get(index); + return optional(proposal && { isOngoing: true, asOngoing: { proposal, end: codec(9_000_000) } }); + }, + }, + preimage: { + preimageFor: async ([preimage, len]: [{ toHex(): string }, unknown]) => { + chain.preimageReads++; + const name = chain.preimages.get(`${preimage.toHex()}:${len}`); + return optional(name === undefined ? undefined : encodeCall(name)); + }, + }, + }, + }); + const api = { + genesisHash: hash(`0x${"5d".repeat(32)}`), + rpc: { + chain: { + getFinalizedHead: async () => { + if (chain.failure) throw chain.failure; + return chain.finalized; + }, + getHeader: async (block: number) => ({ number: codec(block) }), + getBlockHash: async (block: number) => block, + }, + }, + at: async (block: number) => at(block), + }; + return { chain, api: api as unknown as ApiPromise }; +} +type Pendulum = ReturnType["chain"]; + +/** A finalized burn on Pendulum, in block 101 unless stated. */ +function burn(pendulum: Pendulum, nonce: bigint, palletAmount: bigint, block = 101): void { + pendulum.finalized = Math.max(pendulum.finalized, block); + const events = pendulum.events.get(block) ?? []; + events.push({ + section: "tokenMigration", + method: "MigrationInitiated", + data: [codec(nonce), "5Migrator", hash(RECIPIENT), codec(palletAmount)], + }); + pendulum.events.set(block, events); + pendulum.totalMigrated += palletAmount; + pendulum.nextNonce = nonce + 1n; +} + +interface Page { + subject: string; + detail: string; + status?: number; +} + +async function body(request: IncomingMessage): Promise { + let text = ""; + for await (const chunk of request) text += chunk; + return text; +} + +let instances = 0; +let monitorEnv: string[] = []; + +/** A started monitor against fresh stand-ins. `webhook` answers each page + * with an HTTP status; `pages` lists them in arrival order from after + * startup, `startup` the ones before. */ +async function setup( + t: { after(fn: () => void): void }, + env: Record = {}, + webhook: (page: Page) => number | Promise = () => 200, +) { + const base = baseChain(); + const { chain: pendulum, api } = pendulumChain(); + const pages: Page[] = []; + const server = createServer(async (request, response) => { + const text = await body(request); + if (request.url === "/alert") { + const { subject, detail } = JSON.parse(text) as Page; + const page: Page = { subject, detail }; + pages.push(page); + page.status = await webhook(page); + response.writeHead(page.status).end(); + return; + } + const { id, method, params } = JSON.parse(text); + let reply: object; + try { + reply = { result: answer(base, method, params) }; + } catch (error) { + reply = { error: { code: -32603, message: String(error) } }; + } + response.setHeader("content-type", "application/json"); + response.end(JSON.stringify({ jsonrpc: "2.0", id, ...reply })); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + t.after(() => { + server.closeAllConnections(); + server.close(); + }); + const url = `http://127.0.0.1:${(server.address() as AddressInfo).port}`; + const stateFile = join(mkdtempSync(join(tmpdir(), "pen-monitor-test-")), "state.json"); + const variables: Record = { + PENDULUM_WS: "ws://127.0.0.1:1", + BASE_RPC_URL: url, + VAULT_ADDRESS: VAULT, + BASE_START_BLOCK: "1000", + PENDULUM_START_BLOCK: "101", + STATE_FILE: stateFile, + ALERT_WEBHOOK_URL: `${url}/alert`, + ...env, + }; + for (const name of monitorEnv) delete process.env[name]; + monitorEnv = Object.keys(variables); + Object.assign(process.env, variables); + const monitor: typeof import("./main.js") = await import(`./main.js?instance=${++instances}`); + await monitor.start(api); + const startup = pages.splice(0); + return { + base, + pendulum, + api, + pages, + startup, + monitor, + subjects: () => pages.map((page) => page.subject), + persisted: () => JSON.parse(readFileSync(stateFile, "utf8")) as { baseFromBlock: string; migrations: Array<{ nonce: string }> }, + }; +} + +test("a released migration reconciles and the poll pages nothing", async (t) => { + const { base, pendulum, api, monitor, subjects, persisted } = await setup(t); + burn(pendulum, 0n, 100n * PEN); + release(base, 0n, 100n * PEN); + await monitor.poll(api); + assert.deepEqual(subjects(), []); + assert.deepEqual(persisted().migrations, []); + assert.equal(persisted().baseFromBlock, "1011"); +}); + +test("a vault balance deficit pages and asks for the auto-pause", async (t) => { + const { base, api, monitor, subjects } = await setup(t); + base.vaultBalance -= 1n; + await monitor.poll(api); + assert.deepEqual(subjects(), ["VAULT BALANCE DEFICIT", "AUTO-PAUSE UNAVAILABLE"]); +}); diff --git a/monitor/src/main.ts b/monitor/src/main.ts index b1d4ded66..543c6c4c7 100644 --- a/monitor/src/main.ts +++ b/monitor/src/main.ts @@ -1043,14 +1043,15 @@ async function check(api: ApiPromise): Promise { ); } -async function main(): Promise { +/** Startup once the Pendulum API is connected: durable state, the multicall + * probe and the state identity check. */ +export async function start(api: ApiPromise): Promise { state = hydrateState(); const multicallCode = await publicClient.getBytecode({ address: MULTICALL3_ADDRESS }); if (!multicallCode || multicallCode === "0x") { multicallUnavailable = true; await alert("multicall unavailable, using bounded individual reads", `no contract code at ${MULTICALL3_ADDRESS}`); } - const api = await ApiPromise.create({ provider: new WsProvider(config.pendulumWs) }); const pendulumGenesisHash = api.genesisHash.toHex(); if (state.pendulumGenesisHash) { assertMonitorStateIdentity( @@ -1069,17 +1070,30 @@ async function main(): Promise { `polling every ${config.pollIntervalMs}ms; governance queue ` + (config.timelockAddress ? `watched at ${config.timelockAddress}` : "not watched (TIMELOCK_ADDRESS unset)"), ); +} + +/** One poll; a failure pages and the next poll starts afresh. */ +export async function poll(api: ApiPromise): Promise { + try { + await check(api); + } catch (checkError) { + await alert("monitor check failed", `${checkError}`); + } +} + +async function main(): Promise { + const api = await ApiPromise.create({ provider: new WsProvider(config.pendulumWs) }); + await start(api); for (;;) { - try { - await check(api); - } catch (checkError) { - await alert("monitor check failed", `${checkError}`); - } + await poll(api); await new Promise((resolve) => setTimeout(resolve, config.pollIntervalMs)); } } -main().catch(async (error) => { - await alert("monitor startup failed", `${error}`); - process.exit(1); -}); +// main.test.ts sets this before importing the module, to drive single polls. +if (!(globalThis as { penMonitorUnderTest?: boolean }).penMonitorUnderTest) { + main().catch(async (error) => { + await alert("monitor startup failed", `${error}`); + process.exit(1); + }); +} From a91c516c329300bff103f81c6fdb5099d0347964 Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Thu, 22 Oct 2026 10:13:35 +0200 Subject: [PATCH 05/75] fix(monitor): Retry once-only pages until the webhook accepts them alert() now resolves to whether the webhook answered 2xx in time and logs rejected or failed deliveries. Pages that must go out exactly once (scheduled timelock operations, Pendulum governance items, the first sighting of an unverified Base event, unreleasable burns) are keyed and held until delivered, and every poll retries the undelivered ones. --- monitor/src/main.test.ts | 24 ++++++++++++++++++ monitor/src/main.ts | 54 ++++++++++++++++++++++++++++------------ 2 files changed, 62 insertions(+), 16 deletions(-) diff --git a/monitor/src/main.test.ts b/monitor/src/main.test.ts index a62ee82ac..adf8d962f 100644 --- a/monitor/src/main.test.ts +++ b/monitor/src/main.test.ts @@ -28,6 +28,7 @@ import { const VAULT = "0x1111111111111111111111111111111111111111"; const TOKEN = "0x2222222222222222222222222222222222222222"; +const TIMELOCK = "0x3333333333333333333333333333333333333333"; const RECIPIENT = "0x4444444444444444444444444444444444444444"; const ATTESTORS = ["0xa1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1", "0xa2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2", "0xa3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3"]; const CF = 1_000_000n; @@ -361,3 +362,26 @@ test("a vault balance deficit pages and asks for the auto-pause", async (t) => { await monitor.poll(api); assert.deepEqual(subjects(), ["VAULT BALANCE DEFICIT", "AUTO-PAUSE UNAVAILABLE"]); }); + +/** A timelock operation scheduling the vault's `pause()`, ready in 48 h. */ +function schedule(base: Base, id: Hex, blockNumber = 1_005n): void { + const call = { target: VAULT, value: 0n, data: "0x8456cb59", predecessor: toHex(0, { size: 32 }), delay: 172_800n }; + base.logs.push({ address: TIMELOCK, eventName: "CallScheduled", args: { id, index: 0n, ...call }, blockNumber }); + base.readyAt.set(id, BigInt(Math.floor(Date.now() / 1000) + 172_800)); +} + +test("once-only pages the webhook rejects are sent again on the next poll", async (t) => { + const failedOnce = new Set(); + const { base, pendulum, api, monitor, pages } = await setup(t, { TIMELOCK_ADDRESS: TIMELOCK }, ({ subject }) => { + if (failedOnce.has(subject)) return 200; + failedOnce.add(subject); + return 500; + }); + schedule(base, toHex(1, { size: 32 })); + pendulum.publicProps.push([7, inline("remark")]); + await monitor.poll(api); + await monitor.poll(api); + const delivered = pages.filter((page) => page.status === 200).map((page) => page.subject); + assert.ok(delivered.includes("GOVERNANCE OPERATION SCHEDULED"), `delivered: ${delivered}`); + assert.ok(delivered.includes("PENDULUM GOVERNANCE ITEM"), `delivered: ${delivered}`); +}); diff --git a/monitor/src/main.ts b/monitor/src/main.ts index 543c6c4c7..4de05ccd1 100644 --- a/monitor/src/main.ts +++ b/monitor/src/main.ts @@ -327,10 +327,9 @@ let lastIssuanceAlertMs = 0; * throws on failure, so the persisted Base cursor never passes a range it has * not scanned; a restart resumes from that cursor. */ let timelockFromBlock: bigint | undefined; -// ponytail: in-memory dedupe; a restart can re-page operations between the persisted cursor and the last scan. -const pagedTimelockOperations = new Set(); -// ponytail: in-memory dedupe; a restart re-pages the Pendulum governance items still pending. -const pagedGovernanceItems = new Set(); +/** Once-only pages by key, each held until the webhook accepts it. */ +// ponytail: in memory; a restart re-pages what it finds again (timelock operations after the persisted cursor, pending governance items) and drops undelivered pages it does not find again. +const onceOnlyPages = new Map(); let lastGovernanceWatchFailureMs = 0; function log(message: string): void { @@ -343,11 +342,13 @@ function redact(text: string): string { return text.replace(/https?:\/\/[^\s"')]+/gi, ""); } -async function alert(subject: string, detail: string): Promise { +/** Sends one page and resolves to whether it was delivered: the webhook must + * answer 2xx within the timeout. Without a webhook the log line is the page. */ +async function alert(subject: string, detail: string): Promise { console.error(`${new Date().toISOString()} ALERT: ${subject} — ${detail}`); - if (!config.alertWebhookUrl) return; + if (!config.alertWebhookUrl) return true; try { - await fetch(config.alertWebhookUrl, { + const response = await fetch(config.alertWebhookUrl, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ service: "pen-monitor", subject, detail: redact(detail) }), @@ -355,9 +356,27 @@ async function alert(subject: string, detail: string): Promise { // check loop (or the pause that follows a violation) indefinitely. signal: AbortSignal.timeout(10_000), }); + if (response.ok) return true; + console.error(`alert webhook answered ${response.status} for: ${subject}`); } catch (webhookError) { - console.error("alert webhook failed", webhookError); + console.error(`alert webhook failed for: ${subject}: ${redact(`${webhookError}`)}`); } + return false; +} + +/** Pages `key` once. A page the webhook does not accept stays queued, and + * every poll retries it until it is delivered. */ +async function pageOnce(key: string, subject: string, detail: string): Promise { + if (onceOnlyPages.has(key)) return; + const page = { subject, detail, delivered: false }; + onceOnlyPages.set(key, page); + page.delivered = await alert(subject, detail); +} + +async function retryUndeliveredPages(): Promise { + await Promise.all([...onceOnlyPages.values()].filter((page) => !page.delivered).map(async (page) => { + page.delivered = await alert(page.subject, page.detail); + })); } async function waitForBaseFinality( @@ -477,7 +496,8 @@ async function unmatchedDisposition( const firstSeen = draft.unmatchedBaseFirstSeenAt.get(key) ?? observedAt; draft.unmatchedBaseFirstSeenAt.set(key, firstSeen); if (!alreadyKnown) { - await alert( + await pageOnce( + `unverified:${key}`, "UNVERIFIED BASE EVENT", `${kind} nonce ${nonce} at Base block ${eventBlockNumber} has no finalized Pendulum source yet; ` + `waiting up to ${config.unmatchedEventGraceSeconds}s for the source view to reveal its burn`, @@ -523,7 +543,8 @@ async function ingestPendulumEvents(api: ApiPromise, draft: RuntimeState, finali // non-zero forever and page liveness every grace period (and make // RB-7's "zero outstanding nonces" unsatisfiable). Record it once // and leave it out of reconciliation. - await alert( + await pageOnce( + `unreleasable:${nonce}`, "CRITICAL: unreleasable migration burned on Pendulum", `nonce ${nonce} in block ${block} burned ${amountValue.toBigInt()} to ${recipient}; the vault can never release it`, ); @@ -718,7 +739,7 @@ async function watchTimelock(toBlock: bigint): Promise { [config.timelockAddress.toLowerCase()]: "timelock", }; for (const [id, calls] of operations) { - if (pagedTimelockOperations.has(id)) continue; + if (onceOnlyPages.has(`timelock:${id}`)) continue; const readyAt = await publicClient.readContract({ address: config.timelockAddress, abi: timelockAbi, @@ -726,11 +747,11 @@ async function watchTimelock(toBlock: bigint): Promise { args: [id], blockNumber: toBlock, }); - await alert( + await pageOnce( + `timelock:${id}`, "GOVERNANCE OPERATION SCHEDULED", describeTimelockOperation(id, calls, readyAt, Date.now(), labels, KNOWN_FUNCTIONS), ); - pagedTimelockOperations.add(id); } } timelockFromBlock = toBlock + 1n; @@ -817,16 +838,16 @@ async function watchPendulumGovernance(apiAt: ApiAt, finalizedBlock: number): Pr }); } for (const item of items) { - if (pagedGovernanceItems.has(item.key)) continue; + if (onceOnlyPages.has(item.key)) continue; const call = await boundedCall(apiAt, item.bounded); const risky = call.node ? riskyCalls(call.node) : []; - await alert( + await pageOnce( + item.key, risky.length ? "CRITICAL: Pendulum governance item mints, moves or unlocks balances, or changes the runtime" : "PENDULUM GOVERNANCE ITEM", `${item.label}: ${call.text}${risky.length ? ` (flagged: ${risky.join(", ")})` : ""}; ${item.hint}`, ); - pagedGovernanceItems.add(item.key); } } @@ -1079,6 +1100,7 @@ export async function poll(api: ApiPromise): Promise { } catch (checkError) { await alert("monitor check failed", `${checkError}`); } + await retryUndeliveredPages(); } async function main(): Promise { From dd5bd96de3f038840a43f52cb2cfd956b3d9e416 Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Thu, 22 Oct 2026 10:14:36 +0200 Subject: [PATCH 06/75] fix(monitor): Keep the Base-side checks running when Pendulum fails All Pendulum reads and the event ingest now run in readSource(). If it throws, the poll pages PENDULUM SOURCE UNREADABLE (throttled) and still runs the vault balance check and its auto-pause; reconciliation, the released-vs-migrated check and liveness wait for the source. --- monitor/src/main.test.ts | 8 +++++++ monitor/src/main.ts | 47 ++++++++++++++++++++++++++++++++++------ 2 files changed, 48 insertions(+), 7 deletions(-) diff --git a/monitor/src/main.test.ts b/monitor/src/main.test.ts index adf8d962f..9078aad0f 100644 --- a/monitor/src/main.test.ts +++ b/monitor/src/main.test.ts @@ -363,6 +363,14 @@ test("a vault balance deficit pages and asks for the auto-pause", async (t) => { assert.deepEqual(subjects(), ["VAULT BALANCE DEFICIT", "AUTO-PAUSE UNAVAILABLE"]); }); +test("a Pendulum failure pages on its own and the vault balance check still runs", async (t) => { + const { base, pendulum, api, monitor, subjects } = await setup(t); + pendulum.failure = new Error("node unreachable"); + base.vaultBalance -= 1n; + await monitor.poll(api); + assert.deepEqual(subjects(), ["PENDULUM SOURCE UNREADABLE", "VAULT BALANCE DEFICIT", "AUTO-PAUSE UNAVAILABLE"]); +}); + /** A timelock operation scheduling the vault's `pause()`, ready in 48 h. */ function schedule(base: Base, id: Hex, blockNumber = 1_005n): void { const call = { target: VAULT, value: 0n, data: "0x8456cb59", predecessor: toHex(0, { size: 32 }), delay: 172_800n }; diff --git a/monitor/src/main.ts b/monitor/src/main.ts index 4de05ccd1..4683c4d55 100644 --- a/monitor/src/main.ts +++ b/monitor/src/main.ts @@ -322,6 +322,7 @@ function persistState(state: RuntimeState): void { let state: RuntimeState; let multicallUnavailable = false; let lastSourceStallAlertMs = 0; +let lastSourceFailureAlertMs = 0; let lastIssuanceAlertMs = 0; /** The governance-queue scan runs ahead of everything else in a poll and * throws on failure, so the persisted Base cursor never passes a range it has @@ -891,12 +892,15 @@ async function activeApprovalCounts(migrations: MigrationRecord[], blockNumber: return counts; } -async function check(api: ApiPromise): Promise { - const confirmedBase = await publicClient.getBlock({ blockTag: config.baseFinalityTag }); - if (confirmedBase.number === null) throw new Error(`${config.baseFinalityTag} Base block has no number`); - const baseBlock = confirmedBase.number; - await watchTimelock(baseBlock); +interface SourceView { + finalizedBlock: number; + timestampMs: number; + totalMigrated: bigint; +} +/** Reads finalized Pendulum and ingests its new migrations into `draft`. + * Throws on any read, decode or continuity failure. */ +async function readSource(api: ApiPromise, draft: RuntimeState): Promise { // Read Pendulum after choosing the Base snapshot. TotalMigrated can only // grow, so this ordering cannot false-positive if a burn finalizes mid-poll. const finalizedHash = await api.rpc.chain.getFinalizedHead(); @@ -929,7 +933,6 @@ async function check(api: ApiPromise): Promise { await alert("pendulum governance watch failed", `${governanceError}`); } } - const draft = cloneState(state); await ingestPendulumEvents(api, draft, finalizedBlock); const [totalMigratedValue, chainNextNonceValue, totalIssuanceValue] = await Promise.all([ apiAt.query.tokenMigration.totalMigrated(), @@ -965,6 +968,32 @@ async function check(api: ApiPromise): Promise { `source and can be burned against the fixed-supply vault — investigate before it migrates (RB-3)`, ); } + return { finalizedBlock, timestampMs: sourceFinalizedTsMs, totalMigrated }; +} + +async function check(api: ApiPromise): Promise { + const confirmedBase = await publicClient.getBlock({ blockTag: config.baseFinalityTag }); + if (confirmedBase.number === null) throw new Error(`${config.baseFinalityTag} Base block has no number`); + const baseBlock = confirmedBase.number; + await watchTimelock(baseBlock); + + // A Pendulum failure must not switch off the Base-side checks: the vault + // balance check and its pause run without the source. Reconciliation, M2a + // and liveness need the source and wait for it. + let draft = cloneState(state); + let source: SourceView | undefined; + try { + source = await readSource(api, draft); + } catch (sourceError) { + draft = cloneState(state); // drops a half-ingested block; the 200-block checkpoints already moved `state` + if (Date.now() - lastSourceFailureAlertMs > config.unmatchedEventGraceSeconds * 1000) { + lastSourceFailureAlertMs = Date.now(); + await alert( + "PENDULUM SOURCE UNREADABLE", + `${sourceError}; the vault balance check continues, but Base events are not reconciled until the source reads again`, + ); + } + } const [totalReleased, totalSwept, conversionFactor, tokenAddress] = await Promise.all([ publicClient.readContract({ address: config.vaultAddress, abi: vaultAbi, functionName: "totalReleased", blockNumber: baseBlock }), @@ -973,7 +1002,9 @@ async function check(api: ApiPromise): Promise { publicClient.readContract({ address: config.vaultAddress, abi: vaultAbi, functionName: "token", blockNumber: baseBlock }), ]); - const reconciliation = await reconcileBaseEvents(draft, baseBlock, conversionFactor, sourceFinalizedTsMs); + const reconciliation = source + ? await reconcileBaseEvents(draft, baseBlock, conversionFactor, source.timestampMs) + : { violation: undefined, awaitingSource: undefined }; if (reconciliation.violation) { if (await securityViolation("MIGRATION TUPLE VIOLATION", reconciliation.violation)) { state = draft; @@ -1002,6 +1033,8 @@ async function check(api: ApiPromise): Promise { } return; } + if (!source) return; + const { finalizedBlock, totalMigrated } = source; if (reconciliation.awaitingSource) { state = draft; persistState(state); From 7667e3503b992e2fff8616346371ceb9af827035 Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Thu, 22 Oct 2026 10:15:54 +0200 Subject: [PATCH 07/75] fix(monitor): Page an unreadable governance item and watch on A governance item whose call cannot be read now pages as critical on its own and the watch continues with the next item, instead of the whole watch failing on that item every poll. --- monitor/src/main.test.ts | 21 +++++++++++++++++++-- monitor/src/main.ts | 13 ++++++++++++- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/monitor/src/main.test.ts b/monitor/src/main.test.ts index 9078aad0f..0bb383447 100644 --- a/monitor/src/main.test.ts +++ b/monitor/src/main.test.ts @@ -356,11 +356,14 @@ test("a released migration reconciles and the poll pages nothing", async (t) => assert.equal(persisted().baseFromBlock, "1011"); }); +/** Pages sent concurrently can arrive in either order. */ +const sorted = (subjects: string[]) => [...subjects].sort(); + test("a vault balance deficit pages and asks for the auto-pause", async (t) => { const { base, api, monitor, subjects } = await setup(t); base.vaultBalance -= 1n; await monitor.poll(api); - assert.deepEqual(subjects(), ["VAULT BALANCE DEFICIT", "AUTO-PAUSE UNAVAILABLE"]); + assert.deepEqual(sorted(subjects()), ["AUTO-PAUSE UNAVAILABLE", "VAULT BALANCE DEFICIT"]); }); test("a Pendulum failure pages on its own and the vault balance check still runs", async (t) => { @@ -368,7 +371,21 @@ test("a Pendulum failure pages on its own and the vault balance check still runs pendulum.failure = new Error("node unreachable"); base.vaultBalance -= 1n; await monitor.poll(api); - assert.deepEqual(subjects(), ["PENDULUM SOURCE UNREADABLE", "VAULT BALANCE DEFICIT", "AUTO-PAUSE UNAVAILABLE"]); + assert.deepEqual(sorted(subjects()), ["AUTO-PAUSE UNAVAILABLE", "PENDULUM SOURCE UNREADABLE", "VAULT BALANCE DEFICIT"]); +}); + +const UNREADABLE = "CRITICAL: Pendulum governance item cannot be read"; +const RISKY = "CRITICAL: Pendulum governance item mints, moves or unlocks balances, or changes the runtime"; +const pageFor = (pages: Page[], subject: string, label: string) => + pages.find((page) => page.subject === subject && page.detail.startsWith(label)); + +test("an undecodable governance item pages as unreadable and hides nothing after it", async (t) => { + const { pendulum, api, monitor, pages } = await setup(t); + pendulum.publicProps.push([5, inline("garbage")]); + pendulum.referenda.set(9, inline("mint")); + await monitor.poll(api); + assert.ok(pageFor(pages, UNREADABLE, "public proposal #5"), JSON.stringify(pages)); + assert.ok(pageFor(pages, RISKY, "referendum #9"), JSON.stringify(pages)); }); /** A timelock operation scheduling the vault's `pause()`, ready in 48 h. */ diff --git a/monitor/src/main.ts b/monitor/src/main.ts index 4683c4d55..fe7becb59 100644 --- a/monitor/src/main.ts +++ b/monitor/src/main.ts @@ -840,7 +840,18 @@ async function watchPendulumGovernance(apiAt: ApiAt, finalizedBlock: number): Pr } for (const item of items) { if (onceOnlyPages.has(item.key)) continue; - const call = await boundedCall(apiAt, item.bounded); + let call: { node?: CallNode; text: string }; + try { + call = await boundedCall(apiAt, item.bounded); + } catch (readError) { + // One unreadable item must never hide the items after it. + await pageOnce( + item.key, + "CRITICAL: Pendulum governance item cannot be read", + `${item.label}: ${String(readError).slice(0, 300)}; review it manually; ${item.hint}`, + ); + continue; + } const risky = call.node ? riskyCalls(call.node) : []; await pageOnce( item.key, From d5a5c8a3e18443400d62bb294c64151509a4beb9 Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Thu, 22 Oct 2026 10:17:38 +0200 Subject: [PATCH 08/75] fix(monitor): Bound the governance calls the monitor decodes The Pendulum governance watch now reads a preimage's length from its lookup reference and neither fetches nor decodes a call over 64 KiB; such an item pages as unreadable. The call-tree walk stops past 256 calls or 16 levels of nesting. The watch runs after the conservation checks, with its own finalized view and error handling, and sends its pages together. --- monitor/src/checks.test.ts | 11 +++++++++ monitor/src/checks.ts | 26 +++++++++++++++------ monitor/src/main.test.ts | 25 +++++++++++++++++++- monitor/src/main.ts | 48 ++++++++++++++++++++++++++------------ 4 files changed, 87 insertions(+), 23 deletions(-) diff --git a/monitor/src/checks.test.ts b/monitor/src/checks.test.ts index d01eeacce..8a86b4683 100644 --- a/monitor/src/checks.test.ts +++ b/monitor/src/checks.test.ts @@ -162,3 +162,14 @@ test("a governance call tree is walked through batches and minting calls are fla assert.deepEqual(riskyCalls(nested), ["tokenMigration.setPaused"]); assert.deepEqual(riskyCalls(toCallNode({ section: "system", method: "remark", args: ["0x00"] })), []); }); + +test("the call-tree walk is bounded in size and depth", () => { + const remark = { section: "system", method: "remark", args: [] }; + const batch = (calls: unknown[]) => ({ section: "utility", method: "batch", args: [calls] }); + assert.equal(toCallNode(batch(Array(255).fill(remark))).calls.length, 255); + assert.throws(() => toCallNode(batch(Array(256).fill(remark))), /more than 256 calls/); + let nested: { section: string; method: string; args: unknown[] } = remark; + for (let depth = 1; depth < 16; depth++) nested = batch([nested]); + assert.equal(describeCall(toCallNode(nested)).split("(").length, 16); + assert.throws(() => toCallNode(batch([nested])), /more than 16 levels/); +}); diff --git a/monitor/src/checks.ts b/monitor/src/checks.ts index 87f3b4527..aa20b0a43 100644 --- a/monitor/src/checks.ts +++ b/monitor/src/checks.ts @@ -223,14 +223,26 @@ function isCallLike(value: unknown): value is CallLike { return typeof call?.section === "string" && typeof call.method === "string" && Array.isArray(call.args); } -/** Walks a decoded call, descending into arguments that are calls or lists of calls (batch, dispatch_as, ...). */ +/** Bounds on a walked call tree. A governance call is untrusted data; + * legitimate ones stay far below both, and a bigger tree is unreadable. */ +const MAX_CALL_NODES = 256; +const MAX_CALL_DEPTH = 16; + +/** Walks a decoded call, descending into arguments that are calls or lists of + * calls (batch, dispatch_as, ...). Throws past the node or depth bound. */ export function toCallNode(call: CallLike): CallNode { - const calls: CallNode[] = []; - for (const arg of call.args) { - if (isCallLike(arg)) calls.push(toCallNode(arg)); - else if (Array.isArray(arg)) for (const item of arg) if (isCallLike(item)) calls.push(toCallNode(item)); - } - return { name: `${call.section}.${call.method}`, calls }; + let nodes = 0; + const walk = (node: CallLike, depth: number): CallNode => { + if (++nodes > MAX_CALL_NODES) throw new Error(`call tree has more than ${MAX_CALL_NODES} calls`); + if (depth > MAX_CALL_DEPTH) throw new Error(`call tree is nested more than ${MAX_CALL_DEPTH} levels deep`); + const calls: CallNode[] = []; + for (const arg of node.args) { + if (isCallLike(arg)) calls.push(walk(arg, depth + 1)); + else if (Array.isArray(arg)) for (const item of arg) if (isCallLike(item)) calls.push(walk(item, depth + 1)); + } + return { name: `${node.section}.${node.method}`, calls }; + }; + return walk(call, 1); } /** Calls that mint, move or unlock balances outside normal transfers, or change diff --git a/monitor/src/main.test.ts b/monitor/src/main.test.ts index 0bb383447..92ac9d7f0 100644 --- a/monitor/src/main.test.ts +++ b/monitor/src/main.test.ts @@ -371,7 +371,12 @@ test("a Pendulum failure pages on its own and the vault balance check still runs pendulum.failure = new Error("node unreachable"); base.vaultBalance -= 1n; await monitor.poll(api); - assert.deepEqual(sorted(subjects()), ["AUTO-PAUSE UNAVAILABLE", "PENDULUM SOURCE UNREADABLE", "VAULT BALANCE DEFICIT"]); + assert.deepEqual(sorted(subjects()), [ + "AUTO-PAUSE UNAVAILABLE", + "PENDULUM SOURCE UNREADABLE", + "VAULT BALANCE DEFICIT", + "pendulum governance watch failed", + ]); }); const UNREADABLE = "CRITICAL: Pendulum governance item cannot be read"; @@ -388,6 +393,24 @@ test("an undecodable governance item pages as unreadable and hides nothing after assert.ok(pageFor(pages, RISKY, "referendum #9"), JSON.stringify(pages)); }); +test("an oversized governance call pages as unreadable without being fetched", async (t) => { + const { pendulum, api, monitor, pages } = await setup(t); + pendulum.publicProps.push([5, lookup(`0x${"ab".repeat(32)}`, 3_900_000)]); + pendulum.preimages.set(`0x${"ab".repeat(32)}:3900000`, "mint"); + await monitor.poll(api); + assert.match(pageFor(pages, UNREADABLE, "public proposal #5")?.detail ?? "", /3900000 bytes/); + assert.equal(pendulum.preimageReads, 0); +}); + +test("the governance watch runs after the conservation checks", async (t) => { + const { base, pendulum, api, monitor, subjects } = await setup(t); + base.vaultBalance -= 1n; + pendulum.publicProps.push([5, inline("remark")]); + await monitor.poll(api); + assert.deepEqual(subjects().slice(2), ["PENDULUM GOVERNANCE ITEM"]); + assert.deepEqual(sorted(subjects().slice(0, 2)), ["AUTO-PAUSE UNAVAILABLE", "VAULT BALANCE DEFICIT"]); +}); + /** A timelock operation scheduling the vault's `pause()`, ready in 48 h. */ function schedule(base: Base, id: Hex, blockNumber = 1_005n): void { const call = { target: VAULT, value: 0n, data: "0x8456cb59", predecessor: toHex(0, { size: 32 }), delay: 172_800n }; diff --git a/monitor/src/main.ts b/monitor/src/main.ts index fe7becb59..43649fc1b 100644 --- a/monitor/src/main.ts +++ b/monitor/src/main.ts @@ -769,12 +769,24 @@ interface BoundedCall { toString(): string; } +/** + * Largest governance call the watcher fetches and decodes. Anyone can note a + * preimage and propose it, and the decoding cost grows with its size; 64 KiB + * decodes in about 0.15 s even when packed with the smallest calls. + * Legitimate calls stay far below it; a bigger one is never fetched and pages + * as unreadable. + */ +const MAX_GOVERNANCE_CALL_BYTES = 64 * 1024; + /** Decodes a proposal's call; a preimage that is not noted yet cannot be read. */ async function boundedCall(apiAt: ApiAt, bounded: BoundedCall): Promise<{ node?: CallNode; text: string }> { let bytes: Uint8Array; if (bounded.isInline) bytes = bounded.asInline; else if (bounded.isLookup) { const { hash_, len } = bounded.asLookup; + if (len.toNumber() > MAX_GOVERNANCE_CALL_BYTES) { + throw new Error(`the call is ${len.toNumber()} bytes, over the ${MAX_GOVERNANCE_CALL_BYTES} bytes the monitor decodes`); + } const preimage = (await apiAt.query.preimage.preimageFor([hash_, len])) as unknown as { isSome: boolean; unwrap(): Uint8Array; @@ -792,9 +804,13 @@ async function boundedCall(apiAt: ApiAt, bounded: BoundedCall): Promise<{ node?: * Calls that mint, move or unlock balances or change the runtime page as * CRITICAL: minted PEN burned through the migration drains the fixed-supply * vault, and only Pendulum governance can stop the call before enactment. - * Alert-only. + * Alert-only; it runs after the conservation checks and decodes within fixed + * bounds, so it can neither stop nor delay them. */ -async function watchPendulumGovernance(apiAt: ApiAt, finalizedBlock: number): Promise { +async function watchPendulumGovernance(api: ApiPromise): Promise { + const finalizedHash = await api.rpc.chain.getFinalizedHead(); + const finalizedBlock = (await api.rpc.chain.getHeader(finalizedHash)).number.toNumber(); + const apiAt = await api.at(finalizedHash); const democracy = apiAt.query.democracy; const launchPeriod = Number(apiAt.consts.democracy.launchPeriod.toString()); const nextLaunch = (Math.floor(finalizedBlock / launchPeriod) + 1) * launchPeriod; @@ -838,6 +854,8 @@ async function watchPendulumGovernance(apiAt: ApiAt, finalizedBlock: number): Pr bounded: ongoing.proposal, }); } + // Pages go out together, so a burst of new items costs one webhook round trip. + const pages: Promise[] = []; for (const item of items) { if (onceOnlyPages.has(item.key)) continue; let call: { node?: CallNode; text: string }; @@ -845,22 +863,23 @@ async function watchPendulumGovernance(apiAt: ApiAt, finalizedBlock: number): Pr call = await boundedCall(apiAt, item.bounded); } catch (readError) { // One unreadable item must never hide the items after it. - await pageOnce( + pages.push(pageOnce( item.key, "CRITICAL: Pendulum governance item cannot be read", `${item.label}: ${String(readError).slice(0, 300)}; review it manually; ${item.hint}`, - ); + )); continue; } const risky = call.node ? riskyCalls(call.node) : []; - await pageOnce( + pages.push(pageOnce( item.key, risky.length ? "CRITICAL: Pendulum governance item mints, moves or unlocks balances, or changes the runtime" : "PENDULUM GOVERNANCE ITEM", `${item.label}: ${call.text}${risky.length ? ` (flagged: ${risky.join(", ")})` : ""}; ${item.hint}`, - ); + )); } + await Promise.all(pages); } function payloadHash(migration: MigrationRecord): `0x${string}` { @@ -935,15 +954,6 @@ async function readSource(api: ApiPromise, draft: RuntimeState): Promise config.graceSeconds * 1000) { - lastGovernanceWatchFailureMs = Date.now(); - await alert("pendulum governance watch failed", `${governanceError}`); - } - } await ingestPendulumEvents(api, draft, finalizedBlock); const [totalMigratedValue, chainNextNonceValue, totalIssuanceValue] = await Promise.all([ apiAt.query.tokenMigration.totalMigrated(), @@ -1144,6 +1154,14 @@ export async function poll(api: ApiPromise): Promise { } catch (checkError) { await alert("monitor check failed", `${checkError}`); } + try { + await watchPendulumGovernance(api); + } catch (governanceError) { + if (Date.now() - lastGovernanceWatchFailureMs > config.graceSeconds * 1000) { + lastGovernanceWatchFailureMs = Date.now(); + await alert("pendulum governance watch failed", `${governanceError}`); + } + } await retryUndeliveredPages(); } From f7605592d88f3540bf9c89772617667feea73c9c Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Thu, 22 Oct 2026 10:19:18 +0200 Subject: [PATCH 09/75] fix(monitor): Re-check governance calls until they can be read A governance item whose preimage is not noted yet, or whose call could not be fetched, now pages as critical once and is re-checked every poll; once its call can be read it pages again, as critical when the call is risky. Legacy call references are read through the noted preimage's length. A call that is too big or cannot be decoded pages as unreadable. --- monitor/src/main.test.ts | 54 ++++++++++++++++++++++++++- monitor/src/main.ts | 80 +++++++++++++++++++++++++++------------- 2 files changed, 107 insertions(+), 27 deletions(-) diff --git a/monitor/src/main.test.ts b/monitor/src/main.test.ts index 92ac9d7f0..0e90eb5f1 100644 --- a/monitor/src/main.test.ts +++ b/monitor/src/main.test.ts @@ -177,7 +177,14 @@ const lookup = (hex: string, len: number) => ({ isLegacy: false, toString: () => `lookup ${hex}`, }); -type Bounded = ReturnType | ReturnType; +const legacy = (hex: string) => ({ + isInline: false, + isLookup: false, + isLegacy: true, + asLegacy: { hash_: hash(hex) }, + toString: () => `legacy ${hex}`, +}); +type Bounded = ReturnType | ReturnType | ReturnType; interface PendulumEvent { section: string; @@ -198,9 +205,12 @@ function pendulumChain() { nextNonce: 0n, publicProps: [] as Array<[number, Bounded]>, referenda: new Map(), - /** Noted preimages by `${hash}:${len}`. */ + /** Noted preimages by `${hash}:${len}`, and their lengths by hash. */ preimages: new Map(), + notedLengths: new Map(), preimageReads: 0, + /** Preimage reads still to fail, as an RPC error would. */ + preimageFailures: 0, }; const optional = (value: T | undefined) => (value === undefined ? { isSome: false } : { isSome: true, unwrap: () => value }); const at = (block: number) => ({ @@ -227,9 +237,14 @@ function pendulumChain() { preimage: { preimageFor: async ([preimage, len]: [{ toHex(): string }, unknown]) => { chain.preimageReads++; + if (chain.preimageFailures > 0 && chain.preimageFailures--) throw new Error("request timed out"); const name = chain.preimages.get(`${preimage.toHex()}:${len}`); return optional(name === undefined ? undefined : encodeCall(name)); }, + requestStatusFor: async (preimage: { toHex(): string }) => { + const len = chain.notedLengths.get(preimage.toHex()); + return { toJSON: () => (len === undefined ? null : { unrequested: { ticket: ["5Noter", 1], len } }) }; + }, }, }, }); @@ -402,6 +417,41 @@ test("an oversized governance call pages as unreadable without being fetched", a assert.equal(pendulum.preimageReads, 0); }); +test("a governance call that cannot be read yet is re-checked and pages critical once readable", async (t) => { + const { pendulum, api, monitor, pages } = await setup(t); + const preimage = `0x${"cd".repeat(32)}`; + pendulum.referenda.set(9, lookup(preimage, 4)); + await monitor.poll(api); + assert.match(pageFor(pages, UNREADABLE, "referendum #9")?.detail ?? "", /not noted yet/); + pendulum.preimages.set(`${preimage}:4`, "mint"); + await monitor.poll(api); + assert.ok(pageFor(pages, RISKY, "referendum #9"), JSON.stringify(pages)); +}); + +test("a governance item whose read fails is retried and hides nothing after it", async (t) => { + const { pendulum, api, monitor, pages } = await setup(t); + const preimage = `0x${"aa".repeat(32)}`; + pendulum.publicProps.push([5, lookup(preimage, 4)]); + pendulum.preimages.set(`${preimage}:4`, "mint"); + pendulum.referenda.set(9, inline("mint")); + pendulum.preimageFailures = 1; + await monitor.poll(api); + assert.match(pageFor(pages, UNREADABLE, "public proposal #5")?.detail ?? "", /reading the call failed/); + assert.ok(pageFor(pages, RISKY, "referendum #9"), JSON.stringify(pages)); + await monitor.poll(api); + assert.ok(pageFor(pages, RISKY, "public proposal #5"), JSON.stringify(pages)); +}); + +test("a legacy call reference is read through the noted preimage's length", async (t) => { + const { pendulum, api, monitor, pages } = await setup(t); + const preimage = `0x${"ef".repeat(32)}`; + pendulum.referenda.set(9, legacy(preimage)); + pendulum.notedLengths.set(preimage, 4); + pendulum.preimages.set(`${preimage}:4`, "mint"); + await monitor.poll(api); + assert.ok(pageFor(pages, RISKY, "referendum #9"), JSON.stringify(pages)); +}); + test("the governance watch runs after the conservation checks", async (t) => { const { base, pendulum, api, monitor, subjects } = await setup(t); base.vaultBalance -= 1n; diff --git a/monitor/src/main.ts b/monitor/src/main.ts index 43649fc1b..ae03c2833 100644 --- a/monitor/src/main.ts +++ b/monitor/src/main.ts @@ -766,9 +766,18 @@ interface BoundedCall { asInline: Uint8Array; isLookup: boolean; asLookup: { hash_: { toHex(): string }; len: { toNumber(): number } }; + asLegacy: { hash_: { toHex(): string } }; toString(): string; } +/** A governance item's call: decoded, `pending` while it cannot be read yet + * (re-checked every poll), or neither when it is unreadable for good. */ +interface CallRead { + node?: CallNode; + text: string; + pending?: boolean; +} + /** * Largest governance call the watcher fetches and decodes. Anyone can note a * preimage and propose it, and the decoding cost grows with its size; 64 KiB @@ -778,24 +787,38 @@ interface BoundedCall { */ const MAX_GOVERNANCE_CALL_BYTES = 64 * 1024; -/** Decodes a proposal's call; a preimage that is not noted yet cannot be read. */ -async function boundedCall(apiAt: ApiAt, bounded: BoundedCall): Promise<{ node?: CallNode; text: string }> { - let bytes: Uint8Array; - if (bounded.isInline) bytes = bounded.asInline; - else if (bounded.isLookup) { - const { hash_, len } = bounded.asLookup; - if (len.toNumber() > MAX_GOVERNANCE_CALL_BYTES) { - throw new Error(`the call is ${len.toNumber()} bytes, over the ${MAX_GOVERNANCE_CALL_BYTES} bytes the monitor decodes`); - } - const preimage = (await apiAt.query.preimage.preimageFor([hash_, len])) as unknown as { - isSome: boolean; - unwrap(): Uint8Array; - }; - if (!preimage.isSome) return { text: `preimage ${hash_.toHex()} is not noted yet; review the call once it is` }; - bytes = preimage.unwrap(); - } else return { text: `legacy call reference ${bounded.toString()}; review it manually` }; - const node = toCallNode(apiAt.registry.createType("Call", bytes) as unknown as Parameters[0]); - return { node, text: describeCall(node) }; +function decodeCall(apiAt: ApiAt, bytes: Uint8Array): CallRead { + try { + const node = toCallNode(apiAt.registry.createType("Call", bytes) as unknown as Parameters[0]); + return { node, text: describeCall(node) }; + } catch (decodeError) { + return { text: `the call cannot be decoded: ${String(decodeError).slice(0, 300)}` }; + } +} + +/** The length of a noted preimage, which a legacy call reference leaves out. */ +async function notedLength(apiAt: ApiAt, hash: { toHex(): string }): Promise { + const status = (await apiAt.query.preimage.requestStatusFor(hash)).toJSON() as { + unrequested?: { len: number }; + requested?: { maybeLen: number | null }; + } | null; + return status?.unrequested?.len ?? status?.requested?.maybeLen ?? undefined; +} + +/** Reads a proposal's call. Its preimage may be noted at any time before + * enactment, so until then the call is pending, not unreadable. */ +async function boundedCall(apiAt: ApiAt, bounded: BoundedCall): Promise { + if (bounded.isInline) return decodeCall(apiAt, bounded.asInline); + const hash = bounded.isLookup ? bounded.asLookup.hash_ : bounded.asLegacy.hash_; + const len = bounded.isLookup ? bounded.asLookup.len.toNumber() : await notedLength(apiAt, hash); + if (len !== undefined && len > MAX_GOVERNANCE_CALL_BYTES) { + return { text: `the call is ${len} bytes, over the ${MAX_GOVERNANCE_CALL_BYTES} bytes the monitor decodes` }; + } + const preimage = len === undefined + ? undefined + : (await apiAt.query.preimage.preimageFor([hash, len])) as unknown as { isSome: boolean; unwrap(): Uint8Array }; + if (!preimage?.isSome) return { text: `preimage ${hash.toHex()} is not noted yet`, pending: true }; + return decodeCall(apiAt, preimage.unwrap()); } /** @@ -804,8 +827,9 @@ async function boundedCall(apiAt: ApiAt, bounded: BoundedCall): Promise<{ node?: * Calls that mint, move or unlock balances or change the runtime page as * CRITICAL: minted PEN burned through the migration drains the fixed-supply * vault, and only Pendulum governance can stop the call before enactment. - * Alert-only; it runs after the conservation checks and decodes within fixed - * bounds, so it can neither stop nor delay them. + * So does a call the watcher cannot read; until it can, it is re-checked + * every poll. Alert-only; it runs after the conservation checks and decodes + * within fixed bounds, so it can neither stop nor delay them. */ async function watchPendulumGovernance(api: ApiPromise): Promise { const finalizedHash = await api.rpc.chain.getFinalizedHead(); @@ -858,19 +882,25 @@ async function watchPendulumGovernance(api: ApiPromise): Promise { const pages: Promise[] = []; for (const item of items) { if (onceOnlyPages.has(item.key)) continue; - let call: { node?: CallNode; text: string }; + let call: CallRead; try { call = await boundedCall(apiAt, item.bounded); } catch (readError) { - // One unreadable item must never hide the items after it. + // One item must never hide the items after it; a failed read is retried. + call = { text: `reading the call failed: ${String(readError).slice(0, 300)}`, pending: true }; + } + if (!call.node) { + // A call nobody can review is critical. One that cannot be read yet + // is re-checked every poll and pages again once it can be read. pages.push(pageOnce( - item.key, + call.pending ? `${item.key}:unread` : item.key, "CRITICAL: Pendulum governance item cannot be read", - `${item.label}: ${String(readError).slice(0, 300)}; review it manually; ${item.hint}`, + `${item.label}: ${call.text}; ` + + `${call.pending ? "the monitor pages its call once it can read it" : "review it manually"}; ${item.hint}`, )); continue; } - const risky = call.node ? riskyCalls(call.node) : []; + const risky = riskyCalls(call.node); pages.push(pageOnce( item.key, risky.length From d33ab0ce864ba6bedb826f520044fae6039e18be Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Thu, 22 Oct 2026 10:21:33 +0200 Subject: [PATCH 10/75] feat(monitor): Watch council and technical-committee motions The Pendulum governance watch now also decodes every open council and technical-committee motion with the same size bound and risk rules, and the event ingest pages each MigrationPauseSet and TreasuryDestinationSet of the migration pallet as it is finalized. --- monitor/src/main.test.ts | 49 +++++++++++++++++++++++++ monitor/src/main.ts | 78 +++++++++++++++++++++++++++++++--------- 2 files changed, 110 insertions(+), 17 deletions(-) diff --git a/monitor/src/main.test.ts b/monitor/src/main.test.ts index 0e90eb5f1..3152593f8 100644 --- a/monitor/src/main.test.ts +++ b/monitor/src/main.test.ts @@ -211,7 +211,26 @@ function pendulumChain() { preimageReads: 0, /** Preimage reads still to fail, as an RPC error would. */ preimageFailures: 0, + /** Open motions by hash, the stored size where it is not the call's, + * and the motions whose call was fetched. */ + motions: { council: new Map(), technicalCommittee: new Map() }, + motionSizes: new Map(), + motionReads: new Set(), }; + const collective = (motions: Map) => ({ + proposals: async () => [...motions.keys()].map(hash), + proposalOf: Object.assign( + async (motion: { toHex(): string }) => { + chain.motionReads.add(motion.toHex()); + const name = motions.get(motion.toHex()); + return optional(name === undefined ? undefined : decodeCall(encodeCall(name))); + }, + { + size: async (motion: { toHex(): string }) => + codec(chain.motionSizes.get(motion.toHex()) ?? encodeCall(motions.get(motion.toHex()) ?? "").length), + }, + ), + }); const optional = (value: T | undefined) => (value === undefined ? { isSome: false } : { isSome: true, unwrap: () => value }); const at = (block: number) => ({ registry: { createType: (_type: string, bytes: Uint8Array) => decodeCall(bytes) }, @@ -246,6 +265,8 @@ function pendulumChain() { return { toJSON: () => (len === undefined ? null : { unrequested: { ticket: ["5Noter", 1], len } }) }; }, }, + council: collective(chain.motions.council), + technicalCommittee: collective(chain.motions.technicalCommittee), }, }); const api = { @@ -452,6 +473,34 @@ test("a legacy call reference is read through the noted preimage's length", asyn assert.ok(pageFor(pages, RISKY, "referendum #9"), JSON.stringify(pages)); }); +test("council and technical-committee motions are decoded with the same risk rules", async (t) => { + const { pendulum, api, monitor, pages } = await setup(t); + const [anchor, remark, huge] = ["0x01", "0x02", "0x03"].map((prefix) => prefix + "00".repeat(31)); + pendulum.motions.council.set(anchor, "setTreasuryDestination"); + pendulum.motions.technicalCommittee.set(remark, "remark"); + pendulum.motions.council.set(huge, "mint"); + pendulum.motionSizes.set(huge, 100_000); + await monitor.poll(api); + assert.ok(pageFor(pages, RISKY, `council motion ${anchor}`), JSON.stringify(pages)); + assert.ok(pageFor(pages, "PENDULUM GOVERNANCE ITEM", `technical committee motion ${remark}`), JSON.stringify(pages)); + assert.match(pageFor(pages, UNREADABLE, `council motion ${huge}`)?.detail ?? "", /100000 bytes/); + assert.ok(!pendulum.motionReads.has(huge)); +}); + +test("admin events of the migration pallet page from the ingest", async (t) => { + const { pendulum, api, monitor, pages } = await setup(t); + pendulum.finalized = 101; + pendulum.events.set(101, [ + { section: "tokenMigration", method: "TreasuryDestinationSet", data: ["0x5555555555555555555555555555555555555555"] }, + { section: "tokenMigration", method: "MigrationPauseSet", data: [false] }, + ]); + await monitor.poll(api); + const adminPages = pages.filter((page) => page.subject === "MIGRATION PALLET ADMIN EVENT").map((page) => page.detail); + assert.equal(adminPages.length, 2, JSON.stringify(pages)); + assert.match(adminPages[0], /TreasuryDestinationSet\(0x5{40}\) in finalized Pendulum block 101/); + assert.match(adminPages[1], /MigrationPauseSet\(false\)/); +}); + test("the governance watch runs after the conservation checks", async (t) => { const { base, pendulum, api, monitor, subjects } = await setup(t); base.vaultBalance -= 1n; diff --git a/monitor/src/main.ts b/monitor/src/main.ts index ae03c2833..6b6c53d5e 100644 --- a/monitor/src/main.ts +++ b/monitor/src/main.ts @@ -510,6 +510,15 @@ async function unmatchedDisposition( : "expired"; } +/** The migration pallet's admin events, each with what to check. */ +const PALLET_ADMIN_EVENTS = new Map([ + ["MigrationPauseSet", "migrations were paused or resumed; check that it was intended"], + [ + "TreasuryDestinationSet", + "treasury migrations are released to this Base address from now on; check that it is the governance timelock", + ], +]); + async function ingestPendulumEvents(api: ApiPromise, draft: RuntimeState, finalizedBlock: number): Promise { for (let block = draft.lastPendulumBlock + 1; block <= finalizedBlock; block++) { const blockHash = await api.rpc.chain.getBlockHash(block); @@ -518,9 +527,20 @@ async function ingestPendulumEvents(api: ApiPromise, draft: RuntimeState, finali const records = (await apiAt.query.system.events()) as unknown as { event: { section: string; method: string; data: unknown[] }; }[]; - for (const record of records) { + for (const [index, record] of records.entries()) { const { section, method, data } = record.event; - if (section !== "tokenMigration" || method !== "MigrationInitiated") continue; + if (section !== "tokenMigration") continue; + const adminCheck = PALLET_ADMIN_EVENTS.get(method); + if (adminCheck) { + // Every admin change, however fast the motion behind it passed. + await pageOnce( + `pallet:${block}:${index}`, + "MIGRATION PALLET ADMIN EVENT", + `tokenMigration.${method}(${data.join(", ")}) in finalized Pendulum block ${block}: ${adminCheck}`, + ); + continue; + } + if (method !== "MigrationInitiated") continue; if (data.length !== 4) throw new Error(`MigrationInitiated in block ${block} has ${data.length} fields, expected 4`); const [nonceValue, , recipientValue, amountValue] = data as [ { toBigInt(): bigint }, @@ -787,15 +807,19 @@ interface CallRead { */ const MAX_GOVERNANCE_CALL_BYTES = 64 * 1024; -function decodeCall(apiAt: ApiAt, bytes: Uint8Array): CallRead { +function readCall(decode: () => unknown): CallRead { try { - const node = toCallNode(apiAt.registry.createType("Call", bytes) as unknown as Parameters[0]); + const node = toCallNode(decode() as Parameters[0]); return { node, text: describeCall(node) }; } catch (decodeError) { return { text: `the call cannot be decoded: ${String(decodeError).slice(0, 300)}` }; } } +const oversizedCall = (len: number): CallRead => ({ + text: `the call is ${len} bytes, over the ${MAX_GOVERNANCE_CALL_BYTES} bytes the monitor decodes`, +}); + /** The length of a noted preimage, which a legacy call reference leaves out. */ async function notedLength(apiAt: ApiAt, hash: { toHex(): string }): Promise { const status = (await apiAt.query.preimage.requestStatusFor(hash)).toJSON() as { @@ -808,23 +832,32 @@ async function notedLength(apiAt: ApiAt, hash: { toHex(): string }): Promise { - if (bounded.isInline) return decodeCall(apiAt, bounded.asInline); + if (bounded.isInline) return readCall(() => apiAt.registry.createType("Call", bounded.asInline)); const hash = bounded.isLookup ? bounded.asLookup.hash_ : bounded.asLegacy.hash_; const len = bounded.isLookup ? bounded.asLookup.len.toNumber() : await notedLength(apiAt, hash); - if (len !== undefined && len > MAX_GOVERNANCE_CALL_BYTES) { - return { text: `the call is ${len} bytes, over the ${MAX_GOVERNANCE_CALL_BYTES} bytes the monitor decodes` }; - } + if (len !== undefined && len > MAX_GOVERNANCE_CALL_BYTES) return oversizedCall(len); const preimage = len === undefined ? undefined : (await apiAt.query.preimage.preimageFor([hash, len])) as unknown as { isSome: boolean; unwrap(): Uint8Array }; if (!preimage?.isSome) return { text: `preimage ${hash.toHex()} is not noted yet`, pending: true }; - return decodeCall(apiAt, preimage.unwrap()); + return readCall(() => apiAt.registry.createType("Call", preimage.unwrap())); +} + +/** Reads a council or technical-committee motion's call, sized before it is fetched. */ +async function motionCall(apiAt: ApiAt, collective: "council" | "technicalCommittee", hash: { toHex(): string }): Promise { + const proposalOf = apiAt.query[collective].proposalOf; + const len = Number((await proposalOf.size(hash)).toString()); + if (len > MAX_GOVERNANCE_CALL_BYTES) return oversizedCall(len); + const call = (await proposalOf(hash)) as unknown as { isSome: boolean; unwrap(): unknown }; + if (!call.isSome) return { text: "the motion's call is not stored", pending: true }; + return readCall(() => call.unwrap()); } /** - * Page every pending Pendulum democracy item once: public proposals, the - * council's external proposal and ongoing referenda, with the call decoded. - * Calls that mint, move or unlock balances or change the runtime page as + * Page every pending Pendulum governance item once: public proposals, the + * council's external proposal, ongoing referenda and open council and + * technical-committee motions, with the call decoded. Calls that mint, move + * or unlock balances or change the runtime or the migration pallet page as * CRITICAL: minted PEN burned through the migration drains the fixed-supply * vault, and only Pendulum governance can stop the call before enactment. * So does a call the watcher cannot read; until it can, it is re-checked @@ -838,7 +871,7 @@ async function watchPendulumGovernance(api: ApiPromise): Promise { const democracy = apiAt.query.democracy; const launchPeriod = Number(apiAt.consts.democracy.launchPeriod.toString()); const nextLaunch = (Math.floor(finalizedBlock / launchPeriod) + 1) * launchPeriod; - const items: Array<{ key: string; label: string; hint: string; bounded: BoundedCall }> = []; + const items: Array<{ key: string; label: string; hint: string; read(): Promise }> = []; const proposals = (await democracy.publicProps()) as unknown as Array< [{ toNumber(): number }, BoundedCall, { toString(): string }] >; @@ -848,7 +881,7 @@ async function watchPendulumGovernance(api: ApiPromise): Promise { label: `public proposal #${index.toNumber()} from ${proposer.toString()}`, hint: `the best-backed public proposal becomes a referendum at block ${nextLaunch}; ` + "a hostile one needs a council or technical-committee cancel before then, or nay votes after", - bounded, + read: () => boundedCall(apiAt, bounded), }); } const external = (await democracy.nextExternal()) as unknown as { isSome: boolean; unwrap(): [BoundedCall, unknown] }; @@ -858,7 +891,7 @@ async function watchPendulumGovernance(api: ApiPromise): Promise { key: `external:${bounded.toString()}`, label: "the council's external proposal", hint: `it can become a referendum at block ${nextLaunch}`, - bounded, + read: () => boundedCall(apiAt, bounded), }); } const [count, lowest] = (await Promise.all([democracy.referendumCount(), democracy.lowestUnbaked()])).map((value) => @@ -875,16 +908,27 @@ async function watchPendulumGovernance(api: ApiPromise): Promise { key: `referendum:${index}`, label: `referendum #${index}`, hint: `voting ends at block ${ongoing.end.toNumber()}; a hostile one needs nay votes or an emergency cancel`, - bounded: ongoing.proposal, + read: () => boundedCall(apiAt, ongoing.proposal), }); } + for (const collective of ["council", "technicalCommittee"] as const) { + const motions = (await apiAt.query[collective].proposals()) as unknown as Array<{ toHex(): string }>; + for (const hash of motions) { + items.push({ + key: `${collective}:${hash.toHex()}`, + label: `${collective === "council" ? "council" : "technical committee"} motion ${hash.toHex()}`, + hint: "it executes as soon as enough members vote aye; a hostile one needs nay votes", + read: () => motionCall(apiAt, collective, hash), + }); + } + } // Pages go out together, so a burst of new items costs one webhook round trip. const pages: Promise[] = []; for (const item of items) { if (onceOnlyPages.has(item.key)) continue; let call: CallRead; try { - call = await boundedCall(apiAt, item.bounded); + call = await item.read(); } catch (readError) { // One item must never hide the items after it; a failed read is retried. call = { text: `reading the call failed: ${String(readError).slice(0, 300)}`, pending: true }; From 2ad36cf26d1d512c7b9f84ca9c8338680287a514 Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Thu, 22 Oct 2026 10:22:09 +0200 Subject: [PATCH 11/75] feat(monitor): Page when the Base view stops advancing Each poll compares the timestamp of the safe/finalized Base block with the wall clock and pages BASE VIEW STALLED, throttled like the Pendulum stall page, once it is older than BASE_STALE_ALERT_SECONDS (default 600 s for safe, 1800 s for finalized). --- monitor/src/main.test.ts | 7 +++++++ monitor/src/main.ts | 19 +++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/monitor/src/main.test.ts b/monitor/src/main.test.ts index 3152593f8..d9a5d6acd 100644 --- a/monitor/src/main.test.ts +++ b/monitor/src/main.test.ts @@ -415,6 +415,13 @@ test("a Pendulum failure pages on its own and the vault balance check still runs ]); }); +test("a Base view that stops advancing pages", async (t) => { + const { base, api, monitor, pages } = await setup(t); + base.safeAgeSeconds = 601; + await monitor.poll(api); + assert.match(pages.find((page) => page.subject === "BASE VIEW STALLED")?.detail ?? "", /^safe block 1010 is 60\ds old/); +}); + const UNREADABLE = "CRITICAL: Pendulum governance item cannot be read"; const RISKY = "CRITICAL: Pendulum governance item mints, moves or unlocks balances, or changes the runtime"; const pageFor = (pages: Page[], subject: string, label: string) => diff --git a/monitor/src/main.ts b/monitor/src/main.ts index 6b6c53d5e..2349037a2 100644 --- a/monitor/src/main.ts +++ b/monitor/src/main.ts @@ -205,6 +205,10 @@ const config = { /** Optional: the governance TimelockController; every operation it * schedules pages, because the canceller must veto inside the delay. */ timelockAddress: optionalAddress("TIMELOCK_ADDRESS"), + /** Age of the confirmed Base block (vs. wall clock) past which the monitor + * pages that its Base view stopped advancing. `safe` normally trails by + * about a minute, `finalized` by about 20 minutes. */ + baseStaleAlertMs: envNumber("BASE_STALE_ALERT_SECONDS", baseFinalityTag === "finalized" ? 1800 : 600) * 1000, }; const ZERO_ADDRESS = "0x0000000000000000000000000000000000000000"; @@ -323,6 +327,7 @@ let state: RuntimeState; let multicallUnavailable = false; let lastSourceStallAlertMs = 0; let lastSourceFailureAlertMs = 0; +let lastBaseStallAlertMs = 0; let lastIssuanceAlertMs = 0; /** The governance-queue scan runs ahead of everything else in a poll and * throws on failure, so the persisted Base cursor never passes a range it has @@ -1070,6 +1075,20 @@ async function check(api: ApiPromise): Promise { const confirmedBase = await publicClient.getBlock({ blockTag: config.baseFinalityTag }); if (confirmedBase.number === null) throw new Error(`${config.baseFinalityTag} Base block has no number`); const baseBlock = confirmedBase.number; + // A Base view that stops advancing hides every vault event after it while + // all checks keep passing at the old block; page it like a stalled source. + const baseAgeMs = Date.now() - Number(confirmedBase.timestamp) * 1000; + if ( + baseAgeMs > config.baseStaleAlertMs && + Date.now() - lastBaseStallAlertMs > config.unmatchedEventGraceSeconds * 1000 + ) { + lastBaseStallAlertMs = Date.now(); + await alert( + "BASE VIEW STALLED", + `${config.baseFinalityTag} block ${baseBlock} is ${Math.round(baseAgeMs / 1000)}s old; vault events after it ` + + `are not being checked — restore the Base node, or check whether Base's ${config.baseFinalityTag} head stalled`, + ); + } await watchTimelock(baseBlock); // A Pendulum failure must not switch off the Base-side checks: the vault From ea18c2c3483446b8c7cd2e5359d536379983d750 Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Thu, 22 Oct 2026 10:23:57 +0200 Subject: [PATCH 12/75] feat(monitor): Check that the auto-pause would work At startup and on every poll the monitor checks that the Base RPC serves BASE_CHAIN_ID and, with a guardian key configured, that the key parses, is vault.guardian() and holds at least GUARDIAN_MIN_BALANCE_WEI (default 0.001 ETH). Problems page as AUTO-PAUSE NOT READY, throttled; a monitor without a key pages AUTO-PAUSE DISABLED at startup, and the startup log states the auto-pause status. --- monitor/src/main.test.ts | 67 ++++++++++++++++++++++++++++++++++------ monitor/src/main.ts | 53 +++++++++++++++++++++++++++++-- 2 files changed, 109 insertions(+), 11 deletions(-) diff --git a/monitor/src/main.test.ts b/monitor/src/main.test.ts index d9a5d6acd..6eebf5ba8 100644 --- a/monitor/src/main.test.ts +++ b/monitor/src/main.test.ts @@ -23,6 +23,7 @@ import { parseAbi, toHex, } from "viem"; +import { privateKeyToAccount } from "viem/accounts"; (globalThis as { penMonitorUnderTest?: boolean }).penMonitorUnderTest = true; @@ -34,6 +35,8 @@ const ATTESTORS = ["0xa1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1", "0xa2a2a2a2a2a2 const CF = 1_000_000n; const SUPPLY = 150_000_000n * 10n ** 18n; const PEN = 10n ** 12n; +const GUARDIAN_KEY = `0x${"11".repeat(32)}` as const; +const GUARDIAN = privateKeyToAccount(GUARDIAN_KEY).address; const abi = parseAbi([ "function totalReleased() view returns (uint256)", @@ -42,6 +45,7 @@ const abi = parseAbi([ "function token() view returns (address)", "function paused() view returns (bool)", "function threshold() view returns (uint256)", + "function guardian() view returns (address)", "function activeApprovals(bytes32 payload) view returns (uint256)", "function totalSupply() view returns (uint256)", "function balanceOf(address owner) view returns (uint256)", @@ -67,11 +71,14 @@ function baseChain() { safeAgeSeconds: 2, paused: false, threshold: 3n, + guardian: GUARDIAN as string, totalReleased: 0n, totalSwept: 0n, vaultBalance: SUPPLY, approvals: new Map(), readyAt: new Map(), + /** Ether balances by lowercased address. */ + balances: new Map([[GUARDIAN.toLowerCase(), 10n ** 18n]]), logs: [] as Log[], }; } @@ -84,6 +91,8 @@ function answer(base: Base, method: string, params: any[]): unknown { return toHex(base.chainId); case "eth_getCode": return "0x"; + case "eth_getBalance": + return toHex(base.balances.get(String(params[0]).toLowerCase()) ?? 0n); case "eth_getBlockByNumber": { const number = String(params[0]).startsWith("0x") ? BigInt(params[0]) : base.safe; const timestamp = Math.floor(Date.now() / 1000) - base.safeAgeSeconds; @@ -99,6 +108,7 @@ function answer(base: Base, method: string, params: any[]): unknown { token: TOKEN, paused: base.paused, threshold: base.threshold, + guardian: base.guardian, activeApprovals: base.approvals.get(key) ?? 0n, totalSupply: SUPPLY, balanceOf: base.vaultBalance, @@ -316,16 +326,24 @@ async function body(request: IncomingMessage): Promise { let instances = 0; let monitorEnv: string[] = []; -/** A started monitor against fresh stand-ins. `webhook` answers each page - * with an HTTP status; `pages` lists them in arrival order from after - * startup, `startup` the ones before. */ +/** A started monitor against fresh stand-ins, which `prepare` can change + * before startup. `webhook` answers each page with an HTTP status; `pages` + * lists them in arrival order from after startup, `startup` the ones before. */ async function setup( t: { after(fn: () => void): void }, - env: Record = {}, - webhook: (page: Page) => number | Promise = () => 200, + { + env = {}, + webhook = () => 200, + prepare = () => {}, + }: { + env?: Record; + webhook?: (page: Page) => number | Promise; + prepare?: (base: Base, pendulum: Pendulum) => void; + } = {}, ) { const base = baseChain(); const { chain: pendulum, api } = pendulumChain(); + prepare(base, pendulum); const pages: Page[] = []; const server = createServer(async (request, response) => { const text = await body(request); @@ -415,6 +433,34 @@ test("a Pendulum failure pages on its own and the vault balance check still runs ]); }); +test("auto-pause readiness is checked at startup and each problem pages", async (t) => { + const { startup } = await setup(t, { + env: { GUARDIAN_PRIVATE_KEY: GUARDIAN_KEY }, + prepare: (base) => { + base.chainId = 84532; + base.guardian = "0x5555555555555555555555555555555555555555"; + base.balances.clear(); + }, + }); + const detail = startup.find((page) => page.subject === "AUTO-PAUSE NOT READY")?.detail ?? ""; + assert.match(detail, /serves chain 84532, not BASE_CHAIN_ID 8453/); + assert.match(detail, new RegExp(`${GUARDIAN} is not the vault guardian 0x5{40}`)); + assert.match(detail, /holds 0 wei/); +}); + +test("auto-pause readiness is re-checked every poll", async (t) => { + const { base, api, monitor, startup, subjects } = await setup(t, { env: { GUARDIAN_PRIVATE_KEY: GUARDIAN_KEY } }); + assert.ok(!startup.some((page) => page.subject.startsWith("AUTO-PAUSE")), JSON.stringify(startup)); + base.guardian = "0x5555555555555555555555555555555555555555"; + await monitor.poll(api); + assert.deepEqual(subjects(), ["AUTO-PAUSE NOT READY"]); +}); + +test("a monitor without a guardian key says so at startup", async (t) => { + const { startup } = await setup(t); + assert.ok(startup.some((page) => page.subject === "AUTO-PAUSE DISABLED"), JSON.stringify(startup)); +}); + test("a Base view that stops advancing pages", async (t) => { const { base, api, monitor, pages } = await setup(t); base.safeAgeSeconds = 601; @@ -526,10 +572,13 @@ function schedule(base: Base, id: Hex, blockNumber = 1_005n): void { test("once-only pages the webhook rejects are sent again on the next poll", async (t) => { const failedOnce = new Set(); - const { base, pendulum, api, monitor, pages } = await setup(t, { TIMELOCK_ADDRESS: TIMELOCK }, ({ subject }) => { - if (failedOnce.has(subject)) return 200; - failedOnce.add(subject); - return 500; + const { base, pendulum, api, monitor, pages } = await setup(t, { + env: { TIMELOCK_ADDRESS: TIMELOCK }, + webhook: ({ subject }) => { + if (failedOnce.has(subject)) return 200; + failedOnce.add(subject); + return 500; + }, }); schedule(base, toHex(1, { size: 32 })); pendulum.publicProps.push([7, inline("remark")]); diff --git a/monitor/src/main.ts b/monitor/src/main.ts index 2349037a2..2fad27545 100644 --- a/monitor/src/main.ts +++ b/monitor/src/main.ts @@ -52,6 +52,7 @@ const vaultAbi = [ { type: "function", name: "token", stateMutability: "view", inputs: [], outputs: [{ type: "address" }] }, { type: "function", name: "paused", stateMutability: "view", inputs: [], outputs: [{ type: "bool" }] }, { type: "function", name: "threshold", stateMutability: "view", inputs: [], outputs: [{ type: "uint256" }] }, + { type: "function", name: "guardian", stateMutability: "view", inputs: [], outputs: [{ type: "address" }] }, { type: "function", name: "activeApprovals", @@ -209,6 +210,8 @@ const config = { * pages that its Base view stopped advancing. `safe` normally trails by * about a minute, `finalized` by about 20 minutes. */ baseStaleAlertMs: envNumber("BASE_STALE_ALERT_SECONDS", baseFinalityTag === "finalized" ? 1800 : 600) * 1000, + /** Guardian balance (wei) below which the auto-pause counts as not ready. */ + guardianMinBalanceWei: BigInt(process.env.GUARDIAN_MIN_BALANCE_WEI ?? "1000000000000000"), }; const ZERO_ADDRESS = "0x0000000000000000000000000000000000000000"; @@ -328,6 +331,7 @@ let multicallUnavailable = false; let lastSourceStallAlertMs = 0; let lastSourceFailureAlertMs = 0; let lastBaseStallAlertMs = 0; +let lastReadinessAlertMs = 0; let lastIssuanceAlertMs = 0; /** The governance-queue scan runs ahead of everything else in a poll and * throws on failure, so the persisted Base cursor never passes a range it has @@ -462,6 +466,42 @@ async function pauseVault(): Promise { } } +/** What would make an auto-pause fail now: an RPC on another chain, or a key + * that does not parse, is not the vault's guardian or cannot pay for the + * pause. Pages, throttled, so it surfaces before a violation needs it. */ +async function checkAutoPauseReadiness(): Promise { + const problems: string[] = []; + const chainId = await publicClient.getChainId(); + if (chainId !== config.baseChainId) { + problems.push(`the Base RPC serves chain ${chainId}, not BASE_CHAIN_ID ${config.baseChainId}`); + } + if (config.guardianPrivateKey) { + let guardian: `0x${string}` | undefined; + try { + guardian = privateKeyToAccount(config.guardianPrivateKey).address; + } catch { + problems.push("GUARDIAN_PRIVATE_KEY is not a valid private key"); + } + if (guardian) { + const [vaultGuardian, balance] = await Promise.all([ + publicClient.readContract({ address: config.vaultAddress, abi: vaultAbi, functionName: "guardian" }), + publicClient.getBalance({ address: guardian }), + ]); + if (vaultGuardian.toLowerCase() !== guardian.toLowerCase()) { + problems.push(`the key's address ${guardian} is not the vault guardian ${vaultGuardian}`); + } + if (balance < config.guardianMinBalanceWei) { + problems.push(`guardian ${guardian} holds ${balance} wei, less than GUARDIAN_MIN_BALANCE_WEI ${config.guardianMinBalanceWei}`); + } + } + } + if (problems.length && Date.now() - lastReadinessAlertMs > config.graceSeconds * 1000) { + lastReadinessAlertMs = Date.now(); + await alert("AUTO-PAUSE NOT READY", `${problems.join("; ")} — fix it before a violation needs the pause`); + } + return problems; +} + async function securityViolation(subject: string, detail: string): Promise { // Pause FIRST. The alert webhook is an external dependency with its own // latency; nothing may sit between detecting a violation and the pause. @@ -1089,6 +1129,7 @@ async function check(api: ApiPromise): Promise { `are not being checked — restore the Base node, or check whether Base's ${config.baseFinalityTag} head stalled`, ); } + await checkAutoPauseReadiness(); await watchTimelock(baseBlock); // A Pendulum failure must not switch off the Base-side checks: the vault @@ -1212,7 +1253,7 @@ async function check(api: ApiPromise): Promise { } /** Startup once the Pendulum API is connected: durable state, the multicall - * probe and the state identity check. */ + * probe, the state identity check and the auto-pause readiness. */ export async function start(api: ApiPromise): Promise { state = hydrateState(); const multicallCode = await publicClient.getBytecode({ address: MULTICALL3_ADDRESS }); @@ -1233,10 +1274,18 @@ export async function start(api: ApiPromise): Promise { } else { state.pendulumGenesisHash = pendulumGenesisHash; } + if (!config.guardianPrivateKey) { + await alert( + "AUTO-PAUSE DISABLED", + "no GUARDIAN_PRIVATE_KEY configured: violations page, but nothing pauses the vault automatically", + ); + } + const readiness = await checkAutoPauseReadiness(); log( `monitor started; Pendulum after ${state.lastPendulumBlock}, Base from ${state.baseFromBlock}, ` + `polling every ${config.pollIntervalMs}ms; governance queue ` + - (config.timelockAddress ? `watched at ${config.timelockAddress}` : "not watched (TIMELOCK_ADDRESS unset)"), + (config.timelockAddress ? `watched at ${config.timelockAddress}` : "not watched (TIMELOCK_ADDRESS unset)") + + `; auto-pause ${!config.guardianPrivateKey ? "disabled" : readiness.length ? "NOT READY" : "ready"}`, ); } From 16eea8790800f77c08200aca80d2cb22d2cfabf3 Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Thu, 22 Oct 2026 10:25:16 +0200 Subject: [PATCH 13/75] feat(monitor): Page every vault admin event The scan that runs ahead of the vault checks now also reads the vault's admin events (token, pause and unpause, attestors, threshold, caps, guardian, admin transfer, sweep, stale pending cleared) and pages each one once as VAULT ADMIN EVENT, so the persisted Base cursor never passes an unscanned one. --- monitor/src/main.test.ts | 16 +++++++++ monitor/src/main.ts | 73 ++++++++++++++++++++++++++++++---------- 2 files changed, 71 insertions(+), 18 deletions(-) diff --git a/monitor/src/main.test.ts b/monitor/src/main.test.ts index 6eebf5ba8..a7c8fc06b 100644 --- a/monitor/src/main.test.ts +++ b/monitor/src/main.test.ts @@ -52,6 +52,8 @@ const abi = parseAbi([ "function getTimestamp(bytes32 id) view returns (uint256)", "event Approved(uint64 indexed nonce, address indexed recipient, uint256 palletAmount, address indexed attestor)", "event Released(uint64 indexed nonce, address indexed recipient, uint256 palletAmount, uint256 tokenAmount)", + "event GuardianUpdated(address indexed guardian)", + "event CapsUpdated(uint256 perReleaseCap, uint256 dailyCap)", "event CallScheduled(bytes32 indexed id, uint256 indexed index, address target, uint256 value, bytes data, bytes32 predecessor, uint256 delay)", ]); @@ -456,6 +458,20 @@ test("auto-pause readiness is re-checked every poll", async (t) => { assert.deepEqual(subjects(), ["AUTO-PAUSE NOT READY"]); }); +test("vault admin events page once each from the Base scan", async (t) => { + const { base, api, monitor, pages } = await setup(t); + const guardian = "0x5555555555555555555555555555555555555555"; + base.logs.push({ address: VAULT, eventName: "GuardianUpdated", args: { guardian }, blockNumber: 1_004n }); + base.logs.push({ address: VAULT, eventName: "CapsUpdated", args: { perReleaseCap: 5n, dailyCap: 7n }, blockNumber: 1_012n }); + await monitor.poll(api); + base.safe = 1_020n; + await monitor.poll(api); + const adminPages = pages.filter((page) => page.subject === "VAULT ADMIN EVENT").map((page) => page.detail); + assert.equal(adminPages.length, 2, JSON.stringify(pages)); + assert.match(adminPages[0], /^GuardianUpdated\(guardian=0x5{40}\) at Base block 1004/i); + assert.match(adminPages[1], /^CapsUpdated\(perReleaseCap=5, dailyCap=7\) at Base block 1012/); +}); + test("a monitor without a guardian key says so at startup", async (t) => { const { startup } = await setup(t); assert.ok(startup.some((page) => page.subject === "AUTO-PAUSE DISABLED"), JSON.stringify(startup)); diff --git a/monitor/src/main.ts b/monitor/src/main.ts index 2fad27545..76ddeb65c 100644 --- a/monitor/src/main.ts +++ b/monitor/src/main.ts @@ -15,6 +15,7 @@ import { encodeAbiParameters, http, keccak256, + parseAbi, toFunctionSelector, } from "viem"; import { privateKeyToAccount } from "viem/accounts"; @@ -83,6 +84,22 @@ const vaultAbi = [ }, ] as const; +/** The vault events of its admin (and of the guardian's pause). */ +const vaultAdminEventsAbi = parseAbi([ + "event TokenSet(address indexed token)", + "event Paused(address indexed by)", + "event Unpaused(address indexed by)", + "event AttestorAdded(address indexed attestor)", + "event AttestorRemoved(address indexed attestor)", + "event ThresholdUpdated(uint256 threshold)", + "event CapsUpdated(uint256 perReleaseCap, uint256 dailyCap)", + "event GuardianUpdated(address indexed guardian)", + "event AdminTransferStarted(address indexed pendingAdmin)", + "event AdminTransferred(address indexed newAdmin)", + "event RemainderSwept(address indexed to, uint256 amount)", + "event StalePendingCleared(bytes32 indexed payload, uint256 tokenAmount)", +]); + const erc20Abi = [ { type: "function", name: "totalSupply", stateMutability: "view", inputs: [], outputs: [{ type: "uint256" }] }, { @@ -333,12 +350,13 @@ let lastSourceFailureAlertMs = 0; let lastBaseStallAlertMs = 0; let lastReadinessAlertMs = 0; let lastIssuanceAlertMs = 0; -/** The governance-queue scan runs ahead of everything else in a poll and - * throws on failure, so the persisted Base cursor never passes a range it has - * not scanned; a restart resumes from that cursor. */ -let timelockFromBlock: bigint | undefined; +/** The Base admin scan (vault admin events and the governance queue) runs + * ahead of the vault checks in a poll and throws on failure, so the persisted + * Base cursor never passes a range it has not scanned; a restart resumes from + * that cursor. */ +let baseAdminFromBlock: bigint | undefined; /** Once-only pages by key, each held until the webhook accepts it. */ -// ponytail: in memory; a restart re-pages what it finds again (timelock operations after the persisted cursor, pending governance items) and drops undelivered pages it does not find again. +// ponytail: in memory; a restart re-pages what it finds again (Base admin events after the persisted cursor, pending governance items) and drops undelivered pages it does not find again. const onceOnlyPages = new Map(); let lastGovernanceWatchFailureMs = 0; @@ -765,15 +783,33 @@ async function reconcileBaseEvents( return { violation: firstViolation }; } -/** Page every operation the governance Timelock scheduled up to `toBlock`, - * the poll's `safe`/`finalized` Base block. */ -async function watchTimelock(toBlock: bigint): Promise { - if (!config.timelockAddress) return; +/** Page every vault admin event, and every operation the governance Timelock + * scheduled, up to `toBlock`, the poll's `safe`/`finalized` Base block. + * Before the governance handover the vault admin acts without a delay, so + * its events are the only trace of a guardian, attestor or cap change. */ +async function watchBaseAdmin(toBlock: bigint): Promise { + const timelock = config.timelockAddress; const operations = new Map<`0x${string}`, ScheduledCall[]>(); - for (const [start, end] of blockRanges(timelockFromBlock ?? state.baseFromBlock, toBlock, config.baseMaxBlockRange)) { + const pages: Promise[] = []; + for (const [start, end] of blockRanges(baseAdminFromBlock ?? state.baseFromBlock, toBlock, config.baseMaxBlockRange)) { await publicClient.getBlock({ blockNumber: end }); // same lagging-node probe as the vault scan + const adminEvents = await publicClient.getContractEvents({ + address: config.vaultAddress, + abi: vaultAdminEventsAbi, + fromBlock: start, + toBlock: end, + }); + for (const event of adminEvents) { + const args = Object.entries(event.args).map(([name, value]) => `${name}=${value}`).join(", "); + pages.push(pageOnce( + `vault:${event.transactionHash}:${event.logIndex}`, + "VAULT ADMIN EVENT", + `${event.eventName}(${args}) at Base block ${event.blockNumber}, tx ${event.transactionHash}`, + )); + } + if (!timelock) continue; const scheduled = await publicClient.getContractEvents({ - address: config.timelockAddress, + address: timelock, abi: timelockAbi, eventName: "CallScheduled", fromBlock: start, @@ -792,7 +828,7 @@ async function watchTimelock(toBlock: bigint): Promise { operations.set(id, calls); } } - if (operations.size > 0) { + if (timelock && operations.size > 0) { const token = await publicClient.readContract({ address: config.vaultAddress, abi: vaultAbi, @@ -802,25 +838,26 @@ async function watchTimelock(toBlock: bigint): Promise { const labels = { [config.vaultAddress.toLowerCase()]: "vault", [token.toLowerCase()]: "token", - [config.timelockAddress.toLowerCase()]: "timelock", + [timelock.toLowerCase()]: "timelock", }; for (const [id, calls] of operations) { if (onceOnlyPages.has(`timelock:${id}`)) continue; const readyAt = await publicClient.readContract({ - address: config.timelockAddress, + address: timelock, abi: timelockAbi, functionName: "getTimestamp", args: [id], blockNumber: toBlock, }); - await pageOnce( + pages.push(pageOnce( `timelock:${id}`, "GOVERNANCE OPERATION SCHEDULED", describeTimelockOperation(id, calls, readyAt, Date.now(), labels, KNOWN_FUNCTIONS), - ); + )); } } - timelockFromBlock = toBlock + 1n; + await Promise.all(pages); + baseAdminFromBlock = toBlock + 1n; } type ApiAt = Awaited>; @@ -1130,7 +1167,7 @@ async function check(api: ApiPromise): Promise { ); } await checkAutoPauseReadiness(); - await watchTimelock(baseBlock); + await watchBaseAdmin(baseBlock); // A Pendulum failure must not switch off the Base-side checks: the vault // balance check and its pause run without the source. Reconciliation, M2a From f5bd76b3fe8bbf35ae10fa228f61417e40586a38 Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Thu, 22 Oct 2026 10:27:10 +0200 Subject: [PATCH 14/75] fix(monitor): Send one liveness page per kind and poll Stale migrations now page as one summary per kind and poll (count and the first nonces), with the detail per nonce in the log, instead of one awaited page per nonce; a nonce counts as paged only once its summary is delivered. The first sighting of an unmatched Base event pages without being awaited, so reconciliation and the pause decision never wait for the webhook, and the retry of once-only pages waits for one that is still in flight instead of sending it twice. --- monitor/src/main.test.ts | 56 +++++++++++++++++++++++++++ monitor/src/main.ts | 83 ++++++++++++++++++++++++---------------- 2 files changed, 105 insertions(+), 34 deletions(-) diff --git a/monitor/src/main.test.ts b/monitor/src/main.test.ts index a7c8fc06b..93bba5e3f 100644 --- a/monitor/src/main.test.ts +++ b/monitor/src/main.test.ts @@ -20,6 +20,7 @@ import { encodeAbiParameters, encodeEventTopics, type Hex, + keccak256, parseAbi, toHex, } from "viem"; @@ -579,6 +580,61 @@ test("the governance watch runs after the conservation checks", async (t) => { assert.deepEqual(sorted(subjects().slice(0, 2)), ["AUTO-PAUSE UNAVAILABLE", "VAULT BALANCE DEFICIT"]); }); +test("stale migrations page one liveness summary per kind and poll", async (t) => { + const { base, pendulum, api, monitor, pages } = await setup(t, { env: { GRACE_SECONDS: "30" } }); + pendulum.headAgeMs = 60_000; + for (let nonce = 0n; nonce < 27n; nonce++) burn(pendulum, nonce, 100n * PEN); + for (const nonce of [25n, 26n]) { + const payload = keccak256(encodeAbiParameters( + [{ type: "uint64" }, { type: "address" }, { type: "uint256" }], + [nonce, RECIPIENT, 100n * PEN], + )); + base.approvals.set(payload, 3n); + } + await monitor.poll(api); + await monitor.poll(api); + const liveness = pages.filter((page) => page.subject.startsWith("LIVENESS")); + assert.equal(liveness.length, 2, JSON.stringify(liveness)); + assert.match(pageFor(liveness, "LIVENESS: migration below approval quorum", "25 ")!.detail, /nonce 0, 1, .*, 9 and 15 more\)/); + assert.match(pageFor(liveness, "LIVENESS: quorum reached but not released", "2 ")!.detail, /\(nonce 25, 26\)/); +}); + +test("first sightings of unmatched events do not delay the pause", async (t) => { + const events: string[] = []; + let violationArrived = () => {}; + const arrived = new Promise((resolve) => (violationArrived = resolve)); + const { base, pendulum, api, monitor } = await setup(t, { + webhook: async ({ subject }) => { + if (subject === "MIGRATION TUPLE VIOLATION") { + events.push("violation paged"); + violationArrived(); + } + if (subject === "UNVERIFIED BASE EVENT") { + events.push("first sighting paged"); + await Promise.race([arrived, new Promise((resolve) => setTimeout(resolve, 3_000).unref())]); + events.push("first sighting answered"); + } + return 200; + }, + }); + burn(pendulum, 0n, 100n * PEN); + base.logs.push({ + address: VAULT, + eventName: "Approved", + args: { nonce: 5n, recipient: RECIPIENT, palletAmount: 100n * PEN, attestor: ATTESTORS[0] }, + blockNumber: 1_005n, + }); + base.logs.push({ + address: VAULT, + eventName: "Released", + args: { nonce: 0n, recipient: RECIPIENT, palletAmount: 100n * PEN, tokenAmount: 1n }, + blockNumber: 1_006n, + }); + await monitor.poll(api); + while (!events.includes("first sighting answered")) await new Promise((resolve) => setTimeout(resolve, 10)); + assert.deepEqual(events, ["first sighting paged", "violation paged", "first sighting answered"]); +}); + /** A timelock operation scheduling the vault's `pause()`, ready in 48 h. */ function schedule(base: Base, id: Hex, blockNumber = 1_005n): void { const call = { target: VAULT, value: 0n, data: "0x8456cb59", predecessor: toHex(0, { size: 32 }), delay: 172_800n }; diff --git a/monitor/src/main.ts b/monitor/src/main.ts index 76ddeb65c..a4d8f22d1 100644 --- a/monitor/src/main.ts +++ b/monitor/src/main.ts @@ -357,7 +357,7 @@ let lastIssuanceAlertMs = 0; let baseAdminFromBlock: bigint | undefined; /** Once-only pages by key, each held until the webhook accepts it. */ // ponytail: in memory; a restart re-pages what it finds again (Base admin events after the persisted cursor, pending governance items) and drops undelivered pages it does not find again. -const onceOnlyPages = new Map(); +const onceOnlyPages = new Map }>(); let lastGovernanceWatchFailureMs = 0; function log(message: string): void { @@ -394,16 +394,19 @@ async function alert(subject: string, detail: string): Promise { /** Pages `key` once. A page the webhook does not accept stays queued, and * every poll retries it until it is delivered. */ -async function pageOnce(key: string, subject: string, detail: string): Promise { - if (onceOnlyPages.has(key)) return; - const page = { subject, detail, delivered: false }; - onceOnlyPages.set(key, page); - page.delivered = await alert(subject, detail); +function pageOnce(key: string, subject: string, detail: string): Promise { + const queued = onceOnlyPages.get(key); + if (queued) return queued.delivery; + const delivery = alert(subject, detail); + onceOnlyPages.set(key, { subject, detail, delivery }); + return delivery; } async function retryUndeliveredPages(): Promise { - await Promise.all([...onceOnlyPages.values()].filter((page) => !page.delivered).map(async (page) => { - page.delivered = await alert(page.subject, page.detail); + await Promise.all([...onceOnlyPages.values()].map(async (page) => { + if (await page.delivery) return; + page.delivery = alert(page.subject, page.detail); + await page.delivery; })); } @@ -545,9 +548,10 @@ async function securityViolation(subject: string, detail: string): Promise { +): "proven" | "await" | "expired" { const alreadyKnown = draft.unmatchedBaseFirstSeenAt.has(key); const firstSeen = draft.unmatchedBaseFirstSeenAt.get(key) ?? observedAt; draft.unmatchedBaseFirstSeenAt.set(key, firstSeen); if (!alreadyKnown) { - await pageOnce( + void pageOnce( `unverified:${key}`, "UNVERIFIED BASE EVENT", `${kind} nonce ${nonce} at Base block ${eventBlockNumber} has no finalized Pendulum source yet; ` + @@ -699,7 +703,7 @@ async function reconcileBaseEvents( continue; } if (!expected) { - const disposition = await unmatchedDisposition( + const disposition = unmatchedDisposition( "Approved", key, actual.nonce, @@ -735,7 +739,7 @@ async function reconcileBaseEvents( continue; } if (!expected) { - const disposition = await unmatchedDisposition( + const disposition = unmatchedDisposition( "Released", key, actual.nonce, @@ -790,7 +794,7 @@ async function reconcileBaseEvents( async function watchBaseAdmin(toBlock: bigint): Promise { const timelock = config.timelockAddress; const operations = new Map<`0x${string}`, ScheduledCall[]>(); - const pages: Promise[] = []; + const pages: Promise[] = []; for (const [start, end] of blockRanges(baseAdminFromBlock ?? state.baseFromBlock, toBlock, config.baseMaxBlockRange)) { await publicClient.getBlock({ blockNumber: end }); // same lagging-node probe as the vault scan const adminEvents = await publicClient.getContractEvents({ @@ -1005,7 +1009,7 @@ async function watchPendulumGovernance(api: ApiPromise): Promise { } } // Pages go out together, so a burst of new items costs one webhook round trip. - const pages: Promise[] = []; + const pages: Promise[] = []; for (const item of items) { if (onceOnlyPages.has(item.key)) continue; let call: CallRead; @@ -1256,30 +1260,41 @@ async function check(api: ApiPromise): Promise { }); const approvals = await activeApprovalCounts(pending, baseBlock); const now = Date.now(); + // One page per kind and poll however long the backlog, with the detail + // per nonce in the log: a page per nonce floods the receiver and, awaited + // in turn, delays the next poll's checks. + const quorum: MigrationRecord[] = []; + const belowQuorum: MigrationRecord[] = []; for (let index = 0; index < pending.length; index++) { const migration = pending[index]; if (!isStale(migration.firstSeenMs, now, config.graceSeconds)) continue; const lastAlerted = draft.livenessAlertedAt.get(migration.nonce); if (lastAlerted !== undefined && now - lastAlerted < config.graceSeconds * 1000) continue; - draft.livenessAlertedAt.set(migration.nonce, now); - if (approvals[index] >= threshold) { - // Quorum is present but nothing released it: cap-deferred and - // starved of daily allowance, paused, or made releasable by a - // threshold cut without ever emitting ReleasePending (so the - // releaser cannot see it). Silent indefinitely before round 9. - await alert( - "LIVENESS: quorum reached but not released", - `nonce ${migration.nonce} has ${approvals[index]}/${threshold} active approvals and is still unreleased ` + - `after ${config.graceSeconds}s — check the daily allowance / pause state, or call release() directly ` + - `if it qualified through a threshold cut (RB-6)`, - ); - continue; - } - await alert( - "LIVENESS: migration below approval quorum", - `nonce ${migration.nonce} has ${approvals[index]}/${threshold} active approvals after ${config.graceSeconds}s`, - ); + log(`liveness: nonce ${migration.nonce} has ${approvals[index]}/${threshold} active approvals and is unreleased`); + (approvals[index] >= threshold ? quorum : belowQuorum).push(migration); } + const pageLiveness = async (migrations: MigrationRecord[], subject: string, detail: string) => { + if (!migrations.length) return; + const nonces = migrations.slice(0, 10).map((migration) => migration.nonce).join(", ") + + (migrations.length > 10 ? ` and ${migrations.length - 10} more` : ""); + if (await alert(subject, `${migrations.length} migration(s) (nonce ${nonces}) ${detail}`)) { + for (const migration of migrations) draft.livenessAlertedAt.set(migration.nonce, now); + } + }; + // Quorum is present but nothing released it: cap-deferred and starved of + // daily allowance, paused, or made releasable by a threshold cut without + // ever emitting ReleasePending (so the releaser cannot see it). + await pageLiveness( + quorum, + "LIVENESS: quorum reached but not released", + `have ${threshold} or more active approvals and are still unreleased after ${config.graceSeconds}s — check the ` + + `daily allowance / pause state, or call release() directly if one qualified through a threshold cut (RB-6)`, + ); + await pageLiveness( + belowQuorum, + "LIVENESS: migration below approval quorum", + `have fewer than ${threshold} active approvals after ${config.graceSeconds}s`, + ); state = draft; persistState(state); From f124f69ff399d02ba759eb014e8682aed2b6ee02 Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Thu, 22 Oct 2026 10:28:30 +0200 Subject: [PATCH 15/75] fix(monitor): Hold the Base cursor on events still awaiting their source Reconciliation now stops at a block range that holds a Base event whose nonce the source view may still reveal (within the grace or past it) even when the poll found a violation, so the state persisted after the pause replays that range and matches the release once the burn is ingested. Events proven to have no source are still passed after a confirmed pause. --- monitor/src/main.test.ts | 17 +++++++++++++++++ monitor/src/main.ts | 17 ++++++++--------- 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/monitor/src/main.test.ts b/monitor/src/main.test.ts index 93bba5e3f..0bf17b757 100644 --- a/monitor/src/main.test.ts +++ b/monitor/src/main.test.ts @@ -635,6 +635,23 @@ test("first sightings of unmatched events do not delay the pause", async (t) => assert.deepEqual(events, ["first sighting paged", "violation paged", "first sighting answered"]); }); +test("after a violation the Base cursor stays on events still awaiting their source", async (t) => { + const { base, pendulum, api, monitor, subjects, persisted } = await setup(t, { + env: { UNMATCHED_EVENT_GRACE_SECONDS: "0", GUARDIAN_PRIVATE_KEY: GUARDIAN_KEY }, + }); + base.paused = true; // so the pause is confirmed and the poll persists its state + release(base, 0n, 100n * PEN); // the burn is not in the lagging source view yet + await monitor.poll(api); + await new Promise((resolve) => setTimeout(resolve, 5)); + await monitor.poll(api); // the grace has run out: a cannot-verify violation + assert.ok(subjects().includes("MIGRATION TUPLE VIOLATION"), JSON.stringify(subjects())); + assert.equal(persisted().baseFromBlock, "1000"); + burn(pendulum, 0n, 100n * PEN); // the source view catches up + await monitor.poll(api); + assert.deepEqual(persisted().migrations, []); + assert.equal(persisted().baseFromBlock, "1011"); +}); + /** A timelock operation scheduling the vault's `pause()`, ready in 48 h. */ function schedule(base: Base, id: Hex, blockNumber = 1_005n): void { const call = { target: VAULT, value: 0n, data: "0x8456cb59", predecessor: toHex(0, { size: 32 }), delay: 172_800n }; diff --git a/monitor/src/main.ts b/monitor/src/main.ts index a4d8f22d1..ed64a7d7c 100644 --- a/monitor/src/main.ts +++ b/monitor/src/main.ts @@ -712,10 +712,8 @@ async function reconcileBaseEvents( observedAt, sourceFinalizedTsMs, ); - if (disposition === "await") { - awaitingSource ??= `Approved nonce ${actual.nonce} at Base block ${event.blockNumber}`; - continue; - } + if (disposition !== "proven") awaitingSource ??= `Approved nonce ${actual.nonce} at Base block ${event.blockNumber}`; + if (disposition === "await") continue; firstViolation ??= `Approved at Base block ${event.blockNumber}: ${mismatch} (` + (disposition === "proven" ? "source view has passed the event's timestamp — fabricated" @@ -748,10 +746,8 @@ async function reconcileBaseEvents( observedAt, sourceFinalizedTsMs, ); - if (disposition === "await") { - awaitingSource ??= `Released nonce ${actual.nonce} at Base block ${event.blockNumber}`; - continue; - } + if (disposition !== "proven") awaitingSource ??= `Released nonce ${actual.nonce} at Base block ${event.blockNumber}`; + if (disposition === "await") continue; firstViolation ??= `Released at Base block ${event.blockNumber}: ${mismatch} (` + (disposition === "proven" ? "source view has passed the event's timestamp — fabricated" @@ -760,7 +756,10 @@ async function reconcileBaseEvents( } firstViolation ??= `Released at Base block ${event.blockNumber}: ${mismatch}`; } - if (awaitingSource && !firstViolation) return { awaitingSource }; + // A range holding an event the source may still reveal (awaiting it, or + // past the grace) is replayed later even after a violation: passing it + // would lose a legitimate release for good. + if (awaitingSource) return { violation: firstViolation, awaitingSource }; for (const event of releases) { const actual = event.args as { From 139b4cbace6fa5f601b1e74895f852d4fce2551b Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Thu, 22 Oct 2026 10:29:47 +0200 Subject: [PATCH 16/75] feat(monitor): Let operators acknowledge a quantified release excess ACKNOWLEDGED_RELEASE_EXCESS (token units, default 0, validated and logged at startup) is added to the migrated total in the released-vs-migrated check, so once a loss has been quantified and accepted the vault can be unpaused without the check pausing it again; any further excess still fires. --- monitor/src/checks.test.ts | 5 +++++ monitor/src/checks.ts | 7 ++++++- monitor/src/main.test.ts | 12 ++++++++++++ monitor/src/main.ts | 18 +++++++++++++++--- 4 files changed, 38 insertions(+), 4 deletions(-) diff --git a/monitor/src/checks.test.ts b/monitor/src/checks.test.ts index 8a86b4683..f182c7f40 100644 --- a/monitor/src/checks.test.ts +++ b/monitor/src/checks.test.ts @@ -36,6 +36,11 @@ test("M2a: released exceeding migrated fires", () => { assert.equal(releasedExceedsMigrated(6n * CF, 5n, CF), true); }); +test("M2a: an acknowledged loss is tolerated, and nothing beyond it", () => { + assert.equal(releasedExceedsMigrated(6n * CF, 5n, CF, CF), false); + assert.equal(releasedExceedsMigrated(6n * CF + 1n, 5n, CF, CF), true); +}); + test("M2b: exact conservation is not a deficit", () => { // balance + released + swept == totalSupply assert.equal(vaultConservationDeficit(SUPPLY - 30n, 20n, 10n, SUPPLY), false); diff --git a/monitor/src/checks.ts b/monitor/src/checks.ts index aa20b0a43..75f35a0dd 100644 --- a/monitor/src/checks.ts +++ b/monitor/src/checks.ts @@ -13,13 +13,18 @@ * units. It lags `totalMigrated` only via finality delay, never the other way, * because a release requires attestations of an already-finalized burn — so any * excess is the strongest possible signal of attestor compromise. + * + * `acknowledgedExcess` is a loss operators have quantified and accepted: it + * stays in the cumulative totals for good, and without it the check would + * re-pause the vault after every recovery unpause. */ export function releasedExceedsMigrated( totalReleased: bigint, totalMigrated: bigint, conversionFactor: bigint, + acknowledgedExcess = 0n, ): boolean { - return totalReleased > totalMigrated * conversionFactor; + return totalReleased > totalMigrated * conversionFactor + acknowledgedExcess; } /** diff --git a/monitor/src/main.test.ts b/monitor/src/main.test.ts index 0bf17b757..3f419a008 100644 --- a/monitor/src/main.test.ts +++ b/monitor/src/main.test.ts @@ -652,6 +652,18 @@ test("after a violation the Base cursor stays on events still awaiting their sou assert.equal(persisted().baseFromBlock, "1011"); }); +test("the conservation check fires only beyond an acknowledged loss", async (t) => { + const { base, api, monitor, subjects } = await setup(t, { env: { ACKNOWLEDGED_RELEASE_EXCESS: "1000" } }); + base.totalReleased = 1_000n; // released without a burn, quantified and accepted + base.vaultBalance = SUPPLY - 1_000n; + await monitor.poll(api); + assert.deepEqual(subjects(), []); + base.totalReleased += 1n; + base.vaultBalance -= 1n; + await monitor.poll(api); + assert.ok(subjects().includes("CONSERVATION VIOLATION"), JSON.stringify(subjects())); +}); + /** A timelock operation scheduling the vault's `pause()`, ready in 48 h. */ function schedule(base: Base, id: Hex, blockNumber = 1_005n): void { const call = { target: VAULT, value: 0n, data: "0x8456cb59", predecessor: toHex(0, { size: 32 }), delay: 172_800n }; diff --git a/monitor/src/main.ts b/monitor/src/main.ts index ed64a7d7c..177aa9c0f 100644 --- a/monitor/src/main.ts +++ b/monitor/src/main.ts @@ -169,6 +169,13 @@ function envNumber(name: string, fallback: number | undefined): number { return value; } +/** A non-negative integer amount from the environment, validated at startup. */ +function envAmount(name: string): bigint { + const raw = process.env[name] || "0"; + if (!/^\d+$/.test(raw)) throw new Error(`invalid ${name}: ${raw}`); + return BigInt(raw); +} + function optionalAddress(name: string): `0x${string}` | undefined { const value = process.env[name]; if (!value) return undefined; @@ -229,6 +236,9 @@ const config = { baseStaleAlertMs: envNumber("BASE_STALE_ALERT_SECONDS", baseFinalityTag === "finalized" ? 1800 : 600) * 1000, /** Guardian balance (wei) below which the auto-pause counts as not ready. */ guardianMinBalanceWei: BigInt(process.env.GUARDIAN_MIN_BALANCE_WEI ?? "1000000000000000"), + /** Excess of released over migrated, in token units, that operators have + * quantified and accepted after a real loss (RB-3); M2a fires beyond it. */ + acknowledgedReleaseExcess: envAmount("ACKNOWLEDGED_RELEASE_EXCESS"), }; const ZERO_ADDRESS = "0x0000000000000000000000000000000000000000"; @@ -1239,10 +1249,11 @@ async function check(api: ApiPromise): Promise { ); return; } - if (releasedExceedsMigrated(totalReleased, totalMigrated, conversionFactor)) { + if (releasedExceedsMigrated(totalReleased, totalMigrated, conversionFactor, config.acknowledgedReleaseExcess)) { if (await securityViolation( "CONSERVATION VIOLATION", - `released ${totalReleased} > migrated ${totalMigrated * conversionFactor} (token units)`, + `released ${totalReleased} > migrated ${totalMigrated * conversionFactor} + acknowledged excess ` + + `${config.acknowledgedReleaseExcess} (token units)`, )) { state = draft; persistState(state); @@ -1336,7 +1347,8 @@ export async function start(api: ApiPromise): Promise { `monitor started; Pendulum after ${state.lastPendulumBlock}, Base from ${state.baseFromBlock}, ` + `polling every ${config.pollIntervalMs}ms; governance queue ` + (config.timelockAddress ? `watched at ${config.timelockAddress}` : "not watched (TIMELOCK_ADDRESS unset)") + - `; auto-pause ${!config.guardianPrivateKey ? "disabled" : readiness.length ? "NOT READY" : "ready"}`, + `; auto-pause ${!config.guardianPrivateKey ? "disabled" : readiness.length ? "NOT READY" : "ready"}` + + `; acknowledged release excess ${config.acknowledgedReleaseExcess} token units`, ); } From 6c4224b985d3c78e6e42f9805b7701e4f011c6d8 Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Thu, 22 Oct 2026 10:32:21 +0200 Subject: [PATCH 17/75] docs(monitor): Document the new checks, settings and pages Describe the Base view, auto-pause readiness, Pendulum source failure, vault and migration-pallet admin event checks, the motions and decoding bounds of the governance watch, the liveness summaries, webhook delivery and retry, the acknowledged release excess, and list every page subject. --- monitor/README.md | 51 +++++++++++++++++++++++++++++++++++------------ 1 file changed, 38 insertions(+), 13 deletions(-) diff --git a/monitor/README.md b/monitor/README.md index d48b328e5..264ac8bbf 100644 --- a/monitor/README.md +++ b/monitor/README.md @@ -8,13 +8,18 @@ Checks every poll: | Check | Meaning | Reaction | |---|---|---| -| M2 tuple match | Every safe Base `Approved` and `Released` event must match one finalized Pendulum `MigrationInitiated` nonce, recipient, and pallet amount; `Released.tokenAmount` must equal `palletAmount × conversionFactor`. | Alert + auto-pause. A mismatch against a *known* nonce pauses immediately. An *unknown* nonce first pages (`UNVERIFIED BASE EVENT`), then pauses as soon as it is provable — the finalized source view passed the event's own timestamp and the nonce still does not exist — or after `UNMATCHED_EVENT_GRACE_SECONDS` if the source view stays behind (a stalled source also pages on its own, see below) | -| M2a aggregate | `totalReleased <= TotalMigrated × conversionFactor`. This remains independent defence-in-depth around the event reconciliation. | Alert + auto-pause | -| M2b: `balanceOf(vault) + totalReleased + totalSwept >= totalSupply` | Vault-internal conservation. Only a **deficit** alerts: a surplus is a harmless inbound transfer (donation, or a migration whose recipient is the vault) and is ignored, so it cannot false-trigger a pause. | Alert + auto-pause on a deficit | -| M4: every old migration reaches active approval quorum — and then releases | Liveness of the attestor fleet (`below approval quorum`), and of everything after it (`quorum reached but not released`: a release starved of daily allowance, paused, or made releasable by a threshold cut without a `ReleasePending`, which the releaser cannot see). | Alert, throttled per nonce | +| M2 tuple match | Every safe Base `Approved` and `Released` event must match one finalized Pendulum `MigrationInitiated` nonce, recipient, and pallet amount; `Released.tokenAmount` must equal `palletAmount × conversionFactor`. After a violation the Base cursor stays on any block range whose events the source view may still reveal, so a pause caused by a lagging source does not lose those releases. | Alert + auto-pause. A mismatch against a *known* nonce pauses immediately. An *unknown* nonce first pages (`UNVERIFIED BASE EVENT`, without delaying the pause decision), then pauses as soon as it is provable — the finalized source view passed the event's own timestamp and the nonce still does not exist — or after `UNMATCHED_EVENT_GRACE_SECONDS` if the source view stays behind (a stalled source also pages on its own, see below) | +| M2a aggregate | `totalReleased <= TotalMigrated × conversionFactor + ACKNOWLEDGED_RELEASE_EXCESS`. This remains independent defence-in-depth around the event reconciliation. After a real, quantified loss, operators set `ACKNOWLEDGED_RELEASE_EXCESS` to it (RB-3) so the vault can be unpaused; any further excess still fires. | Alert + auto-pause | +| M2b: `balanceOf(vault) + totalReleased + totalSwept >= totalSupply` | Vault-internal conservation. Only a **deficit** alerts: a surplus is a harmless inbound transfer (donation, or a migration whose recipient is the vault) and is ignored, so it cannot false-trigger a pause. | Alert + auto-pause on a deficit, also while Pendulum cannot be read | +| M4: every old migration reaches active approval quorum — and then releases | Liveness of the attestor fleet (`below approval quorum`), and of everything after it (`quorum reached but not released`: a release starved of daily allowance, paused, or made releasable by a threshold cut without a `ReleasePending`, which the releaser cannot see). | One summary page per kind and poll (count and the first nonces; the log lists every nonce), each nonce at most once per `GRACE_SECONDS` | | Source supply | `totalIssuance + TotalMigrated` on Pendulum must never grow past its first-seen anchor. | Alert (`SOURCE SUPPLY GREW`) | | Unreleasable burns | A burn to the zero or vault address can never be approved or released; it is paged once and excluded from the pending count so it cannot page liveness forever or block RB-7. | Alert once | -| Pendulum governance | Every pending public proposal, the council's external proposal and every ongoing referendum, with the call decoded. A call that mints, moves or unlocks balances outside normal transfers, or changes the runtime or the migration pallet, pages as critical: minted PEN burned through the migration drains the fixed-supply vault, and only Pendulum governance can stop the call before enactment. | Alert once per item (`PENDULUM GOVERNANCE ITEM`, or `CRITICAL: Pendulum governance item …`); a restart re-pages the items still pending | +| Pendulum source | The finalized Pendulum view must stay fresh and readable. While it cannot be read (an RPC error, a nonce discontinuity, an event-shape change), the Base-side checks — vault balance and its auto-pause, Base view, auto-pause readiness, vault admin events, governance queue — still run; reconciliation, M2a and liveness wait for the source. | Alert, throttled: `PENDULUM SOURCE VIEW STALLED` once the view is older than `SOURCE_STALE_ALERT_SECONDS`, `PENDULUM SOURCE UNREADABLE` on a failure | +| Base view | The safe/finalized Base block must not be older than `BASE_STALE_ALERT_SECONDS`: a view that stops advancing hides every later vault event while every check still passes at the old block. | Alert (`BASE VIEW STALLED`), throttled | +| Auto-pause readiness | At startup and every poll: the Base RPC serves `BASE_CHAIN_ID` and, with a guardian key configured, the key parses, is `vault.guardian()` and holds at least `GUARDIAN_MIN_BALANCE_WEI`. | Alert (`AUTO-PAUSE NOT READY`), throttled; `AUTO-PAUSE DISABLED` once at startup without a key | +| Pendulum governance | Every pending public proposal, the council's external proposal, every ongoing referendum and every open council and technical-committee motion, with the call decoded. A call that mints, moves or unlocks balances outside normal transfers, or changes the runtime or the migration pallet, pages as critical: minted PEN burned through the migration drains the fixed-supply vault, and only Pendulum governance can stop the call before enactment. A call the monitor cannot read pages as critical too: one over 64 KiB is never fetched, and one that does not decode or nests more than 256 calls or 16 levels deep is unreadable; one whose preimage is not noted yet is re-checked every poll and paged again once it can be read. The watch runs after the conservation checks, so it can neither stop nor delay them. | Alert once per item (`PENDULUM GOVERNANCE ITEM`, `CRITICAL: Pendulum governance item …`); a restart re-pages the items still pending | +| Migration pallet admin events | Every finalized `MigrationPauseSet` and `TreasuryDestinationSet`, however fast the motion behind it passed. | Alert once per event (`MIGRATION PALLET ADMIN EVENT`) | +| Vault admin events | Every admin event of the vault (token, pause, unpause, attestors, threshold, caps, guardian, admin transfer, sweep, stale pending cleared). Before the governance handover the admin acts without a delay, so these are the only trace of such a change. | Alert once per event (`VAULT ADMIN EVENT`); the first run also pages the deployment's own events | | Governance queue | Every operation the governance Timelock schedules (`CallScheduled`), with its calls decoded and the time it becomes executable. Once the delay has passed anyone may execute a ready operation, so the canceller has exactly the delay to review and veto it. | Alert once per operation (`GOVERNANCE OPERATION SCHEDULED`) | ## Configuration (environment) @@ -29,11 +34,14 @@ Checks every poll: | `UNMATCHED_EVENT_GRACE_SECONDS` | Time a lagging Pendulum source view gets to reveal the burn behind a safe Base event before a cannot-verify pause (default 10 min). The first sighting of an unmatched event pages immediately, so this is the window operators have to repair a stalled node. Size it to your node-ops response time | | `SOURCE_CLOCK_SKEW_MARGIN_SECONDS` | Clock-skew allowance for the fabrication proof (default 120s): once the finalized Pendulum view has passed the moment the monitor first observed an unmatched Base event by this margin and the nonce still does not exist, the event is provably fabricated and pauses without waiting out the grace. The anchor is the monitor's own clock, not the Base block timestamp (which trails real time after a sequencer outage); a future-dated source view proves nothing | | `SOURCE_STALE_ALERT_SECONDS` | Age of the finalized Pendulum view (vs. wall clock) past which the monitor pages that it is going blind (default 300s) | +| `BASE_STALE_ALERT_SECONDS` | Age of the safe/finalized Base block (vs. wall clock) past which `BASE VIEW STALLED` pages (default 600s for `safe`, 1800s for `finalized`; Base's safe head normally trails by about a minute, its finalized head by about 20) | | `ISSUANCE_TOLERANCE` | Growth of Pendulum `totalIssuance + TotalMigrated` over its first-seen anchor (12-decimal pallet units) tolerated before `SOURCE SUPPLY GREW` pages (default 0) | +| `ACKNOWLEDGED_RELEASE_EXCESS` | Excess of released over migrated, in token units (18 decimals), that operators quantified and accepted after a real loss; M2a fires only beyond it (default 0). Set it at the RB-3 sign-off and restart; the startup log shows it | | `TIMELOCK_ADDRESS` | Optional: the governance TimelockController. Set it as soon as governance is deployed; the queue is scanned from the monitor's Base cursor onwards, so operations scheduled before the monitor knew the address are not paged | -| `ALERT_WEBHOOK_URL` | Webhook receiving JSON alerts — wire this to paging | -| `GUARDIAN_PRIVATE_KEY` | Optional: a guardian key enabling automatic `pause()` on conservation violations | -| `BASE_CHAIN_ID` | Default 8453 | +| `ALERT_WEBHOOK_URL` | Webhook receiving JSON alerts — wire this to paging. A page counts as delivered only on a 2xx answer within 10s: once-only pages (governance items, admin events, scheduled operations, first sightings) are retried every poll until delivered, and failed deliveries are logged. Without it the log line is the page | +| `GUARDIAN_PRIVATE_KEY` | Optional: a guardian key enabling automatic `pause()` on conservation violations; its readiness is checked at startup and every poll | +| `GUARDIAN_MIN_BALANCE_WEI` | Guardian ETH balance below which `AUTO-PAUSE NOT READY` pages (default 10^15 wei = 0.001 ETH) | +| `BASE_CHAIN_ID` | Default 8453; the Base RPC must serve it | | `PENDULUM_START_BLOCK` | **Required on first run:** first block to scan, normally the pallet activation block or the next finalized block before unpausing | | `PENDULUM_START_NONCE` | Nonce expected at that block (default 0; set explicitly when starting after any migrations) | | `BASE_START_BLOCK` | **Required on first run:** vault deployment block or an earlier block | @@ -41,6 +49,22 @@ Checks every poll: | `BASE_FINALITY_TAG` | `safe` (default) or `finalized`; only events inside this boundary are trusted | | `BASE_MAX_BLOCK_RANGE` | Maximum `eth_getLogs` range (default 10,000 blocks) | +## Pages + +Each page is a JSON `POST` of `{ service, subject, detail }` to `ALERT_WEBHOOK_URL`. + +| Subjects | Meaning | +|---|---| +| `MIGRATION TUPLE VIOLATION`, `VAULT BALANCE DEFICIT`, `CONSERVATION VIOLATION` | A conservation check failed; the auto-pause follows with `vault auto-pause confirmed`, `vault pause confirmed` (already paused), `AUTO-PAUSE PENDING FINALITY`, `AUTO-PAUSE FAILED` or `AUTO-PAUSE UNAVAILABLE` | +| `UNVERIFIED BASE EVENT` | First sighting of a Base event whose nonce the source view does not know yet | +| `LIVENESS: migration below approval quorum`, `LIVENESS: quorum reached but not released` | Stalled migrations, one summary per kind and poll | +| `PENDULUM SOURCE VIEW STALLED`, `PENDULUM SOURCE UNREADABLE`, `BASE VIEW STALLED` | The monitor is going blind on one chain | +| `AUTO-PAUSE NOT READY`, `AUTO-PAUSE DISABLED` | The next violation would not pause the vault automatically | +| `SOURCE SUPPLY GREW`, `CRITICAL: unreleasable migration burned on Pendulum` | Supply problems at the source | +| `PENDULUM GOVERNANCE ITEM`, `CRITICAL: Pendulum governance item mints, moves or unlocks balances, or changes the runtime`, `CRITICAL: Pendulum governance item cannot be read`, `MIGRATION PALLET ADMIN EVENT` | Pendulum governance | +| `GOVERNANCE OPERATION SCHEDULED`, `VAULT ADMIN EVENT` | Base governance and vault administration | +| `monitor check failed`, `pendulum governance watch failed`, `multicall unavailable, …`, `multicall batch failed, …`, `monitor startup failed` | The monitor itself | + ## Run ```sh @@ -60,11 +84,12 @@ silently resetting the security history. archive` (or `archive-canonical`): the monitor catches up block by block through historical state, and a default-pruned node (256 blocks ≈ 51 min) cannot serve that after any monitor outage longer than the pruning horizon — -the monitor then wedges loudly on restart. Recovery is pointing `PENDULUM_WS` -at an archive node, never editing the state file. A monitor whose *running* -source view stalls pages `PENDULUM SOURCE VIEW STALLED` well before the -unmatched-event grace can force a cannot-verify pause; treat that page as -"restore the node now". +the monitor then pages `PENDULUM SOURCE UNREADABLE` on restart. Recovery is +pointing `PENDULUM_WS` at an archive node, never editing the state file. A +monitor whose *running* source view stalls pages `PENDULUM SOURCE VIEW +STALLED` well before the unmatched-event grace can force a cannot-verify +pause; treat that page as "restore the node now". The Base-side checks keep +running while the source cannot be read. An automatic pause is reported as successful only after the transaction is canonical inside the configured Base confirmation boundary and `paused()` is From 9dd408887c887722ed44d480e0ebe6f1e1421877 Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Thu, 22 Oct 2026 11:01:13 +0200 Subject: [PATCH 18/75] fix(monitor): Page a failed readiness read instead of aborting the poll A failed eth_chainId, guardian() or eth_getBalance read now counts as an auto-pause readiness problem and pages, throttled, like the others. The vault balance check and the rest of the poll keep running. --- monitor/README.md | 2 +- monitor/src/main.test.ts | 13 ++++++++++++ monitor/src/main.ts | 45 ++++++++++++++++++++++------------------ 3 files changed, 39 insertions(+), 21 deletions(-) diff --git a/monitor/README.md b/monitor/README.md index 264ac8bbf..7edac419b 100644 --- a/monitor/README.md +++ b/monitor/README.md @@ -16,7 +16,7 @@ Checks every poll: | Unreleasable burns | A burn to the zero or vault address can never be approved or released; it is paged once and excluded from the pending count so it cannot page liveness forever or block RB-7. | Alert once | | Pendulum source | The finalized Pendulum view must stay fresh and readable. While it cannot be read (an RPC error, a nonce discontinuity, an event-shape change), the Base-side checks — vault balance and its auto-pause, Base view, auto-pause readiness, vault admin events, governance queue — still run; reconciliation, M2a and liveness wait for the source. | Alert, throttled: `PENDULUM SOURCE VIEW STALLED` once the view is older than `SOURCE_STALE_ALERT_SECONDS`, `PENDULUM SOURCE UNREADABLE` on a failure | | Base view | The safe/finalized Base block must not be older than `BASE_STALE_ALERT_SECONDS`: a view that stops advancing hides every later vault event while every check still passes at the old block. | Alert (`BASE VIEW STALLED`), throttled | -| Auto-pause readiness | At startup and every poll: the Base RPC serves `BASE_CHAIN_ID` and, with a guardian key configured, the key parses, is `vault.guardian()` and holds at least `GUARDIAN_MIN_BALANCE_WEI`. | Alert (`AUTO-PAUSE NOT READY`), throttled; `AUTO-PAUSE DISABLED` once at startup without a key | +| Auto-pause readiness | At startup and every poll: the Base RPC serves `BASE_CHAIN_ID` and, with a guardian key configured, the key parses, is `vault.guardian()` and holds at least `GUARDIAN_MIN_BALANCE_WEI`. A read that fails pages as well and never stops the poll's other checks. | Alert (`AUTO-PAUSE NOT READY`), throttled; `AUTO-PAUSE DISABLED` once at startup without a key | | Pendulum governance | Every pending public proposal, the council's external proposal, every ongoing referendum and every open council and technical-committee motion, with the call decoded. A call that mints, moves or unlocks balances outside normal transfers, or changes the runtime or the migration pallet, pages as critical: minted PEN burned through the migration drains the fixed-supply vault, and only Pendulum governance can stop the call before enactment. A call the monitor cannot read pages as critical too: one over 64 KiB is never fetched, and one that does not decode or nests more than 256 calls or 16 levels deep is unreadable; one whose preimage is not noted yet is re-checked every poll and paged again once it can be read. The watch runs after the conservation checks, so it can neither stop nor delay them. | Alert once per item (`PENDULUM GOVERNANCE ITEM`, `CRITICAL: Pendulum governance item …`); a restart re-pages the items still pending | | Migration pallet admin events | Every finalized `MigrationPauseSet` and `TreasuryDestinationSet`, however fast the motion behind it passed. | Alert once per event (`MIGRATION PALLET ADMIN EVENT`) | | Vault admin events | Every admin event of the vault (token, pause, unpause, attestors, threshold, caps, guardian, admin transfer, sweep, stale pending cleared). Before the governance handover the admin acts without a delay, so these are the only trace of such a change. | Alert once per event (`VAULT ADMIN EVENT`); the first run also pages the deployment's own events | diff --git a/monitor/src/main.test.ts b/monitor/src/main.test.ts index 3f419a008..bec01af39 100644 --- a/monitor/src/main.test.ts +++ b/monitor/src/main.test.ts @@ -83,12 +83,15 @@ function baseChain() { /** Ether balances by lowercased address. */ balances: new Map([[GUARDIAN.toLowerCase(), 10n ** 18n]]), logs: [] as Log[], + /** Requests to answer with an RPC error. */ + failing: undefined as ((method: string, params: unknown[]) => boolean) | undefined, }; } type Base = ReturnType; // eslint-disable-next-line @typescript-eslint/no-explicit-any function answer(base: Base, method: string, params: any[]): unknown { + if (base.failing?.(method, params)) throw new Error("rate limited"); switch (method) { case "eth_chainId": return toHex(base.chainId); @@ -459,6 +462,16 @@ test("auto-pause readiness is re-checked every poll", async (t) => { assert.deepEqual(subjects(), ["AUTO-PAUSE NOT READY"]); }); +test("a failed readiness read pages and the vault balance check still runs", async (t) => { + const { base, api, monitor, pages, subjects } = await setup(t, { env: { GUARDIAN_PRIVATE_KEY: GUARDIAN_KEY } }); + base.failing = (method) => method === "eth_getBalance"; + base.vaultBalance -= 1n; + base.paused = true; // so the auto-pause only confirms + await monitor.poll(api); + assert.deepEqual(sorted(subjects()), ["AUTO-PAUSE NOT READY", "VAULT BALANCE DEFICIT", "vault pause confirmed"]); + assert.match(pages.find((page) => page.subject === "AUTO-PAUSE NOT READY")?.detail ?? "", /readiness reads failed.*rate limited/s); +}); + test("vault admin events page once each from the Base scan", async (t) => { const { base, api, monitor, pages } = await setup(t); const guardian = "0x5555555555555555555555555555555555555555"; diff --git a/monitor/src/main.ts b/monitor/src/main.ts index 177aa9c0f..d99b1e58f 100644 --- a/monitor/src/main.ts +++ b/monitor/src/main.ts @@ -499,32 +499,37 @@ async function pauseVault(): Promise { /** What would make an auto-pause fail now: an RPC on another chain, or a key * that does not parse, is not the vault's guardian or cannot pay for the - * pause. Pages, throttled, so it surfaces before a violation needs it. */ + * pause. Pages, throttled, so it surfaces before a violation needs it. A read + * that fails is a problem too: it must not stop the poll's other checks. */ async function checkAutoPauseReadiness(): Promise { const problems: string[] = []; - const chainId = await publicClient.getChainId(); - if (chainId !== config.baseChainId) { - problems.push(`the Base RPC serves chain ${chainId}, not BASE_CHAIN_ID ${config.baseChainId}`); - } - if (config.guardianPrivateKey) { - let guardian: `0x${string}` | undefined; - try { - guardian = privateKeyToAccount(config.guardianPrivateKey).address; - } catch { - problems.push("GUARDIAN_PRIVATE_KEY is not a valid private key"); + try { + const chainId = await publicClient.getChainId(); + if (chainId !== config.baseChainId) { + problems.push(`the Base RPC serves chain ${chainId}, not BASE_CHAIN_ID ${config.baseChainId}`); } - if (guardian) { - const [vaultGuardian, balance] = await Promise.all([ - publicClient.readContract({ address: config.vaultAddress, abi: vaultAbi, functionName: "guardian" }), - publicClient.getBalance({ address: guardian }), - ]); - if (vaultGuardian.toLowerCase() !== guardian.toLowerCase()) { - problems.push(`the key's address ${guardian} is not the vault guardian ${vaultGuardian}`); + if (config.guardianPrivateKey) { + let guardian: `0x${string}` | undefined; + try { + guardian = privateKeyToAccount(config.guardianPrivateKey).address; + } catch { + problems.push("GUARDIAN_PRIVATE_KEY is not a valid private key"); } - if (balance < config.guardianMinBalanceWei) { - problems.push(`guardian ${guardian} holds ${balance} wei, less than GUARDIAN_MIN_BALANCE_WEI ${config.guardianMinBalanceWei}`); + if (guardian) { + const [vaultGuardian, balance] = await Promise.all([ + publicClient.readContract({ address: config.vaultAddress, abi: vaultAbi, functionName: "guardian" }), + publicClient.getBalance({ address: guardian }), + ]); + if (vaultGuardian.toLowerCase() !== guardian.toLowerCase()) { + problems.push(`the key's address ${guardian} is not the vault guardian ${vaultGuardian}`); + } + if (balance < config.guardianMinBalanceWei) { + problems.push(`guardian ${guardian} holds ${balance} wei, less than GUARDIAN_MIN_BALANCE_WEI ${config.guardianMinBalanceWei}`); + } } } + } catch (readError) { + problems.push(`the readiness reads failed: ${readError}`); } if (problems.length && Date.now() - lastReadinessAlertMs > config.graceSeconds * 1000) { lastReadinessAlertMs = Date.now(); From ea38185a3aaff547698b504c9e1d0156bc8cb799 Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Thu, 22 Oct 2026 11:02:59 +0200 Subject: [PATCH 19/75] fix(monitor): Keep the vault balance check running when the admin scan fails A failed scan of the vault admin events and the governance queue now pages BASE ADMIN SCAN FAILED, throttled, instead of aborting the poll: the vault balance check and its auto-pause still run. Reconciliation, M2a and liveness wait until the scan reads again, so the persisted Base cursor never passes a range the scan has not covered. --- monitor/README.md | 4 ++-- monitor/src/main.test.ts | 20 ++++++++++++++++++++ monitor/src/main.ts | 31 ++++++++++++++++++++++++------- 3 files changed, 46 insertions(+), 9 deletions(-) diff --git a/monitor/README.md b/monitor/README.md index 7edac419b..877ea26aa 100644 --- a/monitor/README.md +++ b/monitor/README.md @@ -19,7 +19,7 @@ Checks every poll: | Auto-pause readiness | At startup and every poll: the Base RPC serves `BASE_CHAIN_ID` and, with a guardian key configured, the key parses, is `vault.guardian()` and holds at least `GUARDIAN_MIN_BALANCE_WEI`. A read that fails pages as well and never stops the poll's other checks. | Alert (`AUTO-PAUSE NOT READY`), throttled; `AUTO-PAUSE DISABLED` once at startup without a key | | Pendulum governance | Every pending public proposal, the council's external proposal, every ongoing referendum and every open council and technical-committee motion, with the call decoded. A call that mints, moves or unlocks balances outside normal transfers, or changes the runtime or the migration pallet, pages as critical: minted PEN burned through the migration drains the fixed-supply vault, and only Pendulum governance can stop the call before enactment. A call the monitor cannot read pages as critical too: one over 64 KiB is never fetched, and one that does not decode or nests more than 256 calls or 16 levels deep is unreadable; one whose preimage is not noted yet is re-checked every poll and paged again once it can be read. The watch runs after the conservation checks, so it can neither stop nor delay them. | Alert once per item (`PENDULUM GOVERNANCE ITEM`, `CRITICAL: Pendulum governance item …`); a restart re-pages the items still pending | | Migration pallet admin events | Every finalized `MigrationPauseSet` and `TreasuryDestinationSet`, however fast the motion behind it passed. | Alert once per event (`MIGRATION PALLET ADMIN EVENT`) | -| Vault admin events | Every admin event of the vault (token, pause, unpause, attestors, threshold, caps, guardian, admin transfer, sweep, stale pending cleared). Before the governance handover the admin acts without a delay, so these are the only trace of such a change. | Alert once per event (`VAULT ADMIN EVENT`); the first run also pages the deployment's own events | +| Vault admin events | Every admin event of the vault (token, pause, unpause, attestors, threshold, caps, guardian, admin transfer, sweep, stale pending cleared). Before the governance handover the admin acts without a delay, so these are the only trace of such a change. While this scan (shared with the governance queue) fails, the vault balance check and its auto-pause still run; reconciliation, M2a and liveness wait for it, so the persisted Base cursor never passes a range it has not scanned. | Alert once per event (`VAULT ADMIN EVENT`); the first run also pages the deployment's own events; `BASE ADMIN SCAN FAILED`, throttled, on a failure | | Governance queue | Every operation the governance Timelock schedules (`CallScheduled`), with its calls decoded and the time it becomes executable. Once the delay has passed anyone may execute a ready operation, so the canceller has exactly the delay to review and veto it. | Alert once per operation (`GOVERNANCE OPERATION SCHEDULED`) | ## Configuration (environment) @@ -58,7 +58,7 @@ Each page is a JSON `POST` of `{ service, subject, detail }` to `ALERT_WEBHOOK_U | `MIGRATION TUPLE VIOLATION`, `VAULT BALANCE DEFICIT`, `CONSERVATION VIOLATION` | A conservation check failed; the auto-pause follows with `vault auto-pause confirmed`, `vault pause confirmed` (already paused), `AUTO-PAUSE PENDING FINALITY`, `AUTO-PAUSE FAILED` or `AUTO-PAUSE UNAVAILABLE` | | `UNVERIFIED BASE EVENT` | First sighting of a Base event whose nonce the source view does not know yet | | `LIVENESS: migration below approval quorum`, `LIVENESS: quorum reached but not released` | Stalled migrations, one summary per kind and poll | -| `PENDULUM SOURCE VIEW STALLED`, `PENDULUM SOURCE UNREADABLE`, `BASE VIEW STALLED` | The monitor is going blind on one chain | +| `PENDULUM SOURCE VIEW STALLED`, `PENDULUM SOURCE UNREADABLE`, `BASE VIEW STALLED`, `BASE ADMIN SCAN FAILED` | The monitor is going blind on one chain | | `AUTO-PAUSE NOT READY`, `AUTO-PAUSE DISABLED` | The next violation would not pause the vault automatically | | `SOURCE SUPPLY GREW`, `CRITICAL: unreleasable migration burned on Pendulum` | Supply problems at the source | | `PENDULUM GOVERNANCE ITEM`, `CRITICAL: Pendulum governance item mints, moves or unlocks balances, or changes the runtime`, `CRITICAL: Pendulum governance item cannot be read`, `MIGRATION PALLET ADMIN EVENT` | Pendulum governance | diff --git a/monitor/src/main.test.ts b/monitor/src/main.test.ts index bec01af39..1aeb1436f 100644 --- a/monitor/src/main.test.ts +++ b/monitor/src/main.test.ts @@ -22,6 +22,7 @@ import { type Hex, keccak256, parseAbi, + toEventSelector, toHex, } from "viem"; import { privateKeyToAccount } from "viem/accounts"; @@ -486,6 +487,25 @@ test("vault admin events page once each from the Base scan", async (t) => { assert.match(adminPages[1], /^CapsUpdated\(perReleaseCap=5, dailyCap=7\) at Base block 1012/); }); +test("a failed admin scan pages, the vault balance check still runs and the Base cursor waits", async (t) => { + const { base, pendulum, api, monitor, subjects, persisted } = await setup(t, { env: { GUARDIAN_PRIVATE_KEY: GUARDIAN_KEY } }); + burn(pendulum, 0n, 100n * PEN); + release(base, 0n, 100n * PEN); + base.logs.push({ address: VAULT, eventName: "GuardianUpdated", args: { guardian: GUARDIAN }, blockNumber: 1_004n }); + const guardianUpdated = toEventSelector("GuardianUpdated(address)"); // only the admin scan asks for it + base.failing = (method, params) => method === "eth_getLogs" && JSON.stringify(params).includes(guardianUpdated); + base.vaultBalance -= 1n; + base.paused = true; // so the auto-pause only confirms and the poll persists its state + await monitor.poll(api); + assert.deepEqual(sorted(subjects()), ["BASE ADMIN SCAN FAILED", "VAULT BALANCE DEFICIT", "vault pause confirmed"]); + assert.equal(persisted().baseFromBlock, "1000"); + base.failing = undefined; + base.vaultBalance += 1n; + await monitor.poll(api); + assert.ok(subjects().includes("VAULT ADMIN EVENT"), JSON.stringify(subjects())); + assert.equal(persisted().baseFromBlock, "1011"); +}); + test("a monitor without a guardian key says so at startup", async (t) => { const { startup } = await setup(t); assert.ok(startup.some((page) => page.subject === "AUTO-PAUSE DISABLED"), JSON.stringify(startup)); diff --git a/monitor/src/main.ts b/monitor/src/main.ts index d99b1e58f..4233a26fc 100644 --- a/monitor/src/main.ts +++ b/monitor/src/main.ts @@ -359,11 +359,12 @@ let lastSourceStallAlertMs = 0; let lastSourceFailureAlertMs = 0; let lastBaseStallAlertMs = 0; let lastReadinessAlertMs = 0; +let lastAdminScanFailureAlertMs = 0; let lastIssuanceAlertMs = 0; /** The Base admin scan (vault admin events and the governance queue) runs - * ahead of the vault checks in a poll and throws on failure, so the persisted - * Base cursor never passes a range it has not scanned; a restart resumes from - * that cursor. */ + * ahead of reconciliation in a poll, and reconciliation waits while it fails, + * so the persisted Base cursor never passes a range it has not scanned; a + * restart resumes from that cursor. */ let baseAdminFromBlock: bigint | undefined; /** Once-only pages by key, each held until the webhook accepts it. */ // ponytail: in memory; a restart re-pages what it finds again (Base admin events after the persisted cursor, pending governance items) and drops undelivered pages it does not find again. @@ -1185,11 +1186,27 @@ async function check(api: ApiPromise): Promise { ); } await checkAutoPauseReadiness(); - await watchBaseAdmin(baseBlock); + // The admin scan only pages, so its failure must not skip the vault balance + // check. Reconciliation waits for it: a restart resumes the scan from the + // persisted Base cursor, which must never pass a range it has not covered. + let adminScanned = false; + try { + await watchBaseAdmin(baseBlock); + adminScanned = true; + } catch (adminError) { + if (Date.now() - lastAdminScanFailureAlertMs > config.unmatchedEventGraceSeconds * 1000) { + lastAdminScanFailureAlertMs = Date.now(); + await alert( + "BASE ADMIN SCAN FAILED", + `${adminError}; the vault balance check continues, but vault admin events, the governance queue and ` + + "the reconciliation of Approved and Released events wait until the scan reads again", + ); + } + } // A Pendulum failure must not switch off the Base-side checks: the vault // balance check and its pause run without the source. Reconciliation, M2a - // and liveness need the source and wait for it. + // and liveness need the source and the admin scan, and wait for them. let draft = cloneState(state); let source: SourceView | undefined; try { @@ -1212,7 +1229,7 @@ async function check(api: ApiPromise): Promise { publicClient.readContract({ address: config.vaultAddress, abi: vaultAbi, functionName: "token", blockNumber: baseBlock }), ]); - const reconciliation = source + const reconciliation = source && adminScanned ? await reconcileBaseEvents(draft, baseBlock, conversionFactor, source.timestampMs) : { violation: undefined, awaitingSource: undefined }; if (reconciliation.violation) { @@ -1243,7 +1260,7 @@ async function check(api: ApiPromise): Promise { } return; } - if (!source) return; + if (!source || !adminScanned) return; const { finalizedBlock, totalMigrated } = source; if (reconciliation.awaitingSource) { state = draft; From 8746062538a229f741e0f03c22becdd1417fdfec Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Thu, 22 Oct 2026 11:06:58 +0200 Subject: [PATCH 20/75] fix(monitor): Reconcile Base events while Pendulum cannot be read While the source read fails, Base events are now reconciled against the last Pendulum view read in full, as against a lagging source but without the fabrication proof: a mismatch on a known nonce pauses at once, and an event with an unknown nonce pages UNVERIFIED BASE EVENT and pauses once UNMATCHED_EVENT_GRACE_SECONDS has passed. The poll persists that state and skips only M2a and liveness, which need the current source. --- monitor/README.md | 11 +++++++---- monitor/src/main.test.ts | 24 ++++++++++++++++++++++++ monitor/src/main.ts | 23 ++++++++++++++++------- 3 files changed, 47 insertions(+), 11 deletions(-) diff --git a/monitor/README.md b/monitor/README.md index 877ea26aa..40f6bfdcd 100644 --- a/monitor/README.md +++ b/monitor/README.md @@ -8,13 +8,13 @@ Checks every poll: | Check | Meaning | Reaction | |---|---|---| -| M2 tuple match | Every safe Base `Approved` and `Released` event must match one finalized Pendulum `MigrationInitiated` nonce, recipient, and pallet amount; `Released.tokenAmount` must equal `palletAmount × conversionFactor`. After a violation the Base cursor stays on any block range whose events the source view may still reveal, so a pause caused by a lagging source does not lose those releases. | Alert + auto-pause. A mismatch against a *known* nonce pauses immediately. An *unknown* nonce first pages (`UNVERIFIED BASE EVENT`, without delaying the pause decision), then pauses as soon as it is provable — the finalized source view passed the event's own timestamp and the nonce still does not exist — or after `UNMATCHED_EVENT_GRACE_SECONDS` if the source view stays behind (a stalled source also pages on its own, see below) | +| M2 tuple match | Every safe Base `Approved` and `Released` event must match one finalized Pendulum `MigrationInitiated` nonce, recipient, and pallet amount; `Released.tokenAmount` must equal `palletAmount × conversionFactor`. After a violation the Base cursor stays on any block range whose events the source view may still reveal, so a pause caused by a lagging source does not lose those releases. | Alert + auto-pause. A mismatch against a *known* nonce pauses immediately. An *unknown* nonce first pages (`UNVERIFIED BASE EVENT`, without delaying the pause decision), then pauses as soon as it is provable — the finalized source view passed the event's own timestamp and the nonce still does not exist — or after `UNMATCHED_EVENT_GRACE_SECONDS` if the source view stays behind or cannot be read (a stalled or unreadable source also pages on its own, see below) | | M2a aggregate | `totalReleased <= TotalMigrated × conversionFactor + ACKNOWLEDGED_RELEASE_EXCESS`. This remains independent defence-in-depth around the event reconciliation. After a real, quantified loss, operators set `ACKNOWLEDGED_RELEASE_EXCESS` to it (RB-3) so the vault can be unpaused; any further excess still fires. | Alert + auto-pause | | M2b: `balanceOf(vault) + totalReleased + totalSwept >= totalSupply` | Vault-internal conservation. Only a **deficit** alerts: a surplus is a harmless inbound transfer (donation, or a migration whose recipient is the vault) and is ignored, so it cannot false-trigger a pause. | Alert + auto-pause on a deficit, also while Pendulum cannot be read | | M4: every old migration reaches active approval quorum — and then releases | Liveness of the attestor fleet (`below approval quorum`), and of everything after it (`quorum reached but not released`: a release starved of daily allowance, paused, or made releasable by a threshold cut without a `ReleasePending`, which the releaser cannot see). | One summary page per kind and poll (count and the first nonces; the log lists every nonce), each nonce at most once per `GRACE_SECONDS` | | Source supply | `totalIssuance + TotalMigrated` on Pendulum must never grow past its first-seen anchor. | Alert (`SOURCE SUPPLY GREW`) | | Unreleasable burns | A burn to the zero or vault address can never be approved or released; it is paged once and excluded from the pending count so it cannot page liveness forever or block RB-7. | Alert once | -| Pendulum source | The finalized Pendulum view must stay fresh and readable. While it cannot be read (an RPC error, a nonce discontinuity, an event-shape change), the Base-side checks — vault balance and its auto-pause, Base view, auto-pause readiness, vault admin events, governance queue — still run; reconciliation, M2a and liveness wait for the source. | Alert, throttled: `PENDULUM SOURCE VIEW STALLED` once the view is older than `SOURCE_STALE_ALERT_SECONDS`, `PENDULUM SOURCE UNREADABLE` on a failure | +| Pendulum source | The finalized Pendulum view must stay fresh and readable. While it cannot be read (an RPC error, a nonce discontinuity, an event-shape change), the Base-side checks — vault balance and its auto-pause, Base view, auto-pause readiness, vault admin events, governance queue — still run, and Base events are reconciled against the last view read in full: an unreadable source counts as a stalled one, so a mismatch on a known nonce pauses at once and an unknown nonce after `UNMATCHED_EVENT_GRACE_SECONDS`. M2a and liveness wait for the source. | Alert, throttled: `PENDULUM SOURCE VIEW STALLED` once the view is older than `SOURCE_STALE_ALERT_SECONDS`, `PENDULUM SOURCE UNREADABLE` on a failure | | Base view | The safe/finalized Base block must not be older than `BASE_STALE_ALERT_SECONDS`: a view that stops advancing hides every later vault event while every check still passes at the old block. | Alert (`BASE VIEW STALLED`), throttled | | Auto-pause readiness | At startup and every poll: the Base RPC serves `BASE_CHAIN_ID` and, with a guardian key configured, the key parses, is `vault.guardian()` and holds at least `GUARDIAN_MIN_BALANCE_WEI`. A read that fails pages as well and never stops the poll's other checks. | Alert (`AUTO-PAUSE NOT READY`), throttled; `AUTO-PAUSE DISABLED` once at startup without a key | | Pendulum governance | Every pending public proposal, the council's external proposal, every ongoing referendum and every open council and technical-committee motion, with the call decoded. A call that mints, moves or unlocks balances outside normal transfers, or changes the runtime or the migration pallet, pages as critical: minted PEN burned through the migration drains the fixed-supply vault, and only Pendulum governance can stop the call before enactment. A call the monitor cannot read pages as critical too: one over 64 KiB is never fetched, and one that does not decode or nests more than 256 calls or 16 levels deep is unreadable; one whose preimage is not noted yet is re-checked every poll and paged again once it can be read. The watch runs after the conservation checks, so it can neither stop nor delay them. | Alert once per item (`PENDULUM GOVERNANCE ITEM`, `CRITICAL: Pendulum governance item …`); a restart re-pages the items still pending | @@ -31,7 +31,7 @@ Checks every poll: | `VAULT_ADDRESS` | MigrationVault address on Base | | `POLL_INTERVAL_MS` | Poll cadence (default 60s) | | `GRACE_SECONDS` | Liveness alert threshold (default 30 min) | -| `UNMATCHED_EVENT_GRACE_SECONDS` | Time a lagging Pendulum source view gets to reveal the burn behind a safe Base event before a cannot-verify pause (default 10 min). The first sighting of an unmatched event pages immediately, so this is the window operators have to repair a stalled node. Size it to your node-ops response time | +| `UNMATCHED_EVENT_GRACE_SECONDS` | Time a lagging or unreadable Pendulum source view gets to reveal the burn behind a safe Base event before a cannot-verify pause (default 10 min). The first sighting of an unmatched event pages immediately, so this is the window operators have to repair a stalled or failing node. Size it to your node-ops response time | | `SOURCE_CLOCK_SKEW_MARGIN_SECONDS` | Clock-skew allowance for the fabrication proof (default 120s): once the finalized Pendulum view has passed the moment the monitor first observed an unmatched Base event by this margin and the nonce still does not exist, the event is provably fabricated and pauses without waiting out the grace. The anchor is the monitor's own clock, not the Base block timestamp (which trails real time after a sequencer outage); a future-dated source view proves nothing | | `SOURCE_STALE_ALERT_SECONDS` | Age of the finalized Pendulum view (vs. wall clock) past which the monitor pages that it is going blind (default 300s) | | `BASE_STALE_ALERT_SECONDS` | Age of the safe/finalized Base block (vs. wall clock) past which `BASE VIEW STALLED` pages (default 600s for `safe`, 1800s for `finalized`; Base's safe head normally trails by about a minute, its finalized head by about 20) | @@ -89,7 +89,10 @@ pointing `PENDULUM_WS` at an archive node, never editing the state file. A monitor whose *running* source view stalls pages `PENDULUM SOURCE VIEW STALLED` well before the unmatched-event grace can force a cannot-verify pause; treat that page as "restore the node now". The Base-side checks keep -running while the source cannot be read. +running while the source cannot be read, and an unreadable source counts as +a stalled one: once a Base event it cannot verify appears, the monitor pages +`UNVERIFIED BASE EVENT` and pauses when the grace runs out, so treat +`PENDULUM SOURCE UNREADABLE` the same way. An automatic pause is reported as successful only after the transaction is canonical inside the configured Base confirmation boundary and `paused()` is diff --git a/monitor/src/main.test.ts b/monitor/src/main.test.ts index 1aeb1436f..c2a4c5d3f 100644 --- a/monitor/src/main.test.ts +++ b/monitor/src/main.test.ts @@ -440,6 +440,30 @@ test("a Pendulum failure pages on its own and the vault balance check still runs ]); }); +test("while Pendulum cannot be read, a Base event with an unknown nonce pauses after the grace", async (t) => { + const { base, pendulum, api, monitor, subjects } = await setup(t, { env: { UNMATCHED_EVENT_GRACE_SECONDS: "0" } }); + pendulum.failure = new Error("node unreachable"); + release(base, 7n, 5_000n * PEN); // no burn behind it + await monitor.poll(api); + assert.ok(subjects().includes("UNVERIFIED BASE EVENT"), JSON.stringify(subjects())); + assert.ok(!subjects().includes("MIGRATION TUPLE VIOLATION"), JSON.stringify(subjects())); + await new Promise((resolve) => setTimeout(resolve, 5)); + await monitor.poll(api); // the grace has run out + assert.ok(subjects().includes("MIGRATION TUPLE VIOLATION"), JSON.stringify(subjects())); +}); + +test("while Pendulum cannot be read, a mismatch on a known nonce pauses at once", async (t) => { + const { base, pendulum, api, monitor, pages } = await setup(t); + burn(pendulum, 0n, 100n * PEN); + await monitor.poll(api); + pendulum.failure = new Error("node unreachable"); + const approval = { nonce: 0n, recipient: TOKEN, palletAmount: 100n * PEN, attestor: ATTESTORS[0] }; + base.logs.push({ address: VAULT, eventName: "Approved", args: approval, blockNumber: 1_012n }); + base.safe = 1_020n; + await monitor.poll(api); + assert.match(pages.find((page) => page.subject === "MIGRATION TUPLE VIOLATION")?.detail ?? "", /recipient/); +}); + test("auto-pause readiness is checked at startup and each problem pages", async (t) => { const { startup } = await setup(t, { env: { GUARDIAN_PRIVATE_KEY: GUARDIAN_KEY }, diff --git a/monitor/src/main.ts b/monitor/src/main.ts index 4233a26fc..0fd707dcd 100644 --- a/monitor/src/main.ts +++ b/monitor/src/main.ts @@ -1204,9 +1204,12 @@ async function check(api: ApiPromise): Promise { } } - // A Pendulum failure must not switch off the Base-side checks: the vault - // balance check and its pause run without the source. Reconciliation, M2a - // and liveness need the source and the admin scan, and wait for them. + // A Pendulum failure must not switch off the Base-side checks. Base events + // are then reconciled against the last view read in full, as against a + // lagging source but with no fabrication proof: a mismatch on a known nonce + // still pauses at once, an unknown nonce after the grace. The vault balance + // check needs no source; M2a and liveness wait for it. All but the vault + // balance check also wait for the admin scan. let draft = cloneState(state); let source: SourceView | undefined; try { @@ -1217,7 +1220,8 @@ async function check(api: ApiPromise): Promise { lastSourceFailureAlertMs = Date.now(); await alert( "PENDULUM SOURCE UNREADABLE", - `${sourceError}; the vault balance check continues, but Base events are not reconciled until the source reads again`, + `${sourceError}; Base events are reconciled against the last Pendulum view read in full, so one with an ` + + `unknown nonce pauses the vault after ${config.unmatchedEventGraceSeconds}s unless the source reads again`, ); } } @@ -1229,8 +1233,8 @@ async function check(api: ApiPromise): Promise { publicClient.readContract({ address: config.vaultAddress, abi: vaultAbi, functionName: "token", blockNumber: baseBlock }), ]); - const reconciliation = source && adminScanned - ? await reconcileBaseEvents(draft, baseBlock, conversionFactor, source.timestampMs) + const reconciliation = adminScanned + ? await reconcileBaseEvents(draft, baseBlock, conversionFactor, source?.timestampMs ?? 0) : { violation: undefined, awaitingSource: undefined }; if (reconciliation.violation) { if (await securityViolation("MIGRATION TUPLE VIOLATION", reconciliation.violation)) { @@ -1260,7 +1264,12 @@ async function check(api: ApiPromise): Promise { } return; } - if (!source || !adminScanned) return; + if (!adminScanned) return; + if (!source) { + state = draft; // reconciled against the last view read in full + persistState(state); + return; + } const { finalizedBlock, totalMigrated } = source; if (reconciliation.awaitingSource) { state = draft; From 2e9023094411fe14a62948e85c9da14ef7110e86 Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Thu, 22 Oct 2026 11:08:36 +0200 Subject: [PATCH 21/75] fix(monitor): Page a motion or external proposal made again with the same call Council and technical-committee motions are now paged once per motion index instead of once per call hash, so a motion that was closed and proposed again with the same call pages again. The external proposal, which only its call identifies, pages again once it has left the queue and is made again. --- monitor/README.md | 2 +- monitor/src/main.test.ts | 27 ++++++++++++++++++++++++--- monitor/src/main.ts | 10 +++++++++- 3 files changed, 34 insertions(+), 5 deletions(-) diff --git a/monitor/README.md b/monitor/README.md index 40f6bfdcd..1d2796223 100644 --- a/monitor/README.md +++ b/monitor/README.md @@ -17,7 +17,7 @@ Checks every poll: | Pendulum source | The finalized Pendulum view must stay fresh and readable. While it cannot be read (an RPC error, a nonce discontinuity, an event-shape change), the Base-side checks — vault balance and its auto-pause, Base view, auto-pause readiness, vault admin events, governance queue — still run, and Base events are reconciled against the last view read in full: an unreadable source counts as a stalled one, so a mismatch on a known nonce pauses at once and an unknown nonce after `UNMATCHED_EVENT_GRACE_SECONDS`. M2a and liveness wait for the source. | Alert, throttled: `PENDULUM SOURCE VIEW STALLED` once the view is older than `SOURCE_STALE_ALERT_SECONDS`, `PENDULUM SOURCE UNREADABLE` on a failure | | Base view | The safe/finalized Base block must not be older than `BASE_STALE_ALERT_SECONDS`: a view that stops advancing hides every later vault event while every check still passes at the old block. | Alert (`BASE VIEW STALLED`), throttled | | Auto-pause readiness | At startup and every poll: the Base RPC serves `BASE_CHAIN_ID` and, with a guardian key configured, the key parses, is `vault.guardian()` and holds at least `GUARDIAN_MIN_BALANCE_WEI`. A read that fails pages as well and never stops the poll's other checks. | Alert (`AUTO-PAUSE NOT READY`), throttled; `AUTO-PAUSE DISABLED` once at startup without a key | -| Pendulum governance | Every pending public proposal, the council's external proposal, every ongoing referendum and every open council and technical-committee motion, with the call decoded. A call that mints, moves or unlocks balances outside normal transfers, or changes the runtime or the migration pallet, pages as critical: minted PEN burned through the migration drains the fixed-supply vault, and only Pendulum governance can stop the call before enactment. A call the monitor cannot read pages as critical too: one over 64 KiB is never fetched, and one that does not decode or nests more than 256 calls or 16 levels deep is unreadable; one whose preimage is not noted yet is re-checked every poll and paged again once it can be read. The watch runs after the conservation checks, so it can neither stop nor delay them. | Alert once per item (`PENDULUM GOVERNANCE ITEM`, `CRITICAL: Pendulum governance item …`); a restart re-pages the items still pending | +| Pendulum governance | Every pending public proposal, the council's external proposal, every ongoing referendum and every open council and technical-committee motion, with the call decoded. A call that mints, moves or unlocks balances outside normal transfers, or changes the runtime or the migration pallet, pages as critical: minted PEN burned through the migration drains the fixed-supply vault, and only Pendulum governance can stop the call before enactment. A call the monitor cannot read pages as critical too: one over 64 KiB is never fetched, and one that does not decode or nests more than 256 calls or 16 levels deep is unreadable; one whose preimage is not noted yet is re-checked every poll and paged again once it can be read. The watch runs after the conservation checks, so it can neither stop nor delay them. | Alert once per item (`PENDULUM GOVERNANCE ITEM`, `CRITICAL: Pendulum governance item …`); a motion proposed again with the same call (a new motion index), or an external proposal made again after it left the queue, pages again; a restart re-pages the items still pending | | Migration pallet admin events | Every finalized `MigrationPauseSet` and `TreasuryDestinationSet`, however fast the motion behind it passed. | Alert once per event (`MIGRATION PALLET ADMIN EVENT`) | | Vault admin events | Every admin event of the vault (token, pause, unpause, attestors, threshold, caps, guardian, admin transfer, sweep, stale pending cleared). Before the governance handover the admin acts without a delay, so these are the only trace of such a change. While this scan (shared with the governance queue) fails, the vault balance check and its auto-pause still run; reconciliation, M2a and liveness wait for it, so the persisted Base cursor never passes a range it has not scanned. | Alert once per event (`VAULT ADMIN EVENT`); the first run also pages the deployment's own events; `BASE ADMIN SCAN FAILED`, throttled, on a failure | | Governance queue | Every operation the governance Timelock schedules (`CallScheduled`), with its calls decoded and the time it becomes executable. Once the delay has passed anyone may execute a ready operation, so the canceller has exactly the delay to review and veto it. | Alert once per operation (`GOVERNANCE OPERATION SCHEDULED`) | diff --git a/monitor/src/main.test.ts b/monitor/src/main.test.ts index c2a4c5d3f..476d786bd 100644 --- a/monitor/src/main.test.ts +++ b/monitor/src/main.test.ts @@ -221,6 +221,7 @@ function pendulumChain() { totalMigrated: 0n, nextNonce: 0n, publicProps: [] as Array<[number, Bounded]>, + external: undefined as Bounded | undefined, referenda: new Map(), /** Noted preimages by `${hash}:${len}`, and their lengths by hash. */ preimages: new Map(), @@ -228,14 +229,18 @@ function pendulumChain() { preimageReads: 0, /** Preimage reads still to fail, as an RPC error would. */ preimageFailures: 0, - /** Open motions by hash, the stored size where it is not the call's, - * and the motions whose call was fetched. */ + /** Open motions by hash, their index where it is not 0, the stored size + * where it is not the call's, and the motions whose call was fetched. */ motions: { council: new Map(), technicalCommittee: new Map() }, + motionIndices: new Map(), motionSizes: new Map(), motionReads: new Set(), }; const collective = (motions: Map) => ({ proposals: async () => [...motions.keys()].map(hash), + voting: async (motion: { toHex(): string }) => ({ + toJSON: () => (motions.has(motion.toHex()) ? { index: chain.motionIndices.get(motion.toHex()) ?? 0 } : null), + }), proposalOf: Object.assign( async (motion: { toHex(): string }) => { chain.motionReads.add(motion.toHex()); @@ -262,7 +267,7 @@ function pendulumChain() { balances: { totalIssuance: async () => codec(100_000_000n * PEN - chain.totalMigrated) }, democracy: { publicProps: async () => chain.publicProps.map(([index, bounded]) => [codec(index), bounded, "5Proposer"]), - nextExternal: async () => optional(undefined), + nextExternal: async () => optional(chain.external && [chain.external, "SimpleMajority"]), referendumCount: async () => codec(chain.referenda.size ? Math.max(...chain.referenda.keys()) + 1 : 0), lowestUnbaked: async () => codec(chain.referenda.size ? Math.min(...chain.referenda.keys()) : 0), referendumInfoOf: async (index: number) => { @@ -614,6 +619,22 @@ test("council and technical-committee motions are decoded with the same risk rul assert.ok(!pendulum.motionReads.has(huge)); }); +test("a motion or external proposal made again with the same call pages again", async (t) => { + const { pendulum, api, monitor, pages } = await setup(t); + const motion = `0x04${"00".repeat(31)}`; + pendulum.motions.council.set(motion, "remark"); + pendulum.motionIndices.set(motion, 3); + pendulum.external = inline("remark"); + await monitor.poll(api); + pendulum.motionIndices.set(motion, 4); // closed and proposed again between two polls + pendulum.external = undefined; // tabled + await monitor.poll(api); + pendulum.external = inline("remark"); // proposed again + await monitor.poll(api); + assert.equal(pages.filter((page) => page.detail.startsWith(`council motion ${motion}`)).length, 2, JSON.stringify(pages)); + assert.equal(pages.filter((page) => page.detail.startsWith("the council's external proposal")).length, 2, JSON.stringify(pages)); +}); + test("admin events of the migration pallet page from the ingest", async (t) => { const { pendulum, api, monitor, pages } = await setup(t); pendulum.finalized = 101; diff --git a/monitor/src/main.ts b/monitor/src/main.ts index 0fd707dcd..7fb4289f8 100644 --- a/monitor/src/main.ts +++ b/monitor/src/main.ts @@ -994,6 +994,11 @@ async function watchPendulumGovernance(api: ApiPromise): Promise { hint: `it can become a referendum at block ${nextLaunch}`, read: () => boundedCall(apiAt, bounded), }); + } else { + // Only its call identifies an external proposal, so once it has left the + // queue the same call proposed again pages again. + // ponytail: one tabled and proposed again between two polls pages again only as a referendum. + for (const key of onceOnlyPages.keys()) if (key.startsWith("external:")) onceOnlyPages.delete(key); } const [count, lowest] = (await Promise.all([democracy.referendumCount(), democracy.lowestUnbaked()])).map((value) => Number(value.toString()) @@ -1015,8 +1020,11 @@ async function watchPendulumGovernance(api: ApiPromise): Promise { for (const collective of ["council", "technicalCommittee"] as const) { const motions = (await apiAt.query[collective].proposals()) as unknown as Array<{ toHex(): string }>; for (const hash of motions) { + // A closed motion can be proposed again with the same call and hash, + // but never with the same index, so the index keys the page. + const votes = (await apiAt.query[collective].voting(hash)).toJSON() as { index: number } | null; items.push({ - key: `${collective}:${hash.toHex()}`, + key: `${collective}:${votes?.index}:${hash.toHex()}`, label: `${collective === "council" ? "council" : "technical committee"} motion ${hash.toHex()}`, hint: "it executes as soon as enough members vote aye; a hostile one needs nay votes", read: () => motionCall(apiAt, collective, hash), From 5d5a4a0ae3f63b93aeadbb0af226d941a28f5622 Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Thu, 22 Oct 2026 11:09:43 +0200 Subject: [PATCH 22/75] fix(monitor): Validate GUARDIAN_MIN_BALANCE_WEI at startup The guardian balance floor is now parsed like the other amounts: a negative or non-numeric value fails at startup, and an empty one keeps the default, instead of silently switching the balance check off. --- monitor/src/main.test.ts | 10 ++++++++++ monitor/src/main.ts | 6 +++--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/monitor/src/main.test.ts b/monitor/src/main.test.ts index 476d786bd..6100a9420 100644 --- a/monitor/src/main.test.ts +++ b/monitor/src/main.test.ts @@ -484,6 +484,16 @@ test("auto-pause readiness is checked at startup and each problem pages", async assert.match(detail, /holds 0 wei/); }); +test("the guardian balance floor rejects a negative value and keeps its default when empty", async (t) => { + await assert.rejects(setup(t, { env: { GUARDIAN_MIN_BALANCE_WEI: "-1" } }), /invalid GUARDIAN_MIN_BALANCE_WEI: -1/); + const { startup } = await setup(t, { + env: { GUARDIAN_PRIVATE_KEY: GUARDIAN_KEY, GUARDIAN_MIN_BALANCE_WEI: "" }, + prepare: (base) => base.balances.clear(), + }); + const detail = startup.find((page) => page.subject === "AUTO-PAUSE NOT READY")?.detail ?? ""; + assert.match(detail, /holds 0 wei, less than GUARDIAN_MIN_BALANCE_WEI 1000000000000000/); +}); + test("auto-pause readiness is re-checked every poll", async (t) => { const { base, api, monitor, startup, subjects } = await setup(t, { env: { GUARDIAN_PRIVATE_KEY: GUARDIAN_KEY } }); assert.ok(!startup.some((page) => page.subject.startsWith("AUTO-PAUSE")), JSON.stringify(startup)); diff --git a/monitor/src/main.ts b/monitor/src/main.ts index 7fb4289f8..1e28cbb20 100644 --- a/monitor/src/main.ts +++ b/monitor/src/main.ts @@ -170,8 +170,8 @@ function envNumber(name: string, fallback: number | undefined): number { } /** A non-negative integer amount from the environment, validated at startup. */ -function envAmount(name: string): bigint { - const raw = process.env[name] || "0"; +function envAmount(name: string, fallback = "0"): bigint { + const raw = process.env[name] || fallback; if (!/^\d+$/.test(raw)) throw new Error(`invalid ${name}: ${raw}`); return BigInt(raw); } @@ -235,7 +235,7 @@ const config = { * about a minute, `finalized` by about 20 minutes. */ baseStaleAlertMs: envNumber("BASE_STALE_ALERT_SECONDS", baseFinalityTag === "finalized" ? 1800 : 600) * 1000, /** Guardian balance (wei) below which the auto-pause counts as not ready. */ - guardianMinBalanceWei: BigInt(process.env.GUARDIAN_MIN_BALANCE_WEI ?? "1000000000000000"), + guardianMinBalanceWei: envAmount("GUARDIAN_MIN_BALANCE_WEI", "1000000000000000"), /** Excess of released over migrated, in token units, that operators have * quantified and accepted after a real loss (RB-3); M2a fires beyond it. */ acknowledgedReleaseExcess: envAmount("ACKNOWLEDGED_RELEASE_EXCESS"), From 38b501439797d6c29eabf719e1779aad1956f6ea Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Thu, 22 Oct 2026 11:10:18 +0200 Subject: [PATCH 23/75] fix(monitor): Retry the AUTO-PAUSE DISABLED page until it is delivered The startup page that says no guardian key is configured is now a once-only page, so a webhook that is down at startup no longer loses it: every poll sends it again until the webhook accepts it. --- monitor/src/main.test.ts | 1 + monitor/src/main.ts | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/monitor/src/main.test.ts b/monitor/src/main.test.ts index 6100a9420..5ad31907f 100644 --- a/monitor/src/main.test.ts +++ b/monitor/src/main.test.ts @@ -776,4 +776,5 @@ test("once-only pages the webhook rejects are sent again on the next poll", asyn const delivered = pages.filter((page) => page.status === 200).map((page) => page.subject); assert.ok(delivered.includes("GOVERNANCE OPERATION SCHEDULED"), `delivered: ${delivered}`); assert.ok(delivered.includes("PENDULUM GOVERNANCE ITEM"), `delivered: ${delivered}`); + assert.ok(delivered.includes("AUTO-PAUSE DISABLED"), `delivered: ${delivered}`); // first sent at startup }); diff --git a/monitor/src/main.ts b/monitor/src/main.ts index 1e28cbb20..64b4b6b2c 100644 --- a/monitor/src/main.ts +++ b/monitor/src/main.ts @@ -1376,7 +1376,8 @@ export async function start(api: ApiPromise): Promise { state.pendulumGenesisHash = pendulumGenesisHash; } if (!config.guardianPrivateKey) { - await alert( + await pageOnce( + "autopause-disabled", "AUTO-PAUSE DISABLED", "no GUARDIAN_PRIVATE_KEY configured: violations page, but nothing pauses the vault automatically", ); From 895a5ab1ace53524bc1f0fed455ed4c8b760b1da Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Thu, 22 Oct 2026 11:11:24 +0200 Subject: [PATCH 24/75] fix(monitor): Read legacy preimage lengths from the deprecated status map A legacy call reference is sized through preimage.requestStatusFor, but a preimage noted before the pallet's status migration stays in the deprecated statusFor until something touches it. The watcher now falls back to statusFor, so such a call is decoded instead of staying pending. --- monitor/src/main.test.ts | 18 +++++++++++++++++- monitor/src/main.ts | 11 +++++++++-- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/monitor/src/main.test.ts b/monitor/src/main.test.ts index 5ad31907f..217e6df1f 100644 --- a/monitor/src/main.test.ts +++ b/monitor/src/main.test.ts @@ -223,9 +223,11 @@ function pendulumChain() { publicProps: [] as Array<[number, Bounded]>, external: undefined as Bounded | undefined, referenda: new Map(), - /** Noted preimages by `${hash}:${len}`, and their lengths by hash. */ + /** Noted preimages by `${hash}:${len}`, and their lengths by hash (old: + * noted before the pallet's status migration). */ preimages: new Map(), notedLengths: new Map(), + oldNotedLengths: new Map(), preimageReads: 0, /** Preimage reads still to fail, as an RPC error would. */ preimageFailures: 0, @@ -286,6 +288,10 @@ function pendulumChain() { const len = chain.notedLengths.get(preimage.toHex()); return { toJSON: () => (len === undefined ? null : { unrequested: { ticket: ["5Noter", 1], len } }) }; }, + statusFor: async (preimage: { toHex(): string }) => { + const len = chain.oldNotedLengths.get(preimage.toHex()); + return { toJSON: () => (len === undefined ? null : { unrequested: { deposit: ["5Noter", 1], len } }) }; + }, }, council: collective(chain.motions.council), technicalCommittee: collective(chain.motions.technicalCommittee), @@ -615,6 +621,16 @@ test("a legacy call reference is read through the noted preimage's length", asyn assert.ok(pageFor(pages, RISKY, "referendum #9"), JSON.stringify(pages)); }); +test("a legacy call reference to a preimage noted before the status migration is read too", async (t) => { + const { pendulum, api, monitor, pages } = await setup(t); + const preimage = `0x${"fe".repeat(32)}`; + pendulum.referenda.set(9, legacy(preimage)); + pendulum.oldNotedLengths.set(preimage, 4); + pendulum.preimages.set(`${preimage}:4`, "mint"); + await monitor.poll(api); + assert.ok(pageFor(pages, RISKY, "referendum #9"), JSON.stringify(pages)); +}); + test("council and technical-committee motions are decoded with the same risk rules", async (t) => { const { pendulum, api, monitor, pages } = await setup(t); const [anchor, remark, huge] = ["0x01", "0x02", "0x03"].map((prefix) => prefix + "00".repeat(31)); diff --git a/monitor/src/main.ts b/monitor/src/main.ts index 64b4b6b2c..9b485af57 100644 --- a/monitor/src/main.ts +++ b/monitor/src/main.ts @@ -921,13 +921,20 @@ const oversizedCall = (len: number): CallRead => ({ text: `the call is ${len} bytes, over the ${MAX_GOVERNANCE_CALL_BYTES} bytes the monitor decodes`, }); -/** The length of a noted preimage, which a legacy call reference leaves out. */ +/** The length of a noted preimage, which a legacy call reference leaves out. + * One noted before the pallet's status migration stays in the deprecated + * `statusFor` until something touches it. */ async function notedLength(apiAt: ApiAt, hash: { toHex(): string }): Promise { const status = (await apiAt.query.preimage.requestStatusFor(hash)).toJSON() as { unrequested?: { len: number }; requested?: { maybeLen: number | null }; } | null; - return status?.unrequested?.len ?? status?.requested?.maybeLen ?? undefined; + if (status) return status.unrequested?.len ?? status.requested?.maybeLen ?? undefined; + const old = (await apiAt.query.preimage.statusFor(hash)).toJSON() as { + unrequested?: { len: number }; + requested?: { len: number | null }; + } | null; + return old?.unrequested?.len ?? old?.requested?.len ?? undefined; } /** Reads a proposal's call. Its preimage may be noted at any time before From ead5e1180a03c1c53d9e71bfe6b726c74ba8db50 Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Thu, 22 Oct 2026 11:11:47 +0200 Subject: [PATCH 25/75] docs(monitor): Clarify the governance watch cost and the guardian key The governance watch cannot stop or delay a poll's conservation checks, and its cost to the next poll is bounded; the guardian key must be the vault guardian's, and any other key pages AUTO-PAUSE NOT READY. --- monitor/README.md | 4 ++-- monitor/src/main.ts | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/monitor/README.md b/monitor/README.md index 1d2796223..4ab914383 100644 --- a/monitor/README.md +++ b/monitor/README.md @@ -17,7 +17,7 @@ Checks every poll: | Pendulum source | The finalized Pendulum view must stay fresh and readable. While it cannot be read (an RPC error, a nonce discontinuity, an event-shape change), the Base-side checks — vault balance and its auto-pause, Base view, auto-pause readiness, vault admin events, governance queue — still run, and Base events are reconciled against the last view read in full: an unreadable source counts as a stalled one, so a mismatch on a known nonce pauses at once and an unknown nonce after `UNMATCHED_EVENT_GRACE_SECONDS`. M2a and liveness wait for the source. | Alert, throttled: `PENDULUM SOURCE VIEW STALLED` once the view is older than `SOURCE_STALE_ALERT_SECONDS`, `PENDULUM SOURCE UNREADABLE` on a failure | | Base view | The safe/finalized Base block must not be older than `BASE_STALE_ALERT_SECONDS`: a view that stops advancing hides every later vault event while every check still passes at the old block. | Alert (`BASE VIEW STALLED`), throttled | | Auto-pause readiness | At startup and every poll: the Base RPC serves `BASE_CHAIN_ID` and, with a guardian key configured, the key parses, is `vault.guardian()` and holds at least `GUARDIAN_MIN_BALANCE_WEI`. A read that fails pages as well and never stops the poll's other checks. | Alert (`AUTO-PAUSE NOT READY`), throttled; `AUTO-PAUSE DISABLED` once at startup without a key | -| Pendulum governance | Every pending public proposal, the council's external proposal, every ongoing referendum and every open council and technical-committee motion, with the call decoded. A call that mints, moves or unlocks balances outside normal transfers, or changes the runtime or the migration pallet, pages as critical: minted PEN burned through the migration drains the fixed-supply vault, and only Pendulum governance can stop the call before enactment. A call the monitor cannot read pages as critical too: one over 64 KiB is never fetched, and one that does not decode or nests more than 256 calls or 16 levels deep is unreadable; one whose preimage is not noted yet is re-checked every poll and paged again once it can be read. The watch runs after the conservation checks, so it can neither stop nor delay them. | Alert once per item (`PENDULUM GOVERNANCE ITEM`, `CRITICAL: Pendulum governance item …`); a motion proposed again with the same call (a new motion index), or an external proposal made again after it left the queue, pages again; a restart re-pages the items still pending | +| Pendulum governance | Every pending public proposal, the council's external proposal, every ongoing referendum and every open council and technical-committee motion, with the call decoded. A call that mints, moves or unlocks balances outside normal transfers, or changes the runtime or the migration pallet, pages as critical: minted PEN burned through the migration drains the fixed-supply vault, and only Pendulum governance can stop the call before enactment. A call the monitor cannot read pages as critical too: one over 64 KiB is never fetched, and one that does not decode or nests more than 256 calls or 16 levels deep is unreadable; one whose preimage is not noted yet is re-checked every poll and paged again once it can be read. The watch runs after the poll's conservation checks, so it cannot stop or delay them, and it decodes within fixed bounds, so its cost to the next poll is bounded. | Alert once per item (`PENDULUM GOVERNANCE ITEM`, `CRITICAL: Pendulum governance item …`); a motion proposed again with the same call (a new motion index), or an external proposal made again after it left the queue, pages again; a restart re-pages the items still pending | | Migration pallet admin events | Every finalized `MigrationPauseSet` and `TreasuryDestinationSet`, however fast the motion behind it passed. | Alert once per event (`MIGRATION PALLET ADMIN EVENT`) | | Vault admin events | Every admin event of the vault (token, pause, unpause, attestors, threshold, caps, guardian, admin transfer, sweep, stale pending cleared). Before the governance handover the admin acts without a delay, so these are the only trace of such a change. While this scan (shared with the governance queue) fails, the vault balance check and its auto-pause still run; reconciliation, M2a and liveness wait for it, so the persisted Base cursor never passes a range it has not scanned. | Alert once per event (`VAULT ADMIN EVENT`); the first run also pages the deployment's own events; `BASE ADMIN SCAN FAILED`, throttled, on a failure | | Governance queue | Every operation the governance Timelock schedules (`CallScheduled`), with its calls decoded and the time it becomes executable. Once the delay has passed anyone may execute a ready operation, so the canceller has exactly the delay to review and veto it. | Alert once per operation (`GOVERNANCE OPERATION SCHEDULED`) | @@ -39,7 +39,7 @@ Checks every poll: | `ACKNOWLEDGED_RELEASE_EXCESS` | Excess of released over migrated, in token units (18 decimals), that operators quantified and accepted after a real loss; M2a fires only beyond it (default 0). Set it at the RB-3 sign-off and restart; the startup log shows it | | `TIMELOCK_ADDRESS` | Optional: the governance TimelockController. Set it as soon as governance is deployed; the queue is scanned from the monitor's Base cursor onwards, so operations scheduled before the monitor knew the address are not paged | | `ALERT_WEBHOOK_URL` | Webhook receiving JSON alerts — wire this to paging. A page counts as delivered only on a 2xx answer within 10s: once-only pages (governance items, admin events, scheduled operations, first sightings) are retried every poll until delivered, and failed deliveries are logged. Without it the log line is the page | -| `GUARDIAN_PRIVATE_KEY` | Optional: a guardian key enabling automatic `pause()` on conservation violations; its readiness is checked at startup and every poll | +| `GUARDIAN_PRIVATE_KEY` | Optional: the key of `vault.guardian()`, enabling automatic `pause()` on conservation violations; its readiness is checked at startup and every poll, and any other key pages `AUTO-PAUSE NOT READY` (the vault admin may pause too, but its key does not belong on the monitor) | | `GUARDIAN_MIN_BALANCE_WEI` | Guardian ETH balance below which `AUTO-PAUSE NOT READY` pages (default 10^15 wei = 0.001 ETH) | | `BASE_CHAIN_ID` | Default 8453; the Base RPC must serve it | | `PENDULUM_START_BLOCK` | **Required on first run:** first block to scan, normally the pallet activation block or the next finalized block before unpausing | diff --git a/monitor/src/main.ts b/monitor/src/main.ts index 9b485af57..62c44dc9c 100644 --- a/monitor/src/main.ts +++ b/monitor/src/main.ts @@ -969,8 +969,9 @@ async function motionCall(apiAt: ApiAt, collective: "council" | "technicalCommit * CRITICAL: minted PEN burned through the migration drains the fixed-supply * vault, and only Pendulum governance can stop the call before enactment. * So does a call the watcher cannot read; until it can, it is re-checked - * every poll. Alert-only; it runs after the conservation checks and decodes - * within fixed bounds, so it can neither stop nor delay them. + * every poll. Alert-only; it runs after the poll's conservation checks, so it + * cannot stop or delay them, and decodes within fixed bounds, so its cost to + * the next poll is bounded. */ async function watchPendulumGovernance(api: ApiPromise): Promise { const finalizedHash = await api.rpc.chain.getFinalizedHead(); From 508ca2b8551caf0ee7137ddb82f926bd7b879c73 Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Thu, 22 Oct 2026 11:56:38 +0200 Subject: [PATCH 26/75] fix(monitor): Page a scheduled operation whatever its delay or event fields The page for an operation the governance timelock schedules formatted its ready time as a date, which throws for a time past the range a date can hold, and it read the call fields as strings. Such an operation now pages with its ready time in seconds, and missing call fields are shown as they are, so describing one operation never fails the scan or hides the operations after it. --- monitor/src/checks.test.ts | 7 +++++++ monitor/src/checks.ts | 11 +++++++---- monitor/src/main.test.ts | 13 +++++++++++++ 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/monitor/src/checks.test.ts b/monitor/src/checks.test.ts index f182c7f40..c3ee77db7 100644 --- a/monitor/src/checks.test.ts +++ b/monitor/src/checks.test.ts @@ -157,6 +157,13 @@ test("a scheduled operation with unknown parts or a settled timestamp still page assert.match(describeTimelockOperation("0x1", [call], 100n, 200_000, {}, {}), /\(ready now\)/); }); +test("a scheduled operation pages whatever its delay or its event carries", () => { + const call = { index: 0n, target: "0x3333333333333333333333333333333333333333", value: 0n, data: "0x8456cb59" }; + assert.match(describeTimelockOperation("0x1", [call], 10n ** 20n, 0, {}, {}), /from timestamp 100000000000000000000 \(in /); + const undecoded = { index: undefined, target: undefined, value: undefined, data: undefined } as unknown as typeof call; + assert.match(describeTimelockOperation("0x1", [undecoded, call], 5n, 0, {}, {}), /^operation 0x1: #undefined undefined\.unknown function undefined/); +}); + test("a governance call tree is walked through batches and minting calls are flagged", () => { const setBalance = { section: "tokens", method: "setBalance", args: ["who", "Native", 4_000_000n] }; // The recurring attack proposal: utility.batch of tokens.setBalance to the proposer. diff --git a/monitor/src/checks.ts b/monitor/src/checks.ts index 75f35a0dd..9cf2755a9 100644 --- a/monitor/src/checks.ts +++ b/monitor/src/checks.ts @@ -177,7 +177,8 @@ export interface ScheduledCall { * * `readyAtSeconds` is the Timelock's `getTimestamp(id)`, whose sentinels are * 0 (unknown or cancelled) and 1 (executed). `labels` maps lowercased - * addresses and `functions` maps selectors to readable names. + * addresses and `functions` maps selectors to readable names. Never throws, + * whatever the event carries: one operation must not hide the next. */ export function describeTimelockOperation( id: string, @@ -190,8 +191,8 @@ export function describeTimelockOperation( const callText = [...calls] .sort((a, b) => (a.index < b.index ? -1 : a.index > b.index ? 1 : 0)) .map((call) => { - const target = labels[call.target.toLowerCase()] ?? call.target; - const selector = call.data.slice(0, 10).toLowerCase(); + const target = labels[String(call.target).toLowerCase()] ?? call.target; + const selector = String(call.data).slice(0, 10).toLowerCase(); const name = functions[selector] ?? `unknown function ${selector}`; return `#${call.index} ${target}.${name}${call.value ? ` value=${call.value}` : ""} data=${call.data}`; }) @@ -202,8 +203,10 @@ export function describeTimelockOperation( else { const readyMs = Number(readyAtSeconds) * 1000; const leftMinutes = Math.floor((readyMs - nowMs) / 60_000); + // A Date holds at most 8.64e15 ms; a later time is shown in seconds. + const readyText = readyMs <= 8.64e15 ? new Date(readyMs).toISOString() : `timestamp ${readyAtSeconds}`; status = - `executable by anyone from ${new Date(readyMs).toISOString()}` + + `executable by anyone from ${readyText}` + (leftMinutes > 0 ? ` (in ${Math.floor(leftMinutes / 60)}h ${leftMinutes % 60}m)` : " (ready now)") + "; if it is not intended, the canceller must cancel it before then"; } diff --git a/monitor/src/main.test.ts b/monitor/src/main.test.ts index 217e6df1f..aa7767a2d 100644 --- a/monitor/src/main.test.ts +++ b/monitor/src/main.test.ts @@ -775,6 +775,19 @@ function schedule(base: Base, id: Hex, blockNumber = 1_005n): void { base.readyAt.set(id, BigInt(Math.floor(Date.now() / 1000) + 172_800)); } +test("a scheduled operation with an extreme delay pages and hides nothing after it", async (t) => { + const { base, api, monitor, pages } = await setup(t, { env: { TIMELOCK_ADDRESS: TIMELOCK } }); + const [extreme, ordinary] = [toHex(1, { size: 32 }), toHex(2, { size: 32 })]; + schedule(base, extreme, 1_004n); + base.readyAt.set(extreme, 10n ** 20n); + schedule(base, ordinary, 1_006n); + await monitor.poll(api); + const scheduled = pages.filter((page) => page.subject === "GOVERNANCE OPERATION SCHEDULED").map((page) => page.detail); + assert.equal(scheduled.length, 2, JSON.stringify(pages)); + assert.ok(scheduled.some((detail) => detail.startsWith(`operation ${ordinary}`)), JSON.stringify(scheduled)); + assert.ok(!pages.some((page) => page.subject === "BASE ADMIN SCAN FAILED"), JSON.stringify(pages)); +}); + test("once-only pages the webhook rejects are sent again on the next poll", async (t) => { const failedOnce = new Set(); const { base, pendulum, api, monitor, pages } = await setup(t, { From 756a16a200f0fa6a81ed02bc271f8962deb38249 Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Thu, 22 Oct 2026 11:59:40 +0200 Subject: [PATCH 27/75] fix(monitor): Give the Base admin scan its own persisted cursor Reconciliation, M2a and liveness waited for the scan of vault admin events and the governance queue, so a scan that kept failing switched them off. The scan now keeps its own cursor in the state file: while it fails, every other check runs, and it resumes where it stopped, also after a restart. State written before keeps working; its admin cursor starts at the Base cursor, which never passed an unscanned range. The PENDULUM SOURCE UNREADABLE page now says what happens until the source reads again. --- monitor/README.md | 4 ++-- monitor/src/main.test.ts | 29 ++++++++++++++++++-------- monitor/src/main.ts | 43 ++++++++++++++++++--------------------- monitor/src/state.test.ts | 13 ++++++++++++ monitor/src/state.ts | 4 ++++ 5 files changed, 60 insertions(+), 33 deletions(-) diff --git a/monitor/README.md b/monitor/README.md index 4ab914383..6abd36e07 100644 --- a/monitor/README.md +++ b/monitor/README.md @@ -19,7 +19,7 @@ Checks every poll: | Auto-pause readiness | At startup and every poll: the Base RPC serves `BASE_CHAIN_ID` and, with a guardian key configured, the key parses, is `vault.guardian()` and holds at least `GUARDIAN_MIN_BALANCE_WEI`. A read that fails pages as well and never stops the poll's other checks. | Alert (`AUTO-PAUSE NOT READY`), throttled; `AUTO-PAUSE DISABLED` once at startup without a key | | Pendulum governance | Every pending public proposal, the council's external proposal, every ongoing referendum and every open council and technical-committee motion, with the call decoded. A call that mints, moves or unlocks balances outside normal transfers, or changes the runtime or the migration pallet, pages as critical: minted PEN burned through the migration drains the fixed-supply vault, and only Pendulum governance can stop the call before enactment. A call the monitor cannot read pages as critical too: one over 64 KiB is never fetched, and one that does not decode or nests more than 256 calls or 16 levels deep is unreadable; one whose preimage is not noted yet is re-checked every poll and paged again once it can be read. The watch runs after the poll's conservation checks, so it cannot stop or delay them, and it decodes within fixed bounds, so its cost to the next poll is bounded. | Alert once per item (`PENDULUM GOVERNANCE ITEM`, `CRITICAL: Pendulum governance item …`); a motion proposed again with the same call (a new motion index), or an external proposal made again after it left the queue, pages again; a restart re-pages the items still pending | | Migration pallet admin events | Every finalized `MigrationPauseSet` and `TreasuryDestinationSet`, however fast the motion behind it passed. | Alert once per event (`MIGRATION PALLET ADMIN EVENT`) | -| Vault admin events | Every admin event of the vault (token, pause, unpause, attestors, threshold, caps, guardian, admin transfer, sweep, stale pending cleared). Before the governance handover the admin acts without a delay, so these are the only trace of such a change. While this scan (shared with the governance queue) fails, the vault balance check and its auto-pause still run; reconciliation, M2a and liveness wait for it, so the persisted Base cursor never passes a range it has not scanned. | Alert once per event (`VAULT ADMIN EVENT`); the first run also pages the deployment's own events; `BASE ADMIN SCAN FAILED`, throttled, on a failure | +| Vault admin events | Every admin event of the vault (token, pause, unpause, attestors, threshold, caps, guardian, admin transfer, sweep, stale pending cleared). Before the governance handover the admin acts without a delay, so these are the only trace of such a change. This scan (shared with the governance queue) keeps its own persisted cursor: while it fails, every other check still runs, and it resumes where it stopped, also after a restart. | Alert once per event (`VAULT ADMIN EVENT`); the first run also pages the deployment's own events; `BASE ADMIN SCAN FAILED`, throttled, on a failure | | Governance queue | Every operation the governance Timelock schedules (`CallScheduled`), with its calls decoded and the time it becomes executable. Once the delay has passed anyone may execute a ready operation, so the canceller has exactly the delay to review and veto it. | Alert once per operation (`GOVERNANCE OPERATION SCHEDULED`) | ## Configuration (environment) @@ -45,7 +45,7 @@ Checks every poll: | `PENDULUM_START_BLOCK` | **Required on first run:** first block to scan, normally the pallet activation block or the next finalized block before unpausing | | `PENDULUM_START_NONCE` | Nonce expected at that block (default 0; set explicitly when starting after any migrations) | | `BASE_START_BLOCK` | **Required on first run:** vault deployment block or an earlier block | -| `STATE_FILE` | Durable two-chain cursor, pending tuples, and liveness timestamps (default `./monitor-state.json`) | +| `STATE_FILE` | Durable cursors (Pendulum, Base, Base admin scan), pending tuples, and liveness timestamps (default `./monitor-state.json`) | | `BASE_FINALITY_TAG` | `safe` (default) or `finalized`; only events inside this boundary are trusted | | `BASE_MAX_BLOCK_RANGE` | Maximum `eth_getLogs` range (default 10,000 blocks) | diff --git a/monitor/src/main.test.ts b/monitor/src/main.test.ts index aa7767a2d..eaa7c6964 100644 --- a/monitor/src/main.test.ts +++ b/monitor/src/main.test.ts @@ -403,8 +403,12 @@ async function setup( for (const name of monitorEnv) delete process.env[name]; monitorEnv = Object.keys(variables); Object.assign(process.env, variables); - const monitor: typeof import("./main.js") = await import(`./main.js?instance=${++instances}`); - await monitor.start(api); + const launch = async () => { + const monitor: typeof import("./main.js") = await import(`./main.js?instance=${++instances}`); + await monitor.start(api); + return monitor; + }; + const monitor = await launch(); const startup = pages.splice(0); return { base, @@ -413,8 +417,14 @@ async function setup( pages, startup, monitor, + /** A new monitor process on the same state file and stand-ins. */ + restart: launch, subjects: () => pages.map((page) => page.subject), - persisted: () => JSON.parse(readFileSync(stateFile, "utf8")) as { baseFromBlock: string; migrations: Array<{ nonce: string }> }, + persisted: () => JSON.parse(readFileSync(stateFile, "utf8")) as { + baseFromBlock: string; + baseAdminFromBlock: string; + migrations: Array<{ nonce: string }>; + }, }; } @@ -532,8 +542,10 @@ test("vault admin events page once each from the Base scan", async (t) => { assert.match(adminPages[1], /^CapsUpdated\(perReleaseCap=5, dailyCap=7\) at Base block 1012/); }); -test("a failed admin scan pages, the vault balance check still runs and the Base cursor waits", async (t) => { - const { base, pendulum, api, monitor, subjects, persisted } = await setup(t, { env: { GUARDIAN_PRIVATE_KEY: GUARDIAN_KEY } }); +test("a failed admin scan pages, every other check still runs and the scan resumes where it stopped", async (t) => { + const { base, pendulum, api, monitor, restart, subjects, persisted } = await setup(t, { + env: { GUARDIAN_PRIVATE_KEY: GUARDIAN_KEY }, + }); burn(pendulum, 0n, 100n * PEN); release(base, 0n, 100n * PEN); base.logs.push({ address: VAULT, eventName: "GuardianUpdated", args: { guardian: GUARDIAN }, blockNumber: 1_004n }); @@ -543,12 +555,13 @@ test("a failed admin scan pages, the vault balance check still runs and the Base base.paused = true; // so the auto-pause only confirms and the poll persists its state await monitor.poll(api); assert.deepEqual(sorted(subjects()), ["BASE ADMIN SCAN FAILED", "VAULT BALANCE DEFICIT", "vault pause confirmed"]); - assert.equal(persisted().baseFromBlock, "1000"); + assert.equal(persisted().baseFromBlock, "1011"); // reconciled although the admin scan failed + assert.equal(persisted().baseAdminFromBlock, "1000"); base.failing = undefined; base.vaultBalance += 1n; - await monitor.poll(api); + await (await restart()).poll(api); assert.ok(subjects().includes("VAULT ADMIN EVENT"), JSON.stringify(subjects())); - assert.equal(persisted().baseFromBlock, "1011"); + assert.equal(persisted().baseAdminFromBlock, "1011"); }); test("a monitor without a guardian key says so at startup", async (t) => { diff --git a/monitor/src/main.ts b/monitor/src/main.ts index 62c44dc9c..da82c07ba 100644 --- a/monitor/src/main.ts +++ b/monitor/src/main.ts @@ -264,6 +264,8 @@ interface RuntimeState { lastPendulumBlock: number; nextExpectedNonce: bigint; baseFromBlock: bigint; + /** The admin scan's own cursor: it only pages, so nothing waits for it. */ + baseAdminFromBlock: bigint; migrations: Map; livenessAlertedAt: Map; unmatchedBaseFirstSeenAt: Map; @@ -285,6 +287,7 @@ function hydrateState(): RuntimeState { lastPendulumBlock: config.pendulumStartBlock - 1, nextExpectedNonce: config.pendulumStartNonce, baseFromBlock: config.baseStartBlock, + baseAdminFromBlock: config.baseStartBlock, migrations: new Map(), livenessAlertedAt: new Map(), unmatchedBaseFirstSeenAt: new Map(), @@ -298,6 +301,7 @@ function hydrateState(): RuntimeState { lastPendulumBlock: persisted.lastPendulumBlock, nextExpectedNonce: BigInt(persisted.nextExpectedNonce), baseFromBlock: BigInt(persisted.baseFromBlock), + baseAdminFromBlock: BigInt(persisted.baseAdminFromBlock ?? persisted.baseFromBlock), migrations: new Map(persisted.migrations.map((migration) => [ BigInt(migration.nonce), { @@ -322,6 +326,7 @@ function cloneState(source: RuntimeState): RuntimeState { lastPendulumBlock: source.lastPendulumBlock, nextExpectedNonce: source.nextExpectedNonce, baseFromBlock: source.baseFromBlock, + baseAdminFromBlock: source.baseAdminFromBlock, migrations: new Map(source.migrations), livenessAlertedAt: new Map(source.livenessAlertedAt), unmatchedBaseFirstSeenAt: new Map(source.unmatchedBaseFirstSeenAt), @@ -339,6 +344,7 @@ function persistState(state: RuntimeState): void { lastPendulumBlock: state.lastPendulumBlock, nextExpectedNonce: state.nextExpectedNonce.toString(), baseFromBlock: state.baseFromBlock.toString(), + baseAdminFromBlock: state.baseAdminFromBlock.toString(), migrations: [...state.migrations.values()].map((migration) => ({ nonce: migration.nonce.toString(), recipient: migration.recipient as `0x${string}`, @@ -361,11 +367,6 @@ let lastBaseStallAlertMs = 0; let lastReadinessAlertMs = 0; let lastAdminScanFailureAlertMs = 0; let lastIssuanceAlertMs = 0; -/** The Base admin scan (vault admin events and the governance queue) runs - * ahead of reconciliation in a poll, and reconciliation waits while it fails, - * so the persisted Base cursor never passes a range it has not scanned; a - * restart resumes from that cursor. */ -let baseAdminFromBlock: bigint | undefined; /** Once-only pages by key, each held until the webhook accepts it. */ // ponytail: in memory; a restart re-pages what it finds again (Base admin events after the persisted cursor, pending governance items) and drops undelivered pages it does not find again. const onceOnlyPages = new Map }>(); @@ -805,12 +806,14 @@ async function reconcileBaseEvents( /** Page every vault admin event, and every operation the governance Timelock * scheduled, up to `toBlock`, the poll's `safe`/`finalized` Base block. * Before the governance handover the vault admin acts without a delay, so - * its events are the only trace of a guardian, attestor or cap change. */ + * its events are the only trace of a guardian, attestor or cap change. The + * scan keeps its own cursor in `state`, so a restart resumes where it stopped. */ async function watchBaseAdmin(toBlock: bigint): Promise { const timelock = config.timelockAddress; const operations = new Map<`0x${string}`, ScheduledCall[]>(); const pages: Promise[] = []; - for (const [start, end] of blockRanges(baseAdminFromBlock ?? state.baseFromBlock, toBlock, config.baseMaxBlockRange)) { + const fromBlock = state.baseAdminFromBlock; + for (const [start, end] of blockRanges(fromBlock, toBlock, config.baseMaxBlockRange)) { await publicClient.getBlock({ blockNumber: end }); // same lagging-node probe as the vault scan const adminEvents = await publicClient.getContractEvents({ address: config.vaultAddress, @@ -876,7 +879,7 @@ async function watchBaseAdmin(toBlock: bigint): Promise { } } await Promise.all(pages); - baseAdminFromBlock = toBlock + 1n; + if (toBlock >= fromBlock) state.baseAdminFromBlock = toBlock + 1n; } type ApiAt = Awaited>; @@ -1202,20 +1205,17 @@ async function check(api: ApiPromise): Promise { ); } await checkAutoPauseReadiness(); - // The admin scan only pages, so its failure must not skip the vault balance - // check. Reconciliation waits for it: a restart resumes the scan from the - // persisted Base cursor, which must never pass a range it has not covered. - let adminScanned = false; + // The admin scan only pages and keeps its own cursor, so nothing else waits + // for it, and its failure skips no other check. try { await watchBaseAdmin(baseBlock); - adminScanned = true; } catch (adminError) { if (Date.now() - lastAdminScanFailureAlertMs > config.unmatchedEventGraceSeconds * 1000) { lastAdminScanFailureAlertMs = Date.now(); await alert( "BASE ADMIN SCAN FAILED", - `${adminError}; the vault balance check continues, but vault admin events, the governance queue and ` + - "the reconciliation of Approved and Released events wait until the scan reads again", + `${adminError}; vault admin events and the governance queue are not paged until the scan reads again, ` + + "and it then resumes where it stopped; every other check still runs", ); } } @@ -1224,8 +1224,7 @@ async function check(api: ApiPromise): Promise { // are then reconciled against the last view read in full, as against a // lagging source but with no fabrication proof: a mismatch on a known nonce // still pauses at once, an unknown nonce after the grace. The vault balance - // check needs no source; M2a and liveness wait for it. All but the vault - // balance check also wait for the admin scan. + // check needs no source; M2a and liveness wait for it. let draft = cloneState(state); let source: SourceView | undefined; try { @@ -1236,8 +1235,9 @@ async function check(api: ApiPromise): Promise { lastSourceFailureAlertMs = Date.now(); await alert( "PENDULUM SOURCE UNREADABLE", - `${sourceError}; Base events are reconciled against the last Pendulum view read in full, so one with an ` + - `unknown nonce pauses the vault after ${config.unmatchedEventGraceSeconds}s unless the source reads again`, + `${sourceError}; until the source reads again, Base events are checked against the last Pendulum view ` + + `read in full: one whose nonce that view lacks pauses the vault ${config.unmatchedEventGraceSeconds}s ` + + "after its first sighting, and M2a and liveness are skipped", ); } } @@ -1249,9 +1249,7 @@ async function check(api: ApiPromise): Promise { publicClient.readContract({ address: config.vaultAddress, abi: vaultAbi, functionName: "token", blockNumber: baseBlock }), ]); - const reconciliation = adminScanned - ? await reconcileBaseEvents(draft, baseBlock, conversionFactor, source?.timestampMs ?? 0) - : { violation: undefined, awaitingSource: undefined }; + const reconciliation = await reconcileBaseEvents(draft, baseBlock, conversionFactor, source?.timestampMs ?? 0); if (reconciliation.violation) { if (await securityViolation("MIGRATION TUPLE VIOLATION", reconciliation.violation)) { state = draft; @@ -1280,7 +1278,6 @@ async function check(api: ApiPromise): Promise { } return; } - if (!adminScanned) return; if (!source) { state = draft; // reconciled against the last view read in full persistState(state); diff --git a/monitor/src/state.test.ts b/monitor/src/state.test.ts index ba6f96c5b..c7ab42e1a 100644 --- a/monitor/src/state.test.ts +++ b/monitor/src/state.test.ts @@ -69,3 +69,16 @@ test("missing unmatched-event history is rejected rather than resetting its grac rmSync(directory, { recursive: true, force: true }); } }); + +test("optional cursors are checked when present", () => { + const directory = mkdtempSync(join(tmpdir(), "pen-monitor-state-")); + const path = join(directory, "state.json"); + try { + saveMonitorState(path, { ...state, baseAdminFromBlock: "150" }); + assert.equal(loadMonitorState(path)?.baseAdminFromBlock, "150"); + writeFileSync(path, JSON.stringify({ ...state, baseAdminFromBlock: "-1" })); + assert.throws(() => loadMonitorState(path), /invalid or legacy schema/); + } finally { + rmSync(directory, { recursive: true, force: true }); + } +}); diff --git a/monitor/src/state.ts b/monitor/src/state.ts index caf936a90..786de55bd 100644 --- a/monitor/src/state.ts +++ b/monitor/src/state.ts @@ -31,6 +31,9 @@ export interface PersistedMonitorState { /** Pendulum totalIssuance + TotalMigrated when first observed (round 9); * absent in state written before it existed and re-anchored on load. */ issuanceAnchor?: string; + /** Next Base block of the admin scan; absent in older state, which never + * persisted a Base cursor past it, and then `baseFromBlock`. */ + baseAdminFromBlock?: string; } function isMissing(error: unknown): boolean { @@ -86,6 +89,7 @@ export function loadMonitorState(path: string): PersistedMonitorState | undefine !isUint(state.nextExpectedNonce) || !isUint(state.baseFromBlock) || (state.issuanceAnchor !== undefined && !isUint(state.issuanceAnchor)) || + (state.baseAdminFromBlock !== undefined && !isUint(state.baseAdminFromBlock)) || badMigration || badAlerts || badUnmatchedEvents From e7440b7e19d6afe5a78badb982038fb138ace765 Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Thu, 22 Oct 2026 12:00:35 +0200 Subject: [PATCH 28/75] fix(monitor): Run the vault balance check before the Pendulum and log reads The vault balance check (M2b) ran after the Pendulum read and the Base log reconciliation, so a failing log read skipped it and a slow source delayed it. It reads Base only, so it now runs first in every poll, and its pause no longer ends the poll: reconciliation and the other checks still run after it. --- monitor/README.md | 2 +- monitor/src/main.test.ts | 38 +++++++++++++++++++++++++++ monitor/src/main.ts | 56 ++++++++++++++++++++-------------------- 3 files changed, 67 insertions(+), 29 deletions(-) diff --git a/monitor/README.md b/monitor/README.md index 6abd36e07..b3078fa51 100644 --- a/monitor/README.md +++ b/monitor/README.md @@ -10,7 +10,7 @@ Checks every poll: |---|---|---| | M2 tuple match | Every safe Base `Approved` and `Released` event must match one finalized Pendulum `MigrationInitiated` nonce, recipient, and pallet amount; `Released.tokenAmount` must equal `palletAmount × conversionFactor`. After a violation the Base cursor stays on any block range whose events the source view may still reveal, so a pause caused by a lagging source does not lose those releases. | Alert + auto-pause. A mismatch against a *known* nonce pauses immediately. An *unknown* nonce first pages (`UNVERIFIED BASE EVENT`, without delaying the pause decision), then pauses as soon as it is provable — the finalized source view passed the event's own timestamp and the nonce still does not exist — or after `UNMATCHED_EVENT_GRACE_SECONDS` if the source view stays behind or cannot be read (a stalled or unreadable source also pages on its own, see below) | | M2a aggregate | `totalReleased <= TotalMigrated × conversionFactor + ACKNOWLEDGED_RELEASE_EXCESS`. This remains independent defence-in-depth around the event reconciliation. After a real, quantified loss, operators set `ACKNOWLEDGED_RELEASE_EXCESS` to it (RB-3) so the vault can be unpaused; any further excess still fires. | Alert + auto-pause | -| M2b: `balanceOf(vault) + totalReleased + totalSwept >= totalSupply` | Vault-internal conservation. Only a **deficit** alerts: a surplus is a harmless inbound transfer (donation, or a migration whose recipient is the vault) and is ignored, so it cannot false-trigger a pause. | Alert + auto-pause on a deficit, also while Pendulum cannot be read | +| M2b: `balanceOf(vault) + totalReleased + totalSwept >= totalSupply` | Vault-internal conservation. Only a **deficit** alerts: a surplus is a harmless inbound transfer (donation, or a migration whose recipient is the vault) and is ignored, so it cannot false-trigger a pause. It reads Base only and runs first in every poll, so a slow or failing Pendulum source or Base log scan can neither delay nor skip it. | Alert + auto-pause on a deficit | | M4: every old migration reaches active approval quorum — and then releases | Liveness of the attestor fleet (`below approval quorum`), and of everything after it (`quorum reached but not released`: a release starved of daily allowance, paused, or made releasable by a threshold cut without a `ReleasePending`, which the releaser cannot see). | One summary page per kind and poll (count and the first nonces; the log lists every nonce), each nonce at most once per `GRACE_SECONDS` | | Source supply | `totalIssuance + TotalMigrated` on Pendulum must never grow past its first-seen anchor. | Alert (`SOURCE SUPPLY GREW`) | | Unreleasable burns | A burn to the zero or vault address can never be approved or released; it is paged once and excluded from the pending count so it cannot page liveness forever or block RB-7. | Alert once | diff --git a/monitor/src/main.test.ts b/monitor/src/main.test.ts index eaa7c6964..89f28d4fc 100644 --- a/monitor/src/main.test.ts +++ b/monitor/src/main.test.ts @@ -217,6 +217,8 @@ function pendulumChain() { /** Age of the finalized head; earlier blocks are 12 s apart. */ headAgeMs: 0, failure: undefined as Error | undefined, + /** Holds every finalized-head read until it settles, as a slow node would. */ + gate: undefined as Promise | undefined, events: new Map(), totalMigrated: 0n, nextNonce: 0n, @@ -302,6 +304,7 @@ function pendulumChain() { rpc: { chain: { getFinalizedHead: async () => { + await chain.gate; if (chain.failure) throw chain.failure; return chain.finalized; }, @@ -461,6 +464,41 @@ test("a Pendulum failure pages on its own and the vault balance check still runs ]); }); +test("a failing Base log read does not skip the vault balance check", async (t) => { + const { base, pendulum, api, monitor, subjects } = await setup(t, { env: { GUARDIAN_PRIVATE_KEY: GUARDIAN_KEY } }); + pendulum.failure = new Error("node unreachable"); + const released = toEventSelector("Released(uint64,address,uint256,uint256)"); // only reconciliation asks for it + base.failing = (method, params) => method === "eth_getLogs" && JSON.stringify(params).includes(released); + base.vaultBalance -= 1n; + base.paused = true; // so the auto-pause only confirms + await monitor.poll(api); + assert.ok(subjects().includes("VAULT BALANCE DEFICIT"), JSON.stringify(subjects())); + assert.ok(subjects().includes("vault pause confirmed"), JSON.stringify(subjects())); +}); + +/** Waits until `predicate` holds, failing after `ms`. */ +async function until(predicate: () => boolean, label: string, ms = 2_000): Promise { + const deadline = Date.now() + ms; + while (!predicate()) { + if (Date.now() > deadline) assert.fail(`timed out waiting for ${label}`); + await new Promise((resolve) => setTimeout(resolve, 10)); + } +} + +test("a slow Pendulum source does not delay the vault balance check", async (t) => { + const { base, pendulum, api, monitor, subjects } = await setup(t); + let respond = () => {}; + pendulum.gate = new Promise((resolve) => (respond = resolve)); + base.vaultBalance -= 1n; + const polling = monitor.poll(api); + try { + await until(() => subjects().includes("VAULT BALANCE DEFICIT"), "the deficit page while Pendulum is still answering"); + } finally { + respond(); + await polling; + } +}); + test("while Pendulum cannot be read, a Base event with an unknown nonce pauses after the grace", async (t) => { const { base, pendulum, api, monitor, subjects } = await setup(t, { env: { UNMATCHED_EVENT_GRACE_SECONDS: "0" } }); pendulum.failure = new Error("node unreachable"); diff --git a/monitor/src/main.ts b/monitor/src/main.ts index da82c07ba..1da69bd92 100644 --- a/monitor/src/main.ts +++ b/monitor/src/main.ts @@ -1205,6 +1205,34 @@ async function check(api: ApiPromise): Promise { ); } await checkAutoPauseReadiness(); + + // The vault balance check (M2b) reads Base only, so it runs before anything + // that waits on Pendulum or scans Base logs: a slow or failing source, or a + // failing log scan, can neither delay nor skip it. Its pause does not end + // the poll. + const [totalReleased, totalSwept, conversionFactor, tokenAddress] = await Promise.all([ + publicClient.readContract({ address: config.vaultAddress, abi: vaultAbi, functionName: "totalReleased", blockNumber: baseBlock }), + publicClient.readContract({ address: config.vaultAddress, abi: vaultAbi, functionName: "totalSwept", blockNumber: baseBlock }), + publicClient.readContract({ address: config.vaultAddress, abi: vaultAbi, functionName: "conversionFactor", blockNumber: baseBlock }), + publicClient.readContract({ address: config.vaultAddress, abi: vaultAbi, functionName: "token", blockNumber: baseBlock }), + ]); + const [totalSupply, vaultBalance] = await Promise.all([ + publicClient.readContract({ address: tokenAddress, abi: erc20Abi, functionName: "totalSupply", blockNumber: baseBlock }), + publicClient.readContract({ + address: tokenAddress, + abi: erc20Abi, + functionName: "balanceOf", + args: [config.vaultAddress], + blockNumber: baseBlock, + }), + ]); + if (vaultConservationDeficit(vaultBalance, totalReleased, totalSwept, totalSupply)) { + await securityViolation( + "VAULT BALANCE DEFICIT", + `balance ${vaultBalance} + released ${totalReleased} + swept ${totalSwept} < supply ${totalSupply}`, + ); + } + // The admin scan only pages and keeps its own cursor, so nothing else waits // for it, and its failure skips no other check. try { @@ -1242,13 +1270,6 @@ async function check(api: ApiPromise): Promise { } } - const [totalReleased, totalSwept, conversionFactor, tokenAddress] = await Promise.all([ - publicClient.readContract({ address: config.vaultAddress, abi: vaultAbi, functionName: "totalReleased", blockNumber: baseBlock }), - publicClient.readContract({ address: config.vaultAddress, abi: vaultAbi, functionName: "totalSwept", blockNumber: baseBlock }), - publicClient.readContract({ address: config.vaultAddress, abi: vaultAbi, functionName: "conversionFactor", blockNumber: baseBlock }), - publicClient.readContract({ address: config.vaultAddress, abi: vaultAbi, functionName: "token", blockNumber: baseBlock }), - ]); - const reconciliation = await reconcileBaseEvents(draft, baseBlock, conversionFactor, source?.timestampMs ?? 0); if (reconciliation.violation) { if (await securityViolation("MIGRATION TUPLE VIOLATION", reconciliation.violation)) { @@ -1257,27 +1278,6 @@ async function check(api: ApiPromise): Promise { } return; } - - const [totalSupply, vaultBalance] = await Promise.all([ - publicClient.readContract({ address: tokenAddress, abi: erc20Abi, functionName: "totalSupply", blockNumber: baseBlock }), - publicClient.readContract({ - address: tokenAddress, - abi: erc20Abi, - functionName: "balanceOf", - args: [config.vaultAddress], - blockNumber: baseBlock, - }), - ]); - if (vaultConservationDeficit(vaultBalance, totalReleased, totalSwept, totalSupply)) { - if (await securityViolation( - "VAULT BALANCE DEFICIT", - `balance ${vaultBalance} + released ${totalReleased} + swept ${totalSwept} < supply ${totalSupply}`, - )) { - state = draft; - persistState(state); - } - return; - } if (!source) { state = draft; // reconciled against the last view read in full persistState(state); From 215f5160985a6ed58655b6fabe3c006ad11900c5 Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Thu, 22 Oct 2026 12:01:34 +0200 Subject: [PATCH 29/75] fix(monitor): Page an external proposal made again after it was tabled An external proposal is identified only by its call, so the same call proposed again right after it became a referendum kept its key and did not page until it became a referendum too. The key now includes the newest ongoing referendum with the same call, so such a proposal pages again in the next poll. --- monitor/README.md | 2 +- monitor/src/main.test.ts | 6 ++++-- monitor/src/main.ts | 34 +++++++++++++++++++--------------- 3 files changed, 24 insertions(+), 18 deletions(-) diff --git a/monitor/README.md b/monitor/README.md index b3078fa51..384f8ea52 100644 --- a/monitor/README.md +++ b/monitor/README.md @@ -17,7 +17,7 @@ Checks every poll: | Pendulum source | The finalized Pendulum view must stay fresh and readable. While it cannot be read (an RPC error, a nonce discontinuity, an event-shape change), the Base-side checks — vault balance and its auto-pause, Base view, auto-pause readiness, vault admin events, governance queue — still run, and Base events are reconciled against the last view read in full: an unreadable source counts as a stalled one, so a mismatch on a known nonce pauses at once and an unknown nonce after `UNMATCHED_EVENT_GRACE_SECONDS`. M2a and liveness wait for the source. | Alert, throttled: `PENDULUM SOURCE VIEW STALLED` once the view is older than `SOURCE_STALE_ALERT_SECONDS`, `PENDULUM SOURCE UNREADABLE` on a failure | | Base view | The safe/finalized Base block must not be older than `BASE_STALE_ALERT_SECONDS`: a view that stops advancing hides every later vault event while every check still passes at the old block. | Alert (`BASE VIEW STALLED`), throttled | | Auto-pause readiness | At startup and every poll: the Base RPC serves `BASE_CHAIN_ID` and, with a guardian key configured, the key parses, is `vault.guardian()` and holds at least `GUARDIAN_MIN_BALANCE_WEI`. A read that fails pages as well and never stops the poll's other checks. | Alert (`AUTO-PAUSE NOT READY`), throttled; `AUTO-PAUSE DISABLED` once at startup without a key | -| Pendulum governance | Every pending public proposal, the council's external proposal, every ongoing referendum and every open council and technical-committee motion, with the call decoded. A call that mints, moves or unlocks balances outside normal transfers, or changes the runtime or the migration pallet, pages as critical: minted PEN burned through the migration drains the fixed-supply vault, and only Pendulum governance can stop the call before enactment. A call the monitor cannot read pages as critical too: one over 64 KiB is never fetched, and one that does not decode or nests more than 256 calls or 16 levels deep is unreadable; one whose preimage is not noted yet is re-checked every poll and paged again once it can be read. The watch runs after the poll's conservation checks, so it cannot stop or delay them, and it decodes within fixed bounds, so its cost to the next poll is bounded. | Alert once per item (`PENDULUM GOVERNANCE ITEM`, `CRITICAL: Pendulum governance item …`); a motion proposed again with the same call (a new motion index), or an external proposal made again after it left the queue, pages again; a restart re-pages the items still pending | +| Pendulum governance | Every pending public proposal, the council's external proposal, every ongoing referendum and every open council and technical-committee motion, with the call decoded. A call that mints, moves or unlocks balances outside normal transfers, or changes the runtime or the migration pallet, pages as critical: minted PEN burned through the migration drains the fixed-supply vault, and only Pendulum governance can stop the call before enactment. A call the monitor cannot read pages as critical too: one over 64 KiB is never fetched, and one that does not decode or nests more than 256 calls or 16 levels deep is unreadable; one whose preimage is not noted yet is re-checked every poll and paged again once it can be read. The watch runs after the poll's conservation checks, so it cannot stop or delay them, and it decodes within fixed bounds, so its cost to the next poll is bounded. | Alert once per item (`PENDULUM GOVERNANCE ITEM`, `CRITICAL: Pendulum governance item …`); a motion proposed again with the same call (a new motion index), or an external proposal made again after it left the queue or became a referendum, pages again; a restart re-pages the items still pending | | Migration pallet admin events | Every finalized `MigrationPauseSet` and `TreasuryDestinationSet`, however fast the motion behind it passed. | Alert once per event (`MIGRATION PALLET ADMIN EVENT`) | | Vault admin events | Every admin event of the vault (token, pause, unpause, attestors, threshold, caps, guardian, admin transfer, sweep, stale pending cleared). Before the governance handover the admin acts without a delay, so these are the only trace of such a change. This scan (shared with the governance queue) keeps its own persisted cursor: while it fails, every other check still runs, and it resumes where it stopped, also after a restart. | Alert once per event (`VAULT ADMIN EVENT`); the first run also pages the deployment's own events; `BASE ADMIN SCAN FAILED`, throttled, on a failure | | Governance queue | Every operation the governance Timelock schedules (`CallScheduled`), with its calls decoded and the time it becomes executable. Once the delay has passed anyone may execute a ready operation, so the canceller has exactly the delay to review and veto it. | Alert once per operation (`GOVERNANCE OPERATION SCHEDULED`) | diff --git a/monitor/src/main.test.ts b/monitor/src/main.test.ts index 89f28d4fc..fa9293a3a 100644 --- a/monitor/src/main.test.ts +++ b/monitor/src/main.test.ts @@ -704,12 +704,14 @@ test("a motion or external proposal made again with the same call pages again", pendulum.external = inline("remark"); await monitor.poll(api); pendulum.motionIndices.set(motion, 4); // closed and proposed again between two polls - pendulum.external = undefined; // tabled + pendulum.referenda.set(9, inline("remark")); // the external proposal tabled, and proposed again between two polls + await monitor.poll(api); + pendulum.external = undefined; // tabled or vetoed await monitor.poll(api); pendulum.external = inline("remark"); // proposed again await monitor.poll(api); assert.equal(pages.filter((page) => page.detail.startsWith(`council motion ${motion}`)).length, 2, JSON.stringify(pages)); - assert.equal(pages.filter((page) => page.detail.startsWith("the council's external proposal")).length, 2, JSON.stringify(pages)); + assert.equal(pages.filter((page) => page.detail.startsWith("the council's external proposal")).length, 3, JSON.stringify(pages)); }); test("admin events of the migration pallet page from the ingest", async (t) => { diff --git a/monitor/src/main.ts b/monitor/src/main.ts index 1da69bd92..836146fe8 100644 --- a/monitor/src/main.ts +++ b/monitor/src/main.ts @@ -996,24 +996,11 @@ async function watchPendulumGovernance(api: ApiPromise): Promise { read: () => boundedCall(apiAt, bounded), }); } - const external = (await democracy.nextExternal()) as unknown as { isSome: boolean; unwrap(): [BoundedCall, unknown] }; - if (external.isSome) { - const [bounded] = external.unwrap(); - items.push({ - key: `external:${bounded.toString()}`, - label: "the council's external proposal", - hint: `it can become a referendum at block ${nextLaunch}`, - read: () => boundedCall(apiAt, bounded), - }); - } else { - // Only its call identifies an external proposal, so once it has left the - // queue the same call proposed again pages again. - // ponytail: one tabled and proposed again between two polls pages again only as a referendum. - for (const key of onceOnlyPages.keys()) if (key.startsWith("external:")) onceOnlyPages.delete(key); - } const [count, lowest] = (await Promise.all([democracy.referendumCount(), democracy.lowestUnbaked()])).map((value) => Number(value.toString()) ); + /** The newest ongoing referendum of each call. */ + const referendumOf = new Map(); for (let index = lowest; index < count; index++) { const info = (await democracy.referendumInfoOf(index)) as unknown as { isSome: boolean; @@ -1021,6 +1008,7 @@ async function watchPendulumGovernance(api: ApiPromise): Promise { }; if (!info.isSome || !info.unwrap().isOngoing) continue; const ongoing = info.unwrap().asOngoing; + referendumOf.set(ongoing.proposal.toString(), index); items.push({ key: `referendum:${index}`, label: `referendum #${index}`, @@ -1028,6 +1016,22 @@ async function watchPendulumGovernance(api: ApiPromise): Promise { read: () => boundedCall(apiAt, ongoing.proposal), }); } + const external = (await democracy.nextExternal()) as unknown as { isSome: boolean; unwrap(): [BoundedCall, unknown] }; + if (external.isSome) { + const [bounded] = external.unwrap(); + // Only its call identifies an external proposal. A tabled one becomes a + // referendum with the same call, so that referendum keys the page of the + // same call proposed again. + items.push({ + key: `external:${referendumOf.get(bounded.toString()) ?? "-"}:${bounded.toString()}`, + label: "the council's external proposal", + hint: `it can become a referendum at block ${nextLaunch}`, + read: () => boundedCall(apiAt, bounded), + }); + } else { + // Once it has left the queue, the same call proposed again pages again. + for (const key of onceOnlyPages.keys()) if (key.startsWith("external:")) onceOnlyPages.delete(key); + } for (const collective of ["council", "technicalCommittee"] as const) { const motions = (await apiAt.query[collective].proposals()) as unknown as Array<{ toHex(): string }>; for (const hash of motions) { From f4a52054380f09e90275221803d0dc20a3f984bb Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Thu, 22 Oct 2026 12:02:32 +0200 Subject: [PATCH 30/75] fix(monitor): Page a timelock operation scheduled again with the same id A scheduled operation paged once per id, so one that was cancelled and then scheduled again with the identical calls did not page again. The page is now keyed by the id and the block of its CallScheduled events. --- monitor/README.md | 2 +- monitor/src/main.test.ts | 13 +++++++++++++ monitor/src/main.ts | 17 ++++++++++------- 3 files changed, 24 insertions(+), 8 deletions(-) diff --git a/monitor/README.md b/monitor/README.md index 384f8ea52..5b82be130 100644 --- a/monitor/README.md +++ b/monitor/README.md @@ -20,7 +20,7 @@ Checks every poll: | Pendulum governance | Every pending public proposal, the council's external proposal, every ongoing referendum and every open council and technical-committee motion, with the call decoded. A call that mints, moves or unlocks balances outside normal transfers, or changes the runtime or the migration pallet, pages as critical: minted PEN burned through the migration drains the fixed-supply vault, and only Pendulum governance can stop the call before enactment. A call the monitor cannot read pages as critical too: one over 64 KiB is never fetched, and one that does not decode or nests more than 256 calls or 16 levels deep is unreadable; one whose preimage is not noted yet is re-checked every poll and paged again once it can be read. The watch runs after the poll's conservation checks, so it cannot stop or delay them, and it decodes within fixed bounds, so its cost to the next poll is bounded. | Alert once per item (`PENDULUM GOVERNANCE ITEM`, `CRITICAL: Pendulum governance item …`); a motion proposed again with the same call (a new motion index), or an external proposal made again after it left the queue or became a referendum, pages again; a restart re-pages the items still pending | | Migration pallet admin events | Every finalized `MigrationPauseSet` and `TreasuryDestinationSet`, however fast the motion behind it passed. | Alert once per event (`MIGRATION PALLET ADMIN EVENT`) | | Vault admin events | Every admin event of the vault (token, pause, unpause, attestors, threshold, caps, guardian, admin transfer, sweep, stale pending cleared). Before the governance handover the admin acts without a delay, so these are the only trace of such a change. This scan (shared with the governance queue) keeps its own persisted cursor: while it fails, every other check still runs, and it resumes where it stopped, also after a restart. | Alert once per event (`VAULT ADMIN EVENT`); the first run also pages the deployment's own events; `BASE ADMIN SCAN FAILED`, throttled, on a failure | -| Governance queue | Every operation the governance Timelock schedules (`CallScheduled`), with its calls decoded and the time it becomes executable. Once the delay has passed anyone may execute a ready operation, so the canceller has exactly the delay to review and veto it. | Alert once per operation (`GOVERNANCE OPERATION SCHEDULED`) | +| Governance queue | Every operation the governance Timelock schedules (`CallScheduled`), with its calls decoded and the time it becomes executable. Once the delay has passed anyone may execute a ready operation, so the canceller has exactly the delay to review and veto it. | Alert once per operation and scheduling block (`GOVERNANCE OPERATION SCHEDULED`), so an operation cancelled and scheduled again with the same id pages again | ## Configuration (environment) diff --git a/monitor/src/main.test.ts b/monitor/src/main.test.ts index fa9293a3a..2d9dfd7be 100644 --- a/monitor/src/main.test.ts +++ b/monitor/src/main.test.ts @@ -841,6 +841,19 @@ test("a scheduled operation with an extreme delay pages and hides nothing after assert.ok(!pages.some((page) => page.subject === "BASE ADMIN SCAN FAILED"), JSON.stringify(pages)); }); +test("an operation cancelled and scheduled again with the same id pages again", async (t) => { + const { base, api, monitor, pages } = await setup(t, { env: { TIMELOCK_ADDRESS: TIMELOCK } }); + const id = toHex(7, { size: 32 }); + schedule(base, id, 1_005n); + await monitor.poll(api); + base.readyAt.set(id, 0n); // cancelled + schedule(base, id, 1_015n); + base.safe = 1_020n; + await monitor.poll(api); + const scheduled = pages.filter((page) => page.subject === "GOVERNANCE OPERATION SCHEDULED"); + assert.equal(scheduled.length, 2, JSON.stringify(pages)); +}); + test("once-only pages the webhook rejects are sent again on the next poll", async (t) => { const failedOnce = new Set(); const { base, pendulum, api, monitor, pages } = await setup(t, { diff --git a/monitor/src/main.ts b/monitor/src/main.ts index 836146fe8..7535377bb 100644 --- a/monitor/src/main.ts +++ b/monitor/src/main.ts @@ -810,7 +810,9 @@ async function reconcileBaseEvents( * scan keeps its own cursor in `state`, so a restart resumes where it stopped. */ async function watchBaseAdmin(toBlock: bigint): Promise { const timelock = config.timelockAddress; - const operations = new Map<`0x${string}`, ScheduledCall[]>(); + /** Scheduled calls by operation id and block: a cancelled operation can be + * scheduled again with the same id, and then pages again. */ + const operations = new Map(); const pages: Promise[] = []; const fromBlock = state.baseAdminFromBlock; for (const [start, end] of blockRanges(fromBlock, toBlock, config.baseMaxBlockRange)) { @@ -845,9 +847,10 @@ async function watchBaseAdmin(toBlock: bigint): Promise { value: bigint; data: `0x${string}`; }; - const calls = operations.get(id) ?? []; - calls.push({ index, target, value, data }); - operations.set(id, calls); + const key = `timelock:${id}:${event.blockNumber}`; + const operation = operations.get(key) ?? { id, calls: [] }; + operation.calls.push({ index, target, value, data }); + operations.set(key, operation); } } if (timelock && operations.size > 0) { @@ -862,8 +865,8 @@ async function watchBaseAdmin(toBlock: bigint): Promise { [token.toLowerCase()]: "token", [timelock.toLowerCase()]: "timelock", }; - for (const [id, calls] of operations) { - if (onceOnlyPages.has(`timelock:${id}`)) continue; + for (const [key, { id, calls }] of operations) { + if (onceOnlyPages.has(key)) continue; const readyAt = await publicClient.readContract({ address: timelock, abi: timelockAbi, @@ -872,7 +875,7 @@ async function watchBaseAdmin(toBlock: bigint): Promise { blockNumber: toBlock, }); pages.push(pageOnce( - `timelock:${id}`, + key, "GOVERNANCE OPERATION SCHEDULED", describeTimelockOperation(id, calls, readyAt, Date.now(), labels, KNOWN_FUNCTIONS), )); From 2bf7b751194b1d5d59f77698326fe2d5f6791775 Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Thu, 22 Oct 2026 12:08:53 +0200 Subject: [PATCH 31/75] feat(monitor): Send pages as Slack incoming-webhook messages A page was a JSON object of service, subject and detail, which a Slack incoming webhook rejects for lack of a text field. Each page is now a message with a single text field: the severity (CRITICAL or WARNING), the service and the subject, then the detail. Slack's control characters are escaped, so no detail can mention anyone or hide a link, and a long detail is cut short; the log line keeps the full text. --- monitor/README.md | 11 +++++++++-- monitor/src/checks.test.ts | 11 +++++++++++ monitor/src/checks.ts | 18 ++++++++++++++++++ monitor/src/main.test.ts | 28 ++++++++++++++++++++++++++-- monitor/src/main.ts | 21 ++++++++++++++------- 5 files changed, 78 insertions(+), 11 deletions(-) diff --git a/monitor/README.md b/monitor/README.md index 5b82be130..45b00ac33 100644 --- a/monitor/README.md +++ b/monitor/README.md @@ -38,7 +38,7 @@ Checks every poll: | `ISSUANCE_TOLERANCE` | Growth of Pendulum `totalIssuance + TotalMigrated` over its first-seen anchor (12-decimal pallet units) tolerated before `SOURCE SUPPLY GREW` pages (default 0) | | `ACKNOWLEDGED_RELEASE_EXCESS` | Excess of released over migrated, in token units (18 decimals), that operators quantified and accepted after a real loss; M2a fires only beyond it (default 0). Set it at the RB-3 sign-off and restart; the startup log shows it | | `TIMELOCK_ADDRESS` | Optional: the governance TimelockController. Set it as soon as governance is deployed; the queue is scanned from the monitor's Base cursor onwards, so operations scheduled before the monitor knew the address are not paged | -| `ALERT_WEBHOOK_URL` | Webhook receiving JSON alerts — wire this to paging. A page counts as delivered only on a 2xx answer within 10s: once-only pages (governance items, admin events, scheduled operations, first sightings) are retried every poll until delivered, and failed deliveries are logged. Without it the log line is the page | +| `ALERT_WEBHOOK_URL` | Slack incoming webhook receiving the pages (see [Pages](#pages)). A page counts as delivered only on a 2xx answer within 10s: once-only pages (governance items, admin events, scheduled operations, first sightings) are retried every poll until delivered, and failed deliveries are logged. Without it the log line is the page | | `GUARDIAN_PRIVATE_KEY` | Optional: the key of `vault.guardian()`, enabling automatic `pause()` on conservation violations; its readiness is checked at startup and every poll, and any other key pages `AUTO-PAUSE NOT READY` (the vault admin may pause too, but its key does not belong on the monitor) | | `GUARDIAN_MIN_BALANCE_WEI` | Guardian ETH balance below which `AUTO-PAUSE NOT READY` pages (default 10^15 wei = 0.001 ETH) | | `BASE_CHAIN_ID` | Default 8453; the Base RPC must serve it | @@ -51,7 +51,14 @@ Checks every poll: ## Pages -Each page is a JSON `POST` of `{ service, subject, detail }` to `ALERT_WEBHOOK_URL`. +Each page is a Slack incoming-webhook message, a JSON `POST` of `{ "text": … }` +to `ALERT_WEBHOOK_URL`. The text is `[CRITICAL] pen-monitor: SUBJECT` or +`[WARNING] pen-monitor: SUBJECT`, then the detail on the next line. Slack's +control characters `&`, `<` and `>` are escaped, so no detail can mention +anyone or hide a link, and a detail longer than 3,000 characters is cut short +(the monitor's log line keeps the full text). `CRITICAL` marks the conservation +violations, `AUTO-PAUSE FAILED`, `AUTO-PAUSE UNAVAILABLE` and every subject that +starts with `CRITICAL`; everything else is a `WARNING`. | Subjects | Meaning | |---|---| diff --git a/monitor/src/checks.test.ts b/monitor/src/checks.test.ts index c3ee77db7..780d1f837 100644 --- a/monitor/src/checks.test.ts +++ b/monitor/src/checks.test.ts @@ -20,6 +20,7 @@ import { releasedExceedsMigrated, riskyCalls, shouldAwaitSource, + slackText, toCallNode, vaultConservationDeficit, } from "./checks.js"; @@ -164,6 +165,16 @@ test("a scheduled operation pages whatever its delay or its event carries", () = assert.match(describeTimelockOperation("0x1", [undecoded, call], 5n, 0, {}, {}), /^operation 0x1: #undefined undefined\.unknown function undefined/); }); +test("a page is Slack text with its control characters escaped and a long detail cut short", () => { + assert.equal( + slackText("pen-monitor", "critical", "SUBJECT", "to & "), + "[CRITICAL] pen-monitor: SUBJECT\nto <!channel> & <https://x.example|click>", + ); + const text = slackText("pen-monitor", "warning", "SUBJECT", "a".repeat(10_000)); + assert.ok(text.startsWith(`[WARNING] pen-monitor: SUBJECT\n${"a".repeat(3_000)}… (cut short`), text.slice(0, 80)); + assert.ok(text.length < 3_200, String(text.length)); +}); + test("a governance call tree is walked through batches and minting calls are flagged", () => { const setBalance = { section: "tokens", method: "setBalance", args: ["who", "Native", 4_000_000n] }; // The recurring attack proposal: utility.batch of tokens.setBalance to the proposer. diff --git a/monitor/src/checks.ts b/monitor/src/checks.ts index 9cf2755a9..c924dafaa 100644 --- a/monitor/src/checks.ts +++ b/monitor/src/checks.ts @@ -213,6 +213,24 @@ export function describeTimelockOperation( return `operation ${id}: ${callText} — ${status}`; } +export type Severity = "critical" | "warning"; + +/** Longest page detail sent; the monitor's log line keeps the full text. */ +const MAX_PAGE_DETAIL = 3_000; + +/** + * A page as the `text` of a Slack incoming-webhook message: severity, subject + * and detail, with Slack's control characters `&`, `<` and `>` escaped, so + * that no detail can mention anyone or hide a link. + */ +export function slackText(service: string, severity: Severity, subject: string, detail: string): string { + const body = detail.length > MAX_PAGE_DETAIL + ? `${detail.slice(0, MAX_PAGE_DETAIL)}… (cut short; the monitor log has the full text)` + : detail; + const escapes: Record = { "&": "&", "<": "<", ">": ">" }; + return `[${severity.toUpperCase()}] ${service}: ${subject}\n${body}`.replace(/[&<>]/g, (char) => escapes[char]); +} + /** A decoded Pendulum call: `section.method` plus any calls nested in its arguments. */ export interface CallNode { name: string; diff --git a/monitor/src/main.test.ts b/monitor/src/main.test.ts index 2d9dfd7be..ec19351d6 100644 --- a/monitor/src/main.test.ts +++ b/monitor/src/main.test.ts @@ -335,9 +335,20 @@ function burn(pendulum: Pendulum, nonce: bigint, palletAmount: bigint, block = 1 interface Page { subject: string; detail: string; + severity: string; + /** The Slack message as sent. */ + message: Record; status?: number; } +/** A page from the Slack message the monitor sent, with Slack's escapes undone. */ +function readPage(message: Record): Page { + const [, severity, subject, detail] = /^\[(CRITICAL|WARNING)\] pen-monitor: (.*)\n([\s\S]*)$/.exec(String(message.text)) ?? []; + assert.ok(subject, `not a monitor page: ${JSON.stringify(message)}`); + const unescaped = detail.replaceAll("<", "<").replaceAll(">", ">").replaceAll("&", "&"); + return { subject, detail: unescaped, severity, message }; +} + async function body(request: IncomingMessage): Promise { let text = ""; for await (const chunk of request) text += chunk; @@ -369,8 +380,7 @@ async function setup( const server = createServer(async (request, response) => { const text = await body(request); if (request.url === "/alert") { - const { subject, detail } = JSON.parse(text) as Page; - const page: Page = { subject, detail }; + const page = readPage(JSON.parse(text)); pages.push(page); page.status = await webhook(page); response.writeHead(page.status).end(); @@ -444,6 +454,20 @@ test("a released migration reconciles and the poll pages nothing", async (t) => /** Pages sent concurrently can arrive in either order. */ const sorted = (subjects: string[]) => [...subjects].sort(); +test("pages are Slack messages carrying severity, subject and detail in their text only", async (t) => { + const { base, api, monitor, pages } = await setup(t); + base.vaultBalance -= 1n; + base.safeAgeSeconds = 601; + await monitor.poll(api); + assert.ok(pages.length >= 3, JSON.stringify(pages)); + for (const page of pages) assert.deepEqual(Object.keys(page.message), ["text"]); + const severity = (subject: string) => pages.find((page) => page.subject === subject)?.severity; + assert.equal(severity("VAULT BALANCE DEFICIT"), "CRITICAL"); + assert.equal(severity("AUTO-PAUSE UNAVAILABLE"), "CRITICAL"); + assert.equal(severity("BASE VIEW STALLED"), "WARNING"); + assert.match(pages.find((page) => page.subject === "VAULT BALANCE DEFICIT")!.detail, /^balance \d+ \+ released 0 \+ swept 0 < supply/); +}); + test("a vault balance deficit pages and asks for the auto-pause", async (t) => { const { base, api, monitor, subjects } = await setup(t); base.vaultBalance -= 1n; diff --git a/monitor/src/main.ts b/monitor/src/main.ts index 7535377bb..5f7fb9664 100644 --- a/monitor/src/main.ts +++ b/monitor/src/main.ts @@ -32,7 +32,9 @@ import { releasedExceedsMigrated, riskyCalls, type ScheduledCall, + type Severity, shouldAwaitSource, + slackText, toCallNode, type MigrationTuple, vaultConservationDeficit, @@ -382,16 +384,21 @@ function redact(text: string): string { return text.replace(/https?:\/\/[^\s"')]+/gi, ""); } -/** Sends one page and resolves to whether it was delivered: the webhook must - * answer 2xx within the timeout. Without a webhook the log line is the page. */ -async function alert(subject: string, detail: string): Promise { +/** Sends one page, as a Slack incoming-webhook message, and resolves to + * whether it was delivered: the webhook must answer 2xx within the timeout. + * Without a webhook the log line is the page. */ +async function alert( + subject: string, + detail: string, + severity: Severity = subject.startsWith("CRITICAL") ? "critical" : "warning", +): Promise { console.error(`${new Date().toISOString()} ALERT: ${subject} — ${detail}`); if (!config.alertWebhookUrl) return true; try { const response = await fetch(config.alertWebhookUrl, { method: "POST", headers: { "content-type": "application/json" }, - body: JSON.stringify({ service: "pen-monitor", subject, detail: redact(detail) }), + body: JSON.stringify({ text: slackText("pen-monitor", severity, subject, redact(detail)) }), // The webhook is an external dependency; it must never stall the // check loop (or the pause that follows a violation) indefinitely. signal: AbortSignal.timeout(10_000), @@ -446,7 +453,7 @@ async function waitForBaseFinality( async function pauseVault(): Promise { if (!config.guardianPrivateKey) { - await alert("AUTO-PAUSE UNAVAILABLE", "no GUARDIAN_PRIVATE_KEY configured; pause manually NOW"); + await alert("AUTO-PAUSE UNAVAILABLE", "no GUARDIAN_PRIVATE_KEY configured; pause manually NOW", "critical"); return false; } try { @@ -494,7 +501,7 @@ async function pauseVault(): Promise { await alert("vault auto-pause confirmed", `tx=${txHash} ${config.baseFinalityTag}Block=${safeBlock}`); return true; } catch (pauseError) { - await alert("AUTO-PAUSE FAILED", `${pauseError}; pause manually NOW`); + await alert("AUTO-PAUSE FAILED", `${pauseError}; pause manually NOW`, "critical"); return false; } } @@ -543,7 +550,7 @@ async function checkAutoPauseReadiness(): Promise { async function securityViolation(subject: string, detail: string): Promise { // Pause FIRST. The alert webhook is an external dependency with its own // latency; nothing may sit between detecting a violation and the pause. - const notified = alert(subject, detail); + const notified = alert(subject, detail, "critical"); const paused = await pauseVault(); await notified; return paused; From d6e6024a90bb6f774981dc6092221867877673d7 Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Thu, 22 Oct 2026 12:11:20 +0200 Subject: [PATCH 32/75] feat(monitor): Request an optional heartbeat URL after each complete poll With HEARTBEAT_URL set, the monitor sends a GET with a short timeout to it after every poll that ran all checks, for a dead-man's switch such as Healthchecks.io that pages when the requests stop. A poll that could not read Pendulum, scan the vault admin events, reconcile or watch governance sends none. A failed request is logged and never stops the monitor. --- monitor/README.md | 4 +++- monitor/src/main.test.ts | 40 +++++++++++++++++++++++++++++++++++++++- monitor/src/main.ts | 33 +++++++++++++++++++++++++++------ 3 files changed, 69 insertions(+), 8 deletions(-) diff --git a/monitor/README.md b/monitor/README.md index 45b00ac33..d6b847ced 100644 --- a/monitor/README.md +++ b/monitor/README.md @@ -39,6 +39,7 @@ Checks every poll: | `ACKNOWLEDGED_RELEASE_EXCESS` | Excess of released over migrated, in token units (18 decimals), that operators quantified and accepted after a real loss; M2a fires only beyond it (default 0). Set it at the RB-3 sign-off and restart; the startup log shows it | | `TIMELOCK_ADDRESS` | Optional: the governance TimelockController. Set it as soon as governance is deployed; the queue is scanned from the monitor's Base cursor onwards, so operations scheduled before the monitor knew the address are not paged | | `ALERT_WEBHOOK_URL` | Slack incoming webhook receiving the pages (see [Pages](#pages)). A page counts as delivered only on a 2xx answer within 10s: once-only pages (governance items, admin events, scheduled operations, first sightings) are retried every poll until delivered, and failed deliveries are logged. Without it the log line is the page | +| `HEARTBEAT_URL` | Optional: a dead-man's switch URL (Healthchecks.io style), requested with a `GET` (10s timeout) after every poll that ran all checks; a failed request is logged and never stops the monitor. Have the service page when the requests stop: the monitor is down, stuck, or cannot complete its checks (for example while the Pendulum source cannot be read) | | `GUARDIAN_PRIVATE_KEY` | Optional: the key of `vault.guardian()`, enabling automatic `pause()` on conservation violations; its readiness is checked at startup and every poll, and any other key pages `AUTO-PAUSE NOT READY` (the vault admin may pause too, but its key does not belong on the monitor) | | `GUARDIAN_MIN_BALANCE_WEI` | Guardian ETH balance below which `AUTO-PAUSE NOT READY` pages (default 10^15 wei = 0.001 ETH) | | `BASE_CHAIN_ID` | Default 8453; the Base RPC must serve it | @@ -81,7 +82,8 @@ npm start ``` Run it under a process manager and treat "monitor down" itself as a paging -condition: an unwatched migration is the risk model failing silently. The +condition (`HEARTBEAT_URL`): an unwatched migration is the risk model failing +silently. The two-chain cursors, unresolved canonical tuples, their original finalized block timestamps, and alert throttles are atomically persisted. A malformed or Pendulum-genesis/Base-chain/vault-mismatched state file is fatal rather than diff --git a/monitor/src/main.test.ts b/monitor/src/main.test.ts index ec19351d6..af377eb4c 100644 --- a/monitor/src/main.test.ts +++ b/monitor/src/main.test.ts @@ -360,16 +360,19 @@ let monitorEnv: string[] = []; /** A started monitor against fresh stand-ins, which `prepare` can change * before startup. `webhook` answers each page with an HTTP status; `pages` - * lists them in arrival order from after startup, `startup` the ones before. */ + * lists them in arrival order from after startup, `startup` the ones before. + * `heartbeats` lists the heartbeat requests, which `heartbeat` answers. */ async function setup( t: { after(fn: () => void): void }, { env = {}, webhook = () => 200, + heartbeat = () => 200, prepare = () => {}, }: { env?: Record; webhook?: (page: Page) => number | Promise; + heartbeat?: () => number; prepare?: (base: Base, pendulum: Pendulum) => void; } = {}, ) { @@ -377,8 +380,14 @@ async function setup( const { chain: pendulum, api } = pendulumChain(); prepare(base, pendulum); const pages: Page[] = []; + const heartbeats: string[] = []; const server = createServer(async (request, response) => { const text = await body(request); + if (request.url === "/heartbeat") { + heartbeats.push(request.method ?? ""); + response.writeHead(heartbeat()).end(); + return; + } if (request.url === "/alert") { const page = readPage(JSON.parse(text)); pages.push(page); @@ -411,6 +420,7 @@ async function setup( PENDULUM_START_BLOCK: "101", STATE_FILE: stateFile, ALERT_WEBHOOK_URL: `${url}/alert`, + HEARTBEAT_URL: `${url}/heartbeat`, ...env, }; for (const name of monitorEnv) delete process.env[name]; @@ -429,6 +439,7 @@ async function setup( api, pages, startup, + heartbeats, monitor, /** A new monitor process on the same state file and stand-ins. */ restart: launch, @@ -468,6 +479,33 @@ test("pages are Slack messages carrying severity, subject and detail in their te assert.match(pages.find((page) => page.subject === "VAULT BALANCE DEFICIT")!.detail, /^balance \d+ \+ released 0 \+ swept 0 < supply/); }); +test("only a poll that runs every check requests the heartbeat", async (t) => { + const { base, pendulum, api, monitor, heartbeats } = await setup(t); + await monitor.poll(api); + assert.deepEqual(heartbeats, ["GET"]); + pendulum.failure = new Error("node unreachable"); + await monitor.poll(api); + pendulum.failure = undefined; + const guardianUpdated = toEventSelector("GuardianUpdated(address)"); // only the admin scan asks for it + base.failing = (method, params) => method === "eth_getLogs" && JSON.stringify(params).includes(guardianUpdated); + base.safe = 1_020n; // a new range for the admin scan + await monitor.poll(api); + assert.deepEqual(heartbeats, ["GET"]); + base.failing = undefined; + await monitor.poll(api); + assert.deepEqual(heartbeats, ["GET", "GET"]); +}); + +test("a failing heartbeat never stops the monitor", async (t) => { + const { api, monitor, heartbeats, subjects } = await setup(t, { heartbeat: () => 500 }); + await monitor.poll(api); + await monitor.poll(api); + assert.deepEqual(heartbeats, ["GET", "GET"]); + const unreachable = await setup(t, { env: { HEARTBEAT_URL: "http://127.0.0.1:1/heartbeat" } }); + await unreachable.monitor.poll(unreachable.api); + assert.deepEqual([...subjects(), ...unreachable.subjects()], []); +}); + test("a vault balance deficit pages and asks for the auto-pause", async (t) => { const { base, api, monitor, subjects } = await setup(t); base.vaultBalance -= 1n; diff --git a/monitor/src/main.ts b/monitor/src/main.ts index 5f7fb9664..3d5e367e1 100644 --- a/monitor/src/main.ts +++ b/monitor/src/main.ts @@ -201,6 +201,8 @@ const config = { graceSeconds: envNumber("GRACE_SECONDS", 1800), unmatchedEventGraceSeconds: envNumber("UNMATCHED_EVENT_GRACE_SECONDS", 600), alertWebhookUrl: process.env.ALERT_WEBHOOK_URL, + /** Optional: requested after every poll that ran all checks. */ + heartbeatUrl: process.env.HEARTBEAT_URL, guardianPrivateKey: process.env.GUARDIAN_PRIVATE_KEY as `0x${string}` | undefined, baseChainId: envNumber("BASE_CHAIN_ID", 8453), baseFinalityTag, @@ -1200,7 +1202,8 @@ async function readSource(api: ApiPromise, draft: RuntimeState): Promise { +/** One pass of the checks; resolves to whether every one of them ran. */ +async function check(api: ApiPromise): Promise { const confirmedBase = await publicClient.getBlock({ blockTag: config.baseFinalityTag }); if (confirmedBase.number === null) throw new Error(`${config.baseFinalityTag} Base block has no number`); const baseBlock = confirmedBase.number; @@ -1249,9 +1252,11 @@ async function check(api: ApiPromise): Promise { // The admin scan only pages and keeps its own cursor, so nothing else waits // for it, and its failure skips no other check. + let adminScanned = true; try { await watchBaseAdmin(baseBlock); } catch (adminError) { + adminScanned = false; if (Date.now() - lastAdminScanFailureAlertMs > config.unmatchedEventGraceSeconds * 1000) { lastAdminScanFailureAlertMs = Date.now(); await alert( @@ -1290,12 +1295,12 @@ async function check(api: ApiPromise): Promise { state = draft; persistState(state); } - return; + return false; } if (!source) { state = draft; // reconciled against the last view read in full persistState(state); - return; + return false; } const { finalizedBlock, totalMigrated } = source; if (reconciliation.awaitingSource) { @@ -1305,7 +1310,7 @@ async function check(api: ApiPromise): Promise { `waiting up to ${config.unmatchedEventGraceSeconds}s for Pendulum RPC to catch up with ` + reconciliation.awaitingSource, ); - return; + return false; } if (releasedExceedsMigrated(totalReleased, totalMigrated, conversionFactor, config.acknowledgedReleaseExcess)) { if (await securityViolation( @@ -1316,7 +1321,7 @@ async function check(api: ApiPromise): Promise { state = draft; persistState(state); } - return; + return false; } const pending = [...draft.migrations.values()]; @@ -1370,6 +1375,19 @@ async function check(api: ApiPromise): Promise { `ok: migrated=${totalMigrated} released=${totalReleased} swept=${totalSwept} ` + `pending=${state.migrations.size} vaultBalance=${vaultBalance} base=${baseBlock} pendulum=${finalizedBlock}`, ); + return adminScanned; +} + +/** Tells an external dead-man's switch (Healthchecks.io style) that a poll + * ran every check; it pages when these requests stop. Never fatal. */ +async function heartbeat(): Promise { + if (!config.heartbeatUrl) return; + try { + const response = await fetch(config.heartbeatUrl, { signal: AbortSignal.timeout(10_000) }); + if (!response.ok) console.error(`heartbeat answered ${response.status}`); + } catch (heartbeatError) { + console.error(`heartbeat failed: ${redact(`${heartbeatError}`)}`); + } } /** Startup once the Pendulum API is connected: durable state, the multicall @@ -1413,20 +1431,23 @@ export async function start(api: ApiPromise): Promise { /** One poll; a failure pages and the next poll starts afresh. */ export async function poll(api: ApiPromise): Promise { + let complete = false; try { - await check(api); + complete = await check(api); } catch (checkError) { await alert("monitor check failed", `${checkError}`); } try { await watchPendulumGovernance(api); } catch (governanceError) { + complete = false; if (Date.now() - lastGovernanceWatchFailureMs > config.graceSeconds * 1000) { lastGovernanceWatchFailureMs = Date.now(); await alert("pendulum governance watch failed", `${governanceError}`); } } await retryUndeliveredPages(); + if (complete) await heartbeat(); } async function main(): Promise { From f0bf444f185f4845d7d4cd3ccf62c8a4a7fd530b Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Thu, 22 Oct 2026 12:16:42 +0200 Subject: [PATCH 33/75] feat(monitor): Pause on disagreeing approvals only from two attestors The monitor paused the vault on the first approval that disagreed with the finalized Pendulum burn, although one approval can never release. A single attestor's disagreeing approval now pages CRITICAL: MISMATCHED APPROVAL and does not pause. Disagreeing approvals of one nonce from two or more distinct attestors still pause, also when they arrive in different polls: they are kept in the state file until the nonce is released, and one the source view later confirms no longer counts. Any mismatching release pauses as before. --- monitor/README.md | 5 ++-- monitor/src/checks.test.ts | 15 +++++++++++ monitor/src/checks.ts | 18 +++++++++++++ monitor/src/main.test.ts | 53 +++++++++++++++++++++++++++++++++++--- monitor/src/main.ts | 52 +++++++++++++++++++++++++++++++------ monitor/src/state.test.ts | 19 ++++++++++---- monitor/src/state.ts | 19 ++++++++++++++ 7 files changed, 163 insertions(+), 18 deletions(-) diff --git a/monitor/README.md b/monitor/README.md index d6b847ced..caacdfa96 100644 --- a/monitor/README.md +++ b/monitor/README.md @@ -8,13 +8,13 @@ Checks every poll: | Check | Meaning | Reaction | |---|---|---| -| M2 tuple match | Every safe Base `Approved` and `Released` event must match one finalized Pendulum `MigrationInitiated` nonce, recipient, and pallet amount; `Released.tokenAmount` must equal `palletAmount × conversionFactor`. After a violation the Base cursor stays on any block range whose events the source view may still reveal, so a pause caused by a lagging source does not lose those releases. | Alert + auto-pause. A mismatch against a *known* nonce pauses immediately. An *unknown* nonce first pages (`UNVERIFIED BASE EVENT`, without delaying the pause decision), then pauses as soon as it is provable — the finalized source view passed the event's own timestamp and the nonce still does not exist — or after `UNMATCHED_EVENT_GRACE_SECONDS` if the source view stays behind or cannot be read (a stalled or unreadable source also pages on its own, see below) | +| M2 tuple match | Every safe Base `Approved` and `Released` event must match one finalized Pendulum `MigrationInitiated` nonce, recipient, and pallet amount; `Released.tokenAmount` must equal `palletAmount × conversionFactor`. After a violation the Base cursor stays on any block range whose events the source view may still reveal, so a pause caused by a lagging source does not lose those releases. | Alert + auto-pause on any mismatching `Released`, and on approvals of one nonce that disagree with the finalized Pendulum view from two or more distinct attestors, also across polls and restarts. A single attestor's disagreeing approval pages `CRITICAL: MISMATCHED APPROVAL` instead: one approval can never release. A mismatch against a *known* nonce counts immediately. An *unknown* nonce first pages (`UNVERIFIED BASE EVENT`, without delaying the pause decision), then counts as a mismatch as soon as it is provable — the finalized source view passed the event's own timestamp and the nonce still does not exist — or after `UNMATCHED_EVENT_GRACE_SECONDS` if the source view stays behind or cannot be read (a stalled or unreadable source also pages on its own, see below) | | M2a aggregate | `totalReleased <= TotalMigrated × conversionFactor + ACKNOWLEDGED_RELEASE_EXCESS`. This remains independent defence-in-depth around the event reconciliation. After a real, quantified loss, operators set `ACKNOWLEDGED_RELEASE_EXCESS` to it (RB-3) so the vault can be unpaused; any further excess still fires. | Alert + auto-pause | | M2b: `balanceOf(vault) + totalReleased + totalSwept >= totalSupply` | Vault-internal conservation. Only a **deficit** alerts: a surplus is a harmless inbound transfer (donation, or a migration whose recipient is the vault) and is ignored, so it cannot false-trigger a pause. It reads Base only and runs first in every poll, so a slow or failing Pendulum source or Base log scan can neither delay nor skip it. | Alert + auto-pause on a deficit | | M4: every old migration reaches active approval quorum — and then releases | Liveness of the attestor fleet (`below approval quorum`), and of everything after it (`quorum reached but not released`: a release starved of daily allowance, paused, or made releasable by a threshold cut without a `ReleasePending`, which the releaser cannot see). | One summary page per kind and poll (count and the first nonces; the log lists every nonce), each nonce at most once per `GRACE_SECONDS` | | Source supply | `totalIssuance + TotalMigrated` on Pendulum must never grow past its first-seen anchor. | Alert (`SOURCE SUPPLY GREW`) | | Unreleasable burns | A burn to the zero or vault address can never be approved or released; it is paged once and excluded from the pending count so it cannot page liveness forever or block RB-7. | Alert once | -| Pendulum source | The finalized Pendulum view must stay fresh and readable. While it cannot be read (an RPC error, a nonce discontinuity, an event-shape change), the Base-side checks — vault balance and its auto-pause, Base view, auto-pause readiness, vault admin events, governance queue — still run, and Base events are reconciled against the last view read in full: an unreadable source counts as a stalled one, so a mismatch on a known nonce pauses at once and an unknown nonce after `UNMATCHED_EVENT_GRACE_SECONDS`. M2a and liveness wait for the source. | Alert, throttled: `PENDULUM SOURCE VIEW STALLED` once the view is older than `SOURCE_STALE_ALERT_SECONDS`, `PENDULUM SOURCE UNREADABLE` on a failure | +| Pendulum source | The finalized Pendulum view must stay fresh and readable. While it cannot be read (an RPC error, a nonce discontinuity, an event-shape change), the Base-side checks — vault balance and its auto-pause, Base view, auto-pause readiness, vault admin events, governance queue — still run, and Base events are reconciled against the last view read in full: an unreadable source counts as a stalled one, so a mismatch on a known nonce counts at once and an unknown nonce after `UNMATCHED_EVENT_GRACE_SECONDS`. M2a and liveness wait for the source. | Alert, throttled: `PENDULUM SOURCE VIEW STALLED` once the view is older than `SOURCE_STALE_ALERT_SECONDS`, `PENDULUM SOURCE UNREADABLE` on a failure | | Base view | The safe/finalized Base block must not be older than `BASE_STALE_ALERT_SECONDS`: a view that stops advancing hides every later vault event while every check still passes at the old block. | Alert (`BASE VIEW STALLED`), throttled | | Auto-pause readiness | At startup and every poll: the Base RPC serves `BASE_CHAIN_ID` and, with a guardian key configured, the key parses, is `vault.guardian()` and holds at least `GUARDIAN_MIN_BALANCE_WEI`. A read that fails pages as well and never stops the poll's other checks. | Alert (`AUTO-PAUSE NOT READY`), throttled; `AUTO-PAUSE DISABLED` once at startup without a key | | Pendulum governance | Every pending public proposal, the council's external proposal, every ongoing referendum and every open council and technical-committee motion, with the call decoded. A call that mints, moves or unlocks balances outside normal transfers, or changes the runtime or the migration pallet, pages as critical: minted PEN burned through the migration drains the fixed-supply vault, and only Pendulum governance can stop the call before enactment. A call the monitor cannot read pages as critical too: one over 64 KiB is never fetched, and one that does not decode or nests more than 256 calls or 16 levels deep is unreadable; one whose preimage is not noted yet is re-checked every poll and paged again once it can be read. The watch runs after the poll's conservation checks, so it cannot stop or delay them, and it decodes within fixed bounds, so its cost to the next poll is bounded. | Alert once per item (`PENDULUM GOVERNANCE ITEM`, `CRITICAL: Pendulum governance item …`); a motion proposed again with the same call (a new motion index), or an external proposal made again after it left the queue or became a referendum, pages again; a restart re-pages the items still pending | @@ -65,6 +65,7 @@ starts with `CRITICAL`; everything else is a `WARNING`. |---|---| | `MIGRATION TUPLE VIOLATION`, `VAULT BALANCE DEFICIT`, `CONSERVATION VIOLATION` | A conservation check failed; the auto-pause follows with `vault auto-pause confirmed`, `vault pause confirmed` (already paused), `AUTO-PAUSE PENDING FINALITY`, `AUTO-PAUSE FAILED` or `AUTO-PAUSE UNAVAILABLE` | | `UNVERIFIED BASE EVENT` | First sighting of a Base event whose nonce the source view does not know yet | +| `CRITICAL: MISMATCHED APPROVAL` | One attestor approved a tuple that disagrees with the finalized Pendulum view; check its key and its Pendulum node. A second attestor doing so for the same nonce pauses the vault | | `LIVENESS: migration below approval quorum`, `LIVENESS: quorum reached but not released` | Stalled migrations, one summary per kind and poll | | `PENDULUM SOURCE VIEW STALLED`, `PENDULUM SOURCE UNREADABLE`, `BASE VIEW STALLED`, `BASE ADMIN SCAN FAILED` | The monitor is going blind on one chain | | `AUTO-PAUSE NOT READY`, `AUTO-PAUSE DISABLED` | The next violation would not pause the vault automatically | diff --git a/monitor/src/checks.test.ts b/monitor/src/checks.test.ts index 780d1f837..2c3975786 100644 --- a/monitor/src/checks.test.ts +++ b/monitor/src/checks.test.ts @@ -14,6 +14,7 @@ import { chunks, describeCall, describeTimelockOperation, + disagreeingAttestors, isStale, provablyUnsourced, releaseMismatch, @@ -123,6 +124,20 @@ test("M2 tuple matching rejects missing, wrong-recipient and wrong-amount approv assert.equal(approvalMismatch(expected, { ...expected, recipient: expected.recipient.toUpperCase() }), undefined); }); +test("approvals against the burn count once per attestor and only while they disagree", () => { + const burn = { nonce: 7n, recipient: "0x1111111111111111111111111111111111111111", palletAmount: 100n }; + const [a, b] = ["0xA1a1a1a1a1A1a1a1a1a1a1A1a1a1a1a1a1a1A1a1", "0xa2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2"]; + const approvals = [ + { ...burn, palletAmount: 99n, attestor: a }, + { ...burn, palletAmount: 98n, attestor: a.toLowerCase() }, + { ...burn, attestor: b }, // agrees + { ...burn, nonce: 8n, palletAmount: 99n, attestor: b }, // another nonce + ]; + assert.deepEqual(disagreeingAttestors(burn, 7n, approvals), [a.toLowerCase()]); + assert.deepEqual(disagreeingAttestors(undefined, 7n, approvals), [a.toLowerCase(), b]); // no burn known + assert.deepEqual(disagreeingAttestors({ ...burn, palletAmount: 99n }, 7n, approvals), [a.toLowerCase(), b]); +}); + test("M2 release matching includes the one-and-only decimal conversion", () => { const expected = { nonce: 7n, recipient: "0x1111111111111111111111111111111111111111", palletAmount: 100n }; assert.equal(releaseMismatch(expected, { ...expected, tokenAmount: 100n * CF }, CF), undefined); diff --git a/monitor/src/checks.ts b/monitor/src/checks.ts index c924dafaa..273272e19 100644 --- a/monitor/src/checks.ts +++ b/monitor/src/checks.ts @@ -131,6 +131,24 @@ export function approvalMismatch(expected: MigrationTuple | undefined, actual: M return undefined; } +/** + * The distinct attestors whose recorded approvals of `nonce` disagree with its + * finalized burn, or with its absence. One attestor's approval can never + * release (the threshold is higher), so one pages; two pause the vault before + * a third could complete a quorum. Approvals that the source view has since + * confirmed no longer count. + */ +export function disagreeingAttestors( + expected: MigrationTuple | undefined, + nonce: bigint, + approvals: Array, +): string[] { + const attestors = approvals + .filter((approval) => approval.nonce === nonce && approvalMismatch(expected, approval)) + .map((approval) => approval.attestor.toLowerCase()); + return [...new Set(attestors)]; +} + export function releaseMismatch( expected: MigrationTuple | undefined, actual: MigrationTuple & { tokenAmount: bigint }, diff --git a/monitor/src/main.test.ts b/monitor/src/main.test.ts index af377eb4c..83c47c159 100644 --- a/monitor/src/main.test.ts +++ b/monitor/src/main.test.ts @@ -448,6 +448,7 @@ async function setup( baseFromBlock: string; baseAdminFromBlock: string; migrations: Array<{ nonce: string }>; + mismatchedApprovals: Array<{ nonce: string; attestor: string }>; }, }; } @@ -573,18 +574,64 @@ test("while Pendulum cannot be read, a Base event with an unknown nonce pauses a assert.ok(subjects().includes("MIGRATION TUPLE VIOLATION"), JSON.stringify(subjects())); }); -test("while Pendulum cannot be read, a mismatch on a known nonce pauses at once", async (t) => { +test("while Pendulum cannot be read, a mismatching release of a known nonce pauses at once", async (t) => { const { base, pendulum, api, monitor, pages } = await setup(t); burn(pendulum, 0n, 100n * PEN); await monitor.poll(api); pendulum.failure = new Error("node unreachable"); - const approval = { nonce: 0n, recipient: TOKEN, palletAmount: 100n * PEN, attestor: ATTESTORS[0] }; - base.logs.push({ address: VAULT, eventName: "Approved", args: approval, blockNumber: 1_012n }); + const tokenAmount = 100n * PEN * CF; + const released = { nonce: 0n, recipient: TOKEN, palletAmount: 100n * PEN, tokenAmount }; + base.logs.push({ address: VAULT, eventName: "Released", args: released, blockNumber: 1_012n }); + base.totalReleased += tokenAmount; + base.vaultBalance -= tokenAmount; base.safe = 1_020n; await monitor.poll(api); assert.match(pages.find((page) => page.subject === "MIGRATION TUPLE VIOLATION")?.detail ?? "", /recipient/); }); +/** An approval of `nonce` for `recipient` from attestor `attestor`. */ +function approve(base: Base, nonce: bigint, recipient: string, attestor: string, blockNumber: bigint): void { + base.logs.push({ address: VAULT, eventName: "Approved", args: { nonce, recipient, palletAmount: 100n * PEN, attestor }, blockNumber }); +} + +test("the release of a nonce forgets the approvals of it that disagreed", async (t) => { + const { base, pendulum, api, monitor, persisted } = await setup(t); + burn(pendulum, 0n, 100n * PEN); + approve(base, 0n, TOKEN, ATTESTORS[0], 1_004n); + await monitor.poll(api); + assert.equal(persisted().mismatchedApprovals.length, 1); + release(base, 0n, 100n * PEN, 1_012n); + base.safe = 1_020n; + await monitor.poll(api); + assert.deepEqual(persisted().mismatchedApprovals, []); +}); + +test("one attestor approving against the burn pages critical; a second one pauses, also polls later", async (t) => { + const { base, pendulum, api, monitor, restart, pages, subjects } = await setup(t); + burn(pendulum, 0n, 100n * PEN); + await monitor.poll(api); + approve(base, 0n, TOKEN, ATTESTORS[0], 1_012n); + base.safe = 1_020n; + await monitor.poll(api); + const page = pages.find((page) => page.subject === "CRITICAL: MISMATCHED APPROVAL"); + assert.equal(page?.severity, "CRITICAL", JSON.stringify(pages)); + assert.match(page.detail, new RegExp(`by attestor ${ATTESTORS[0]}: nonce 0 recipient ${TOKEN}`, "i")); + assert.ok(!subjects().includes("MIGRATION TUPLE VIOLATION"), JSON.stringify(subjects())); + approve(base, 0n, RECIPIENT, ATTESTORS[1], 1_022n); // agrees with the burn + approve(base, 0n, "0x5555555555555555555555555555555555555555", ATTESTORS[0], 1_023n); // the same attestor again + base.safe = 1_030n; + await monitor.poll(api); + assert.ok(!subjects().includes("MIGRATION TUPLE VIOLATION"), JSON.stringify(subjects())); + const restarted = await restart(); + approve(base, 0n, TOKEN, ATTESTORS[2], 1_032n); + base.safe = 1_040n; + await restarted.poll(api); + assert.match( + pages.find((page) => page.subject === "MIGRATION TUPLE VIOLATION")?.detail ?? "", + new RegExp(`attestors ${ATTESTORS[0]}, ${ATTESTORS[2]} approved tuples for nonce 0`, "i"), + ); +}); + test("auto-pause readiness is checked at startup and each problem pages", async (t) => { const { startup } = await setup(t, { env: { GUARDIAN_PRIVATE_KEY: GUARDIAN_KEY }, diff --git a/monitor/src/main.ts b/monitor/src/main.ts index 3d5e367e1..006a19eea 100644 --- a/monitor/src/main.ts +++ b/monitor/src/main.ts @@ -26,6 +26,7 @@ import { chunks, describeCall, describeTimelockOperation, + disagreeingAttestors, isStale, provablyUnsourced, releaseMismatch, @@ -273,6 +274,9 @@ interface RuntimeState { migrations: Map; livenessAlertedAt: Map; unmatchedBaseFirstSeenAt: Map; + /** Approvals that disagreed with the finalized burn, or found none, kept + * until their nonce is released (see `disagreeingAttestors`). */ + mismatchedApprovals: Array; /** Pendulum totalIssuance + TotalMigrated at first observation. Under every * legitimate flow this sum can only fall (teleport-out) and later recover * to at most its original value (teleport-in); growth means PEN was minted @@ -295,6 +299,7 @@ function hydrateState(): RuntimeState { migrations: new Map(), livenessAlertedAt: new Map(), unmatchedBaseFirstSeenAt: new Map(), + mismatchedApprovals: [], issuanceAnchor: undefined, }; } @@ -318,6 +323,11 @@ function hydrateState(): RuntimeState { ])), livenessAlertedAt: new Map(persisted.livenessAlertedAt.map(([nonce, at]) => [BigInt(nonce), at])), unmatchedBaseFirstSeenAt: new Map(persisted.unmatchedBaseFirstSeenAt), + mismatchedApprovals: (persisted.mismatchedApprovals ?? []).map((approval) => ({ + ...approval, + nonce: BigInt(approval.nonce), + palletAmount: BigInt(approval.palletAmount), + })), issuanceAnchor: persisted.issuanceAnchor === undefined ? undefined : BigInt(persisted.issuanceAnchor), }; } @@ -334,6 +344,7 @@ function cloneState(source: RuntimeState): RuntimeState { migrations: new Map(source.migrations), livenessAlertedAt: new Map(source.livenessAlertedAt), unmatchedBaseFirstSeenAt: new Map(source.unmatchedBaseFirstSeenAt), + mismatchedApprovals: [...source.mismatchedApprovals], issuanceAnchor: source.issuanceAnchor, }; } @@ -358,6 +369,12 @@ function persistState(state: RuntimeState): void { })), livenessAlertedAt: [...state.livenessAlertedAt].map(([nonce, at]) => [nonce.toString(), at]), unmatchedBaseFirstSeenAt: [...state.unmatchedBaseFirstSeenAt], + mismatchedApprovals: state.mismatchedApprovals.map((approval) => ({ + nonce: approval.nonce.toString(), + recipient: approval.recipient as `0x${string}`, + palletAmount: approval.palletAmount.toString(), + attestor: approval.attestor as `0x${string}`, + })), issuanceAnchor: state.issuanceAnchor?.toString(), }; saveMonitorState(config.stateFile, persisted); @@ -720,9 +737,9 @@ async function reconcileBaseEvents( // Preflight the complete range before deleting released migrations. If a // source RPC is merely behind, the range must be replayed intact later. for (const event of approvals) { - const actual = event.args as { nonce: bigint; recipient: `0x${string}`; palletAmount: bigint }; + const actual = event.args as { nonce: bigint; recipient: `0x${string}`; palletAmount: bigint; attestor: string }; const expected = draft.migrations.get(actual.nonce); - const mismatch = approvalMismatch(expected, actual); + let mismatch = approvalMismatch(expected, actual); const key = `Approved:${actual.nonce}`; if (!mismatch) { draft.unmatchedBaseFirstSeenAt.delete(key); @@ -740,13 +757,31 @@ async function reconcileBaseEvents( ); if (disposition !== "proven") awaitingSource ??= `Approved nonce ${actual.nonce} at Base block ${event.blockNumber}`; if (disposition === "await") continue; - firstViolation ??= `Approved at Base block ${event.blockNumber}: ${mismatch} (` + - (disposition === "proven" - ? "source view has passed the event's timestamp — fabricated" - : "source view still behind after the full grace period") + ")"; + mismatch += disposition === "proven" + ? " (source view has passed the event's timestamp — fabricated)" + : " (source view still behind after the full grace period)"; + } + // One attestor's approval cannot release: it pages. Two attestors + // approving tuples that disagree with the burn pause the vault. + const { nonce, recipient, palletAmount, attestor } = actual; + const seen = draft.mismatchedApprovals.some((approval) => + approval.nonce === nonce && approval.attestor.toLowerCase() === attestor.toLowerCase() && + !approvalMismatch(approval, actual) + ); + if (!seen) draft.mismatchedApprovals.push({ nonce, recipient, palletAmount, attestor }); + const attestors = disagreeingAttestors(expected, nonce, draft.mismatchedApprovals); + if (attestors.length >= 2) { + firstViolation ??= `Approved at Base block ${event.blockNumber}: ${mismatch}; attestors ` + + `${attestors.join(", ")} approved tuples for nonce ${nonce} that disagree with the finalized Pendulum view`; continue; } - firstViolation ??= `Approved at Base block ${event.blockNumber}: ${mismatch}`; + void pageOnce( + `approval:${event.transactionHash}:${event.logIndex}`, + "CRITICAL: MISMATCHED APPROVAL", + `Approved at Base block ${event.blockNumber} by attestor ${attestor}: ${mismatch}; one attestor's approval ` + + "cannot release, so the vault is not paused — check that attestor's key and Pendulum node; a second " + + "attestor approving this nonce against its burn pauses the vault", + ); } for (const event of releases) { const actual = event.args as { @@ -806,6 +841,7 @@ async function reconcileBaseEvents( releasedInScan.add(actual.nonce); draft.migrations.delete(actual.nonce); draft.livenessAlertedAt.delete(actual.nonce); + draft.mismatchedApprovals = draft.mismatchedApprovals.filter((approval) => approval.nonce !== actual.nonce); } draft.baseFromBlock = end + 1n; } @@ -1270,7 +1306,7 @@ async function check(api: ApiPromise): Promise { // A Pendulum failure must not switch off the Base-side checks. Base events // are then reconciled against the last view read in full, as against a // lagging source but with no fabrication proof: a mismatch on a known nonce - // still pauses at once, an unknown nonce after the grace. The vault balance + // is judged at once, an unknown nonce after the grace. The vault balance // check needs no source; M2a and liveness wait for it. let draft = cloneState(state); let source: SourceView | undefined; diff --git a/monitor/src/state.test.ts b/monitor/src/state.test.ts index c7ab42e1a..c37e73eba 100644 --- a/monitor/src/state.test.ts +++ b/monitor/src/state.test.ts @@ -70,14 +70,23 @@ test("missing unmatched-event history is rejected rather than resetting its grac } }); -test("optional cursors are checked when present", () => { +test("optional fields are checked when present", () => { const directory = mkdtempSync(join(tmpdir(), "pen-monitor-state-")); const path = join(directory, "state.json"); + const approval = { + nonce: "2", + recipient: "0x3333333333333333333333333333333333333333", + palletAmount: "100", + attestor: "0x4444444444444444444444444444444444444444", + } as const; try { - saveMonitorState(path, { ...state, baseAdminFromBlock: "150" }); - assert.equal(loadMonitorState(path)?.baseAdminFromBlock, "150"); - writeFileSync(path, JSON.stringify({ ...state, baseAdminFromBlock: "-1" })); - assert.throws(() => loadMonitorState(path), /invalid or legacy schema/); + const complete = { ...state, baseAdminFromBlock: "150", mismatchedApprovals: [approval] }; + saveMonitorState(path, complete); + assert.deepEqual(loadMonitorState(path), complete); + for (const bad of [{ baseAdminFromBlock: "-1" }, { mismatchedApprovals: [{ ...approval, attestor: "0x44" }] }]) { + writeFileSync(path, JSON.stringify({ ...state, ...bad })); + assert.throws(() => loadMonitorState(path), /invalid or legacy schema/); + } } finally { rmSync(directory, { recursive: true, force: true }); } diff --git a/monitor/src/state.ts b/monitor/src/state.ts index 786de55bd..22a3a16d0 100644 --- a/monitor/src/state.ts +++ b/monitor/src/state.ts @@ -34,8 +34,20 @@ export interface PersistedMonitorState { /** Next Base block of the admin scan; absent in older state, which never * persisted a Base cursor past it, and then `baseFromBlock`. */ baseAdminFromBlock?: string; + /** Approvals that disagreed with the finalized burn, until their nonce is + * released; absent in older state. */ + mismatchedApprovals?: PersistedApproval[]; } +export interface PersistedApproval { + nonce: string; + recipient: `0x${string}`; + palletAmount: string; + attestor: `0x${string}`; +} + +const isAddress = (value: unknown) => typeof value === "string" && /^0x[0-9a-fA-F]{40}$/.test(value); + function isMissing(error: unknown): boolean { return (error as NodeJS.ErrnoException)?.code === "ENOENT"; } @@ -70,6 +82,12 @@ export function loadMonitorState(path: string): PersistedMonitorState | undefine const badAlerts = !Array.isArray(state.livenessAlertedAt) || state.livenessAlertedAt.some( (entry) => !Array.isArray(entry) || entry.length !== 2 || !isUint(entry[0]) || !Number.isSafeInteger(entry[1]), ); + const badApprovals = state.mismatchedApprovals !== undefined && ( + !Array.isArray(state.mismatchedApprovals) || state.mismatchedApprovals.some((approval) => + !isUint(approval?.nonce) || !isAddress(approval?.recipient) || !isUint(approval?.palletAmount) || + !isAddress(approval?.attestor) + ) + ); const badUnmatchedEvents = !Array.isArray(state.unmatchedBaseFirstSeenAt) || state.unmatchedBaseFirstSeenAt.some( (entry) => !Array.isArray(entry) || @@ -92,6 +110,7 @@ export function loadMonitorState(path: string): PersistedMonitorState | undefine (state.baseAdminFromBlock !== undefined && !isUint(state.baseAdminFromBlock)) || badMigration || badAlerts || + badApprovals || badUnmatchedEvents ) { throw new Error(`monitor state ${path} has an invalid or legacy schema; refusing to reset security history`); From 81c3f35ad89ab853404a3df92514c34a6cc82435 Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Thu, 22 Oct 2026 12:28:19 +0200 Subject: [PATCH 34/75] feat(monitor): Check each scanned Base range against the vault's totals The end-block probe and the log query are separate requests, so they can reach different backends. For every scanned range the Released amounts must now add up to the growth of totalReleased, and the RemainderSwept amounts to that of totalSwept, between the block before the range and its last block. The vault events of a range are read in one query. A range whose logs do not account for the totals is read once more, then held: the Base cursor stays on it, M2a and liveness wait, and it pages BASE LOG SCAN INCOMPLETE. Totals before the vault existed count as zero. --- monitor/README.md | 5 ++- monitor/src/main.test.ts | 80 +++++++++++++++++++++++++++++++++++-- monitor/src/main.ts | 86 +++++++++++++++++++++++++++++++--------- 3 files changed, 147 insertions(+), 24 deletions(-) diff --git a/monitor/README.md b/monitor/README.md index caacdfa96..f26bbe375 100644 --- a/monitor/README.md +++ b/monitor/README.md @@ -15,6 +15,7 @@ Checks every poll: | Source supply | `totalIssuance + TotalMigrated` on Pendulum must never grow past its first-seen anchor. | Alert (`SOURCE SUPPLY GREW`) | | Unreleasable burns | A burn to the zero or vault address can never be approved or released; it is paged once and excluded from the pending count so it cannot page liveness forever or block RB-7. | Alert once | | Pendulum source | The finalized Pendulum view must stay fresh and readable. While it cannot be read (an RPC error, a nonce discontinuity, an event-shape change), the Base-side checks — vault balance and its auto-pause, Base view, auto-pause readiness, vault admin events, governance queue — still run, and Base events are reconciled against the last view read in full: an unreadable source counts as a stalled one, so a mismatch on a known nonce counts at once and an unknown nonce after `UNMATCHED_EVENT_GRACE_SECONDS`. M2a and liveness wait for the source. | Alert, throttled: `PENDULUM SOURCE VIEW STALLED` once the view is older than `SOURCE_STALE_ALERT_SECONDS`, `PENDULUM SOURCE UNREADABLE` on a failure | +| Base log completeness | In every scanned block range the `Released` amounts must add up to the growth of `totalReleased`, and the `RemainderSwept` amounts to that of `totalSwept`, between the block before the range and its last block. A range whose logs do not (a backend serving truncated logs) is read once more, then held: the Base cursor stays on it and M2a and liveness wait until a read accounts for the totals. | Alert (`BASE LOG SCAN INCOMPLETE`), throttled | | Base view | The safe/finalized Base block must not be older than `BASE_STALE_ALERT_SECONDS`: a view that stops advancing hides every later vault event while every check still passes at the old block. | Alert (`BASE VIEW STALLED`), throttled | | Auto-pause readiness | At startup and every poll: the Base RPC serves `BASE_CHAIN_ID` and, with a guardian key configured, the key parses, is `vault.guardian()` and holds at least `GUARDIAN_MIN_BALANCE_WEI`. A read that fails pages as well and never stops the poll's other checks. | Alert (`AUTO-PAUSE NOT READY`), throttled; `AUTO-PAUSE DISABLED` once at startup without a key | | Pendulum governance | Every pending public proposal, the council's external proposal, every ongoing referendum and every open council and technical-committee motion, with the call decoded. A call that mints, moves or unlocks balances outside normal transfers, or changes the runtime or the migration pallet, pages as critical: minted PEN burned through the migration drains the fixed-supply vault, and only Pendulum governance can stop the call before enactment. A call the monitor cannot read pages as critical too: one over 64 KiB is never fetched, and one that does not decode or nests more than 256 calls or 16 levels deep is unreadable; one whose preimage is not noted yet is re-checked every poll and paged again once it can be read. The watch runs after the poll's conservation checks, so it cannot stop or delay them, and it decodes within fixed bounds, so its cost to the next poll is bounded. | Alert once per item (`PENDULUM GOVERNANCE ITEM`, `CRITICAL: Pendulum governance item …`); a motion proposed again with the same call (a new motion index), or an external proposal made again after it left the queue or became a referendum, pages again; a restart re-pages the items still pending | @@ -27,7 +28,7 @@ Checks every poll: | Variable | Meaning | |---|---| | `PENDULUM_WS` | WebSocket of the monitor's own Pendulum node, run with `--state-pruning archive` (see below) | -| `BASE_RPC_URL` | Base JSON-RPC endpoint — a **single dedicated node**, and a different provider than the attestors use. Load-balanced pools can serve `eth_getLogs` from a lagging replica; each scan range is probed for its end block first, which turns a behind-node into a loud replay instead of a silent gap, but a dedicated node removes the hazard entirely | +| `BASE_RPC_URL` | Base JSON-RPC endpoint — a **single dedicated node**, and a different provider than the attestors use. Load-balanced pools can serve `eth_getLogs` from a lagging replica; each scan range is probed for its end block first and its logs must account for the vault's running totals, which turns a behind-node into a loud replay instead of a silent gap, but a dedicated node removes the hazard entirely | | `VAULT_ADDRESS` | MigrationVault address on Base | | `POLL_INTERVAL_MS` | Poll cadence (default 60s) | | `GRACE_SECONDS` | Liveness alert threshold (default 30 min) | @@ -67,7 +68,7 @@ starts with `CRITICAL`; everything else is a `WARNING`. | `UNVERIFIED BASE EVENT` | First sighting of a Base event whose nonce the source view does not know yet | | `CRITICAL: MISMATCHED APPROVAL` | One attestor approved a tuple that disagrees with the finalized Pendulum view; check its key and its Pendulum node. A second attestor doing so for the same nonce pauses the vault | | `LIVENESS: migration below approval quorum`, `LIVENESS: quorum reached but not released` | Stalled migrations, one summary per kind and poll | -| `PENDULUM SOURCE VIEW STALLED`, `PENDULUM SOURCE UNREADABLE`, `BASE VIEW STALLED`, `BASE ADMIN SCAN FAILED` | The monitor is going blind on one chain | +| `PENDULUM SOURCE VIEW STALLED`, `PENDULUM SOURCE UNREADABLE`, `BASE VIEW STALLED`, `BASE ADMIN SCAN FAILED`, `BASE LOG SCAN INCOMPLETE` | The monitor is going blind on one chain | | `AUTO-PAUSE NOT READY`, `AUTO-PAUSE DISABLED` | The next violation would not pause the vault automatically | | `SOURCE SUPPLY GREW`, `CRITICAL: unreleasable migration burned on Pendulum` | Supply problems at the source | | `PENDULUM GOVERNANCE ITEM`, `CRITICAL: Pendulum governance item mints, moves or unlocks balances, or changes the runtime`, `CRITICAL: Pendulum governance item cannot be read`, `MIGRATION PALLET ADMIN EVENT` | Pendulum governance | diff --git a/monitor/src/main.test.ts b/monitor/src/main.test.ts index 83c47c159..864657ab0 100644 --- a/monitor/src/main.test.ts +++ b/monitor/src/main.test.ts @@ -54,6 +54,7 @@ const abi = parseAbi([ "function getTimestamp(bytes32 id) view returns (uint256)", "event Approved(uint64 indexed nonce, address indexed recipient, uint256 palletAmount, address indexed attestor)", "event Released(uint64 indexed nonce, address indexed recipient, uint256 palletAmount, uint256 tokenAmount)", + "event RemainderSwept(address indexed to, uint256 amount)", "event GuardianUpdated(address indexed guardian)", "event CapsUpdated(uint256 perReleaseCap, uint256 dailyCap)", "event CallScheduled(bytes32 indexed id, uint256 indexed index, address target, uint256 value, bytes data, bytes32 predecessor, uint256 delay)", @@ -84,10 +85,20 @@ function baseChain() { /** Ether balances by lowercased address. */ balances: new Map([[GUARDIAN.toLowerCase(), 10n ** 18n]]), logs: [] as Log[], + /** Where set, log queries answer as a backend whose head is this block. */ + logsHead: undefined as bigint | undefined, + /** Where set, the vault has no code before this block. */ + deployedAt: undefined as bigint | undefined, /** Requests to answer with an RPC error. */ failing: undefined as ((method: string, params: unknown[]) => boolean) | undefined, }; } + +/** A vault total at `block`: its current value less what later events added. */ +function totalAt(base: Base, total: bigint, eventName: string, field: string, block: bigint | undefined): bigint { + const later = base.logs.filter((log) => log.eventName === eventName && block !== undefined && log.blockNumber > block); + return later.reduce((sum, log) => sum - (log.args[field] as bigint), total); +} type Base = ReturnType; // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -96,8 +107,11 @@ function answer(base: Base, method: string, params: any[]): unknown { switch (method) { case "eth_chainId": return toHex(base.chainId); - case "eth_getCode": - return "0x"; + case "eth_getCode": { + const block = String(params[1]).startsWith("0x") ? BigInt(params[1]) : undefined; + const deployed = base.deployedAt !== undefined && (block === undefined || block >= base.deployedAt); + return String(params[0]).toLowerCase() === VAULT && deployed ? "0x6001" : "0x"; + } case "eth_getBalance": return toHex(base.balances.get(String(params[0]).toLowerCase()) ?? 0n); case "eth_getBlockByNumber": { @@ -108,9 +122,12 @@ function answer(base: Base, method: string, params: any[]): unknown { case "eth_call": { const { functionName, args } = decodeFunctionData({ abi, data: params[0].data }); const key = (args as readonly unknown[] | undefined)?.[0] as string; + const block = String(params[1]).startsWith("0x") ? BigInt(params[1]) : undefined; + const undeployed = base.deployedAt !== undefined && block !== undefined && block < base.deployedAt; + if (params[0].to.toLowerCase() === VAULT && undeployed) return "0x"; const result = { - totalReleased: base.totalReleased, - totalSwept: base.totalSwept, + totalReleased: totalAt(base, base.totalReleased, "Released", "tokenAmount", block), + totalSwept: totalAt(base, base.totalSwept, "RemainderSwept", "amount", block), conversionFactor: CF, token: TOKEN, paused: base.paused, @@ -130,6 +147,7 @@ function answer(base: Base, method: string, params: any[]): unknown { return base.logs.flatMap((log, index) => { if (log.address.toLowerCase() !== String(address).toLowerCase()) return []; if (log.blockNumber < BigInt(fromBlock) || log.blockNumber > BigInt(toBlock)) return []; + if (base.logsHead !== undefined && log.blockNumber > base.logsHead) return []; const logTopics = encodeEventTopics({ abi, eventName: log.eventName, args: log.args } as never) as Hex[]; if (wanted.length && !wanted.includes(logTopics[0])) return []; const { inputs } = abi.find((item) => item.type === "event" && item.name === log.eventName) as AbiEvent; @@ -589,6 +607,58 @@ test("while Pendulum cannot be read, a mismatching release of a known nonce paus assert.match(pages.find((page) => page.subject === "MIGRATION TUPLE VIOLATION")?.detail ?? "", /recipient/); }); +test("logs that miss a release hold the Base cursor and page until a read accounts for the totals", async (t) => { + const { base, pendulum, api, monitor, pages, persisted } = await setup(t); + burn(pendulum, 0n, 100n * PEN); + release(base, 0n, 100n * PEN, 1_005n); + base.logsHead = 1_003n; // logs from a backend that has not reached the release + await monitor.poll(api); + await until(() => pages.some((page) => page.subject === "BASE LOG SCAN INCOMPLETE"), "the incomplete scan page"); + assert.match( + pages.find((page) => page.subject === "BASE LOG SCAN INCOMPLETE")!.detail, + /in Base blocks 1000-1010 the Released events add up to 0 .* totalReleased grew by 100000000000000000000/, + ); + assert.equal(persisted().baseFromBlock, "1000"); + assert.deepEqual(persisted().migrations.map((migration) => migration.nonce), ["0"]); + base.logsHead = undefined; + await monitor.poll(api); + assert.equal(persisted().baseFromBlock, "1011"); + assert.deepEqual(persisted().migrations, []); +}); + +test("logs that miss a sweep hold the Base cursor too", async (t) => { + const { base, api, monitor, pages, persisted } = await setup(t); + base.logs.push({ address: VAULT, eventName: "RemainderSwept", args: { to: TOKEN, amount: 5n }, blockNumber: 1_005n }); + base.totalSwept += 5n; + base.vaultBalance -= 5n; + base.logsHead = 1_003n; + await monitor.poll(api); + await until(() => pages.some((page) => page.subject === "BASE LOG SCAN INCOMPLETE"), "the incomplete scan page"); + assert.match(pages.find((page) => page.subject === "BASE LOG SCAN INCOMPLETE")!.detail, /RemainderSwept events to 0, .* totalSwept by 5/); + assert.equal(persisted().baseFromBlock, "1000"); + base.logsHead = undefined; + await monitor.poll(api); + assert.equal(persisted().baseFromBlock, "1011"); +}); + +test("a release in the first block of a range accounts for the totals", async (t) => { + const { base, pendulum, api, monitor, subjects, persisted } = await setup(t); + burn(pendulum, 0n, 100n * PEN); + release(base, 0n, 100n * PEN, 1_000n); + await monitor.poll(api); + assert.deepEqual(subjects(), []); + assert.equal(persisted().baseFromBlock, "1011"); +}); + +test("the totals count as zero before the vault existed", async (t) => { + const { base, pendulum, api, monitor, subjects, persisted } = await setup(t, { prepare: (base) => (base.deployedAt = 1_000n) }); + burn(pendulum, 0n, 100n * PEN); + release(base, 0n, 100n * PEN, 1_005n); + await monitor.poll(api); + assert.deepEqual(subjects(), []); + assert.equal(persisted().baseFromBlock, "1011"); +}); + /** An approval of `nonce` for `recipient` from attestor `attestor`. */ function approve(base: Base, nonce: bigint, recipient: string, attestor: string, blockNumber: bigint): void { base.logs.push({ address: VAULT, eventName: "Approved", args: { nonce, recipient, palletAmount: 100n * PEN, attestor }, blockNumber }); @@ -896,6 +966,8 @@ test("first sightings of unmatched events do not delay the pause", async (t) => args: { nonce: 0n, recipient: RECIPIENT, palletAmount: 100n * PEN, tokenAmount: 1n }, blockNumber: 1_006n, }); + base.totalReleased += 1n; + base.vaultBalance -= 1n; await monitor.poll(api); while (!events.includes("first sighting answered")) await new Promise((resolve) => setTimeout(resolve, 10)); assert.deepEqual(events, ["first sighting paged", "violation paged", "first sighting answered"]); diff --git a/monitor/src/main.ts b/monitor/src/main.ts index 006a19eea..4d2566299 100644 --- a/monitor/src/main.ts +++ b/monitor/src/main.ts @@ -85,6 +85,14 @@ const vaultAbi = [ { name: "tokenAmount", type: "uint256", indexed: false }, ], }, + { + type: "event", + name: "RemainderSwept", + inputs: [ + { name: "to", type: "address", indexed: true }, + { name: "amount", type: "uint256", indexed: false }, + ], + }, ] as const; /** The vault events of its admin (and of the guardian's pause). */ @@ -387,6 +395,7 @@ let lastSourceFailureAlertMs = 0; let lastBaseStallAlertMs = 0; let lastReadinessAlertMs = 0; let lastAdminScanFailureAlertMs = 0; +let lastLogScanAlertMs = 0; let lastIssuanceAlertMs = 0; /** Once-only pages by key, each held until the webhook accepts it. */ // ponytail: in memory; a restart re-pages what it finds again (Base admin events after the persisted cursor, pending governance items) and drops undelivered pages it does not find again. @@ -702,12 +711,48 @@ async function ingestPendulumEvents(api: ApiPromise, draft: RuntimeState, finali } } +/** A vault total at `blockNumber`: zero before the vault existed. */ +async function vaultTotalAt(functionName: "totalReleased" | "totalSwept", blockNumber: bigint): Promise { + if (blockNumber < 0n) return 0n; + try { + return await publicClient.readContract({ address: config.vaultAddress, abi: vaultAbi, functionName, blockNumber }); + } catch (readError) { + if (!(await publicClient.getBytecode({ address: config.vaultAddress, blockNumber }))) return 0n; + throw readError; + } +} + +/** The vault's events in one block range, and `gap` when they do not account + * for the change of its running totals over the range: the Released amounts + * must add up to the growth of totalReleased, the RemainderSwept amounts to + * that of totalSwept. A backend that serves a truncated log result fails it. */ +async function readVaultRange(start: bigint, end: bigint) { + const [events, releasedBefore, sweptBefore, releasedAfter, sweptAfter] = await Promise.all([ + publicClient.getContractEvents({ address: config.vaultAddress, abi: vaultAbi, fromBlock: start, toBlock: end }), + vaultTotalAt("totalReleased", start - 1n), + vaultTotalAt("totalSwept", start - 1n), + vaultTotalAt("totalReleased", end), + vaultTotalAt("totalSwept", end), + ]); + const approvals = events.filter((event) => event.eventName === "Approved"); + const releases = events.filter((event) => event.eventName === "Released"); + const released = releases.reduce((sum, event) => sum + (event.args as { tokenAmount: bigint }).tokenAmount, 0n); + const swept = events + .filter((event) => event.eventName === "RemainderSwept") + .reduce((sum, event) => sum + (event.args as { amount: bigint }).amount, 0n); + const gap = released !== releasedAfter - releasedBefore || swept !== sweptAfter - sweptBefore + ? `in Base blocks ${start}-${end} the Released events add up to ${released} and the RemainderSwept events to ` + + `${swept}, but totalReleased grew by ${releasedAfter - releasedBefore} and totalSwept by ${sweptAfter - sweptBefore}` + : undefined; + return { approvals, releases, gap }; +} + async function reconcileBaseEvents( draft: RuntimeState, toBlock: bigint, conversionFactor: bigint, sourceFinalizedTsMs: number, -): Promise<{ violation?: string; awaitingSource?: string }> { +): Promise<{ violation?: string; awaitingSource?: string; incomplete?: string }> { const releasedInScan = new Set(); let firstViolation: string | undefined; for (const [start, end] of blockRanges(draft.baseFromBlock, toBlock, config.baseMaxBlockRange)) { @@ -717,20 +762,23 @@ async function reconcileBaseEvents( // would advance the cursor past events this monitor never reconciled. A // lagging node fails this probe loudly instead, and the cycle replays. await publicClient.getBlock({ blockNumber: end }); - const approvals = await publicClient.getContractEvents({ - address: config.vaultAddress, - abi: vaultAbi, - eventName: "Approved", - fromBlock: start, - toBlock: end, - }); - const releases = await publicClient.getContractEvents({ - address: config.vaultAddress, - abi: vaultAbi, - eventName: "Released", - fromBlock: start, - toBlock: end, - }); + // The probe and the log query can still reach different backends, so the + // logs must also account for the vault's totals; a range whose logs do + // not is read once more, then held for the next poll. + let range = await readVaultRange(start, end); + if (range.gap) range = await readVaultRange(start, end); + if (range.gap) { + if (Date.now() - lastLogScanAlertMs > config.unmatchedEventGraceSeconds * 1000) { + lastLogScanAlertMs = Date.now(); + void alert( + "BASE LOG SCAN INCOMPLETE", + `${range.gap}; the Base RPC served logs that miss events, so the range is read again every poll ` + + `and the Base cursor stays at block ${start} — check the Base RPC`, + ); + } + return { violation: firstViolation, incomplete: range.gap }; + } + const { approvals, releases } = range; const observedAt = Date.now(); let awaitingSource: string | undefined; @@ -1339,12 +1387,14 @@ async function check(api: ApiPromise): Promise { return false; } const { finalizedBlock, totalMigrated } = source; - if (reconciliation.awaitingSource) { + if (reconciliation.awaitingSource || reconciliation.incomplete) { state = draft; persistState(state); log( - `waiting up to ${config.unmatchedEventGraceSeconds}s for Pendulum RPC to catch up with ` + - reconciliation.awaitingSource, + reconciliation.incomplete + ? `holding the Base cursor at ${draft.baseFromBlock}: ${reconciliation.incomplete}` + : `waiting up to ${config.unmatchedEventGraceSeconds}s for Pendulum RPC to catch up with ` + + reconciliation.awaitingSource, ); return false; } From fc59f490081f0e1fe9ab4844e1b7cb533a732cf9 Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Thu, 22 Oct 2026 12:32:37 +0200 Subject: [PATCH 35/75] feat(monitor): Ingest Pendulum blocks along a verified parent-hash chain Each poll now walks the parent hashes down from the finalized head to the block ingested last, through headers fetched from the node and hashed by the monitor, and reads every new block at the hash from that walk. The walk must arrive at the hash of the last ingested block, which the state file now keeps. A source whose chain does not, or whose header does not hash to its block hash, pages CRITICAL: PENDULUM SOURCE CONTRADICTED and is treated as unreadable, so nothing from it is ingested. --- monitor/README.md | 4 +- monitor/src/main.test.ts | 59 +++++++++++++++++++++++++--- monitor/src/main.ts | 81 ++++++++++++++++++++++++++++++++++----- monitor/src/state.test.ts | 13 ++++++- monitor/src/state.ts | 4 ++ 5 files changed, 144 insertions(+), 17 deletions(-) diff --git a/monitor/README.md b/monitor/README.md index f26bbe375..4d4f29dc6 100644 --- a/monitor/README.md +++ b/monitor/README.md @@ -15,6 +15,7 @@ Checks every poll: | Source supply | `totalIssuance + TotalMigrated` on Pendulum must never grow past its first-seen anchor. | Alert (`SOURCE SUPPLY GREW`) | | Unreleasable burns | A burn to the zero or vault address can never be approved or released; it is paged once and excluded from the pending count so it cannot page liveness forever or block RB-7. | Alert once | | Pendulum source | The finalized Pendulum view must stay fresh and readable. While it cannot be read (an RPC error, a nonce discontinuity, an event-shape change), the Base-side checks — vault balance and its auto-pause, Base view, auto-pause readiness, vault admin events, governance queue — still run, and Base events are reconciled against the last view read in full: an unreadable source counts as a stalled one, so a mismatch on a known nonce counts at once and an unknown nonce after `UNMATCHED_EVENT_GRACE_SECONDS`. M2a and liveness wait for the source. | Alert, throttled: `PENDULUM SOURCE VIEW STALLED` once the view is older than `SOURCE_STALE_ALERT_SECONDS`, `PENDULUM SOURCE UNREADABLE` on a failure | +| Pendulum chain | Every finalized block the monitor ingests is read at a hash it verified: it walks the parent hashes down from the finalized head through headers it fetches from the node and hashes itself, and the walk must arrive at the hash of the block it ingested last (kept in the state file). | Alert (`CRITICAL: PENDULUM SOURCE CONTRADICTED`), throttled; nothing from the source is ingested, and it is treated as unreadable (see Pendulum source) until it verifies again | | Base log completeness | In every scanned block range the `Released` amounts must add up to the growth of `totalReleased`, and the `RemainderSwept` amounts to that of `totalSwept`, between the block before the range and its last block. A range whose logs do not (a backend serving truncated logs) is read once more, then held: the Base cursor stays on it and M2a and liveness wait until a read accounts for the totals. | Alert (`BASE LOG SCAN INCOMPLETE`), throttled | | Base view | The safe/finalized Base block must not be older than `BASE_STALE_ALERT_SECONDS`: a view that stops advancing hides every later vault event while every check still passes at the old block. | Alert (`BASE VIEW STALLED`), throttled | | Auto-pause readiness | At startup and every poll: the Base RPC serves `BASE_CHAIN_ID` and, with a guardian key configured, the key parses, is `vault.guardian()` and holds at least `GUARDIAN_MIN_BALANCE_WEI`. A read that fails pages as well and never stops the poll's other checks. | Alert (`AUTO-PAUSE NOT READY`), throttled; `AUTO-PAUSE DISABLED` once at startup without a key | @@ -47,7 +48,7 @@ Checks every poll: | `PENDULUM_START_BLOCK` | **Required on first run:** first block to scan, normally the pallet activation block or the next finalized block before unpausing | | `PENDULUM_START_NONCE` | Nonce expected at that block (default 0; set explicitly when starting after any migrations) | | `BASE_START_BLOCK` | **Required on first run:** vault deployment block or an earlier block | -| `STATE_FILE` | Durable cursors (Pendulum, Base, Base admin scan), pending tuples, and liveness timestamps (default `./monitor-state.json`) | +| `STATE_FILE` | Durable cursors (Pendulum with the hash of its last block, Base, Base admin scan), pending tuples, and liveness timestamps (default `./monitor-state.json`) | | `BASE_FINALITY_TAG` | `safe` (default) or `finalized`; only events inside this boundary are trusted | | `BASE_MAX_BLOCK_RANGE` | Maximum `eth_getLogs` range (default 10,000 blocks) | @@ -69,6 +70,7 @@ starts with `CRITICAL`; everything else is a `WARNING`. | `CRITICAL: MISMATCHED APPROVAL` | One attestor approved a tuple that disagrees with the finalized Pendulum view; check its key and its Pendulum node. A second attestor doing so for the same nonce pauses the vault | | `LIVENESS: migration below approval quorum`, `LIVENESS: quorum reached but not released` | Stalled migrations, one summary per kind and poll | | `PENDULUM SOURCE VIEW STALLED`, `PENDULUM SOURCE UNREADABLE`, `BASE VIEW STALLED`, `BASE ADMIN SCAN FAILED`, `BASE LOG SCAN INCOMPLETE` | The monitor is going blind on one chain | +| `CRITICAL: PENDULUM SOURCE CONTRADICTED` | The Pendulum source contradicts the chain the monitor verified; stop trusting that node | | `AUTO-PAUSE NOT READY`, `AUTO-PAUSE DISABLED` | The next violation would not pause the vault automatically | | `SOURCE SUPPLY GREW`, `CRITICAL: unreleasable migration burned on Pendulum` | Supply problems at the source | | `PENDULUM GOVERNANCE ITEM`, `CRITICAL: Pendulum governance item mints, moves or unlocks balances, or changes the runtime`, `CRITICAL: Pendulum governance item cannot be read`, `MIGRATION PALLET ADMIN EVENT` | Pendulum governance | diff --git a/monitor/src/main.test.ts b/monitor/src/main.test.ts index 864657ab0..513aa110f 100644 --- a/monitor/src/main.test.ts +++ b/monitor/src/main.test.ts @@ -228,9 +228,14 @@ interface PendulumEvent { } /** The Pendulum side: finalized blocks with their events, the migration - * pallet's totals and the governance queues. Block numbers double as hashes. */ + * pallet's totals and the governance queues. A block hash encodes the block + * number, and whether it is on the fork the node serves from `forkFrom` on. */ function pendulumChain() { const chain = { + /** Where set, the node serves another chain from this block on. */ + forkFrom: undefined as number | undefined, + /** Blocks whose header the node alters, so it no longer hashes to its hash. */ + alteredHeaders: new Set(), finalized: 100, /** Age of the finalized head; earlier blocks are 12 s apart. */ headAgeMs: 0, @@ -324,16 +329,31 @@ function pendulumChain() { getFinalizedHead: async () => { await chain.gate; if (chain.failure) throw chain.failure; - return chain.finalized; + return hash(pendulumHash(chain, chain.finalized)); }, - getHeader: async (block: number) => ({ number: codec(block) }), - getBlockHash: async (block: number) => block, + getHeader: async (value: string | { toHex(): string }) => { + const requested = typeof value === "string" ? value : value.toHex(); + const block = blockOf(requested); + return { + number: codec(block), + parentHash: hash(pendulumHash(chain, block - 1)), + hash: hash(chain.alteredHeaders.has(block) ? pendulumHash(chain, block + 1_000_000) : requested), + }; + }, + getBlockHash: async (block: number) => hash(pendulumHash(chain, block)), }, }, - at: async (block: number) => at(block), + at: async (value: string | { toHex(): string }) => at(blockOf(typeof value === "string" ? value : value.toHex())), }; return { chain, api: api as unknown as ApiPromise }; } + +/** The hash of `block` on the chain the node serves. */ +function pendulumHash(chain: { forkFrom?: number }, block: number): string { + const fork = chain.forkFrom !== undefined && block >= chain.forkFrom ? 1n << 64n : 0n; + return toHex(fork + BigInt(block), { size: 32 }); +} +const blockOf = (blockHash: string) => Number(BigInt(blockHash) & 0xffff_ffff_ffff_ffffn); type Pendulum = ReturnType["chain"]; /** A finalized burn on Pendulum, in block 101 unless stated. */ @@ -463,6 +483,8 @@ async function setup( restart: launch, subjects: () => pages.map((page) => page.subject), persisted: () => JSON.parse(readFileSync(stateFile, "utf8")) as { + lastPendulumBlock: number; + lastPendulumHash: string; baseFromBlock: string; baseAdminFromBlock: string; migrations: Array<{ nonce: string }>; @@ -580,6 +602,33 @@ test("a slow Pendulum source does not delay the vault balance check", async (t) } }); +test("a source whose chain leaves the blocks ingested so far pages critical and ingests nothing", async (t) => { + const { pendulum, api, monitor, restart, pages, subjects, persisted } = await setup(t); + burn(pendulum, 0n, 100n * PEN); + await monitor.poll(api); + const restarted = await restart(); + pendulum.forkFrom = 101; // the node now serves another chain from block 101 on + burn(pendulum, 1n, 100n * PEN, 102); + await restarted.poll(api); + const page = pages.find((page) => page.subject === "CRITICAL: PENDULUM SOURCE CONTRADICTED"); + assert.equal(page?.severity, "CRITICAL", JSON.stringify(pages)); + assert.match(page.detail, /chain has 0x0+10+65 at Pendulum block 101, where the monitor ingested 0x0+65;/); + assert.ok(!subjects().includes("PENDULUM SOURCE UNREADABLE"), JSON.stringify(subjects())); + assert.equal(persisted().lastPendulumBlock, 101); + assert.deepEqual(persisted().migrations.map((migration) => migration.nonce), ["0"]); +}); + +test("a source header that does not hash to its block hash pages critical", async (t) => { + const { pendulum, api, monitor, pages } = await setup(t); + burn(pendulum, 0n, 100n * PEN, 102); + pendulum.alteredHeaders.add(102); + await monitor.poll(api); + assert.match( + pages.find((page) => page.subject === "CRITICAL: PENDULUM SOURCE CONTRADICTED")?.detail ?? "", + /header for Pendulum block 102 does not hash to 0x0+66;/, + ); +}); + test("while Pendulum cannot be read, a Base event with an unknown nonce pauses after the grace", async (t) => { const { base, pendulum, api, monitor, subjects } = await setup(t, { env: { UNMATCHED_EVENT_GRACE_SECONDS: "0" } }); pendulum.failure = new Error("node unreachable"); diff --git a/monitor/src/main.ts b/monitor/src/main.ts index 4d2566299..9a9249958 100644 --- a/monitor/src/main.ts +++ b/monitor/src/main.ts @@ -275,6 +275,8 @@ interface RuntimeState { vaultAddress: `0x${string}`; pendulumGenesisHash: `0x${string}` | undefined; lastPendulumBlock: number; + /** The hash of `lastPendulumBlock`: the next blocks must descend from it. */ + lastPendulumHash: string | undefined; nextExpectedNonce: bigint; baseFromBlock: bigint; /** The admin scan's own cursor: it only pages, so nothing waits for it. */ @@ -301,6 +303,7 @@ function hydrateState(): RuntimeState { vaultAddress: config.vaultAddress, pendulumGenesisHash: undefined, lastPendulumBlock: config.pendulumStartBlock - 1, + lastPendulumHash: undefined, nextExpectedNonce: config.pendulumStartNonce, baseFromBlock: config.baseStartBlock, baseAdminFromBlock: config.baseStartBlock, @@ -316,6 +319,7 @@ function hydrateState(): RuntimeState { vaultAddress: persisted.vaultAddress, pendulumGenesisHash: persisted.pendulumGenesisHash, lastPendulumBlock: persisted.lastPendulumBlock, + lastPendulumHash: persisted.lastPendulumHash, nextExpectedNonce: BigInt(persisted.nextExpectedNonce), baseFromBlock: BigInt(persisted.baseFromBlock), baseAdminFromBlock: BigInt(persisted.baseAdminFromBlock ?? persisted.baseFromBlock), @@ -346,6 +350,7 @@ function cloneState(source: RuntimeState): RuntimeState { vaultAddress: source.vaultAddress, pendulumGenesisHash: source.pendulumGenesisHash, lastPendulumBlock: source.lastPendulumBlock, + lastPendulumHash: source.lastPendulumHash, nextExpectedNonce: source.nextExpectedNonce, baseFromBlock: source.baseFromBlock, baseAdminFromBlock: source.baseAdminFromBlock, @@ -365,6 +370,7 @@ function persistState(state: RuntimeState): void { vaultAddress: state.vaultAddress, pendulumGenesisHash: state.pendulumGenesisHash, lastPendulumBlock: state.lastPendulumBlock, + lastPendulumHash: state.lastPendulumHash as `0x${string}` | undefined, nextExpectedNonce: state.nextExpectedNonce.toString(), baseFromBlock: state.baseFromBlock.toString(), baseAdminFromBlock: state.baseAdminFromBlock.toString(), @@ -392,6 +398,7 @@ let state: RuntimeState; let multicallUnavailable = false; let lastSourceStallAlertMs = 0; let lastSourceFailureAlertMs = 0; +let lastSourceIntegrityAlertMs = 0; let lastBaseStallAlertMs = 0; let lastReadinessAlertMs = 0; let lastAdminScanFailureAlertMs = 0; @@ -629,6 +636,46 @@ function unmatchedDisposition( : "expired"; } +interface BlockRef { + number: number; + hash: string; +} + +/** The Pendulum source contradicts the chain it must be on: the one ingested + * so far, the second node's or the relay chain's. Handled as unreadable. */ +class SourceIntegrityError extends Error {} + +/** Walks parent hashes down from `top` to `bottom` through headers fetched + * from the node, recomputing each hash here, so that the result is one + * chain the node cannot fake: block hashes by height, `bottom` included. */ +async function walkHeaders(api: ApiPromise, top: BlockRef, bottom: number): Promise> { + const hashes = new Map([[top.number, top.hash]]); + let hash = top.hash; + for (let height = top.number; height > bottom; height--) { + const header = await api.rpc.chain.getHeader(hash); + if (header.hash.toHex() !== hash || header.number.toNumber() !== height) { + throw new SourceIntegrityError(`the source's header for Pendulum block ${height} does not hash to ${hash}`); + } + hash = header.parentHash.toHex(); + hashes.set(height - 1, hash); + } + return hashes; +} + +/** The hashes of the finalized blocks to ingest, on one chain with every + * block ingested before. */ +async function sourceHashes(api: ApiPromise, draft: RuntimeState, finalized: BlockRef): Promise> { + const hashes = await walkHeaders(api, finalized, draft.lastPendulumBlock); + const hash = hashes.get(draft.lastPendulumBlock); + if (draft.lastPendulumHash && hash !== undefined && hash !== draft.lastPendulumHash) { + throw new SourceIntegrityError( + `the source's chain has ${hash} at Pendulum block ${draft.lastPendulumBlock}, ` + + `where the monitor ingested ${draft.lastPendulumHash}`, + ); + } + return hashes; +} + /** The migration pallet's admin events, each with what to check. */ const PALLET_ADMIN_EVENTS = new Map([ ["MigrationPauseSet", "migrations were paused or resumed; check that it was intended"], @@ -638,9 +685,17 @@ const PALLET_ADMIN_EVENTS = new Map([ ], ]); -async function ingestPendulumEvents(api: ApiPromise, draft: RuntimeState, finalizedBlock: number): Promise { +/** Ingests the blocks after the cursor up to `finalizedBlock`, each read at + * its hash in `hashes`: the chain that `sourceHashes` verified. */ +async function ingestPendulumEvents( + api: ApiPromise, + draft: RuntimeState, + finalizedBlock: number, + hashes: Map, +): Promise { for (let block = draft.lastPendulumBlock + 1; block <= finalizedBlock; block++) { - const blockHash = await api.rpc.chain.getBlockHash(block); + const blockHash = hashes.get(block); + if (!blockHash) throw new Error(`no verified hash for Pendulum block ${block}`); const apiAt = await api.at(blockHash); const firstSeenMs = Number((await apiAt.query.timestamp.now()).toString().replaceAll(",", "")); const records = (await apiAt.query.system.events()) as unknown as { @@ -699,6 +754,7 @@ async function ingestPendulumEvents(api: ApiPromise, draft: RuntimeState, finali }); } draft.lastPendulumBlock = block; + draft.lastPendulumHash = blockHash; // Checkpoint a long catch-up as it goes: the Pendulum-side cursor is // self-consistent on its own (the Base cursor only ever trails it), so a // transient RPC error hours into a replay must not discard the progress @@ -1248,7 +1304,8 @@ async function readSource(api: ApiPromise, draft: RuntimeState): Promise { source = await readSource(api, draft); } catch (sourceError) { draft = cloneState(state); // drops a half-ingested block; the 200-block checkpoints already moved `state` - if (Date.now() - lastSourceFailureAlertMs > config.unmatchedEventGraceSeconds * 1000) { - lastSourceFailureAlertMs = Date.now(); + const contradicted = sourceError instanceof SourceIntegrityError; + const consequence = `Base events are checked against the last Pendulum view read in full: one whose nonce ` + + `that view lacks pauses the vault ${config.unmatchedEventGraceSeconds}s after its first sighting, and M2a ` + + "and liveness are skipped"; + if (contradicted && Date.now() - lastSourceIntegrityAlertMs > config.unmatchedEventGraceSeconds * 1000) { + lastSourceIntegrityAlertMs = Date.now(); await alert( - "PENDULUM SOURCE UNREADABLE", - `${sourceError}; until the source reads again, Base events are checked against the last Pendulum view ` + - `read in full: one whose nonce that view lacks pauses the vault ${config.unmatchedEventGraceSeconds}s ` + - "after its first sighting, and M2a and liveness are skipped", + "CRITICAL: PENDULUM SOURCE CONTRADICTED", + `${sourceError}; nothing from the source is ingested until it verifies again, and until then ` + + `${consequence} — stop trusting the source node and compare it with other nodes and the relay chain`, ); + } else if (!contradicted && Date.now() - lastSourceFailureAlertMs > config.unmatchedEventGraceSeconds * 1000) { + lastSourceFailureAlertMs = Date.now(); + await alert("PENDULUM SOURCE UNREADABLE", `${sourceError}; until the source reads again, ${consequence}`); } } diff --git a/monitor/src/state.test.ts b/monitor/src/state.test.ts index c37e73eba..c3b188808 100644 --- a/monitor/src/state.test.ts +++ b/monitor/src/state.test.ts @@ -80,10 +80,19 @@ test("optional fields are checked when present", () => { attestor: "0x4444444444444444444444444444444444444444", } as const; try { - const complete = { ...state, baseAdminFromBlock: "150", mismatchedApprovals: [approval] }; + const complete = { + ...state, + baseAdminFromBlock: "150", + mismatchedApprovals: [approval], + lastPendulumHash: `0x${"55".repeat(32)}` as const, + }; saveMonitorState(path, complete); assert.deepEqual(loadMonitorState(path), complete); - for (const bad of [{ baseAdminFromBlock: "-1" }, { mismatchedApprovals: [{ ...approval, attestor: "0x44" }] }]) { + for (const bad of [ + { baseAdminFromBlock: "-1" }, + { mismatchedApprovals: [{ ...approval, attestor: "0x44" }] }, + { lastPendulumHash: "0x55" }, + ]) { writeFileSync(path, JSON.stringify({ ...state, ...bad })); assert.throws(() => loadMonitorState(path), /invalid or legacy schema/); } diff --git a/monitor/src/state.ts b/monitor/src/state.ts index 22a3a16d0..b24635719 100644 --- a/monitor/src/state.ts +++ b/monitor/src/state.ts @@ -37,6 +37,8 @@ export interface PersistedMonitorState { /** Approvals that disagreed with the finalized burn, until their nonce is * released; absent in older state. */ mismatchedApprovals?: PersistedApproval[]; + /** The hash of `lastPendulumBlock`; absent in older state. */ + lastPendulumHash?: `0x${string}`; } export interface PersistedApproval { @@ -47,6 +49,7 @@ export interface PersistedApproval { } const isAddress = (value: unknown) => typeof value === "string" && /^0x[0-9a-fA-F]{40}$/.test(value); +const isHash = (value: unknown) => typeof value === "string" && /^0x[0-9a-fA-F]{64}$/.test(value); function isMissing(error: unknown): boolean { return (error as NodeJS.ErrnoException)?.code === "ENOENT"; @@ -108,6 +111,7 @@ export function loadMonitorState(path: string): PersistedMonitorState | undefine !isUint(state.baseFromBlock) || (state.issuanceAnchor !== undefined && !isUint(state.issuanceAnchor)) || (state.baseAdminFromBlock !== undefined && !isUint(state.baseAdminFromBlock)) || + (state.lastPendulumHash !== undefined && !isHash(state.lastPendulumHash)) || badMigration || badAlerts || badApprovals || From c9c471a5ed42901878faa20a681e17f266aa0935 Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Thu, 22 Oct 2026 12:36:26 +0200 Subject: [PATCH 36/75] feat(monitor): Cross-check the Pendulum source with an optional second node With PENDULUM_WS_CROSSCHECK set, every poll compares the source's finalized head with the second node's block hash at that height before anything is ingested; the parent-hash walk ties every ingested block to it. A different hash pages CRITICAL: PENDULUM SOURCE CONTRADICTED and the source is treated as unreadable. A second node that cannot answer, or has not reached that block yet, pages PENDULUM CROSS-CHECK UNAVAILABLE, throttled, and the poll goes on. Pages now also strip WebSocket URLs. --- monitor/README.md | 4 +- monitor/src/main.test.ts | 62 +++++++++++++++++++++++++++-- monitor/src/main.ts | 84 +++++++++++++++++++++++++++++++++++++--- 3 files changed, 141 insertions(+), 9 deletions(-) diff --git a/monitor/README.md b/monitor/README.md index 4d4f29dc6..5142061bd 100644 --- a/monitor/README.md +++ b/monitor/README.md @@ -15,7 +15,7 @@ Checks every poll: | Source supply | `totalIssuance + TotalMigrated` on Pendulum must never grow past its first-seen anchor. | Alert (`SOURCE SUPPLY GREW`) | | Unreleasable burns | A burn to the zero or vault address can never be approved or released; it is paged once and excluded from the pending count so it cannot page liveness forever or block RB-7. | Alert once | | Pendulum source | The finalized Pendulum view must stay fresh and readable. While it cannot be read (an RPC error, a nonce discontinuity, an event-shape change), the Base-side checks — vault balance and its auto-pause, Base view, auto-pause readiness, vault admin events, governance queue — still run, and Base events are reconciled against the last view read in full: an unreadable source counts as a stalled one, so a mismatch on a known nonce counts at once and an unknown nonce after `UNMATCHED_EVENT_GRACE_SECONDS`. M2a and liveness wait for the source. | Alert, throttled: `PENDULUM SOURCE VIEW STALLED` once the view is older than `SOURCE_STALE_ALERT_SECONDS`, `PENDULUM SOURCE UNREADABLE` on a failure | -| Pendulum chain | Every finalized block the monitor ingests is read at a hash it verified: it walks the parent hashes down from the finalized head through headers it fetches from the node and hashes itself, and the walk must arrive at the hash of the block it ingested last (kept in the state file). | Alert (`CRITICAL: PENDULUM SOURCE CONTRADICTED`), throttled; nothing from the source is ingested, and it is treated as unreadable (see Pendulum source) until it verifies again | +| Pendulum chain | Every finalized block the monitor ingests is read at a hash it verified: it walks the parent hashes down from the finalized head through headers it fetches from the node and hashes itself, and the walk must arrive at the hash of the block it ingested last (kept in the state file). With `PENDULUM_WS_CROSSCHECK` set, the second node must have the same hash at the finalized head's height in every poll; the walk ties every block below to it. A second node that cannot answer, or has not reached that block yet, only warns. | Alert (`CRITICAL: PENDULUM SOURCE CONTRADICTED`), throttled; nothing from the source is ingested, and it is treated as unreadable (see Pendulum source) until it verifies again. `PENDULUM CROSS-CHECK UNAVAILABLE`, throttled, while the second node does not answer | | Base log completeness | In every scanned block range the `Released` amounts must add up to the growth of `totalReleased`, and the `RemainderSwept` amounts to that of `totalSwept`, between the block before the range and its last block. A range whose logs do not (a backend serving truncated logs) is read once more, then held: the Base cursor stays on it and M2a and liveness wait until a read accounts for the totals. | Alert (`BASE LOG SCAN INCOMPLETE`), throttled | | Base view | The safe/finalized Base block must not be older than `BASE_STALE_ALERT_SECONDS`: a view that stops advancing hides every later vault event while every check still passes at the old block. | Alert (`BASE VIEW STALLED`), throttled | | Auto-pause readiness | At startup and every poll: the Base RPC serves `BASE_CHAIN_ID` and, with a guardian key configured, the key parses, is `vault.guardian()` and holds at least `GUARDIAN_MIN_BALANCE_WEI`. A read that fails pages as well and never stops the poll's other checks. | Alert (`AUTO-PAUSE NOT READY`), throttled; `AUTO-PAUSE DISABLED` once at startup without a key | @@ -29,6 +29,7 @@ Checks every poll: | Variable | Meaning | |---|---| | `PENDULUM_WS` | WebSocket of the monitor's own Pendulum node, run with `--state-pruning archive` (see below) | +| `PENDULUM_WS_CROSSCHECK` | Optional: WebSocket of a second Pendulum node, not the one behind `PENDULUM_WS`; its block hash at the source's finalized head must match in every poll | | `BASE_RPC_URL` | Base JSON-RPC endpoint — a **single dedicated node**, and a different provider than the attestors use. Load-balanced pools can serve `eth_getLogs` from a lagging replica; each scan range is probed for its end block first and its logs must account for the vault's running totals, which turns a behind-node into a loud replay instead of a silent gap, but a dedicated node removes the hazard entirely | | `VAULT_ADDRESS` | MigrationVault address on Base | | `POLL_INTERVAL_MS` | Poll cadence (default 60s) | @@ -71,6 +72,7 @@ starts with `CRITICAL`; everything else is a `WARNING`. | `LIVENESS: migration below approval quorum`, `LIVENESS: quorum reached but not released` | Stalled migrations, one summary per kind and poll | | `PENDULUM SOURCE VIEW STALLED`, `PENDULUM SOURCE UNREADABLE`, `BASE VIEW STALLED`, `BASE ADMIN SCAN FAILED`, `BASE LOG SCAN INCOMPLETE` | The monitor is going blind on one chain | | `CRITICAL: PENDULUM SOURCE CONTRADICTED` | The Pendulum source contradicts the chain the monitor verified; stop trusting that node | +| `PENDULUM CROSS-CHECK UNAVAILABLE` | The second Pendulum node does not answer, so the source is not compared with it | | `AUTO-PAUSE NOT READY`, `AUTO-PAUSE DISABLED` | The next violation would not pause the vault automatically | | `SOURCE SUPPLY GREW`, `CRITICAL: unreleasable migration burned on Pendulum` | Supply problems at the source | | `PENDULUM GOVERNANCE ITEM`, `CRITICAL: Pendulum governance item mints, moves or unlocks balances, or changes the runtime`, `CRITICAL: Pendulum governance item cannot be read`, `MIGRATION PALLET ADMIN EVENT` | Pendulum governance | diff --git a/monitor/src/main.test.ts b/monitor/src/main.test.ts index 513aa110f..c394c0d41 100644 --- a/monitor/src/main.test.ts +++ b/monitor/src/main.test.ts @@ -203,7 +203,7 @@ const codec = (value: number | bigint) => ({ toNumber: () => Number(value), toBigInt: () => BigInt(value), }); -const hash = (hex: string) => ({ toHex: () => hex }); +const hash = (hex: string) => ({ toHex: () => hex, isEmpty: /^0x0*$/.test(hex) }); const inline = (name: string) => ({ isInline: true, asInline: encodeCall(name), isLookup: false, isLegacy: false, toString: () => `inline ${name}` }); const lookup = (hex: string, len: number) => ({ isInline: false, @@ -340,7 +340,7 @@ function pendulumChain() { hash: hash(chain.alteredHeaders.has(block) ? pendulumHash(chain, block + 1_000_000) : requested), }; }, - getBlockHash: async (block: number) => hash(pendulumHash(chain, block)), + getBlockHash: async (block: number) => hash(block > chain.finalized ? toHex(0, { size: 32 }) : pendulumHash(chain, block)), }, }, at: async (value: string | { toHex(): string }) => at(blockOf(typeof value === "string" ? value : value.toHex())), @@ -407,11 +407,14 @@ async function setup( webhook = () => 200, heartbeat = () => 200, prepare = () => {}, + connect, }: { env?: Record; webhook?: (page: Page) => number | Promise; heartbeat?: () => number; prepare?: (base: Base, pendulum: Pendulum) => void; + /** Reaches the other nodes: the second Pendulum node and relay RPCs. */ + connect?: (url: string) => Promise; } = {}, ) { const base = baseChain(); @@ -466,7 +469,7 @@ async function setup( Object.assign(process.env, variables); const launch = async () => { const monitor: typeof import("./main.js") = await import(`./main.js?instance=${++instances}`); - await monitor.start(api); + await monitor.start(api, connect ?? (async (url) => assert.fail(`unexpected connection to ${url}`))); return monitor; }; const monitor = await launch(); @@ -629,6 +632,59 @@ test("a source header that does not hash to its block hash pages critical", asyn ); }); +const SECOND_NODE = "ws://second-node.test:9944?apikey=secret"; + +test("a second node that disagrees on the newest block pages critical and nothing is ingested", async (t) => { + const second = pendulumChain(); + second.chain.finalized = 200; + const { pendulum, api, monitor, pages, persisted } = await setup(t, { + env: { PENDULUM_WS_CROSSCHECK: SECOND_NODE }, + connect: async (url) => (url === SECOND_NODE ? second.api : assert.fail(url)), + }); + burn(pendulum, 0n, 100n * PEN); + await monitor.poll(api); + assert.equal(pages.length, 0, JSON.stringify(pages)); + assert.equal(persisted().lastPendulumBlock, 101); + second.chain.forkFrom = 102; + burn(pendulum, 1n, 100n * PEN, 102); + await monitor.poll(api); + assert.match( + pages.find((page) => page.subject === "CRITICAL: PENDULUM SOURCE CONTRADICTED")?.detail ?? "", + /second Pendulum node has 0x0+10+66 at block 102, the source 0x0+66;/, + ); + assert.equal(persisted().lastPendulumBlock, 101); +}); + +test("a second node that cannot answer only warns", async (t) => { + const { pendulum, api, monitor, pages, persisted, heartbeats } = await setup(t, { + env: { PENDULUM_WS_CROSSCHECK: SECOND_NODE }, + connect: async (url) => { + throw new Error(`cannot connect to ${url}`); + }, + }); + burn(pendulum, 0n, 100n * PEN); + await monitor.poll(api); + assert.deepEqual(pages.map((page) => [page.subject, page.severity]), [["PENDULUM CROSS-CHECK UNAVAILABLE", "WARNING"]]); + assert.match(pages[0].detail, /cannot be read: Error: cannot connect to /); + assert.equal(persisted().lastPendulumBlock, 101); + assert.deepEqual(heartbeats, ["GET"]); + await monitor.poll(api); + assert.equal(pages.length, 1); // throttled +}); + +test("a second node that has not reached the newest block only warns", async (t) => { + const second = pendulumChain(); // finalized at 100 + const { pendulum, api, monitor, pages, persisted } = await setup(t, { + env: { PENDULUM_WS_CROSSCHECK: SECOND_NODE }, + connect: async () => second.api, + }); + burn(pendulum, 0n, 100n * PEN); + await monitor.poll(api); + assert.deepEqual(pages.map((page) => page.subject), ["PENDULUM CROSS-CHECK UNAVAILABLE"]); + assert.match(pages[0].detail, /has no block 101 yet;/); + assert.equal(persisted().lastPendulumBlock, 101); +}); + test("while Pendulum cannot be read, a Base event with an unknown nonce pauses after the grace", async (t) => { const { base, pendulum, api, monitor, subjects } = await setup(t, { env: { UNMATCHED_EVENT_GRACE_SECONDS: "0" } }); pendulum.failure = new Error("node unreachable"); diff --git a/monitor/src/main.ts b/monitor/src/main.ts index 9a9249958..a5af65e08 100644 --- a/monitor/src/main.ts +++ b/monitor/src/main.ts @@ -204,6 +204,8 @@ const baseFinalityTag = finalityTag(); const config = { pendulumWs: required("PENDULUM_WS"), + /** Optional: a second Pendulum node whose block hashes must agree. */ + pendulumWsCrossCheck: process.env.PENDULUM_WS_CROSSCHECK, baseRpcUrl: required("BASE_RPC_URL"), vaultAddress: required("VAULT_ADDRESS") as `0x${string}`, pollIntervalMs: envNumber("POLL_INTERVAL_MS", 60_000), @@ -399,6 +401,7 @@ let multicallUnavailable = false; let lastSourceStallAlertMs = 0; let lastSourceFailureAlertMs = 0; let lastSourceIntegrityAlertMs = 0; +let lastCrossCheckAlertMs = 0; let lastBaseStallAlertMs = 0; let lastReadinessAlertMs = 0; let lastAdminScanFailureAlertMs = 0; @@ -414,9 +417,9 @@ function log(message: string): void { } /** Strip URLs before anything leaves the process: RPC endpoints commonly embed - * API keys, and viem error texts quote the endpoint verbatim. */ + * API keys, and viem and polkadot.js error texts quote the endpoint verbatim. */ function redact(text: string): string { - return text.replace(/https?:\/\/[^\s"')]+/gi, ""); + return text.replace(/(?:https?|wss?):\/\/[^\s"')]+/gi, ""); } /** Sends one page, as a Slack incoming-webhook message, and resolves to @@ -645,6 +648,72 @@ interface BlockRef { * so far, the second node's or the relay chain's. Handled as unreadable. */ class SourceIntegrityError extends Error {} +/** Bound on connecting to, and on each read from, the nodes of the integrity checks. */ +const NODE_TIMEOUT_MS = 20_000; + +function withTimeout(promise: Promise, ms: number): Promise { + let timer: NodeJS.Timeout | undefined; + const timeout = new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error(`no answer within ${ms / 1000}s`)), ms); + }); + return Promise.race([promise, timeout]).finally(() => clearTimeout(timer)); +} + +/** Connects to a node of the integrity checks; tests replace it. */ +let connectNode = async (url: string): Promise => { + const provider = new WsProvider(url, 2_500, {}, NODE_TIMEOUT_MS); + try { + return await withTimeout(ApiPromise.create({ provider, throwOnConnect: true, noInitWarn: true }), NODE_TIMEOUT_MS); + } catch (connectError) { + await provider.disconnect().catch(() => {}); + throw connectError; + } +}; +const nodeConnections = new Map>(); + +/** Runs `read` against the node at `url` over a kept connection, bounded in + * time; a failure drops the connection, so the next poll connects afresh. */ +async function onNode(url: string, read: (node: ApiPromise) => Promise): Promise { + let connection = nodeConnections.get(url); + if (!connection) nodeConnections.set(url, (connection = connectNode(url))); + try { + return await withTimeout(connection.then(read), NODE_TIMEOUT_MS); + } catch (nodeError) { + nodeConnections.delete(url); + connection.then((node) => node.disconnect()).catch(() => {}); + throw nodeError; + } +} + +/** The second node must have the source's finalized head at its height. One + * that cannot answer only warns, throttled: it is a check, not a source. */ +async function crossCheckSource(finalized: BlockRef): Promise { + const url = config.pendulumWsCrossCheck; + if (!url) return; + let reason: string; + try { + const other = await onNode(url, (node) => node.rpc.chain.getBlockHash(finalized.number)); + if (!other.isEmpty && other.toHex() !== finalized.hash) { + throw new SourceIntegrityError( + `the second Pendulum node has ${other.toHex()} at block ${finalized.number}, the source ${finalized.hash}`, + ); + } + if (!other.isEmpty) return; + reason = `has no block ${finalized.number} yet`; + } catch (crossCheckError) { + if (crossCheckError instanceof SourceIntegrityError) throw crossCheckError; + reason = `cannot be read: ${crossCheckError}`; + } + if (Date.now() - lastCrossCheckAlertMs > config.graceSeconds * 1000) { + lastCrossCheckAlertMs = Date.now(); + await alert( + "PENDULUM CROSS-CHECK UNAVAILABLE", + `the second Pendulum node (PENDULUM_WS_CROSSCHECK) ${reason}; the source's blocks are not compared ` + + "with it until it answers — restore it", + ); + } +} + /** Walks parent hashes down from `top` to `bottom` through headers fetched * from the node, recomputing each hash here, so that the result is one * chain the node cannot fake: block hashes by height, `bottom` included. */ @@ -1304,7 +1373,9 @@ async function readSource(api: ApiPromise, draft: RuntimeState): Promise { } /** Startup once the Pendulum API is connected: durable state, the multicall - * probe, the state identity check and the auto-pause readiness. */ -export async function start(api: ApiPromise): Promise { + * probe, the state identity check and the auto-pause readiness. `connect` + * replaces how the nodes of the integrity checks are reached (tests). */ +export async function start(api: ApiPromise, connect?: (url: string) => Promise): Promise { + if (connect) connectNode = connect; state = hydrateState(); const multicallCode = await publicClient.getBytecode({ address: MULTICALL3_ADDRESS }); if (!multicallCode || multicallCode === "0x") { @@ -1573,6 +1646,7 @@ export async function start(api: ApiPromise): Promise { `monitor started; Pendulum after ${state.lastPendulumBlock}, Base from ${state.baseFromBlock}, ` + `polling every ${config.pollIntervalMs}ms; governance queue ` + (config.timelockAddress ? `watched at ${config.timelockAddress}` : "not watched (TIMELOCK_ADDRESS unset)") + + `; second Pendulum node ${config.pendulumWsCrossCheck ? "cross-checked" : "not set"}` + `; auto-pause ${!config.guardianPrivateKey ? "disabled" : readiness.length ? "NOT READY" : "ready"}` + `; acknowledged release excess ${config.acknowledgedReleaseExcess} token units`, ); From d6ada286b4945e68f0fef850bbfc923a3a9e5dcb Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Thu, 22 Oct 2026 12:44:11 +0200 Subject: [PATCH 37/75] feat(monitor): Anchor the Pendulum source to the relay chain With POLKADOT_WS set (third-party Polkadot RPCs, tried in order), every poll reads the Pendulum head that the relay chain recorded at its finalized block (paras.heads of PENDULUM_PARA_ID, default 2094) and decodes it as a Pendulum header. The source must have that hash at that height, and the parent-hash walk must pass through it, also for a head past the source's finalized one; the newest such head is kept in the state file and later chains must keep passing through it. A contradiction pages CRITICAL: PENDULUM SOURCE CONTRADICTED and the source is treated as unreadable. When no relay RPC answers, the monitor pages RELAY ANCHOR UNAVAILABLE, throttled, and goes on with the source alone; blocks ingested meanwhile are tied to the next head by the walk. Once no head was anchored for RELAY_ANCHOR_GRACE_SECONDS (default one hour), whatever the reason, it pages CRITICAL: RELAY ANCHOR STALE. --- monitor/README.md | 41 ++++++++- monitor/src/main.test.ts | 181 +++++++++++++++++++++++++++++++++++++- monitor/src/main.ts | 122 +++++++++++++++++++++++-- monitor/src/state.test.ts | 3 + monitor/src/state.ts | 5 ++ 5 files changed, 339 insertions(+), 13 deletions(-) diff --git a/monitor/README.md b/monitor/README.md index 5142061bd..5c2960758 100644 --- a/monitor/README.md +++ b/monitor/README.md @@ -15,7 +15,7 @@ Checks every poll: | Source supply | `totalIssuance + TotalMigrated` on Pendulum must never grow past its first-seen anchor. | Alert (`SOURCE SUPPLY GREW`) | | Unreleasable burns | A burn to the zero or vault address can never be approved or released; it is paged once and excluded from the pending count so it cannot page liveness forever or block RB-7. | Alert once | | Pendulum source | The finalized Pendulum view must stay fresh and readable. While it cannot be read (an RPC error, a nonce discontinuity, an event-shape change), the Base-side checks — vault balance and its auto-pause, Base view, auto-pause readiness, vault admin events, governance queue — still run, and Base events are reconciled against the last view read in full: an unreadable source counts as a stalled one, so a mismatch on a known nonce counts at once and an unknown nonce after `UNMATCHED_EVENT_GRACE_SECONDS`. M2a and liveness wait for the source. | Alert, throttled: `PENDULUM SOURCE VIEW STALLED` once the view is older than `SOURCE_STALE_ALERT_SECONDS`, `PENDULUM SOURCE UNREADABLE` on a failure | -| Pendulum chain | Every finalized block the monitor ingests is read at a hash it verified: it walks the parent hashes down from the finalized head through headers it fetches from the node and hashes itself, and the walk must arrive at the hash of the block it ingested last (kept in the state file). With `PENDULUM_WS_CROSSCHECK` set, the second node must have the same hash at the finalized head's height in every poll; the walk ties every block below to it. A second node that cannot answer, or has not reached that block yet, only warns. | Alert (`CRITICAL: PENDULUM SOURCE CONTRADICTED`), throttled; nothing from the source is ingested, and it is treated as unreadable (see Pendulum source) until it verifies again. `PENDULUM CROSS-CHECK UNAVAILABLE`, throttled, while the second node does not answer | +| Pendulum chain | Every finalized block the monitor ingests is read at a hash it verified against the blocks it ingested before and, where configured, a second node and the relay chain (see [Pendulum chain verification](#pendulum-chain-verification)). | Alert (`CRITICAL: PENDULUM SOURCE CONTRADICTED`), throttled; nothing from the source is ingested, and it is treated as unreadable (see Pendulum source) until it verifies again. A second node or relay RPC that does not answer only warns, throttled (`PENDULUM CROSS-CHECK UNAVAILABLE`, `RELAY ANCHOR UNAVAILABLE`); `CRITICAL: RELAY ANCHOR STALE` once no head was anchored for `RELAY_ANCHOR_GRACE_SECONDS` | | Base log completeness | In every scanned block range the `Released` amounts must add up to the growth of `totalReleased`, and the `RemainderSwept` amounts to that of `totalSwept`, between the block before the range and its last block. A range whose logs do not (a backend serving truncated logs) is read once more, then held: the Base cursor stays on it and M2a and liveness wait until a read accounts for the totals. | Alert (`BASE LOG SCAN INCOMPLETE`), throttled | | Base view | The safe/finalized Base block must not be older than `BASE_STALE_ALERT_SECONDS`: a view that stops advancing hides every later vault event while every check still passes at the old block. | Alert (`BASE VIEW STALLED`), throttled | | Auto-pause readiness | At startup and every poll: the Base RPC serves `BASE_CHAIN_ID` and, with a guardian key configured, the key parses, is `vault.guardian()` and holds at least `GUARDIAN_MIN_BALANCE_WEI`. A read that fails pages as well and never stops the poll's other checks. | Alert (`AUTO-PAUSE NOT READY`), throttled; `AUTO-PAUSE DISABLED` once at startup without a key | @@ -24,12 +24,46 @@ Checks every poll: | Vault admin events | Every admin event of the vault (token, pause, unpause, attestors, threshold, caps, guardian, admin transfer, sweep, stale pending cleared). Before the governance handover the admin acts without a delay, so these are the only trace of such a change. This scan (shared with the governance queue) keeps its own persisted cursor: while it fails, every other check still runs, and it resumes where it stopped, also after a restart. | Alert once per event (`VAULT ADMIN EVENT`); the first run also pages the deployment's own events; `BASE ADMIN SCAN FAILED`, throttled, on a failure | | Governance queue | Every operation the governance Timelock schedules (`CallScheduled`), with its calls decoded and the time it becomes executable. Once the delay has passed anyone may execute a ready operation, so the canceller has exactly the delay to review and veto it. | Alert once per operation and scheduling block (`GOVERNANCE OPERATION SCHEDULED`), so an operation cancelled and scheduled again with the same id pages again | +## Pendulum chain verification + +The monitor reads every new finalized Pendulum block at a hash it verified +itself: + +- **Parent-hash chain.** Each poll walks the parent hashes down from the + finalized head to the block ingested last, through headers fetched from the + node and hashed by the monitor. The walk must arrive at the hash of the block + ingested last, which the state file keeps, so every ingested block lies on + one chain. +- **Second node** (`PENDULUM_WS_CROSSCHECK`). The second node must have the + same hash at the finalized head's height. +- **Relay-chain anchor** (`POLKADOT_WS`). Each poll reads the Pendulum head + that the Polkadot relay chain recorded at its finalized block + (`paras.heads(PENDULUM_PARA_ID)`), from the first relay RPC that answers. The + source must have that hash at that height, and its chain must pass through + it; the newest such head is kept in the state file, and the chain must keep + passing through it. Blocks ingested while no relay RPC answered are tied to + the next head that arrives by the same walk. + +A contradiction pages `CRITICAL: PENDULUM SOURCE CONTRADICTED`, nothing from +the source is ingested, and the source counts as unreadable until it verifies +again. A second node or relay RPC that cannot answer only warns, and the +monitor goes on with the source alone; `CRITICAL: RELAY ANCHOR STALE` pages +once no head was anchored for `RELAY_ANCHOR_GRACE_SECONDS`, whatever the reason +(relay RPCs down, the source behind the relay chain, a relay head that does not +advance). + +These checks bind block hashes only: the events and storage the source returns +for a verified block are not proof-checked against the block's state root. + ## Configuration (environment) | Variable | Meaning | |---|---| | `PENDULUM_WS` | WebSocket of the monitor's own Pendulum node, run with `--state-pruning archive` (see below) | | `PENDULUM_WS_CROSSCHECK` | Optional: WebSocket of a second Pendulum node, not the one behind `PENDULUM_WS`; its block hash at the source's finalized head must match in every poll | +| `POLKADOT_WS` | Optional: comma-separated WebSocket RPCs of the Polkadot relay chain, run by third parties and tried in order (for example `wss://rpc.polkadot.io`); turns on the relay-chain anchor | +| `PENDULUM_PARA_ID` | Pendulum's parachain id on the relay chain (default 2094) | +| `RELAY_ANCHOR_GRACE_SECONDS` | How long the source may go without a relay-anchored head before `CRITICAL: RELAY ANCHOR STALE` pages (default 3600) | | `BASE_RPC_URL` | Base JSON-RPC endpoint — a **single dedicated node**, and a different provider than the attestors use. Load-balanced pools can serve `eth_getLogs` from a lagging replica; each scan range is probed for its end block first and its logs must account for the vault's running totals, which turns a behind-node into a loud replay instead of a silent gap, but a dedicated node removes the hazard entirely | | `VAULT_ADDRESS` | MigrationVault address on Base | | `POLL_INTERVAL_MS` | Poll cadence (default 60s) | @@ -49,7 +83,7 @@ Checks every poll: | `PENDULUM_START_BLOCK` | **Required on first run:** first block to scan, normally the pallet activation block or the next finalized block before unpausing | | `PENDULUM_START_NONCE` | Nonce expected at that block (default 0; set explicitly when starting after any migrations) | | `BASE_START_BLOCK` | **Required on first run:** vault deployment block or an earlier block | -| `STATE_FILE` | Durable cursors (Pendulum with the hash of its last block, Base, Base admin scan), pending tuples, and liveness timestamps (default `./monitor-state.json`) | +| `STATE_FILE` | Durable cursors (Pendulum with the hash of its last block, Base, Base admin scan), the newest relay-anchored Pendulum head, pending tuples, and liveness timestamps (default `./monitor-state.json`) | | `BASE_FINALITY_TAG` | `safe` (default) or `finalized`; only events inside this boundary are trusted | | `BASE_MAX_BLOCK_RANGE` | Maximum `eth_getLogs` range (default 10,000 blocks) | @@ -72,7 +106,8 @@ starts with `CRITICAL`; everything else is a `WARNING`. | `LIVENESS: migration below approval quorum`, `LIVENESS: quorum reached but not released` | Stalled migrations, one summary per kind and poll | | `PENDULUM SOURCE VIEW STALLED`, `PENDULUM SOURCE UNREADABLE`, `BASE VIEW STALLED`, `BASE ADMIN SCAN FAILED`, `BASE LOG SCAN INCOMPLETE` | The monitor is going blind on one chain | | `CRITICAL: PENDULUM SOURCE CONTRADICTED` | The Pendulum source contradicts the chain the monitor verified; stop trusting that node | -| `PENDULUM CROSS-CHECK UNAVAILABLE` | The second Pendulum node does not answer, so the source is not compared with it | +| `PENDULUM CROSS-CHECK UNAVAILABLE`, `RELAY ANCHOR UNAVAILABLE` | The second Pendulum node or every relay RPC does not answer, so the source is used without that check | +| `CRITICAL: RELAY ANCHOR STALE` | No Pendulum head was checked against the relay chain for `RELAY_ANCHOR_GRACE_SECONDS` | | `AUTO-PAUSE NOT READY`, `AUTO-PAUSE DISABLED` | The next violation would not pause the vault automatically | | `SOURCE SUPPLY GREW`, `CRITICAL: unreleasable migration burned on Pendulum` | Supply problems at the source | | `PENDULUM GOVERNANCE ITEM`, `CRITICAL: Pendulum governance item mints, moves or unlocks balances, or changes the runtime`, `CRITICAL: Pendulum governance item cannot be read`, `MIGRATION PALLET ADMIN EVENT` | Pendulum governance | diff --git a/monitor/src/main.test.ts b/monitor/src/main.test.ts index c394c0d41..416fbe910 100644 --- a/monitor/src/main.test.ts +++ b/monitor/src/main.test.ts @@ -236,7 +236,11 @@ function pendulumChain() { forkFrom: undefined as number | undefined, /** Blocks whose header the node alters, so it no longer hashes to its hash. */ alteredHeaders: new Set(), + /** Whether the node names block hashes by number from the chain without the fork. */ + canonicalBlockHashes: false, finalized: 100, + /** The newest block the node imported, where it is past the finalized one. */ + best: undefined as number | undefined, /** Age of the finalized head; earlier blocks are 12 s apart. */ headAgeMs: 0, failure: undefined as Error | undefined, @@ -334,20 +338,54 @@ function pendulumChain() { getHeader: async (value: string | { toHex(): string }) => { const requested = typeof value === "string" ? value : value.toHex(); const block = blockOf(requested); + const onFork = BigInt(requested) >> 64n !== 0n; return { number: codec(block), - parentHash: hash(pendulumHash(chain, block - 1)), + parentHash: hash(pendulumHash(onFork ? chain : {}, block - 1)), hash: hash(chain.alteredHeaders.has(block) ? pendulumHash(chain, block + 1_000_000) : requested), }; }, - getBlockHash: async (block: number) => hash(block > chain.finalized ? toHex(0, { size: 32 }) : pendulumHash(chain, block)), + getBlockHash: async (block: number) => + hash( + block > (chain.best ?? chain.finalized) + ? toHex(0, { size: 32 }) + : pendulumHash(chain.canonicalBlockHashes ? {} : chain, block), + ), }, }, + /** Decodes the head data of the stand-in relay chain. */ + createType: (_type: string, bytes: Uint8Array) => { + const header = JSON.parse(new TextDecoder().decode(bytes)) as { number: number; hash: string }; + return { number: codec(header.number), hash: hash(header.hash) }; + }, at: async (value: string | { toHex(): string }) => at(blockOf(typeof value === "string" ? value : value.toHex())), }; return { chain, api: api as unknown as ApiPromise }; } +/** A relay chain whose finalized block records `head()` as the Pendulum head; + * every read fails while `failing()` holds. */ +function relayChain(head: () => { number: number; hash: string }, failing = () => false): ApiPromise { + const check = () => { + if (failing()) throw new Error("relay RPC unavailable"); + }; + const relay = { + rpc: { chain: { getFinalizedHead: async () => (check(), hash(`0x${"77".repeat(32)}`)) } }, + at: async () => ({ + query: { + paras: { + heads: async (paraId: number) => { + check(); + const data = new TextEncoder().encode(JSON.stringify(head())); + return paraId === 2094 ? { isSome: true, unwrap: () => ({ toU8a: () => data }) } : { isSome: false }; + }, + }, + }, + }), + }; + return relay as unknown as ApiPromise; +} + /** The hash of `block` on the chain the node serves. */ function pendulumHash(chain: { forkFrom?: number }, block: number): string { const fork = chain.forkFrom !== undefined && block >= chain.forkFrom ? 1n << 64n : 0n; @@ -488,6 +526,7 @@ async function setup( persisted: () => JSON.parse(readFileSync(stateFile, "utf8")) as { lastPendulumBlock: number; lastPendulumHash: string; + relayAnchor?: { number: number; hash: string }; baseFromBlock: string; baseAdminFromBlock: string; migrations: Array<{ nonce: string }>; @@ -685,6 +724,144 @@ test("a second node that has not reached the newest block only warns", async (t) assert.equal(persisted().lastPendulumBlock, 101); }); +const RELAY = "wss://relay.test"; +type Head = { number: number; hash: string }; + +/** A monitor with the relay-chain anchor, against a relay chain recording + * `head` (by default the source's finalized head, on the chain without a fork). */ +async function relaySetup( + t: { after(fn: () => void): void }, + { head, failing, env = {} }: { head?: (pendulum: Pendulum) => Head; failing?: () => boolean; env?: Record } = {}, +) { + let chain: Pendulum | undefined; + const honest = (pendulum: Pendulum) => ({ number: pendulum.finalized, hash: pendulumHash({}, pendulum.finalized) }); + const relay = relayChain(() => (head ?? honest)(chain!), failing); + return setup(t, { + env: { POLKADOT_WS: RELAY, ...env }, + prepare: (_base, pendulum) => (chain = pendulum), + connect: async (url) => (url === RELAY ? relay : assert.fail(url)), + }); +} + +test("a relay-chain record of the source's head anchors it and is kept", async (t) => { + const { pendulum, api, monitor, pages, persisted } = await relaySetup(t); + burn(pendulum, 0n, 100n * PEN); + await monitor.poll(api); + assert.equal(pages.length, 0, JSON.stringify(pages)); + assert.deepEqual(persisted().relayAnchor, { number: 101, hash: pendulumHash({}, 101) }); + assert.equal(persisted().lastPendulumBlock, 101); +}); + +test("a source that names another hash than the relay chain's record pages critical", async (t) => { + const { pendulum, api, monitor, pages, persisted } = await relaySetup(t, { + head: (pendulum) => ({ number: pendulum.finalized, hash: pendulumHash({ forkFrom: 0 }, pendulum.finalized) }), + }); + burn(pendulum, 0n, 100n * PEN); + await monitor.poll(api); + assert.match( + pages.find((page) => page.subject === "CRITICAL: PENDULUM SOURCE CONTRADICTED")?.detail ?? "", + /relay chain records 0x0+10+65 as Pendulum block 101, the source has 0x0+65;/, + ); + assert.equal(persisted().lastPendulumBlock, 100); +}); + +test("a source whose chain avoids the relay chain's record pages critical, whatever hash it names", async (t) => { + const { pendulum, api, monitor, pages } = await relaySetup(t, { + head: () => ({ number: 101, hash: pendulumHash({}, 101) }), + }); + pendulum.forkFrom = 101; + pendulum.canonicalBlockHashes = true; // it names the relay chain's hash for block 101 + burn(pendulum, 0n, 100n * PEN, 102); + await monitor.poll(api); + assert.match( + pages.find((page) => page.subject === "CRITICAL: PENDULUM SOURCE CONTRADICTED")?.detail ?? "", + /chain has 0x0+10+65 at Pendulum block 101, where the relay chain records 0x0+65;/, + ); +}); + +test("a relay record past the source's finalized head anchors it, and its chain must keep passing through it", async (t) => { + let relayDown = false; + const { pendulum, api, monitor, pages, persisted } = await relaySetup(t, { + head: (pendulum) => ({ number: 102, hash: pendulumHash(pendulum, 102) }), + failing: () => relayDown, + }); + burn(pendulum, 0n, 100n * PEN); + pendulum.best = 102; // imported, not yet finalized by the source + await monitor.poll(api); + assert.equal(pages.length, 0, JSON.stringify(pages)); + assert.deepEqual(persisted().relayAnchor, { number: 102, hash: pendulumHash({}, 102) }); + assert.equal(persisted().lastPendulumBlock, 101); + relayDown = true; + pendulum.forkFrom = 102; // the source moves to another chain above its last ingested block + burn(pendulum, 1n, 100n * PEN, 103); + await monitor.poll(api); + assert.match( + pages.find((page) => page.subject === "CRITICAL: PENDULUM SOURCE CONTRADICTED")?.detail ?? "", + /chain has 0x0+10+66 at Pendulum block 102, where the relay chain recorded 0x0+66;/, + ); +}); + +test("a relay record past the source's finalized head must descend from that head", async (t) => { + const { pendulum, api, monitor, pages, persisted } = await relaySetup(t, { + head: () => ({ number: 102, hash: pendulumHash({}, 102) }), + }); + burn(pendulum, 0n, 100n * PEN); + pendulum.forkFrom = 101; // its finalized block 101 is on another chain... + pendulum.canonicalBlockHashes = true; // ...though it names the relay chain's hash for block 102 + pendulum.best = 102; + await monitor.poll(api); + assert.match( + pages.find((page) => page.subject === "CRITICAL: PENDULUM SOURCE CONTRADICTED")?.detail ?? "", + /chain has 0x0+65 at Pendulum block 101, where the source's finalized head is 0x0+10+65;/, + ); + assert.equal(persisted().lastPendulumBlock, 100); +}); + +test("an unreadable relay chain warns, the source is used, and it turns critical after the grace", async (t) => { + const { pendulum, api, monitor, pages, persisted } = await relaySetup(t, { failing: () => true }); + burn(pendulum, 0n, 100n * PEN); + await monitor.poll(api); + assert.deepEqual(pages.map((page) => [page.subject, page.severity]), [["RELAY ANCHOR UNAVAILABLE", "WARNING"]]); + assert.match(pages[0].detail, /relay RPC 1: Error: relay RPC unavailable;/); + assert.equal(persisted().lastPendulumBlock, 101); + const late = await relaySetup(t, { failing: () => true, env: { RELAY_ANCHOR_GRACE_SECONDS: "0" } }); + burn(late.pendulum, 0n, 100n * PEN); + await new Promise((resolve) => setTimeout(resolve, 5)); + await late.monitor.poll(late.api); + assert.ok(late.subjects().includes("CRITICAL: RELAY ANCHOR STALE"), JSON.stringify(late.subjects())); +}); + +test("a source behind the relay chain is not anchored, and pages only after the grace", async (t) => { + const ahead = (pendulum: Pendulum) => ({ number: pendulum.finalized + 1, hash: pendulumHash(pendulum, pendulum.finalized + 1) }); + const { pendulum, api, monitor, pages, persisted } = await relaySetup(t, { head: ahead }); + burn(pendulum, 0n, 100n * PEN); + await monitor.poll(api); + assert.equal(pages.length, 0, JSON.stringify(pages)); + assert.equal(persisted().relayAnchor, undefined); + assert.equal(persisted().lastPendulumBlock, 101); + const late = await relaySetup(t, { head: ahead, env: { RELAY_ANCHOR_GRACE_SECONDS: "0" } }); + await new Promise((resolve) => setTimeout(resolve, 5)); + await late.monitor.poll(late.api); + assert.match( + late.pages.find((page) => page.subject === "CRITICAL: RELAY ANCHOR STALE")?.detail ?? "", + /the source has not imported block 101, the relay chain's Pendulum head, yet/, + ); +}); + +test("the relay RPCs are tried in order", async (t) => { + let chain: Pendulum | undefined; + const relay = relayChain(() => ({ number: chain!.finalized, hash: pendulumHash({}, chain!.finalized) })); + const { pendulum, api, monitor, pages, persisted } = await setup(t, { + env: { POLKADOT_WS: "wss://down.test, wss://up.test" }, + prepare: (_base, pendulum) => (chain = pendulum), + connect: async (url) => (url === "wss://up.test" ? relay : assert.fail(`cannot reach ${url}`)), + }); + burn(pendulum, 0n, 100n * PEN); + await monitor.poll(api); + assert.equal(pages.length, 0, JSON.stringify(pages)); + assert.equal(persisted().relayAnchor?.number, 101); +}); + test("while Pendulum cannot be read, a Base event with an unknown nonce pauses after the grace", async (t) => { const { base, pendulum, api, monitor, subjects } = await setup(t, { env: { UNMATCHED_EVENT_GRACE_SECONDS: "0" } }); pendulum.failure = new Error("node unreachable"); diff --git a/monitor/src/main.ts b/monitor/src/main.ts index a5af65e08..f17adb9fb 100644 --- a/monitor/src/main.ts +++ b/monitor/src/main.ts @@ -206,6 +206,12 @@ const config = { pendulumWs: required("PENDULUM_WS"), /** Optional: a second Pendulum node whose block hashes must agree. */ pendulumWsCrossCheck: process.env.PENDULUM_WS_CROSSCHECK, + /** Optional: third-party Polkadot RPCs, tried in order, whose record of + * the Pendulum head anchors the source. */ + polkadotWs: (process.env.POLKADOT_WS ?? "").split(",").map((url) => url.trim()).filter(Boolean), + pendulumParaId: envNumber("PENDULUM_PARA_ID", 2094), + /** How long the source may go without a relay-anchored head before that pages critical. */ + relayAnchorGraceMs: envNumber("RELAY_ANCHOR_GRACE_SECONDS", 3600) * 1000, baseRpcUrl: required("BASE_RPC_URL"), vaultAddress: required("VAULT_ADDRESS") as `0x${string}`, pollIntervalMs: envNumber("POLL_INTERVAL_MS", 60_000), @@ -279,6 +285,9 @@ interface RuntimeState { lastPendulumBlock: number; /** The hash of `lastPendulumBlock`: the next blocks must descend from it. */ lastPendulumHash: string | undefined; + /** The newest Pendulum head the relay chain recorded that the source was + * verified against; the source's chain must keep passing through it. */ + relayAnchor: { number: number; hash: string } | undefined; nextExpectedNonce: bigint; baseFromBlock: bigint; /** The admin scan's own cursor: it only pages, so nothing waits for it. */ @@ -306,6 +315,7 @@ function hydrateState(): RuntimeState { pendulumGenesisHash: undefined, lastPendulumBlock: config.pendulumStartBlock - 1, lastPendulumHash: undefined, + relayAnchor: undefined, nextExpectedNonce: config.pendulumStartNonce, baseFromBlock: config.baseStartBlock, baseAdminFromBlock: config.baseStartBlock, @@ -322,6 +332,7 @@ function hydrateState(): RuntimeState { pendulumGenesisHash: persisted.pendulumGenesisHash, lastPendulumBlock: persisted.lastPendulumBlock, lastPendulumHash: persisted.lastPendulumHash, + relayAnchor: persisted.relayAnchor, nextExpectedNonce: BigInt(persisted.nextExpectedNonce), baseFromBlock: BigInt(persisted.baseFromBlock), baseAdminFromBlock: BigInt(persisted.baseAdminFromBlock ?? persisted.baseFromBlock), @@ -353,6 +364,7 @@ function cloneState(source: RuntimeState): RuntimeState { pendulumGenesisHash: source.pendulumGenesisHash, lastPendulumBlock: source.lastPendulumBlock, lastPendulumHash: source.lastPendulumHash, + relayAnchor: source.relayAnchor, nextExpectedNonce: source.nextExpectedNonce, baseFromBlock: source.baseFromBlock, baseAdminFromBlock: source.baseAdminFromBlock, @@ -373,6 +385,7 @@ function persistState(state: RuntimeState): void { pendulumGenesisHash: state.pendulumGenesisHash, lastPendulumBlock: state.lastPendulumBlock, lastPendulumHash: state.lastPendulumHash as `0x${string}` | undefined, + relayAnchor: state.relayAnchor as { number: number; hash: `0x${string}` } | undefined, nextExpectedNonce: state.nextExpectedNonce.toString(), baseFromBlock: state.baseFromBlock.toString(), baseAdminFromBlock: state.baseAdminFromBlock.toString(), @@ -402,6 +415,10 @@ let lastSourceStallAlertMs = 0; let lastSourceFailureAlertMs = 0; let lastSourceIntegrityAlertMs = 0; let lastCrossCheckAlertMs = 0; +let lastRelayAlertMs = 0; +let lastRelayStaleAlertMs = 0; +/** When the source last verified against a new relay-chain record (or the start). */ +let lastRelayAnchorMs = Date.now(); let lastBaseStallAlertMs = 0; let lastReadinessAlertMs = 0; let lastAdminScanFailureAlertMs = 0; @@ -731,16 +748,105 @@ async function walkHeaders(api: ApiPromise, top: BlockRef, bottom: number): Prom return hashes; } +/** The Pendulum head that the relay chain recorded at its finalized block, + * from the first relay RPC in POLKADOT_WS that answers. */ +async function relayHead(api: ApiPromise): Promise { + const failures: string[] = []; + for (const [index, url] of config.polkadotWs.entries()) { + try { + return await onNode(url, async (relay) => { + const relayFinalized = await relay.rpc.chain.getFinalizedHead(); + const head = (await (await relay.at(relayFinalized)).query.paras.heads(config.pendulumParaId)) as unknown as { + isSome: boolean; + unwrap(): { toU8a(bare: boolean): Uint8Array }; + }; + if (!head.isSome) throw new Error(`no head recorded for para ${config.pendulumParaId}`); + // The head data is the Pendulum header; its hash is computed here. + const header = api.createType("Header", head.unwrap().toU8a(true)) as unknown as { + number: { toNumber(): number }; + hash: { toHex(): string }; + }; + return { number: header.number.toNumber(), hash: header.hash.toHex() }; + }); + } catch (relayError) { + failures.push(`relay RPC ${index + 1}: ${relayError}`); + } + } + throw new Error(failures.join("; ")); +} + +/** The relay chain's record of a Pendulum head newer than the last one + * verified, if this poll can check the source against it. Pages while there + * is none: a warning when the relay RPCs fail, critical once no head was + * anchored for RELAY_ANCHOR_GRACE_SECONDS. Both throttled. */ +async function relayAnchor(api: ApiPromise, draft: RuntimeState, finalized: BlockRef): Promise { + if (!config.polkadotWs.length) return undefined; + let reason: string; + try { + const anchor = await relayHead(api); + const known = draft.relayAnchor; + if (known?.number === anchor.number && known.hash !== anchor.hash) { + throw new SourceIntegrityError( + `the relay chain now records ${anchor.hash} as Pendulum block ${anchor.number}, where it recorded ${known.hash}`, + ); + } + const sourceHash = await api.rpc.chain.getBlockHash(anchor.number); + if (!sourceHash.isEmpty && sourceHash.toHex() !== anchor.hash) { + throw new SourceIntegrityError( + `the relay chain records ${anchor.hash} as Pendulum block ${anchor.number}, the source has ${sourceHash.toHex()}`, + ); + } + if (known && anchor.number <= known.number) reason = `the relay chain's Pendulum head ${anchor.number} is not newer`; + else if (sourceHash.isEmpty && anchor.number > finalized.number) { + reason = `the source has not imported block ${anchor.number}, the relay chain's Pendulum head, yet`; + } else return anchor; + } catch (relayError) { + if (relayError instanceof SourceIntegrityError) throw relayError; + reason = `the relay chain cannot be read: ${relayError}`; + if (Date.now() - lastRelayAlertMs > config.graceSeconds * 1000) { + lastRelayAlertMs = Date.now(); + await alert( + "RELAY ANCHOR UNAVAILABLE", + `${reason}; the Pendulum source is used without the relay-chain anchor until a relay RPC answers, ` + + `and this turns critical after ${config.relayAnchorGraceMs / 1000}s`, + ); + } + } + const unanchoredMs = Date.now() - lastRelayAnchorMs; + if (unanchoredMs > config.relayAnchorGraceMs && Date.now() - lastRelayStaleAlertMs > config.unmatchedEventGraceSeconds * 1000) { + lastRelayStaleAlertMs = Date.now(); + await alert( + "CRITICAL: RELAY ANCHOR STALE", + `no Pendulum head was checked against the relay chain for ${Math.round(unanchoredMs / 1000)}s (${reason}); ` + + "the source is trusted without the anchor — restore a relay RPC, or the source node if it lags", + ); + } + return undefined; +} + /** The hashes of the finalized blocks to ingest, on one chain with every - * block ingested before. */ + * block ingested before and, where available, with the relay chain's record + * of the Pendulum head. */ async function sourceHashes(api: ApiPromise, draft: RuntimeState, finalized: BlockRef): Promise> { - const hashes = await walkHeaders(api, finalized, draft.lastPendulumBlock); - const hash = hashes.get(draft.lastPendulumBlock); - if (draft.lastPendulumHash && hash !== undefined && hash !== draft.lastPendulumHash) { - throw new SourceIntegrityError( - `the source's chain has ${hash} at Pendulum block ${draft.lastPendulumBlock}, ` + - `where the monitor ingested ${draft.lastPendulumHash}`, - ); + const anchor = await relayAnchor(api, draft, finalized); + const top = anchor && anchor.number > finalized.number ? anchor : finalized; + const hashes = await walkHeaders(api, top, Math.min(draft.lastPendulumBlock, anchor?.number ?? Infinity)); + const ingested = draft.lastPendulumHash ? { number: draft.lastPendulumBlock, hash: draft.lastPendulumHash } : undefined; + const expected: Array<[BlockRef | undefined, string]> = [ + [finalized, "the source's finalized head is"], + [ingested, "the monitor ingested"], + [anchor, "the relay chain records"], + [draft.relayAnchor, "the relay chain recorded"], + ]; + for (const [block, what] of expected) { + const hash = block && hashes.get(block.number); + if (block && hash !== undefined && hash !== block.hash) { + throw new SourceIntegrityError(`the source's chain has ${hash} at Pendulum block ${block.number}, where ${what} ${block.hash}`); + } + } + if (anchor) { + draft.relayAnchor = anchor; + lastRelayAnchorMs = Date.now(); } return hashes; } diff --git a/monitor/src/state.test.ts b/monitor/src/state.test.ts index c3b188808..017bb11fb 100644 --- a/monitor/src/state.test.ts +++ b/monitor/src/state.test.ts @@ -85,6 +85,7 @@ test("optional fields are checked when present", () => { baseAdminFromBlock: "150", mismatchedApprovals: [approval], lastPendulumHash: `0x${"55".repeat(32)}` as const, + relayAnchor: { number: 100, hash: `0x${"66".repeat(32)}` as const }, }; saveMonitorState(path, complete); assert.deepEqual(loadMonitorState(path), complete); @@ -92,6 +93,8 @@ test("optional fields are checked when present", () => { { baseAdminFromBlock: "-1" }, { mismatchedApprovals: [{ ...approval, attestor: "0x44" }] }, { lastPendulumHash: "0x55" }, + { relayAnchor: { number: "100", hash: `0x${"66".repeat(32)}` } }, + { relayAnchor: { number: 100, hash: "0x66" } }, ]) { writeFileSync(path, JSON.stringify({ ...state, ...bad })); assert.throws(() => loadMonitorState(path), /invalid or legacy schema/); diff --git a/monitor/src/state.ts b/monitor/src/state.ts index b24635719..3627d380e 100644 --- a/monitor/src/state.ts +++ b/monitor/src/state.ts @@ -39,6 +39,9 @@ export interface PersistedMonitorState { mismatchedApprovals?: PersistedApproval[]; /** The hash of `lastPendulumBlock`; absent in older state. */ lastPendulumHash?: `0x${string}`; + /** The newest relay-chain record of the Pendulum head the source was + * verified against; absent without the relay-chain anchor. */ + relayAnchor?: { number: number; hash: `0x${string}` }; } export interface PersistedApproval { @@ -112,6 +115,8 @@ export function loadMonitorState(path: string): PersistedMonitorState | undefine (state.issuanceAnchor !== undefined && !isUint(state.issuanceAnchor)) || (state.baseAdminFromBlock !== undefined && !isUint(state.baseAdminFromBlock)) || (state.lastPendulumHash !== undefined && !isHash(state.lastPendulumHash)) || + (state.relayAnchor !== undefined && + (!Number.isSafeInteger(state.relayAnchor?.number) || !isHash(state.relayAnchor?.hash))) || badMigration || badAlerts || badApprovals || From 2b865d3c513918a7f0c48fdbc739260c75187121 Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Thu, 22 Oct 2026 12:45:38 +0200 Subject: [PATCH 38/75] fix(monitor): Say why a connection to a checking node failed A WebSocket connection that cannot be opened rejects with an event rather than an Error, so the second-node and relay pages showed "[object ErrorEvent]". Such a failure is now turned into an Error with the event's message. --- monitor/src/main.test.ts | 10 ++++++++++ monitor/src/main.ts | 3 ++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/monitor/src/main.test.ts b/monitor/src/main.test.ts index 416fbe910..7a5c499a6 100644 --- a/monitor/src/main.test.ts +++ b/monitor/src/main.test.ts @@ -711,6 +711,16 @@ test("a second node that cannot answer only warns", async (t) => { assert.equal(pages.length, 1); // throttled }); +test("a node connection that fails with an event says why", async (t) => { + const { pendulum, api, monitor, pages } = await setup(t, { + env: { PENDULUM_WS_CROSSCHECK: SECOND_NODE }, + connect: () => Promise.reject({ type: "error", message: "getaddrinfo ENOTFOUND second-node.test" }), + }); + burn(pendulum, 0n, 100n * PEN); + await monitor.poll(api); + assert.match(pages[0]?.detail ?? "", /cannot be read: Error: getaddrinfo ENOTFOUND second-node\.test;/); +}); + test("a second node that has not reached the newest block only warns", async (t) => { const second = pendulumChain(); // finalized at 100 const { pendulum, api, monitor, pages, persisted } = await setup(t, { diff --git a/monitor/src/main.ts b/monitor/src/main.ts index f17adb9fb..263bf6cf2 100644 --- a/monitor/src/main.ts +++ b/monitor/src/main.ts @@ -698,7 +698,8 @@ async function onNode(url: string, read: (node: ApiPromise) => Promise): P } catch (nodeError) { nodeConnections.delete(url); connection.then((node) => node.disconnect()).catch(() => {}); - throw nodeError; + // A failed WebSocket connection rejects with an event, not an Error. + throw nodeError instanceof Error ? nodeError : new Error(String((nodeError as { message?: unknown })?.message ?? nodeError)); } } From 970dbf2a115b40d0502457b6f1409ad1deb4d49e Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Thu, 22 Oct 2026 12:48:01 +0200 Subject: [PATCH 39/75] feat(monitor): Pause when a treasury migration pays an unexpected address With EXPECTED_TREASURY_DESTINATION set, every finalized MigrationInitiated whose who is the treasury account (PalletId py/trsry, the account migrate_treasury burns from) must pay that Base address. Any other pages TREASURY DESTINATION VIOLATION and pauses the vault when the burn is ingested, before the attestors can release it. While the pause is not confirmed, the monitor does not move past that block, so the next poll tries again. --- monitor/README.md | 8 ++++--- monitor/src/main.test.ts | 47 +++++++++++++++++++++++++++++++++++++--- monitor/src/main.ts | 24 ++++++++++++++++++-- 3 files changed, 71 insertions(+), 8 deletions(-) diff --git a/monitor/README.md b/monitor/README.md index 5c2960758..3454632c6 100644 --- a/monitor/README.md +++ b/monitor/README.md @@ -14,6 +14,7 @@ Checks every poll: | M4: every old migration reaches active approval quorum — and then releases | Liveness of the attestor fleet (`below approval quorum`), and of everything after it (`quorum reached but not released`: a release starved of daily allowance, paused, or made releasable by a threshold cut without a `ReleasePending`, which the releaser cannot see). | One summary page per kind and poll (count and the first nonces; the log lists every nonce), each nonce at most once per `GRACE_SECONDS` | | Source supply | `totalIssuance + TotalMigrated` on Pendulum must never grow past its first-seen anchor. | Alert (`SOURCE SUPPLY GREW`) | | Unreleasable burns | A burn to the zero or vault address can never be approved or released; it is paged once and excluded from the pending count so it cannot page liveness forever or block RB-7. | Alert once | +| Treasury destination | With `EXPECTED_TREASURY_DESTINATION` set, every finalized treasury migration (a `MigrationInitiated` whose `who` is the treasury account, `PalletId(*b"py/trsry")`) must pay that Base address. It is checked when the burn is ingested, before the attestors can release it; while the pause is not confirmed, the monitor does not move past that block. | Alert + auto-pause (`TREASURY DESTINATION VIOLATION`) | | Pendulum source | The finalized Pendulum view must stay fresh and readable. While it cannot be read (an RPC error, a nonce discontinuity, an event-shape change), the Base-side checks — vault balance and its auto-pause, Base view, auto-pause readiness, vault admin events, governance queue — still run, and Base events are reconciled against the last view read in full: an unreadable source counts as a stalled one, so a mismatch on a known nonce counts at once and an unknown nonce after `UNMATCHED_EVENT_GRACE_SECONDS`. M2a and liveness wait for the source. | Alert, throttled: `PENDULUM SOURCE VIEW STALLED` once the view is older than `SOURCE_STALE_ALERT_SECONDS`, `PENDULUM SOURCE UNREADABLE` on a failure | | Pendulum chain | Every finalized block the monitor ingests is read at a hash it verified against the blocks it ingested before and, where configured, a second node and the relay chain (see [Pendulum chain verification](#pendulum-chain-verification)). | Alert (`CRITICAL: PENDULUM SOURCE CONTRADICTED`), throttled; nothing from the source is ingested, and it is treated as unreadable (see Pendulum source) until it verifies again. A second node or relay RPC that does not answer only warns, throttled (`PENDULUM CROSS-CHECK UNAVAILABLE`, `RELAY ANCHOR UNAVAILABLE`); `CRITICAL: RELAY ANCHOR STALE` once no head was anchored for `RELAY_ANCHOR_GRACE_SECONDS` | | Base log completeness | In every scanned block range the `Released` amounts must add up to the growth of `totalReleased`, and the `RemainderSwept` amounts to that of `totalSwept`, between the block before the range and its last block. A range whose logs do not (a backend serving truncated logs) is read once more, then held: the Base cursor stays on it and M2a and liveness wait until a read accounts for the totals. | Alert (`BASE LOG SCAN INCOMPLETE`), throttled | @@ -74,6 +75,7 @@ for a verified block are not proof-checked against the block's state root. | `BASE_STALE_ALERT_SECONDS` | Age of the safe/finalized Base block (vs. wall clock) past which `BASE VIEW STALLED` pages (default 600s for `safe`, 1800s for `finalized`; Base's safe head normally trails by about a minute, its finalized head by about 20) | | `ISSUANCE_TOLERANCE` | Growth of Pendulum `totalIssuance + TotalMigrated` over its first-seen anchor (12-decimal pallet units) tolerated before `SOURCE SUPPLY GREW` pages (default 0) | | `ACKNOWLEDGED_RELEASE_EXCESS` | Excess of released over migrated, in token units (18 decimals), that operators quantified and accepted after a real loss; M2a fires only beyond it (default 0). Set it at the RB-3 sign-off and restart; the startup log shows it | +| `EXPECTED_TREASURY_DESTINATION` | Optional: the Base address every treasury migration must pay, the one set with `tokenMigration.setTreasuryDestination`; any other pauses the vault | | `TIMELOCK_ADDRESS` | Optional: the governance TimelockController. Set it as soon as governance is deployed; the queue is scanned from the monitor's Base cursor onwards, so operations scheduled before the monitor knew the address are not paged | | `ALERT_WEBHOOK_URL` | Slack incoming webhook receiving the pages (see [Pages](#pages)). A page counts as delivered only on a 2xx answer within 10s: once-only pages (governance items, admin events, scheduled operations, first sightings) are retried every poll until delivered, and failed deliveries are logged. Without it the log line is the page | | `HEARTBEAT_URL` | Optional: a dead-man's switch URL (Healthchecks.io style), requested with a `GET` (10s timeout) after every poll that ran all checks; a failed request is logged and never stops the monitor. Have the service page when the requests stop: the monitor is down, stuck, or cannot complete its checks (for example while the Pendulum source cannot be read) | @@ -94,13 +96,13 @@ to `ALERT_WEBHOOK_URL`. The text is `[CRITICAL] pen-monitor: SUBJECT` or `[WARNING] pen-monitor: SUBJECT`, then the detail on the next line. Slack's control characters `&`, `<` and `>` are escaped, so no detail can mention anyone or hide a link, and a detail longer than 3,000 characters is cut short -(the monitor's log line keeps the full text). `CRITICAL` marks the conservation -violations, `AUTO-PAUSE FAILED`, `AUTO-PAUSE UNAVAILABLE` and every subject that +(the monitor's log line keeps the full text). `CRITICAL` marks the violations +that pause the vault, `AUTO-PAUSE FAILED`, `AUTO-PAUSE UNAVAILABLE` and every subject that starts with `CRITICAL`; everything else is a `WARNING`. | Subjects | Meaning | |---|---| -| `MIGRATION TUPLE VIOLATION`, `VAULT BALANCE DEFICIT`, `CONSERVATION VIOLATION` | A conservation check failed; the auto-pause follows with `vault auto-pause confirmed`, `vault pause confirmed` (already paused), `AUTO-PAUSE PENDING FINALITY`, `AUTO-PAUSE FAILED` or `AUTO-PAUSE UNAVAILABLE` | +| `MIGRATION TUPLE VIOLATION`, `VAULT BALANCE DEFICIT`, `CONSERVATION VIOLATION`, `TREASURY DESTINATION VIOLATION` | A conservation or treasury check failed; the auto-pause follows with `vault auto-pause confirmed`, `vault pause confirmed` (already paused), `AUTO-PAUSE PENDING FINALITY`, `AUTO-PAUSE FAILED` or `AUTO-PAUSE UNAVAILABLE` | | `UNVERIFIED BASE EVENT` | First sighting of a Base event whose nonce the source view does not know yet | | `CRITICAL: MISMATCHED APPROVAL` | One attestor approved a tuple that disagrees with the finalized Pendulum view; check its key and its Pendulum node. A second attestor doing so for the same nonce pauses the vault | | `LIVENESS: migration below approval quorum`, `LIVENESS: quorum reached but not released` | Stalled migrations, one summary per kind and poll | diff --git a/monitor/src/main.test.ts b/monitor/src/main.test.ts index 7a5c499a6..5de47fa44 100644 --- a/monitor/src/main.test.ts +++ b/monitor/src/main.test.ts @@ -394,14 +394,23 @@ function pendulumHash(chain: { forkFrom?: number }, block: number): string { const blockOf = (blockHash: string) => Number(BigInt(blockHash) & 0xffff_ffff_ffff_ffffn); type Pendulum = ReturnType["chain"]; -/** A finalized burn on Pendulum, in block 101 unless stated. */ -function burn(pendulum: Pendulum, nonce: bigint, palletAmount: bigint, block = 101): void { +const MIGRATOR = `0x${"ab".repeat(32)}`; +const TREASURY_ACCOUNT = `0x${Buffer.from("modlpy/trsry").toString("hex")}${"00".repeat(20)}`; + +/** A finalized burn on Pendulum by `who` for `recipient`, in block 101 unless stated. */ +function burn( + pendulum: Pendulum, + nonce: bigint, + palletAmount: bigint, + block = 101, + { who = MIGRATOR, recipient = RECIPIENT } = {}, +): void { pendulum.finalized = Math.max(pendulum.finalized, block); const events = pendulum.events.get(block) ?? []; events.push({ section: "tokenMigration", method: "MigrationInitiated", - data: [codec(nonce), "5Migrator", hash(RECIPIENT), codec(palletAmount)], + data: [codec(nonce), hash(who), hash(recipient), codec(palletAmount)], }); pendulum.events.set(block, events); pendulum.totalMigrated += palletAmount; @@ -1185,6 +1194,38 @@ test("a motion or external proposal made again with the same call pages again", assert.equal(pages.filter((page) => page.detail.startsWith("the council's external proposal")).length, 3, JSON.stringify(pages)); }); +const TREASURY_DESTINATION = "0x5555555555555555555555555555555555555555"; + +test("a treasury migration to any but the expected address pages and pauses", async (t) => { + const { base, pendulum, api, monitor, pages, persisted } = await setup(t, { + env: { EXPECTED_TREASURY_DESTINATION: TREASURY_DESTINATION, GUARDIAN_PRIVATE_KEY: GUARDIAN_KEY }, + }); + base.paused = true; // so the auto-pause only confirms + burn(pendulum, 0n, 100n * PEN, 101, { who: TREASURY_ACCOUNT, recipient: TREASURY_DESTINATION }); + burn(pendulum, 1n, 100n * PEN, 101, { recipient: TOKEN }); // anyone else may migrate anywhere + await monitor.poll(api); + assert.equal(pages.length, 0, JSON.stringify(pages)); + burn(pendulum, 2n, 7n * PEN, 102, { who: TREASURY_ACCOUNT, recipient: TOKEN }); + await monitor.poll(api); + const page = pages.find((page) => page.subject === "TREASURY DESTINATION VIOLATION"); + assert.equal(page?.severity, "CRITICAL", JSON.stringify(pages)); + assert.match(page.detail, new RegExp(`nonce 2 in finalized Pendulum block 102 pays 7000000000000 pallet units to ${TOKEN}`, "i")); + assert.ok(pages.some((page) => page.subject === "vault pause confirmed"), JSON.stringify(pages)); + assert.deepEqual(persisted().migrations.map((migration) => migration.nonce), ["0", "1", "2"]); +}); + +test("while the pause after a treasury violation is not confirmed, the monitor holds at its block", async (t) => { + const { pendulum, api, monitor, subjects, persisted } = await setup(t, { + env: { EXPECTED_TREASURY_DESTINATION: TREASURY_DESTINATION }, + }); + burn(pendulum, 0n, 100n * PEN, 101, { who: TREASURY_ACCOUNT, recipient: TOKEN }); + await monitor.poll(api); + await monitor.poll(api); + assert.equal(subjects().filter((subject) => subject === "TREASURY DESTINATION VIOLATION").length, 2); + assert.equal(persisted().lastPendulumBlock, 100); + assert.ok(subjects().includes("PENDULUM SOURCE UNREADABLE"), JSON.stringify(subjects())); +}); + test("admin events of the migration pallet page from the ingest", async (t) => { const { pendulum, api, monitor, pages } = await setup(t); pendulum.finalized = 101; diff --git a/monitor/src/main.ts b/monitor/src/main.ts index 263bf6cf2..5ed9cb0ff 100644 --- a/monitor/src/main.ts +++ b/monitor/src/main.ts @@ -251,6 +251,8 @@ const config = { /** Optional: the governance TimelockController; every operation it * schedules pages, because the canceller must veto inside the delay. */ timelockAddress: optionalAddress("TIMELOCK_ADDRESS"), + /** Optional: the only Base address a treasury migration may pay. */ + expectedTreasuryDestination: optionalAddress("EXPECTED_TREASURY_DESTINATION"), /** Age of the confirmed Base block (vs. wall clock) past which the monitor * pages that its Base view stopped advancing. `safe` normally trails by * about a minute, `finalized` by about 20 minutes. */ @@ -263,6 +265,9 @@ const config = { }; const ZERO_ADDRESS = "0x0000000000000000000000000000000000000000"; +/** The runtime's treasury account, `PalletId(*b"py/trsry")` as an account + * ("modl" + id, zero-padded): the `who` of every treasury migration. */ +const PENDULUM_TREASURY_ACCOUNT = `0x6d6f646c70792f7472737279${"00".repeat(20)}`; const baseChain = defineChain({ id: config.baseChainId, @@ -892,9 +897,9 @@ async function ingestPendulumEvents( } if (method !== "MigrationInitiated") continue; if (data.length !== 4) throw new Error(`MigrationInitiated in block ${block} has ${data.length} fields, expected 4`); - const [nonceValue, , recipientValue, amountValue] = data as [ + const [nonceValue, whoValue, recipientValue, amountValue] = data as [ { toBigInt(): bigint }, - unknown, + { toHex(): string }, { toHex(): string }, { toBigInt(): bigint }, ]; @@ -907,6 +912,21 @@ async function ingestPendulumEvents( throw new Error(`migration nonce gap: finalized event ${nonce}, expected ${draft.nextExpectedNonce}`); } draft.nextExpectedNonce++; + const expectedTreasury = config.expectedTreasuryDestination; + if ( + expectedTreasury && + whoValue.toHex().toLowerCase() === PENDULUM_TREASURY_ACCOUNT && + recipient.toLowerCase() !== expectedTreasury.toLowerCase() + ) { + // The vault is paused before the attestors can release it; while + // the pause is not confirmed, the monitor does not move past it. + const paused = await securityViolation( + "TREASURY DESTINATION VIOLATION", + `treasury migration nonce ${nonce} in finalized Pendulum block ${block} pays ${amountValue.toBigInt()} ` + + `pallet units to ${recipient}, not EXPECTED_TREASURY_DESTINATION ${expectedTreasury}`, + ); + if (!paused) throw new Error(`the vault is not confirmed paused after treasury migration nonce ${nonce}`); + } if (recipient === ZERO_ADDRESS || recipient.toLowerCase() === config.vaultAddress.toLowerCase()) { // The vault deterministically rejects this tuple (ZeroAddress / // RecipientIsVault) and the attestors skip it, so it can never be From a7bb4237958f6d2eb600feab406406b6d8a34fd0 Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Thu, 22 Oct 2026 12:53:36 +0200 Subject: [PATCH 40/75] test(monitor): Cover more of the pause decisions at the call site Adds poll tests for decisions that changed nothing in the suite when altered by hand: a mismatching release must ask for the pause, a second release of one nonce is a violation, a release whose burn the source never shows pauses after the grace on its own, and vault events past the confirmed Base block must not trip the balance check. The stand-in vault now answers its balance per block. --- monitor/src/main.test.ts | 47 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 43 insertions(+), 4 deletions(-) diff --git a/monitor/src/main.test.ts b/monitor/src/main.test.ts index 5de47fa44..c84fb7a90 100644 --- a/monitor/src/main.test.ts +++ b/monitor/src/main.test.ts @@ -94,10 +94,11 @@ function baseChain() { }; } -/** A vault total at `block`: its current value less what later events added. */ -function totalAt(base: Base, total: bigint, eventName: string, field: string, block: bigint | undefined): bigint { +/** A vault total at `block`: its current value less what later events added + * (`sign` -1 for the balance, which those events lowered). */ +function totalAt(base: Base, total: bigint, eventName: string, field: string, block: bigint | undefined, sign = 1n): bigint { const later = base.logs.filter((log) => log.eventName === eventName && block !== undefined && log.blockNumber > block); - return later.reduce((sum, log) => sum - (log.args[field] as bigint), total); + return later.reduce((sum, log) => sum - sign * (log.args[field] as bigint), total); } type Base = ReturnType; @@ -135,7 +136,14 @@ function answer(base: Base, method: string, params: any[]): unknown { guardian: base.guardian, activeApprovals: base.approvals.get(key) ?? 0n, totalSupply: SUPPLY, - balanceOf: base.vaultBalance, + balanceOf: totalAt( + base, + totalAt(base, base.vaultBalance, "Released", "tokenAmount", block, -1n), + "RemainderSwept", + "amount", + block, + -1n, + ), getTimestamp: base.readyAt.get(key) ?? 0n, }[functionName]; const { outputs } = abi.find((item) => item.type === "function" && item.name === functionName) as AbiFunction; @@ -906,6 +914,37 @@ test("while Pendulum cannot be read, a mismatching release of a known nonce paus base.safe = 1_020n; await monitor.poll(api); assert.match(pages.find((page) => page.subject === "MIGRATION TUPLE VIOLATION")?.detail ?? "", /recipient/); + assert.ok(pages.some((page) => page.subject === "AUTO-PAUSE UNAVAILABLE"), JSON.stringify(pages)); // the pause was asked for +}); + +test("a release the source never shows pauses after the grace, with no approval in view", async (t) => { + const { base, pendulum, api, monitor, pages } = await setup(t, { env: { UNMATCHED_EVENT_GRACE_SECONDS: "0" } }); + pendulum.failure = new Error("node unreachable"); + const tokenAmount = 5n * PEN * CF; + base.logs.push({ address: VAULT, eventName: "Released", args: { nonce: 7n, recipient: RECIPIENT, palletAmount: 5n * PEN, tokenAmount }, blockNumber: 1_005n }); + base.totalReleased += tokenAmount; + base.vaultBalance -= tokenAmount; + await monitor.poll(api); + await new Promise((resolve) => setTimeout(resolve, 5)); + await monitor.poll(api); // the grace has run out + assert.match(pages.find((page) => page.subject === "MIGRATION TUPLE VIOLATION")?.detail ?? "", /^Released at Base block 1005: nonce 7 has no finalized/); +}); + +test("a second release of one nonce is a violation", async (t) => { + const { base, pendulum, api, monitor, pages } = await setup(t); + burn(pendulum, 0n, 100n * PEN); + release(base, 0n, 100n * PEN, 1_004n); + release(base, 0n, 100n * PEN, 1_006n); + await monitor.poll(api); + assert.match(pages.find((page) => page.subject === "MIGRATION TUPLE VIOLATION")?.detail ?? "", /duplicate Released event for nonce 0/); +}); + +test("vault events past the confirmed Base block do not trip the balance check", async (t) => { + const { base, pendulum, api, monitor, pages } = await setup(t); + burn(pendulum, 0n, 100n * PEN); + release(base, 0n, 100n * PEN, 1_015n); // after the safe block 1010 + await monitor.poll(api); + assert.equal(pages.length, 0, JSON.stringify(pages)); }); test("logs that miss a release hold the Base cursor and page until a read accounts for the totals", async (t) => { From c609077ae96ed98fb96b309b8c541c12c181a5ba Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Thu, 22 Oct 2026 13:07:46 +0200 Subject: [PATCH 41/75] fix(monitor): Validate the URLs of the checking nodes at startup PENDULUM_WS_CROSSCHECK must be one ws:// or wss:// URL and POLKADOT_WS a list of them; anything else stops the monitor at startup, so a typo fails loudly instead of paging that the node cannot be reached. --- monitor/src/main.test.ts | 8 ++++++++ monitor/src/main.ts | 14 ++++++++++++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/monitor/src/main.test.ts b/monitor/src/main.test.ts index c84fb7a90..70bd9bd4a 100644 --- a/monitor/src/main.test.ts +++ b/monitor/src/main.test.ts @@ -728,6 +728,14 @@ test("a second node that cannot answer only warns", async (t) => { assert.equal(pages.length, 1); // throttled }); +test("the URLs of the checking nodes are validated at startup", async (t) => { + await assert.rejects(setup(t, { env: { POLKADOT_WS: "wss://relay.test, https://relay.test" } }), /invalid POLKADOT_WS/); + await assert.rejects( + setup(t, { env: { PENDULUM_WS_CROSSCHECK: "wss://a.test,wss://b.test" } }), + /invalid PENDULUM_WS_CROSSCHECK: expected one ws/, + ); +}); + test("a node connection that fails with an event says why", async (t) => { const { pendulum, api, monitor, pages } = await setup(t, { env: { PENDULUM_WS_CROSSCHECK: SECOND_NODE }, diff --git a/monitor/src/main.ts b/monitor/src/main.ts index 5ed9cb0ff..9468712a6 100644 --- a/monitor/src/main.ts +++ b/monitor/src/main.ts @@ -194,6 +194,16 @@ function optionalAddress(name: string): `0x${string}` | undefined { return value as `0x${string}`; } +/** Comma-separated WebSocket URLs from the environment, at most `most`, + * validated at startup. */ +function wsUrls(name: string, most = Infinity): string[] { + const urls = (process.env[name] ?? "").split(",").map((url) => url.trim()).filter(Boolean); + if (urls.length > most || urls.some((url) => !/^wss?:\/\//i.test(url))) { + throw new Error(`invalid ${name}: expected ${most === 1 ? "one ws:// or wss:// URL" : "ws:// or wss:// URLs"}`); + } + return urls; +} + function finalityTag(): "safe" | "finalized" { const value = process.env.BASE_FINALITY_TAG ?? "safe"; if (value !== "safe" && value !== "finalized") throw new Error(`invalid BASE_FINALITY_TAG ${value}`); @@ -205,10 +215,10 @@ const baseFinalityTag = finalityTag(); const config = { pendulumWs: required("PENDULUM_WS"), /** Optional: a second Pendulum node whose block hashes must agree. */ - pendulumWsCrossCheck: process.env.PENDULUM_WS_CROSSCHECK, + pendulumWsCrossCheck: wsUrls("PENDULUM_WS_CROSSCHECK", 1)[0] as string | undefined, /** Optional: third-party Polkadot RPCs, tried in order, whose record of * the Pendulum head anchors the source. */ - polkadotWs: (process.env.POLKADOT_WS ?? "").split(",").map((url) => url.trim()).filter(Boolean), + polkadotWs: wsUrls("POLKADOT_WS"), pendulumParaId: envNumber("PENDULUM_PARA_ID", 2094), /** How long the source may go without a relay-anchored head before that pages critical. */ relayAnchorGraceMs: envNumber("RELAY_ANCHOR_GRACE_SECONDS", 3600) * 1000, From 6abfe67fd87f1b958bb1e6ea48c470ee8de3f48a Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Thu, 22 Oct 2026 13:08:06 +0200 Subject: [PATCH 42/75] docs(monitor): Note the Base node's state needs and the walk's bound The log completeness check reads the vault's totals at the boundaries of each scanned range, so after an outage the Base node must serve old state. The parent-hash walk keeps a catch-up in memory in one pass. Also rewraps two README paragraphs. --- monitor/README.md | 14 ++++++++------ monitor/src/main.ts | 1 + 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/monitor/README.md b/monitor/README.md index 3454632c6..54d1dca4a 100644 --- a/monitor/README.md +++ b/monitor/README.md @@ -97,8 +97,8 @@ to `ALERT_WEBHOOK_URL`. The text is `[CRITICAL] pen-monitor: SUBJECT` or control characters `&`, `<` and `>` are escaped, so no detail can mention anyone or hide a link, and a detail longer than 3,000 characters is cut short (the monitor's log line keeps the full text). `CRITICAL` marks the violations -that pause the vault, `AUTO-PAUSE FAILED`, `AUTO-PAUSE UNAVAILABLE` and every subject that -starts with `CRITICAL`; everything else is a `WARNING`. +that pause the vault, `AUTO-PAUSE FAILED`, `AUTO-PAUSE UNAVAILABLE` and every +subject that starts with `CRITICAL`; everything else is a `WARNING`. | Subjects | Meaning | |---|---| @@ -126,9 +126,8 @@ npm start Run it under a process manager and treat "monitor down" itself as a paging condition (`HEARTBEAT_URL`): an unwatched migration is the risk model failing -silently. The -two-chain cursors, unresolved canonical tuples, their original finalized block -timestamps, and alert throttles are atomically persisted. A malformed or +silently. The two-chain cursors, unresolved canonical tuples, their original +finalized block timestamps, and alert throttles are atomically persisted. A malformed or Pendulum-genesis/Base-chain/vault-mismatched state file is fatal rather than silently resetting the security history. @@ -144,7 +143,10 @@ pause; treat that page as "restore the node now". The Base-side checks keep running while the source cannot be read, and an unreadable source counts as a stalled one: once a Base event it cannot verify appears, the monitor pages `UNVERIFIED BASE EVENT` and pauses when the grace runs out, so treat -`PENDULUM SOURCE UNREADABLE` the same way. +`PENDULUM SOURCE UNREADABLE` the same way. The Base node must answer +`eth_call` at the block before each scanned range and at its last block (the +log completeness check); after a monitor outage those are old blocks, so a +node that keeps only recent state stops the scan until it is an archive node. An automatic pause is reported as successful only after the transaction is canonical inside the configured Base confirmation boundary and `paused()` is diff --git a/monitor/src/main.ts b/monitor/src/main.ts index 9468712a6..ae4c69bc1 100644 --- a/monitor/src/main.ts +++ b/monitor/src/main.ts @@ -751,6 +751,7 @@ async function crossCheckSource(finalized: BlockRef): Promise { * from the node, recomputing each hash here, so that the result is one * chain the node cannot fake: block hashes by height, `bottom` included. */ async function walkHeaders(api: ApiPromise, top: BlockRef, bottom: number): Promise> { + // ponytail: one pass, held in memory and started over after a failed read; make it resumable if catch-ups grow to weeks. const hashes = new Map([[top.number, top.hash]]); let hash = top.hash; for (let height = top.number; height > bottom; height--) { From f047d1a0b4ff898a03a15ffa9affd5d3a672d11a Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Thu, 22 Oct 2026 13:48:17 +0200 Subject: [PATCH 43/75] docs(monitor): Trim the Pendulum chain verification section --- monitor/README.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/monitor/README.md b/monitor/README.md index 54d1dca4a..9d738c4f4 100644 --- a/monitor/README.md +++ b/monitor/README.md @@ -53,9 +53,6 @@ once no head was anchored for `RELAY_ANCHOR_GRACE_SECONDS`, whatever the reason (relay RPCs down, the source behind the relay chain, a relay head that does not advance). -These checks bind block hashes only: the events and storage the source returns -for a verified block are not proof-checked against the block's state root. - ## Configuration (environment) | Variable | Meaning | From b170f5e0fb0eb2f20be1c00f52efe43cdd0902c2 Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Thu, 22 Oct 2026 13:48:32 +0200 Subject: [PATCH 44/75] docs(monitor): Drop history notes from code comments --- monitor/src/checks.test.ts | 6 +++--- monitor/src/checks.ts | 5 ++--- monitor/src/state.ts | 4 ++-- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/monitor/src/checks.test.ts b/monitor/src/checks.test.ts index 2c3975786..e5cd838e4 100644 --- a/monitor/src/checks.test.ts +++ b/monitor/src/checks.test.ts @@ -1,9 +1,9 @@ /** * Unit tests for the monitor's conservation/liveness predicates. * - * Run with `npm test` (compiles, then `node --test`). These lock in the - * round-5 fixes: the M2b check must ignore a token surplus (donation / vault - * recipient) and only fire on a real deficit. + * Run with `npm test` (compiles, then `node --test`). Among others, these lock + * in that the M2b check ignores a token surplus (donation / vault recipient) + * and only fires on a real deficit. */ import assert from "node:assert/strict"; diff --git a/monitor/src/checks.ts b/monitor/src/checks.ts index 273272e19..619f6a619 100644 --- a/monitor/src/checks.ts +++ b/monitor/src/checks.ts @@ -2,8 +2,7 @@ * Pure conservation/liveness predicates for the invariant monitor. * * These are isolated from the chain-client plumbing in `main.ts` so they can be - * unit-tested exhaustively (see checks.test.ts). Every subtlety that has bitten - * a review round lives here. + * unit-tested exhaustively (see checks.test.ts). */ /** @@ -95,7 +94,7 @@ export function shouldAwaitSource( * timestamp: OP-stack L2 block timestamps trail real time by the length of any * sequencer outage while it catches up, and an artificially old event * timestamp would let a merely-lagging source "prove" a legitimate event - * fabricated (review round 9). A lagging source never satisfies this predicate + * fabricated. A lagging source never satisfies this predicate * (its head timestamp trails the observation), and a future-dated source view * (a collator clock ahead of real time) proves nothing either and is excluded, * so neither can turn node lag into a false pause. diff --git a/monitor/src/state.ts b/monitor/src/state.ts index 3627d380e..320ae7cff 100644 --- a/monitor/src/state.ts +++ b/monitor/src/state.ts @@ -28,8 +28,8 @@ export interface PersistedMonitorState { migrations: PersistedMigration[]; livenessAlertedAt: Array<[string, number]>; unmatchedBaseFirstSeenAt: Array<[string, number]>; - /** Pendulum totalIssuance + TotalMigrated when first observed (round 9); - * absent in state written before it existed and re-anchored on load. */ + /** Pendulum totalIssuance + TotalMigrated when first observed; absent in + * state written before it existed and re-anchored on load. */ issuanceAnchor?: string; /** Next Base block of the admin scan; absent in older state, which never * persisted a Base cursor past it, and then `baseFromBlock`. */ From 43e8187bd9ea97ec5005b92779814a24e7ade51a Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Thu, 22 Oct 2026 13:49:27 +0200 Subject: [PATCH 45/75] fix(monitor): Keep the source bound to a relay head past its finalized head The parent-hash walk now starts at the newest of the source's finalized head, the relay head of this poll and the one kept from an earlier poll. A kept relay head above the finalized head therefore binds the source's chain also in polls that bring no newer one. --- monitor/src/main.test.ts | 24 ++++++++++++++++++++++++ monitor/src/main.ts | 4 +++- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/monitor/src/main.test.ts b/monitor/src/main.test.ts index 70bd9bd4a..2e4847929 100644 --- a/monitor/src/main.test.ts +++ b/monitor/src/main.test.ts @@ -836,6 +836,30 @@ test("a relay record past the source's finalized head anchors it, and its chain ); }); +test("a kept relay record past the source's finalized head binds the source while no newer one arrives", async (t) => { + let relayDown = false; + const { pendulum, api, monitor, pages, persisted } = await relaySetup(t, { + head: () => ({ number: 103, hash: pendulumHash({}, 103) }), + failing: () => relayDown, + }); + burn(pendulum, 0n, 100n * PEN); + pendulum.best = 103; // imported, not yet finalized by the source + await monitor.poll(api); + assert.equal(pages.length, 0, JSON.stringify(pages)); + assert.deepEqual(persisted().relayAnchor, { number: 103, hash: pendulumHash({}, 103) }); + assert.equal(persisted().lastPendulumBlock, 101); + relayDown = true; + pendulum.forkFrom = 102; // the source finalizes another chain below the kept record... + burn(pendulum, 1n, 999n * PEN, 102); // ...with a burn on it + await monitor.poll(api); + assert.match( + pages.find((page) => page.subject === "CRITICAL: PENDULUM SOURCE CONTRADICTED")?.detail ?? "", + /chain has 0x0+66 at Pendulum block 102, where the source's finalized head is 0x0+10+66;/, + ); + assert.equal(persisted().lastPendulumBlock, 101); + assert.deepEqual(persisted().migrations.map((migration) => migration.nonce), ["0"]); +}); + test("a relay record past the source's finalized head must descend from that head", async (t) => { const { pendulum, api, monitor, pages, persisted } = await relaySetup(t, { head: () => ({ number: 102, hash: pendulumHash({}, 102) }), diff --git a/monitor/src/main.ts b/monitor/src/main.ts index ae4c69bc1..32737eaac 100644 --- a/monitor/src/main.ts +++ b/monitor/src/main.ts @@ -846,7 +846,9 @@ async function relayAnchor(api: ApiPromise, draft: RuntimeState, finalized: Bloc * of the Pendulum head. */ async function sourceHashes(api: ApiPromise, draft: RuntimeState, finalized: BlockRef): Promise> { const anchor = await relayAnchor(api, draft, finalized); - const top = anchor && anchor.number > finalized.number ? anchor : finalized; + // A relay-chain record past the finalized head binds the source too, also + // the one kept from an earlier poll: the walk starts at the newest of them. + const top = [anchor, draft.relayAnchor].reduce((high, ref) => (ref && ref.number > high.number ? ref : high), finalized); const hashes = await walkHeaders(api, top, Math.min(draft.lastPendulumBlock, anchor?.number ?? Infinity)); const ingested = draft.lastPendulumHash ? { number: draft.lastPendulumBlock, hash: draft.lastPendulumHash } : undefined; const expected: Array<[BlockRef | undefined, string]> = [ From 58cd9231208f6712f1dd9e1fdd05239e79558caa Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Thu, 22 Oct 2026 13:51:32 +0200 Subject: [PATCH 46/75] fix(monitor): Let a relay head above the finalized head wait for the source Above its finalized head, the source's best chain may briefly hold another block at the height the relay chain recorded. That no longer counts as a contradiction: the record anchors the source once its best chain holds it, and the record's chain must then pass through the source's finalized head. The source's own chain is walked and checked first; the relay chain's record is then compared with it. --- monitor/src/main.test.ts | 35 +++++++++++-- monitor/src/main.ts | 106 ++++++++++++++++++++++++++------------- 2 files changed, 104 insertions(+), 37 deletions(-) diff --git a/monitor/src/main.test.ts b/monitor/src/main.test.ts index 2e4847929..6b41c76fd 100644 --- a/monitor/src/main.test.ts +++ b/monitor/src/main.test.ts @@ -795,7 +795,7 @@ test("a source that names another hash than the relay chain's record pages criti await monitor.poll(api); assert.match( pages.find((page) => page.subject === "CRITICAL: PENDULUM SOURCE CONTRADICTED")?.detail ?? "", - /relay chain records 0x0+10+65 as Pendulum block 101, the source has 0x0+65;/, + /relay chain records 0x0+10+65 as Pendulum block 101, the source's chain has 0x0+65;/, ); assert.equal(persisted().lastPendulumBlock, 100); }); @@ -810,7 +810,7 @@ test("a source whose chain avoids the relay chain's record pages critical, whate await monitor.poll(api); assert.match( pages.find((page) => page.subject === "CRITICAL: PENDULUM SOURCE CONTRADICTED")?.detail ?? "", - /chain has 0x0+10+65 at Pendulum block 101, where the relay chain records 0x0+65;/, + /relay chain records 0x0+65 as Pendulum block 101, the source's chain has 0x0+10+65;/, ); }); @@ -871,11 +871,40 @@ test("a relay record past the source's finalized head must descend from that hea await monitor.poll(api); assert.match( pages.find((page) => page.subject === "CRITICAL: PENDULUM SOURCE CONTRADICTED")?.detail ?? "", - /chain has 0x0+65 at Pendulum block 101, where the source's finalized head is 0x0+10+65;/, + /records 0x0+66 as Pendulum block 102, whose chain has 0x0+65 at block 101, where the source has 0x0+10+65;/, ); assert.equal(persisted().lastPendulumBlock, 100); }); +test("a relay record past the source's finalized head that its best chain does not hold yet only waits", async (t) => { + const { pendulum, api, monitor, pages, persisted } = await relaySetup(t, { + head: () => ({ number: 102, hash: pendulumHash({}, 102) }), + }); + burn(pendulum, 0n, 100n * PEN); + pendulum.forkFrom = 102; // its best chain holds another block 102 for now + pendulum.best = 102; + await monitor.poll(api); + assert.equal(pages.length, 0, JSON.stringify(pages)); + assert.equal(persisted().relayAnchor, undefined); + assert.equal(persisted().lastPendulumBlock, 101); +}); + +test("a relay chain that records another hash at a height it anchored pages critical", async (t) => { + let recorded = pendulumHash({}, 101); + const { pendulum, api, monitor, pages, persisted } = await relaySetup(t, { + head: () => ({ number: 101, hash: recorded }), + }); + burn(pendulum, 0n, 100n * PEN); + await monitor.poll(api); + assert.deepEqual(persisted().relayAnchor, { number: 101, hash: pendulumHash({}, 101) }); + recorded = pendulumHash({ forkFrom: 0 }, 101); + await monitor.poll(api); + assert.match( + pages.find((page) => page.subject === "CRITICAL: PENDULUM SOURCE CONTRADICTED")?.detail ?? "", + /relay chain now records 0x0+10+65 as Pendulum block 101, where it recorded 0x0+65;/, + ); +}); + test("an unreadable relay chain warns, the source is used, and it turns critical after the grace", async (t) => { const { pendulum, api, monitor, pages, persisted } = await relaySetup(t, { failing: () => true }); burn(pendulum, 0n, 100n * PEN); diff --git a/monitor/src/main.ts b/monitor/src/main.ts index 32737eaac..d63f64626 100644 --- a/monitor/src/main.ts +++ b/monitor/src/main.ts @@ -792,33 +792,68 @@ async function relayHead(api: ApiPromise): Promise { throw new Error(failures.join("; ")); } -/** The relay chain's record of a Pendulum head newer than the last one - * verified, if this poll can check the source against it. Pages while there - * is none: a warning when the relay RPCs fail, critical once no head was - * anchored for RELAY_ANCHOR_GRACE_SECONDS. Both throttled. */ -async function relayAnchor(api: ApiPromise, draft: RuntimeState, finalized: BlockRef): Promise { - if (!config.polkadotWs.length) return undefined; - let reason: string; - try { - const anchor = await relayHead(api); - const known = draft.relayAnchor; - if (known?.number === anchor.number && known.hash !== anchor.hash) { +/** Why the relay chain's record `head` does not anchor the source this poll, + * or undefined when it does. Throws when it contradicts the source's chain: + * `hashes`, walked down from `top` to `bottom`. */ +async function relayVerdict( + api: ApiPromise, + draft: RuntimeState, + hashes: Map, + top: BlockRef, + bottom: BlockRef, + head: BlockRef, +): Promise { + const known = draft.relayAnchor; + if (known && head.number <= known.number) { + if (head.number === known.number && head.hash !== known.hash) { throw new SourceIntegrityError( - `the relay chain now records ${anchor.hash} as Pendulum block ${anchor.number}, where it recorded ${known.hash}`, + `the relay chain now records ${head.hash} as Pendulum block ${head.number}, where it recorded ${known.hash}`, ); } - const sourceHash = await api.rpc.chain.getBlockHash(anchor.number); - if (!sourceHash.isEmpty && sourceHash.toHex() !== anchor.hash) { + return `the relay chain's Pendulum head ${head.number} is not newer`; + } + if (head.number > top.number) { + // Past the finalized head, the source's best chain may still hold another + // block at that height. The record anchors the source once it holds the + // record, whose chain must then pass through `top`. + if ((await api.rpc.chain.getBlockHash(head.number)).toHex() !== head.hash) { + return `the source has not imported block ${head.number}, the relay chain's Pendulum head, yet`; + } + const hash = (await walkHeaders(api, head, top.number)).get(top.number); + if (hash !== top.hash) { throw new SourceIntegrityError( - `the relay chain records ${anchor.hash} as Pendulum block ${anchor.number}, the source has ${sourceHash.toHex()}`, + `the relay chain records ${head.hash} as Pendulum block ${head.number}, whose chain has ${hash} at block ` + + `${top.number}, where the source has ${top.hash}`, ); } - if (known && anchor.number <= known.number) reason = `the relay chain's Pendulum head ${anchor.number} is not newer`; - else if (sourceHash.isEmpty && anchor.number > finalized.number) { - reason = `the source has not imported block ${anchor.number}, the relay chain's Pendulum head, yet`; - } else return anchor; + return undefined; + } + const hash = head.number >= bottom.number + ? hashes.get(head.number) + : (await walkHeaders(api, bottom, head.number)).get(head.number); + if (hash !== head.hash) { + throw new SourceIntegrityError(`the relay chain records ${head.hash} as Pendulum block ${head.number}, the source's chain has ${hash}`); + } + return undefined; +} + +/** Anchors the source to the relay chain's record of the Pendulum head, kept + * as `draft.relayAnchor`. Pages while no new head is anchored: a warning when + * the relay RPCs fail, critical once none was for RELAY_ANCHOR_GRACE_SECONDS. + * Both throttled. */ +async function anchorToRelay( + api: ApiPromise, + draft: RuntimeState, + hashes: Map, + top: BlockRef, + bottom: BlockRef, +): Promise { + if (!config.polkadotWs.length) return; + let head: BlockRef | undefined; + let reason: string | undefined; + try { + head = await relayHead(api); } catch (relayError) { - if (relayError instanceof SourceIntegrityError) throw relayError; reason = `the relay chain cannot be read: ${relayError}`; if (Date.now() - lastRelayAlertMs > config.graceSeconds * 1000) { lastRelayAlertMs = Date.now(); @@ -829,6 +864,14 @@ async function relayAnchor(api: ApiPromise, draft: RuntimeState, finalized: Bloc ); } } + if (head) { + reason = await relayVerdict(api, draft, hashes, top, bottom, head); + if (reason === undefined) { + draft.relayAnchor = head; + lastRelayAnchorMs = Date.now(); + return; + } + } const unanchoredMs = Date.now() - lastRelayAnchorMs; if (unanchoredMs > config.relayAnchorGraceMs && Date.now() - lastRelayStaleAlertMs > config.unmatchedEventGraceSeconds * 1000) { lastRelayStaleAlertMs = Date.now(); @@ -838,24 +881,21 @@ async function relayAnchor(api: ApiPromise, draft: RuntimeState, finalized: Bloc "the source is trusted without the anchor — restore a relay RPC, or the source node if it lags", ); } - return undefined; } /** The hashes of the finalized blocks to ingest, on one chain with every - * block ingested before and, where available, with the relay chain's record - * of the Pendulum head. */ + * block ingested before and with the relay-chain record kept from an earlier + * poll; the source is then anchored to the relay chain's newest record. */ async function sourceHashes(api: ApiPromise, draft: RuntimeState, finalized: BlockRef): Promise> { - const anchor = await relayAnchor(api, draft, finalized); - // A relay-chain record past the finalized head binds the source too, also - // the one kept from an earlier poll: the walk starts at the newest of them. - const top = [anchor, draft.relayAnchor].reduce((high, ref) => (ref && ref.number > high.number ? ref : high), finalized); - const hashes = await walkHeaders(api, top, Math.min(draft.lastPendulumBlock, anchor?.number ?? Infinity)); + // A kept record past the finalized head binds the source too: the walk starts there. + const known = draft.relayAnchor; + const top = known && known.number > finalized.number ? known : finalized; + const hashes = await walkHeaders(api, top, draft.lastPendulumBlock); const ingested = draft.lastPendulumHash ? { number: draft.lastPendulumBlock, hash: draft.lastPendulumHash } : undefined; const expected: Array<[BlockRef | undefined, string]> = [ [finalized, "the source's finalized head is"], [ingested, "the monitor ingested"], - [anchor, "the relay chain records"], - [draft.relayAnchor, "the relay chain recorded"], + [known, "the relay chain recorded"], ]; for (const [block, what] of expected) { const hash = block && hashes.get(block.number); @@ -863,10 +903,8 @@ async function sourceHashes(api: ApiPromise, draft: RuntimeState, finalized: Blo throw new SourceIntegrityError(`the source's chain has ${hash} at Pendulum block ${block.number}, where ${what} ${block.hash}`); } } - if (anchor) { - draft.relayAnchor = anchor; - lastRelayAnchorMs = Date.now(); - } + const lowest = Math.min(top.number, draft.lastPendulumBlock); + await anchorToRelay(api, draft, hashes, top, { number: lowest, hash: hashes.get(lowest) as string }); return hashes; } From d113eb33d8066a0630756e464c9d0dd1dc33c915 Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Thu, 22 Oct 2026 13:53:42 +0200 Subject: [PATCH 47/75] feat(monitor): Confirm a relay-chain contradiction with a second relay RPC A relay head that contradicts the Pendulum source now counts only when the next relay RPC in POLKADOT_WS that answers contradicts the source too, or when POLKADOT_WS lists a single RPC. When the next one does not, RELAY RPCS DISAGREE pages and the poll goes on without the anchor; when no other one answers, RELAY ANCHOR UNAVAILABLE pages with the unconfirmed record. --- monitor/README.md | 20 +++++--- monitor/src/main.test.ts | 41 +++++++++++++++ monitor/src/main.ts | 105 ++++++++++++++++++++++++--------------- 3 files changed, 118 insertions(+), 48 deletions(-) diff --git a/monitor/README.md b/monitor/README.md index 9d738c4f4..714517aa6 100644 --- a/monitor/README.md +++ b/monitor/README.md @@ -16,7 +16,7 @@ Checks every poll: | Unreleasable burns | A burn to the zero or vault address can never be approved or released; it is paged once and excluded from the pending count so it cannot page liveness forever or block RB-7. | Alert once | | Treasury destination | With `EXPECTED_TREASURY_DESTINATION` set, every finalized treasury migration (a `MigrationInitiated` whose `who` is the treasury account, `PalletId(*b"py/trsry")`) must pay that Base address. It is checked when the burn is ingested, before the attestors can release it; while the pause is not confirmed, the monitor does not move past that block. | Alert + auto-pause (`TREASURY DESTINATION VIOLATION`) | | Pendulum source | The finalized Pendulum view must stay fresh and readable. While it cannot be read (an RPC error, a nonce discontinuity, an event-shape change), the Base-side checks — vault balance and its auto-pause, Base view, auto-pause readiness, vault admin events, governance queue — still run, and Base events are reconciled against the last view read in full: an unreadable source counts as a stalled one, so a mismatch on a known nonce counts at once and an unknown nonce after `UNMATCHED_EVENT_GRACE_SECONDS`. M2a and liveness wait for the source. | Alert, throttled: `PENDULUM SOURCE VIEW STALLED` once the view is older than `SOURCE_STALE_ALERT_SECONDS`, `PENDULUM SOURCE UNREADABLE` on a failure | -| Pendulum chain | Every finalized block the monitor ingests is read at a hash it verified against the blocks it ingested before and, where configured, a second node and the relay chain (see [Pendulum chain verification](#pendulum-chain-verification)). | Alert (`CRITICAL: PENDULUM SOURCE CONTRADICTED`), throttled; nothing from the source is ingested, and it is treated as unreadable (see Pendulum source) until it verifies again. A second node or relay RPC that does not answer only warns, throttled (`PENDULUM CROSS-CHECK UNAVAILABLE`, `RELAY ANCHOR UNAVAILABLE`); `CRITICAL: RELAY ANCHOR STALE` once no head was anchored for `RELAY_ANCHOR_GRACE_SECONDS` | +| Pendulum chain | Every finalized block the monitor ingests is read at a hash it verified against the blocks it ingested before and, where configured, a second node and the relay chain (see [Pendulum chain verification](#pendulum-chain-verification)). | Alert (`CRITICAL: PENDULUM SOURCE CONTRADICTED`), throttled; nothing from the source is ingested, and it is treated as unreadable (see Pendulum source) until it verifies again. A second node or relay RPC that does not answer only warns, throttled (`PENDULUM CROSS-CHECK UNAVAILABLE`, `RELAY ANCHOR UNAVAILABLE`), and so do relay RPCs that disagree (`RELAY RPCS DISAGREE`); `CRITICAL: RELAY ANCHOR STALE` once no head was anchored for `RELAY_ANCHOR_GRACE_SECONDS` | | Base log completeness | In every scanned block range the `Released` amounts must add up to the growth of `totalReleased`, and the `RemainderSwept` amounts to that of `totalSwept`, between the block before the range and its last block. A range whose logs do not (a backend serving truncated logs) is read once more, then held: the Base cursor stays on it and M2a and liveness wait until a read accounts for the totals. | Alert (`BASE LOG SCAN INCOMPLETE`), throttled | | Base view | The safe/finalized Base block must not be older than `BASE_STALE_ALERT_SECONDS`: a view that stops advancing hides every later vault event while every check still passes at the old block. | Alert (`BASE VIEW STALLED`), throttled | | Auto-pause readiness | At startup and every poll: the Base RPC serves `BASE_CHAIN_ID` and, with a guardian key configured, the key parses, is `vault.guardian()` and holds at least `GUARDIAN_MIN_BALANCE_WEI`. A read that fails pages as well and never stops the poll's other checks. | Alert (`AUTO-PAUSE NOT READY`), throttled; `AUTO-PAUSE DISABLED` once at startup without a key | @@ -40,10 +40,15 @@ itself: - **Relay-chain anchor** (`POLKADOT_WS`). Each poll reads the Pendulum head that the Polkadot relay chain recorded at its finalized block (`paras.heads(PENDULUM_PARA_ID)`), from the first relay RPC that answers. The - source must have that hash at that height, and its chain must pass through - it; the newest such head is kept in the state file, and the chain must keep - passing through it. Blocks ingested while no relay RPC answered are tied to - the next head that arrives by the same walk. + source's chain must have that hash at that height. A head above the source's + finalized head counts once the source's best chain holds it, and its chain + must then pass through the finalized head. The newest such head is kept in + the state file, and the chain must keep passing through it, also while it + lies above the finalized head. Blocks ingested while no relay RPC answered + are tied to the next head that arrives by the same walk. A head that + contradicts the source counts only when the next relay RPC that answers + contradicts it too, or when `POLKADOT_WS` lists one RPC; otherwise + `RELAY RPCS DISAGREE` pages and the poll goes on without the anchor. A contradiction pages `CRITICAL: PENDULUM SOURCE CONTRADICTED`, nothing from the source is ingested, and the source counts as unreadable until it verifies @@ -59,7 +64,7 @@ advance). |---|---| | `PENDULUM_WS` | WebSocket of the monitor's own Pendulum node, run with `--state-pruning archive` (see below) | | `PENDULUM_WS_CROSSCHECK` | Optional: WebSocket of a second Pendulum node, not the one behind `PENDULUM_WS`; its block hash at the source's finalized head must match in every poll | -| `POLKADOT_WS` | Optional: comma-separated WebSocket RPCs of the Polkadot relay chain, run by third parties and tried in order (for example `wss://rpc.polkadot.io`); turns on the relay-chain anchor | +| `POLKADOT_WS` | Optional: comma-separated WebSocket RPCs of the Polkadot relay chain, run by third parties and tried in order (for example `wss://rpc.polkadot.io`); turns on the relay-chain anchor. List at least two: a record that contradicts the source needs a second relay RPC's confirmation, unless only one is listed | | `PENDULUM_PARA_ID` | Pendulum's parachain id on the relay chain (default 2094) | | `RELAY_ANCHOR_GRACE_SECONDS` | How long the source may go without a relay-anchored head before `CRITICAL: RELAY ANCHOR STALE` pages (default 3600) | | `BASE_RPC_URL` | Base JSON-RPC endpoint — a **single dedicated node**, and a different provider than the attestors use. Load-balanced pools can serve `eth_getLogs` from a lagging replica; each scan range is probed for its end block first and its logs must account for the vault's running totals, which turns a behind-node into a loud replay instead of a silent gap, but a dedicated node removes the hazard entirely | @@ -105,7 +110,8 @@ subject that starts with `CRITICAL`; everything else is a `WARNING`. | `LIVENESS: migration below approval quorum`, `LIVENESS: quorum reached but not released` | Stalled migrations, one summary per kind and poll | | `PENDULUM SOURCE VIEW STALLED`, `PENDULUM SOURCE UNREADABLE`, `BASE VIEW STALLED`, `BASE ADMIN SCAN FAILED`, `BASE LOG SCAN INCOMPLETE` | The monitor is going blind on one chain | | `CRITICAL: PENDULUM SOURCE CONTRADICTED` | The Pendulum source contradicts the chain the monitor verified; stop trusting that node | -| `PENDULUM CROSS-CHECK UNAVAILABLE`, `RELAY ANCHOR UNAVAILABLE` | The second Pendulum node or every relay RPC does not answer, so the source is used without that check | +| `PENDULUM CROSS-CHECK UNAVAILABLE`, `RELAY ANCHOR UNAVAILABLE` | The second Pendulum node or every relay RPC does not answer, or the only relay RPC that answers records a head that contradicts the source, so the source is used without that check | +| `RELAY RPCS DISAGREE` | One relay RPC records a Pendulum head that contradicts the source and the next one that answers does not; the source is used without the anchor until they agree. Check the first one | | `CRITICAL: RELAY ANCHOR STALE` | No Pendulum head was checked against the relay chain for `RELAY_ANCHOR_GRACE_SECONDS` | | `AUTO-PAUSE NOT READY`, `AUTO-PAUSE DISABLED` | The next violation would not pause the vault automatically | | `SOURCE SUPPLY GREW`, `CRITICAL: unreleasable migration burned on Pendulum` | Supply problems at the source | diff --git a/monitor/src/main.test.ts b/monitor/src/main.test.ts index 6b41c76fd..8ad0ac8f1 100644 --- a/monitor/src/main.test.ts +++ b/monitor/src/main.test.ts @@ -950,6 +950,47 @@ test("the relay RPCs are tried in order", async (t) => { assert.equal(persisted().relayAnchor?.number, 101); }); +/** A monitor with two relay RPCs; each records `head` or, without one, is down. */ +async function twoRelaySetup(t: { after(fn: () => void): void }, heads: Array<((pendulum: Pendulum) => Head) | undefined>) { + let chain: Pendulum | undefined; + const urls = ["wss://first.test", "wss://second.test"]; + const relays = heads.map((head) => relayChain(() => head!(chain!), () => !head)); + return setup(t, { + env: { POLKADOT_WS: urls.join(",") }, + prepare: (_base, pendulum) => (chain = pendulum), + connect: async (url) => relays[urls.indexOf(url)] ?? assert.fail(url), + }); +} +const honestHead = (pendulum: Pendulum) => ({ number: pendulum.finalized, hash: pendulumHash({}, pendulum.finalized) }); +const forkedHead = (pendulum: Pendulum) => ({ number: pendulum.finalized, hash: pendulumHash({ forkFrom: 0 }, pendulum.finalized) }); + +test("a relay contradiction the next relay RPC does not confirm only warns, without the anchor", async (t) => { + const { pendulum, api, monitor, pages, persisted } = await twoRelaySetup(t, [forkedHead, honestHead]); + burn(pendulum, 0n, 100n * PEN); + await monitor.poll(api); + assert.deepEqual(pages.map((page) => [page.subject, page.severity]), [["RELAY RPCS DISAGREE", "WARNING"]]); + assert.match(pages[0].detail, /^relay RPC 1 reports that the relay chain records 0x0+10+65 as Pendulum block 101, .*; relay RPC 2 does not;/); + assert.equal(persisted().relayAnchor, undefined); + assert.equal(persisted().lastPendulumBlock, 101); +}); + +test("a relay contradiction the next relay RPC confirms pages critical", async (t) => { + const { pendulum, api, monitor, pages, persisted } = await twoRelaySetup(t, [forkedHead, forkedHead]); + burn(pendulum, 0n, 100n * PEN); + await monitor.poll(api); + assert.ok(pages.some((page) => page.subject === "CRITICAL: PENDULUM SOURCE CONTRADICTED"), JSON.stringify(pages)); + assert.equal(persisted().lastPendulumBlock, 100); +}); + +test("a relay contradiction no other relay RPC can confirm only warns", async (t) => { + const { pendulum, api, monitor, pages, persisted } = await twoRelaySetup(t, [forkedHead, undefined]); + burn(pendulum, 0n, 100n * PEN); + await monitor.poll(api); + assert.deepEqual(pages.map((page) => page.subject), ["RELAY ANCHOR UNAVAILABLE"]); + assert.match(pages[0].detail, /and no other relay RPC answered to confirm it \(relay RPC 2: Error: relay RPC unavailable\)/); + assert.equal(persisted().lastPendulumBlock, 101); +}); + test("while Pendulum cannot be read, a Base event with an unknown nonce pauses after the grace", async (t) => { const { base, pendulum, api, monitor, subjects } = await setup(t, { env: { UNMATCHED_EVENT_GRACE_SECONDS: "0" } }); pendulum.failure = new Error("node unreachable"); diff --git a/monitor/src/main.ts b/monitor/src/main.ts index d63f64626..bcfed6bba 100644 --- a/monitor/src/main.ts +++ b/monitor/src/main.ts @@ -431,6 +431,7 @@ let lastSourceFailureAlertMs = 0; let lastSourceIntegrityAlertMs = 0; let lastCrossCheckAlertMs = 0; let lastRelayAlertMs = 0; +let lastRelayDisagreementAlertMs = 0; let lastRelayStaleAlertMs = 0; /** When the source last verified against a new relay-chain record (or the start). */ let lastRelayAnchorMs = Date.now(); @@ -765,31 +766,23 @@ async function walkHeaders(api: ApiPromise, top: BlockRef, bottom: number): Prom return hashes; } -/** The Pendulum head that the relay chain recorded at its finalized block, - * from the first relay RPC in POLKADOT_WS that answers. */ -async function relayHead(api: ApiPromise): Promise { - const failures: string[] = []; - for (const [index, url] of config.polkadotWs.entries()) { - try { - return await onNode(url, async (relay) => { - const relayFinalized = await relay.rpc.chain.getFinalizedHead(); - const head = (await (await relay.at(relayFinalized)).query.paras.heads(config.pendulumParaId)) as unknown as { - isSome: boolean; - unwrap(): { toU8a(bare: boolean): Uint8Array }; - }; - if (!head.isSome) throw new Error(`no head recorded for para ${config.pendulumParaId}`); - // The head data is the Pendulum header; its hash is computed here. - const header = api.createType("Header", head.unwrap().toU8a(true)) as unknown as { - number: { toNumber(): number }; - hash: { toHex(): string }; - }; - return { number: header.number.toNumber(), hash: header.hash.toHex() }; - }); - } catch (relayError) { - failures.push(`relay RPC ${index + 1}: ${relayError}`); - } - } - throw new Error(failures.join("; ")); +/** The Pendulum head that the relay chain behind the RPC at `url` recorded + * at its finalized block. */ +function relayHead(api: ApiPromise, url: string): Promise { + return onNode(url, async (relay) => { + const relayFinalized = await relay.rpc.chain.getFinalizedHead(); + const head = (await (await relay.at(relayFinalized)).query.paras.heads(config.pendulumParaId)) as unknown as { + isSome: boolean; + unwrap(): { toU8a(bare: boolean): Uint8Array }; + }; + if (!head.isSome) throw new Error(`no head recorded for para ${config.pendulumParaId}`); + // The head data is the Pendulum header; its hash is computed here. + const header = api.createType("Header", head.unwrap().toU8a(true)) as unknown as { + number: { toNumber(): number }; + hash: { toHex(): string }; + }; + return { number: header.number.toNumber(), hash: header.hash.toHex() }; + }); } /** Why the relay chain's record `head` does not anchor the source this poll, @@ -838,9 +831,12 @@ async function relayVerdict( } /** Anchors the source to the relay chain's record of the Pendulum head, kept - * as `draft.relayAnchor`. Pages while no new head is anchored: a warning when - * the relay RPCs fail, critical once none was for RELAY_ANCHOR_GRACE_SECONDS. - * Both throttled. */ + * as `draft.relayAnchor`, from the first relay RPC in POLKADOT_WS that + * answers. A record that contradicts the source counts only when the next + * relay RPC that answers contradicts it too, or when POLKADOT_WS lists one + * RPC; otherwise the poll goes on without the anchor. Pages while no new head + * is anchored: a warning when the relay RPCs fail or disagree, critical once + * none was for RELAY_ANCHOR_GRACE_SECONDS. All throttled. */ async function anchorToRelay( api: ApiPromise, draft: RuntimeState, @@ -849,12 +845,47 @@ async function anchorToRelay( bottom: BlockRef, ): Promise { if (!config.polkadotWs.length) return; - let head: BlockRef | undefined; + const failures: string[] = []; + let contradiction: { rpc: number; error: SourceIntegrityError } | undefined; let reason: string | undefined; - try { - head = await relayHead(api); - } catch (relayError) { - reason = `the relay chain cannot be read: ${relayError}`; + for (const [index, url] of config.polkadotWs.entries()) { + let head: BlockRef; + try { + head = await relayHead(api, url); + } catch (relayError) { + failures.push(`relay RPC ${index + 1}: ${relayError}`); + continue; + } + try { + reason = await relayVerdict(api, draft, hashes, top, bottom, head); + } catch (verdictError) { + if (!(verdictError instanceof SourceIntegrityError)) throw verdictError; + if (contradiction || config.polkadotWs.length === 1) throw contradiction?.error ?? verdictError; + contradiction = { rpc: index + 1, error: verdictError }; + continue; + } + if (contradiction) { + reason = `relay RPC ${contradiction.rpc} reports that ${contradiction.error.message}; relay RPC ${index + 1} does not`; + if (Date.now() - lastRelayDisagreementAlertMs > config.graceSeconds * 1000) { + lastRelayDisagreementAlertMs = Date.now(); + await alert( + "RELAY RPCS DISAGREE", + `${reason}; the Pendulum source is used without the relay-chain anchor until they agree — check relay ` + + `RPC ${contradiction.rpc}`, + ); + } + } else if (reason === undefined) { + draft.relayAnchor = head; + lastRelayAnchorMs = Date.now(); + return; + } + break; + } + if (reason === undefined) { + reason = contradiction + ? `relay RPC ${contradiction.rpc} reports that ${contradiction.error.message}, and no other relay RPC ` + + `answered to confirm it (${failures.join("; ")})` + : `the relay chain cannot be read: ${failures.join("; ")}`; if (Date.now() - lastRelayAlertMs > config.graceSeconds * 1000) { lastRelayAlertMs = Date.now(); await alert( @@ -864,14 +895,6 @@ async function anchorToRelay( ); } } - if (head) { - reason = await relayVerdict(api, draft, hashes, top, bottom, head); - if (reason === undefined) { - draft.relayAnchor = head; - lastRelayAnchorMs = Date.now(); - return; - } - } const unanchoredMs = Date.now() - lastRelayAnchorMs; if (unanchoredMs > config.relayAnchorGraceMs && Date.now() - lastRelayStaleAlertMs > config.unmatchedEventGraceSeconds * 1000) { lastRelayStaleAlertMs = Date.now(); From 7e044310ed99ab7d4e0d78602021e8749c18f1ec Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Thu, 22 Oct 2026 13:54:44 +0200 Subject: [PATCH 48/75] fix(monitor): Compare the second node's hash only at a height it finalized Above its finalized head, a node's block at a height can still be one that loses. The second node's hash at the source's finalized head is now compared once the second node has finalized that height; before that it pages PENDULUM CROSS-CHECK UNAVAILABLE, throttled. --- monitor/README.md | 3 ++- monitor/src/main.test.ts | 16 +++++++++++++- monitor/src/main.ts | 45 ++++++++++++++++++++++------------------ 3 files changed, 42 insertions(+), 22 deletions(-) diff --git a/monitor/README.md b/monitor/README.md index 714517aa6..77cc9af57 100644 --- a/monitor/README.md +++ b/monitor/README.md @@ -36,7 +36,8 @@ itself: ingested last, which the state file keeps, so every ingested block lies on one chain. - **Second node** (`PENDULUM_WS_CROSSCHECK`). The second node must have the - same hash at the finalized head's height. + same hash at the finalized head's height, compared once it has finalized + that height itself. - **Relay-chain anchor** (`POLKADOT_WS`). Each poll reads the Pendulum head that the Polkadot relay chain recorded at its finalized block (`paras.heads(PENDULUM_PARA_ID)`), from the first relay RPC that answers. The diff --git a/monitor/src/main.test.ts b/monitor/src/main.test.ts index 8ad0ac8f1..bc4dac007 100644 --- a/monitor/src/main.test.ts +++ b/monitor/src/main.test.ts @@ -755,7 +755,21 @@ test("a second node that has not reached the newest block only warns", async (t) burn(pendulum, 0n, 100n * PEN); await monitor.poll(api); assert.deepEqual(pages.map((page) => page.subject), ["PENDULUM CROSS-CHECK UNAVAILABLE"]); - assert.match(pages[0].detail, /has no block 101 yet;/); + assert.match(pages[0].detail, /has not finalized block 101 yet;/); + assert.equal(persisted().lastPendulumBlock, 101); +}); + +test("a second node whose best chain holds another newest block, not yet finalized, only warns", async (t) => { + const second = pendulumChain(); // finalized at 100 + second.chain.best = 101; + second.chain.forkFrom = 101; + const { pendulum, api, monitor, pages, persisted } = await setup(t, { + env: { PENDULUM_WS_CROSSCHECK: SECOND_NODE }, + connect: async () => second.api, + }); + burn(pendulum, 0n, 100n * PEN); + await monitor.poll(api); + assert.deepEqual(pages.map((page) => page.subject), ["PENDULUM CROSS-CHECK UNAVAILABLE"]); assert.equal(persisted().lastPendulumBlock, 101); }); diff --git a/monitor/src/main.ts b/monitor/src/main.ts index bcfed6bba..e6e58c644 100644 --- a/monitor/src/main.ts +++ b/monitor/src/main.ts @@ -719,35 +719,40 @@ async function onNode(url: string, read: (node: ApiPromise) => Promise): P } } -/** The second node must have the source's finalized head at its height. One - * that cannot answer only warns, throttled: it is a check, not a source. */ +/** The second node must have finalized the source's finalized head, with the + * same hash. One that cannot answer, or has not finalized that block yet, + * only warns, throttled: it is a check, not a source. */ async function crossCheckSource(finalized: BlockRef): Promise { const url = config.pendulumWsCrossCheck; if (!url) return; - let reason: string; + let other: string | undefined; try { - const other = await onNode(url, (node) => node.rpc.chain.getBlockHash(finalized.number)); - if (!other.isEmpty && other.toHex() !== finalized.hash) { - throw new SourceIntegrityError( - `the second Pendulum node has ${other.toHex()} at block ${finalized.number}, the source ${finalized.hash}`, - ); - } - if (!other.isEmpty) return; - reason = `has no block ${finalized.number} yet`; + other = await onNode(url, async (node) => { + // Above its finalized head, a node's block at a height may still lose. + const head = await node.rpc.chain.getHeader(await node.rpc.chain.getFinalizedHead()); + if (head.number.toNumber() < finalized.number) return undefined; + return (await node.rpc.chain.getBlockHash(finalized.number)).toHex(); + }); } catch (crossCheckError) { - if (crossCheckError instanceof SourceIntegrityError) throw crossCheckError; - reason = `cannot be read: ${crossCheckError}`; + return crossCheckUnavailable(`cannot be read: ${crossCheckError}`); } - if (Date.now() - lastCrossCheckAlertMs > config.graceSeconds * 1000) { - lastCrossCheckAlertMs = Date.now(); - await alert( - "PENDULUM CROSS-CHECK UNAVAILABLE", - `the second Pendulum node (PENDULUM_WS_CROSSCHECK) ${reason}; the source's blocks are not compared ` + - "with it until it answers — restore it", - ); + if (other === undefined) return crossCheckUnavailable(`has not finalized block ${finalized.number} yet`); + if (other !== finalized.hash) { + throw new SourceIntegrityError(`the second Pendulum node has ${other} at block ${finalized.number}, the source ${finalized.hash}`); } } +/** Pages, throttled, that the source cannot be compared with the second node. */ +async function crossCheckUnavailable(reason: string): Promise { + if (Date.now() - lastCrossCheckAlertMs <= config.graceSeconds * 1000) return; + lastCrossCheckAlertMs = Date.now(); + await alert( + "PENDULUM CROSS-CHECK UNAVAILABLE", + `the second Pendulum node (PENDULUM_WS_CROSSCHECK) ${reason}; the source's blocks are not compared ` + + "with it until it answers — restore it", + ); +} + /** Walks parent hashes down from `top` to `bottom` through headers fetched * from the node, recomputing each hash here, so that the result is one * chain the node cannot fake: block hashes by height, `bottom` included. */ From 7a43a5e74b72c71f1f266513e7505855ce70dca4 Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Thu, 22 Oct 2026 13:57:42 +0200 Subject: [PATCH 49/75] feat(monitor): Compare migration totals and events with the second node With PENDULUM_WS_CROSSCHECK set, the second node must also agree with the source on tokenMigration.nextNonce and totalMigrated at the finalized head, and on the migration pallet events of every ingested block that has some. A difference pages CRITICAL: PENDULUM SOURCE CONTRADICTED and nothing from the source is ingested; a second node that cannot answer only warns. --- monitor/README.md | 8 ++-- monitor/src/main.test.ts | 88 ++++++++++++++++++++++++++++++---------- monitor/src/main.ts | 85 ++++++++++++++++++++++++++++++-------- 3 files changed, 139 insertions(+), 42 deletions(-) diff --git a/monitor/README.md b/monitor/README.md index 77cc9af57..2fbcc922d 100644 --- a/monitor/README.md +++ b/monitor/README.md @@ -16,7 +16,7 @@ Checks every poll: | Unreleasable burns | A burn to the zero or vault address can never be approved or released; it is paged once and excluded from the pending count so it cannot page liveness forever or block RB-7. | Alert once | | Treasury destination | With `EXPECTED_TREASURY_DESTINATION` set, every finalized treasury migration (a `MigrationInitiated` whose `who` is the treasury account, `PalletId(*b"py/trsry")`) must pay that Base address. It is checked when the burn is ingested, before the attestors can release it; while the pause is not confirmed, the monitor does not move past that block. | Alert + auto-pause (`TREASURY DESTINATION VIOLATION`) | | Pendulum source | The finalized Pendulum view must stay fresh and readable. While it cannot be read (an RPC error, a nonce discontinuity, an event-shape change), the Base-side checks — vault balance and its auto-pause, Base view, auto-pause readiness, vault admin events, governance queue — still run, and Base events are reconciled against the last view read in full: an unreadable source counts as a stalled one, so a mismatch on a known nonce counts at once and an unknown nonce after `UNMATCHED_EVENT_GRACE_SECONDS`. M2a and liveness wait for the source. | Alert, throttled: `PENDULUM SOURCE VIEW STALLED` once the view is older than `SOURCE_STALE_ALERT_SECONDS`, `PENDULUM SOURCE UNREADABLE` on a failure | -| Pendulum chain | Every finalized block the monitor ingests is read at a hash it verified against the blocks it ingested before and, where configured, a second node and the relay chain (see [Pendulum chain verification](#pendulum-chain-verification)). | Alert (`CRITICAL: PENDULUM SOURCE CONTRADICTED`), throttled; nothing from the source is ingested, and it is treated as unreadable (see Pendulum source) until it verifies again. A second node or relay RPC that does not answer only warns, throttled (`PENDULUM CROSS-CHECK UNAVAILABLE`, `RELAY ANCHOR UNAVAILABLE`), and so do relay RPCs that disagree (`RELAY RPCS DISAGREE`); `CRITICAL: RELAY ANCHOR STALE` once no head was anchored for `RELAY_ANCHOR_GRACE_SECONDS` | +| Pendulum chain | Every finalized block the monitor ingests is read at a hash it verified against the blocks it ingested before and, where configured, a second node and the relay chain; a second node must also agree on the migration totals and on the migration events of each ingested block (see [Pendulum chain verification](#pendulum-chain-verification)). | Alert (`CRITICAL: PENDULUM SOURCE CONTRADICTED`), throttled; nothing from the source is ingested, and it is treated as unreadable (see Pendulum source) until it verifies again. A second node or relay RPC that does not answer only warns, throttled (`PENDULUM CROSS-CHECK UNAVAILABLE`, `RELAY ANCHOR UNAVAILABLE`), and so do relay RPCs that disagree (`RELAY RPCS DISAGREE`); `CRITICAL: RELAY ANCHOR STALE` once no head was anchored for `RELAY_ANCHOR_GRACE_SECONDS` | | Base log completeness | In every scanned block range the `Released` amounts must add up to the growth of `totalReleased`, and the `RemainderSwept` amounts to that of `totalSwept`, between the block before the range and its last block. A range whose logs do not (a backend serving truncated logs) is read once more, then held: the Base cursor stays on it and M2a and liveness wait until a read accounts for the totals. | Alert (`BASE LOG SCAN INCOMPLETE`), throttled | | Base view | The safe/finalized Base block must not be older than `BASE_STALE_ALERT_SECONDS`: a view that stops advancing hides every later vault event while every check still passes at the old block. | Alert (`BASE VIEW STALLED`), throttled | | Auto-pause readiness | At startup and every poll: the Base RPC serves `BASE_CHAIN_ID` and, with a guardian key configured, the key parses, is `vault.guardian()` and holds at least `GUARDIAN_MIN_BALANCE_WEI`. A read that fails pages as well and never stops the poll's other checks. | Alert (`AUTO-PAUSE NOT READY`), throttled; `AUTO-PAUSE DISABLED` once at startup without a key | @@ -37,7 +37,9 @@ itself: one chain. - **Second node** (`PENDULUM_WS_CROSSCHECK`). The second node must have the same hash at the finalized head's height, compared once it has finalized - that height itself. + that height itself, and the same migration totals there (`nextNonce`, + `totalMigrated`). Every ingested block with migration pallet events must + carry the same ones on the second node. - **Relay-chain anchor** (`POLKADOT_WS`). Each poll reads the Pendulum head that the Polkadot relay chain recorded at its finalized block (`paras.heads(PENDULUM_PARA_ID)`), from the first relay RPC that answers. The @@ -64,7 +66,7 @@ advance). | Variable | Meaning | |---|---| | `PENDULUM_WS` | WebSocket of the monitor's own Pendulum node, run with `--state-pruning archive` (see below) | -| `PENDULUM_WS_CROSSCHECK` | Optional: WebSocket of a second Pendulum node, not the one behind `PENDULUM_WS`; its block hash at the source's finalized head must match in every poll | +| `PENDULUM_WS_CROSSCHECK` | Optional: WebSocket of a second Pendulum node, not the one behind `PENDULUM_WS`; it must agree with the source on the finalized head's hash and migration totals in every poll, and on the migration events of every ingested block that has some. Run it with `--state-pruning archive` too, or those events cannot be compared after a monitor outage longer than its pruning horizon (`PENDULUM CROSS-CHECK UNAVAILABLE`) | | `POLKADOT_WS` | Optional: comma-separated WebSocket RPCs of the Polkadot relay chain, run by third parties and tried in order (for example `wss://rpc.polkadot.io`); turns on the relay-chain anchor. List at least two: a record that contradicts the source needs a second relay RPC's confirmation, unless only one is listed | | `PENDULUM_PARA_ID` | Pendulum's parachain id on the relay chain (default 2094) | | `RELAY_ANCHOR_GRACE_SECONDS` | How long the source may go without a relay-anchored head before `CRITICAL: RELAY ANCHOR STALE` pages (default 3600) | diff --git a/monitor/src/main.test.ts b/monitor/src/main.test.ts index bc4dac007..b80fb89ff 100644 --- a/monitor/src/main.test.ts +++ b/monitor/src/main.test.ts @@ -211,7 +211,7 @@ const codec = (value: number | bigint) => ({ toNumber: () => Number(value), toBigInt: () => BigInt(value), }); -const hash = (hex: string) => ({ toHex: () => hex, isEmpty: /^0x0*$/.test(hex) }); +const hash = (hex: string) => ({ toHex: () => hex, toString: () => hex, isEmpty: /^0x0*$/.test(hex) }); const inline = (name: string) => ({ isInline: true, asInline: encodeCall(name), isLookup: false, isLegacy: false, toString: () => `inline ${name}` }); const lookup = (hex: string, len: number) => ({ isInline: false, @@ -238,8 +238,8 @@ interface PendulumEvent { /** The Pendulum side: finalized blocks with their events, the migration * pallet's totals and the governance queues. A block hash encodes the block * number, and whether it is on the fork the node serves from `forkFrom` on. */ -function pendulumChain() { - const chain = { +function pendulumState() { + return { /** Where set, the node serves another chain from this block on. */ forkFrom: undefined as number | undefined, /** Blocks whose header the node alters, so it no longer hashes to its hash. */ @@ -275,6 +275,11 @@ function pendulumChain() { motionSizes: new Map(), motionReads: new Set(), }; +} +type Pendulum = ReturnType; + +/** A node serving `chain`, as much of the polkadot.js API as the monitor uses. */ +function pendulumChain(chain: Pendulum = pendulumState()) { const collective = (motions: Map) => ({ proposals: async () => [...motions.keys()].map(hash), voting: async (motion: { toHex(): string }) => ({ @@ -400,7 +405,6 @@ function pendulumHash(chain: { forkFrom?: number }, block: number): string { return toHex(fork + BigInt(block), { size: 32 }); } const blockOf = (blockHash: string) => Number(BigInt(blockHash) & 0xffff_ffff_ffff_ffffn); -type Pendulum = ReturnType["chain"]; const MIGRATOR = `0x${"ab".repeat(32)}`; const TREASURY_ACCOUNT = `0x${Buffer.from("modlpy/trsry").toString("hex")}${"00".repeat(20)}`; @@ -690,18 +694,29 @@ test("a source header that does not hash to its block hash pages critical", asyn const SECOND_NODE = "ws://second-node.test:9944?apikey=secret"; -test("a second node that disagrees on the newest block pages critical and nothing is ingested", async (t) => { - const second = pendulumChain(); - second.chain.finalized = 200; - const { pendulum, api, monitor, pages, persisted } = await setup(t, { +/** A monitor whose second Pendulum node serves the source's chain, except for + * the values in `own`, which the test may change later. */ +async function crossCheckSetup(t: { after(fn: () => void): void }, own: Partial = {}) { + let source: Pendulum | undefined; + const chain = new Proxy({} as Pendulum, { + get: (_target, key) => (key in own ? own[key as keyof Pendulum] : source![key as keyof Pendulum]), + }); + const second = pendulumChain(chain); + const monitor = await setup(t, { env: { PENDULUM_WS_CROSSCHECK: SECOND_NODE }, + prepare: (_base, pendulum) => (source = pendulum), connect: async (url) => (url === SECOND_NODE ? second.api : assert.fail(url)), }); + return { ...monitor, own }; +} + +test("a second node that disagrees on the newest block pages critical and nothing is ingested", async (t) => { + const { pendulum, api, monitor, pages, persisted, own } = await crossCheckSetup(t, { finalized: 200 }); burn(pendulum, 0n, 100n * PEN); await monitor.poll(api); assert.equal(pages.length, 0, JSON.stringify(pages)); assert.equal(persisted().lastPendulumBlock, 101); - second.chain.forkFrom = 102; + own.forkFrom = 102; burn(pendulum, 1n, 100n * PEN, 102); await monitor.poll(api); assert.match( @@ -747,11 +762,7 @@ test("a node connection that fails with an event says why", async (t) => { }); test("a second node that has not reached the newest block only warns", async (t) => { - const second = pendulumChain(); // finalized at 100 - const { pendulum, api, monitor, pages, persisted } = await setup(t, { - env: { PENDULUM_WS_CROSSCHECK: SECOND_NODE }, - connect: async () => second.api, - }); + const { pendulum, api, monitor, pages, persisted } = await crossCheckSetup(t, { finalized: 100 }); burn(pendulum, 0n, 100n * PEN); await monitor.poll(api); assert.deepEqual(pages.map((page) => page.subject), ["PENDULUM CROSS-CHECK UNAVAILABLE"]); @@ -760,16 +771,51 @@ test("a second node that has not reached the newest block only warns", async (t) }); test("a second node whose best chain holds another newest block, not yet finalized, only warns", async (t) => { - const second = pendulumChain(); // finalized at 100 - second.chain.best = 101; - second.chain.forkFrom = 101; - const { pendulum, api, monitor, pages, persisted } = await setup(t, { - env: { PENDULUM_WS_CROSSCHECK: SECOND_NODE }, - connect: async () => second.api, - }); + const { pendulum, api, monitor, pages, persisted } = await crossCheckSetup(t, { finalized: 100, best: 101, forkFrom: 101 }); + burn(pendulum, 0n, 100n * PEN); + await monitor.poll(api); + assert.deepEqual(pages.map((page) => page.subject), ["PENDULUM CROSS-CHECK UNAVAILABLE"]); + assert.equal(persisted().lastPendulumBlock, 101); +}); + +test("a second node with other migration totals at the finalized head pages critical and nothing is ingested", async (t) => { + const { pendulum, api, monitor, pages, persisted } = await crossCheckSetup(t, { nextNonce: 1n, totalMigrated: 7n * PEN }); + burn(pendulum, 0n, 100n * PEN); + await monitor.poll(api); + assert.match( + pages.find((page) => page.subject === "CRITICAL: PENDULUM SOURCE CONTRADICTED")?.detail ?? "", + /at Pendulum block 101 the second node has nextNonce 1 and totalMigrated 7000000000000, the source 1 and 100000000000000;/, + ); + assert.equal(persisted().lastPendulumBlock, 100); +}); + +test("a second node with other migration events in a block pages critical and nothing is ingested", async (t) => { + const { pendulum, api, monitor, pages, persisted, own } = await crossCheckSetup(t); + burn(pendulum, 0n, 100n * PEN); + const events = new Map(pendulum.events); + own.events = events; + burn(pendulum, 1n, 100n * PEN, 102); + events.set(102, [{ ...pendulum.events.get(102)![0], data: [codec(1n), hash(MIGRATOR), hash(TOKEN), codec(100n * PEN)] }]); + await monitor.poll(api); + assert.match( + pages.find((page) => page.subject === "CRITICAL: PENDULUM SOURCE CONTRADICTED")?.detail ?? "", + new RegExp(`in Pendulum block 102 the second node reports the migration events MigrationInitiated\\(1, ${MIGRATOR}, ${TOKEN}, `, "i"), + ); + assert.equal(persisted().lastPendulumBlock, 100); + assert.deepEqual(persisted().migrations, []); +}); + +test("a second node that cannot read a block's events only warns", async (t) => { + const failing = { + get: () => { + throw new Error("state discarded"); + }, + } as unknown as Pendulum["events"]; + const { pendulum, api, monitor, pages, persisted } = await crossCheckSetup(t, { events: failing }); burn(pendulum, 0n, 100n * PEN); await monitor.poll(api); assert.deepEqual(pages.map((page) => page.subject), ["PENDULUM CROSS-CHECK UNAVAILABLE"]); + assert.match(pages[0].detail, /cannot be read: .*state discarded/); assert.equal(persisted().lastPendulumBlock, 101); }); diff --git a/monitor/src/main.ts b/monitor/src/main.ts index e6e58c644..c7a2f2386 100644 --- a/monitor/src/main.ts +++ b/monitor/src/main.ts @@ -719,26 +719,80 @@ async function onNode(url: string, read: (node: ApiPromise) => Promise): P } } +interface MigrationTotals { + nextNonce: bigint; + totalMigrated: bigint; +} + +async function migrationTotals(apiAt: ApiAt): Promise { + const [nextNonce, totalMigrated] = await Promise.all([ + apiAt.query.tokenMigration.nextNonce(), + apiAt.query.tokenMigration.totalMigrated(), + ]); + return { + nextNonce: BigInt(nextNonce.toString().replaceAll(",", "")), + totalMigrated: BigInt(totalMigrated.toString().replaceAll(",", "")), + }; +} + /** The second node must have finalized the source's finalized head, with the - * same hash. One that cannot answer, or has not finalized that block yet, - * only warns, throttled: it is a check, not a source. */ -async function crossCheckSource(finalized: BlockRef): Promise { + * same hash and the same migration totals there. One that cannot answer, or + * has not finalized that block yet, only warns, throttled: it is a check, + * not a source. */ +async function crossCheckSource(finalized: BlockRef, totals: MigrationTotals): Promise { const url = config.pendulumWsCrossCheck; if (!url) return; - let other: string | undefined; + let other: { hash: string; totals?: MigrationTotals } | undefined; try { other = await onNode(url, async (node) => { // Above its finalized head, a node's block at a height may still lose. const head = await node.rpc.chain.getHeader(await node.rpc.chain.getFinalizedHead()); if (head.number.toNumber() < finalized.number) return undefined; - return (await node.rpc.chain.getBlockHash(finalized.number)).toHex(); + const hash = (await node.rpc.chain.getBlockHash(finalized.number)).toHex(); + return { hash, totals: hash === finalized.hash ? await migrationTotals(await node.at(hash)) : undefined }; }); } catch (crossCheckError) { return crossCheckUnavailable(`cannot be read: ${crossCheckError}`); } if (other === undefined) return crossCheckUnavailable(`has not finalized block ${finalized.number} yet`); - if (other !== finalized.hash) { - throw new SourceIntegrityError(`the second Pendulum node has ${other} at block ${finalized.number}, the source ${finalized.hash}`); + if (other.hash !== finalized.hash) { + throw new SourceIntegrityError(`the second Pendulum node has ${other.hash} at block ${finalized.number}, the source ${finalized.hash}`); + } + if (other.totals?.nextNonce !== totals.nextNonce || other.totals.totalMigrated !== totals.totalMigrated) { + throw new SourceIntegrityError( + `at Pendulum block ${finalized.number} the second node has nextNonce ${other.totals?.nextNonce} and ` + + `totalMigrated ${other.totals?.totalMigrated}, the source ${totals.nextNonce} and ${totals.totalMigrated}`, + ); + } +} + +type EventRecords = Array<{ event: { section: string; method: string; data: unknown[] } }>; + +/** The migration pallet's events in a block, as one text; empty without any. */ +function migrationEventText(records: EventRecords): string { + return records + .filter(({ event }) => event.section === "tokenMigration") + .map(({ event }) => `${event.method}(${event.data.join(", ")})`) + .join("; "); +} + +/** The second node must report the same migration events in the block at + * `blockHash`. One that cannot answer only warns, throttled. */ +async function crossCheckEvents(block: number, blockHash: string, events: string): Promise { + const url = config.pendulumWsCrossCheck; + if (!url) return; + let other: string; + try { + other = await onNode(url, async (node) => + migrationEventText((await (await node.at(blockHash)).query.system.events()) as unknown as EventRecords) + ); + } catch (crossCheckError) { + return crossCheckUnavailable(`cannot be read: ${crossCheckError}`); + } + if (other !== events) { + throw new SourceIntegrityError( + `in Pendulum block ${block} the second node reports the migration events ${other || "none"}, the source ${events}`, + ); } } @@ -958,9 +1012,9 @@ async function ingestPendulumEvents( if (!blockHash) throw new Error(`no verified hash for Pendulum block ${block}`); const apiAt = await api.at(blockHash); const firstSeenMs = Number((await apiAt.query.timestamp.now()).toString().replaceAll(",", "")); - const records = (await apiAt.query.system.events()) as unknown as { - event: { section: string; method: string; data: unknown[] }; - }[]; + const records = (await apiAt.query.system.events()) as unknown as EventRecords; + const migrationEvents = migrationEventText(records); + if (migrationEvents) await crossCheckEvents(block, blockHash, migrationEvents); for (const [index, record] of records.entries()) { const { section, method, data } = record.event; if (section !== "tokenMigration") continue; @@ -1580,16 +1634,11 @@ async function readSource(api: ApiPromise, draft: RuntimeState): Promise Date: Thu, 22 Oct 2026 13:59:11 +0200 Subject: [PATCH 50/75] fix(monitor): Run the conservation check while a Base log range is held M2a needs only totalReleased at the poll block and TotalMigrated, so a log range held for missing events no longer skips it. It still waits while a Base event's burn is not in the source view yet, where it would fire falsely. --- monitor/README.md | 4 ++-- monitor/src/main.test.ts | 18 ++++++++++++++++++ monitor/src/main.ts | 28 +++++++++++++++++----------- 3 files changed, 37 insertions(+), 13 deletions(-) diff --git a/monitor/README.md b/monitor/README.md index 2fbcc922d..4ea260e54 100644 --- a/monitor/README.md +++ b/monitor/README.md @@ -9,7 +9,7 @@ Checks every poll: | Check | Meaning | Reaction | |---|---|---| | M2 tuple match | Every safe Base `Approved` and `Released` event must match one finalized Pendulum `MigrationInitiated` nonce, recipient, and pallet amount; `Released.tokenAmount` must equal `palletAmount × conversionFactor`. After a violation the Base cursor stays on any block range whose events the source view may still reveal, so a pause caused by a lagging source does not lose those releases. | Alert + auto-pause on any mismatching `Released`, and on approvals of one nonce that disagree with the finalized Pendulum view from two or more distinct attestors, also across polls and restarts. A single attestor's disagreeing approval pages `CRITICAL: MISMATCHED APPROVAL` instead: one approval can never release. A mismatch against a *known* nonce counts immediately. An *unknown* nonce first pages (`UNVERIFIED BASE EVENT`, without delaying the pause decision), then counts as a mismatch as soon as it is provable — the finalized source view passed the event's own timestamp and the nonce still does not exist — or after `UNMATCHED_EVENT_GRACE_SECONDS` if the source view stays behind or cannot be read (a stalled or unreadable source also pages on its own, see below) | -| M2a aggregate | `totalReleased <= TotalMigrated × conversionFactor + ACKNOWLEDGED_RELEASE_EXCESS`. This remains independent defence-in-depth around the event reconciliation. After a real, quantified loss, operators set `ACKNOWLEDGED_RELEASE_EXCESS` to it (RB-3) so the vault can be unpaused; any further excess still fires. | Alert + auto-pause | +| M2a aggregate | `totalReleased <= TotalMigrated × conversionFactor + ACKNOWLEDGED_RELEASE_EXCESS`. This remains independent defence-in-depth around the event reconciliation: it also runs while a Base log range is held, and waits only while a Base event's burn is not in the source view yet. After a real, quantified loss, operators set `ACKNOWLEDGED_RELEASE_EXCESS` to it (RB-3) so the vault can be unpaused; any further excess still fires. | Alert + auto-pause | | M2b: `balanceOf(vault) + totalReleased + totalSwept >= totalSupply` | Vault-internal conservation. Only a **deficit** alerts: a surplus is a harmless inbound transfer (donation, or a migration whose recipient is the vault) and is ignored, so it cannot false-trigger a pause. It reads Base only and runs first in every poll, so a slow or failing Pendulum source or Base log scan can neither delay nor skip it. | Alert + auto-pause on a deficit | | M4: every old migration reaches active approval quorum — and then releases | Liveness of the attestor fleet (`below approval quorum`), and of everything after it (`quorum reached but not released`: a release starved of daily allowance, paused, or made releasable by a threshold cut without a `ReleasePending`, which the releaser cannot see). | One summary page per kind and poll (count and the first nonces; the log lists every nonce), each nonce at most once per `GRACE_SECONDS` | | Source supply | `totalIssuance + TotalMigrated` on Pendulum must never grow past its first-seen anchor. | Alert (`SOURCE SUPPLY GREW`) | @@ -17,7 +17,7 @@ Checks every poll: | Treasury destination | With `EXPECTED_TREASURY_DESTINATION` set, every finalized treasury migration (a `MigrationInitiated` whose `who` is the treasury account, `PalletId(*b"py/trsry")`) must pay that Base address. It is checked when the burn is ingested, before the attestors can release it; while the pause is not confirmed, the monitor does not move past that block. | Alert + auto-pause (`TREASURY DESTINATION VIOLATION`) | | Pendulum source | The finalized Pendulum view must stay fresh and readable. While it cannot be read (an RPC error, a nonce discontinuity, an event-shape change), the Base-side checks — vault balance and its auto-pause, Base view, auto-pause readiness, vault admin events, governance queue — still run, and Base events are reconciled against the last view read in full: an unreadable source counts as a stalled one, so a mismatch on a known nonce counts at once and an unknown nonce after `UNMATCHED_EVENT_GRACE_SECONDS`. M2a and liveness wait for the source. | Alert, throttled: `PENDULUM SOURCE VIEW STALLED` once the view is older than `SOURCE_STALE_ALERT_SECONDS`, `PENDULUM SOURCE UNREADABLE` on a failure | | Pendulum chain | Every finalized block the monitor ingests is read at a hash it verified against the blocks it ingested before and, where configured, a second node and the relay chain; a second node must also agree on the migration totals and on the migration events of each ingested block (see [Pendulum chain verification](#pendulum-chain-verification)). | Alert (`CRITICAL: PENDULUM SOURCE CONTRADICTED`), throttled; nothing from the source is ingested, and it is treated as unreadable (see Pendulum source) until it verifies again. A second node or relay RPC that does not answer only warns, throttled (`PENDULUM CROSS-CHECK UNAVAILABLE`, `RELAY ANCHOR UNAVAILABLE`), and so do relay RPCs that disagree (`RELAY RPCS DISAGREE`); `CRITICAL: RELAY ANCHOR STALE` once no head was anchored for `RELAY_ANCHOR_GRACE_SECONDS` | -| Base log completeness | In every scanned block range the `Released` amounts must add up to the growth of `totalReleased`, and the `RemainderSwept` amounts to that of `totalSwept`, between the block before the range and its last block. A range whose logs do not (a backend serving truncated logs) is read once more, then held: the Base cursor stays on it and M2a and liveness wait until a read accounts for the totals. | Alert (`BASE LOG SCAN INCOMPLETE`), throttled | +| Base log completeness | In every scanned block range the `Released` amounts must add up to the growth of `totalReleased`, and the `RemainderSwept` amounts to that of `totalSwept`, between the block before the range and its last block. A range whose logs do not (a backend serving truncated logs) is read once more, then held: the Base cursor stays on it and liveness waits until a read accounts for the totals; M2a still runs. | Alert (`BASE LOG SCAN INCOMPLETE`), throttled | | Base view | The safe/finalized Base block must not be older than `BASE_STALE_ALERT_SECONDS`: a view that stops advancing hides every later vault event while every check still passes at the old block. | Alert (`BASE VIEW STALLED`), throttled | | Auto-pause readiness | At startup and every poll: the Base RPC serves `BASE_CHAIN_ID` and, with a guardian key configured, the key parses, is `vault.guardian()` and holds at least `GUARDIAN_MIN_BALANCE_WEI`. A read that fails pages as well and never stops the poll's other checks. | Alert (`AUTO-PAUSE NOT READY`), throttled; `AUTO-PAUSE DISABLED` once at startup without a key | | Pendulum governance | Every pending public proposal, the council's external proposal, every ongoing referendum and every open council and technical-committee motion, with the call decoded. A call that mints, moves or unlocks balances outside normal transfers, or changes the runtime or the migration pallet, pages as critical: minted PEN burned through the migration drains the fixed-supply vault, and only Pendulum governance can stop the call before enactment. A call the monitor cannot read pages as critical too: one over 64 KiB is never fetched, and one that does not decode or nests more than 256 calls or 16 levels deep is unreadable; one whose preimage is not noted yet is re-checked every poll and paged again once it can be read. The watch runs after the poll's conservation checks, so it cannot stop or delay them, and it decodes within fixed bounds, so its cost to the next poll is bounded. | Alert once per item (`PENDULUM GOVERNANCE ITEM`, `CRITICAL: Pendulum governance item …`); a motion proposed again with the same call (a new motion index), or an external proposal made again after it left the queue or became a referendum, pages again; a restart re-pages the items still pending | diff --git a/monitor/src/main.test.ts b/monitor/src/main.test.ts index b80fb89ff..b5acd6171 100644 --- a/monitor/src/main.test.ts +++ b/monitor/src/main.test.ts @@ -1143,6 +1143,24 @@ test("logs that miss a sweep hold the Base cursor too", async (t) => { assert.equal(persisted().baseFromBlock, "1011"); }); +test("the conservation check still runs while a Base log range is held", async (t) => { + const { base, pendulum, api, monitor, subjects } = await setup(t); + burn(pendulum, 0n, 100n * PEN); + release(base, 0n, 100n * PEN, 1_005n); + release(base, 0n, 100n * PEN, 1_007n); // a second release lifts totalReleased past TotalMigrated... + base.logsHead = 1_006n; // ...while the log backend drops its block + await monitor.poll(api); + assert.ok(subjects().includes("CONSERVATION VIOLATION"), JSON.stringify(subjects())); + assert.ok(subjects().includes("AUTO-PAUSE UNAVAILABLE"), JSON.stringify(subjects())); // the pause was asked for +}); + +test("the conservation check waits while a Base event's burn is not in the source view yet", async (t) => { + const { base, api, monitor, subjects } = await setup(t); + release(base, 0n, 100n * PEN); // the burn is not in the lagging source view yet + await monitor.poll(api); + assert.deepEqual([...new Set(subjects())], ["UNVERIFIED BASE EVENT"]); +}); + test("a release in the first block of a range accounts for the totals", async (t) => { const { base, pendulum, api, monitor, subjects, persisted } = await setup(t); burn(pendulum, 0n, 100n * PEN); diff --git a/monitor/src/main.ts b/monitor/src/main.ts index c7a2f2386..d243a1d8b 100644 --- a/monitor/src/main.ts +++ b/monitor/src/main.ts @@ -1776,6 +1776,23 @@ async function check(api: ApiPromise): Promise { return false; } const { finalizedBlock, totalMigrated } = source; + // M2a needs only the totals, so a held log range does not stop it. A Base + // event whose burn the source view does not show yet would make it fire + // falsely, so it waits for that. + if ( + !reconciliation.awaitingSource && + releasedExceedsMigrated(totalReleased, totalMigrated, conversionFactor, config.acknowledgedReleaseExcess) + ) { + if (await securityViolation( + "CONSERVATION VIOLATION", + `released ${totalReleased} > migrated ${totalMigrated * conversionFactor} + acknowledged excess ` + + `${config.acknowledgedReleaseExcess} (token units)`, + )) { + state = draft; + persistState(state); + } + return false; + } if (reconciliation.awaitingSource || reconciliation.incomplete) { state = draft; persistState(state); @@ -1787,17 +1804,6 @@ async function check(api: ApiPromise): Promise { ); return false; } - if (releasedExceedsMigrated(totalReleased, totalMigrated, conversionFactor, config.acknowledgedReleaseExcess)) { - if (await securityViolation( - "CONSERVATION VIOLATION", - `released ${totalReleased} > migrated ${totalMigrated * conversionFactor} + acknowledged excess ` + - `${config.acknowledgedReleaseExcess} (token units)`, - )) { - state = draft; - persistState(state); - } - return false; - } const pending = [...draft.migrations.values()]; const threshold = await publicClient.readContract({ From 3a427134ac3097da276231c2127a1893156c23d8 Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Thu, 22 Oct 2026 14:00:10 +0200 Subject: [PATCH 51/75] feat(monitor): Page critical when a Base log range stays held While a block range is held for missing events, no later Base event is matched against its burn. Once a range has been held for longer than UNMATCHED_EVENT_GRACE_SECONDS, the throttled page turns into CRITICAL: BASE LOG SCAN INCOMPLETE; a range that reads in full ends the hold. --- monitor/README.md | 3 ++- monitor/src/main.test.ts | 26 ++++++++++++++++++++++++++ monitor/src/main.ts | 13 +++++++++++-- 3 files changed, 39 insertions(+), 3 deletions(-) diff --git a/monitor/README.md b/monitor/README.md index 4ea260e54..ddcb7a05c 100644 --- a/monitor/README.md +++ b/monitor/README.md @@ -17,7 +17,7 @@ Checks every poll: | Treasury destination | With `EXPECTED_TREASURY_DESTINATION` set, every finalized treasury migration (a `MigrationInitiated` whose `who` is the treasury account, `PalletId(*b"py/trsry")`) must pay that Base address. It is checked when the burn is ingested, before the attestors can release it; while the pause is not confirmed, the monitor does not move past that block. | Alert + auto-pause (`TREASURY DESTINATION VIOLATION`) | | Pendulum source | The finalized Pendulum view must stay fresh and readable. While it cannot be read (an RPC error, a nonce discontinuity, an event-shape change), the Base-side checks — vault balance and its auto-pause, Base view, auto-pause readiness, vault admin events, governance queue — still run, and Base events are reconciled against the last view read in full: an unreadable source counts as a stalled one, so a mismatch on a known nonce counts at once and an unknown nonce after `UNMATCHED_EVENT_GRACE_SECONDS`. M2a and liveness wait for the source. | Alert, throttled: `PENDULUM SOURCE VIEW STALLED` once the view is older than `SOURCE_STALE_ALERT_SECONDS`, `PENDULUM SOURCE UNREADABLE` on a failure | | Pendulum chain | Every finalized block the monitor ingests is read at a hash it verified against the blocks it ingested before and, where configured, a second node and the relay chain; a second node must also agree on the migration totals and on the migration events of each ingested block (see [Pendulum chain verification](#pendulum-chain-verification)). | Alert (`CRITICAL: PENDULUM SOURCE CONTRADICTED`), throttled; nothing from the source is ingested, and it is treated as unreadable (see Pendulum source) until it verifies again. A second node or relay RPC that does not answer only warns, throttled (`PENDULUM CROSS-CHECK UNAVAILABLE`, `RELAY ANCHOR UNAVAILABLE`), and so do relay RPCs that disagree (`RELAY RPCS DISAGREE`); `CRITICAL: RELAY ANCHOR STALE` once no head was anchored for `RELAY_ANCHOR_GRACE_SECONDS` | -| Base log completeness | In every scanned block range the `Released` amounts must add up to the growth of `totalReleased`, and the `RemainderSwept` amounts to that of `totalSwept`, between the block before the range and its last block. A range whose logs do not (a backend serving truncated logs) is read once more, then held: the Base cursor stays on it and liveness waits until a read accounts for the totals; M2a still runs. | Alert (`BASE LOG SCAN INCOMPLETE`), throttled | +| Base log completeness | In every scanned block range the `Released` amounts must add up to the growth of `totalReleased`, and the `RemainderSwept` amounts to that of `totalSwept`, between the block before the range and its last block. A range whose logs do not (a backend serving truncated logs) is read once more, then held: the Base cursor stays on it and liveness waits until a read accounts for the totals; M2a still runs. | Alert (`BASE LOG SCAN INCOMPLETE`), throttled; `CRITICAL: BASE LOG SCAN INCOMPLETE` once a range has been held for longer than `UNMATCHED_EVENT_GRACE_SECONDS`, since no later Base event is matched against its burn meanwhile | | Base view | The safe/finalized Base block must not be older than `BASE_STALE_ALERT_SECONDS`: a view that stops advancing hides every later vault event while every check still passes at the old block. | Alert (`BASE VIEW STALLED`), throttled | | Auto-pause readiness | At startup and every poll: the Base RPC serves `BASE_CHAIN_ID` and, with a guardian key configured, the key parses, is `vault.guardian()` and holds at least `GUARDIAN_MIN_BALANCE_WEI`. A read that fails pages as well and never stops the poll's other checks. | Alert (`AUTO-PAUSE NOT READY`), throttled; `AUTO-PAUSE DISABLED` once at startup without a key | | Pendulum governance | Every pending public proposal, the council's external proposal, every ongoing referendum and every open council and technical-committee motion, with the call decoded. A call that mints, moves or unlocks balances outside normal transfers, or changes the runtime or the migration pallet, pages as critical: minted PEN burned through the migration drains the fixed-supply vault, and only Pendulum governance can stop the call before enactment. A call the monitor cannot read pages as critical too: one over 64 KiB is never fetched, and one that does not decode or nests more than 256 calls or 16 levels deep is unreadable; one whose preimage is not noted yet is re-checked every poll and paged again once it can be read. The watch runs after the poll's conservation checks, so it cannot stop or delay them, and it decodes within fixed bounds, so its cost to the next poll is bounded. | Alert once per item (`PENDULUM GOVERNANCE ITEM`, `CRITICAL: Pendulum governance item …`); a motion proposed again with the same call (a new motion index), or an external proposal made again after it left the queue or became a referendum, pages again; a restart re-pages the items still pending | @@ -112,6 +112,7 @@ subject that starts with `CRITICAL`; everything else is a `WARNING`. | `CRITICAL: MISMATCHED APPROVAL` | One attestor approved a tuple that disagrees with the finalized Pendulum view; check its key and its Pendulum node. A second attestor doing so for the same nonce pauses the vault | | `LIVENESS: migration below approval quorum`, `LIVENESS: quorum reached but not released` | Stalled migrations, one summary per kind and poll | | `PENDULUM SOURCE VIEW STALLED`, `PENDULUM SOURCE UNREADABLE`, `BASE VIEW STALLED`, `BASE ADMIN SCAN FAILED`, `BASE LOG SCAN INCOMPLETE` | The monitor is going blind on one chain | +| `CRITICAL: BASE LOG SCAN INCOMPLETE` | A Base block range has been held for longer than `UNMATCHED_EVENT_GRACE_SECONDS`: no later Base event is matched against its burn until it reads in full; fix the Base RPC | | `CRITICAL: PENDULUM SOURCE CONTRADICTED` | The Pendulum source contradicts the chain the monitor verified; stop trusting that node | | `PENDULUM CROSS-CHECK UNAVAILABLE`, `RELAY ANCHOR UNAVAILABLE` | The second Pendulum node or every relay RPC does not answer, or the only relay RPC that answers records a head that contradicts the source, so the source is used without that check | | `RELAY RPCS DISAGREE` | One relay RPC records a Pendulum head that contradicts the source and the next one that answers does not; the source is used without the anchor until they agree. Check the first one | diff --git a/monitor/src/main.test.ts b/monitor/src/main.test.ts index b5acd6171..3caf91dd4 100644 --- a/monitor/src/main.test.ts +++ b/monitor/src/main.test.ts @@ -1143,6 +1143,32 @@ test("logs that miss a sweep hold the Base cursor too", async (t) => { assert.equal(persisted().baseFromBlock, "1011"); }); +test("a Base log range held past the grace pages critical", async (t) => { + const { base, pendulum, api, monitor, pages } = await setup(t, { env: { UNMATCHED_EVENT_GRACE_SECONDS: "0" } }); + burn(pendulum, 0n, 100n * PEN); + release(base, 0n, 100n * PEN, 1_005n); + base.logsHead = 1_003n; + await monitor.poll(api); + await new Promise((resolve) => setTimeout(resolve, 5)); + await monitor.poll(api); + const held = () => pages.filter((page) => page.subject.endsWith("BASE LOG SCAN INCOMPLETE")); + await until(() => held().length === 2, "two incomplete scan pages"); + assert.deepEqual(held().map((page) => [page.subject, page.severity]), [ + ["BASE LOG SCAN INCOMPLETE", "WARNING"], + ["CRITICAL: BASE LOG SCAN INCOMPLETE", "CRITICAL"], + ]); + assert.match(held()[1].detail, /stays at block 1000 \(for \d+s\): no later Base event is matched against its burn/); + base.logsHead = undefined; + await monitor.poll(api); // the range reads in full + burn(pendulum, 1n, 100n * PEN, 102); + release(base, 1n, 100n * PEN, 1_015n); + base.logsHead = 1_012n; + base.safe = 1_020n; + await monitor.poll(api); // a new hold starts as a warning + await until(() => held().length === 3, "the next incomplete scan page"); + assert.equal(held()[2].subject, "BASE LOG SCAN INCOMPLETE"); +}); + test("the conservation check still runs while a Base log range is held", async (t) => { const { base, pendulum, api, monitor, subjects } = await setup(t); burn(pendulum, 0n, 100n * PEN); diff --git a/monitor/src/main.ts b/monitor/src/main.ts index d243a1d8b..99a7006b6 100644 --- a/monitor/src/main.ts +++ b/monitor/src/main.ts @@ -439,6 +439,8 @@ let lastBaseStallAlertMs = 0; let lastReadinessAlertMs = 0; let lastAdminScanFailureAlertMs = 0; let lastLogScanAlertMs = 0; +/** Since when the Base log scan holds a range, while it does. */ +let logScanHeldSinceMs: number | undefined; let lastIssuanceAlertMs = 0; /** Once-only pages by key, each held until the webhook accepts it. */ // ponytail: in memory; a restart re-pages what it finds again (Base admin events after the persisted cursor, pending governance items) and drops undelivered pages it does not find again. @@ -1153,16 +1155,23 @@ async function reconcileBaseEvents( let range = await readVaultRange(start, end); if (range.gap) range = await readVaultRange(start, end); if (range.gap) { + const heldMs = logScanHeldSinceMs === undefined ? 0 : Date.now() - logScanHeldSinceMs; + logScanHeldSinceMs ??= Date.now(); if (Date.now() - lastLogScanAlertMs > config.unmatchedEventGraceSeconds * 1000) { lastLogScanAlertMs = Date.now(); + // Held past the grace, no later Base event has been matched against its burn for that long. + const critical = heldMs > config.unmatchedEventGraceSeconds * 1000; void alert( - "BASE LOG SCAN INCOMPLETE", + `${critical ? "CRITICAL: " : ""}BASE LOG SCAN INCOMPLETE`, `${range.gap}; the Base RPC served logs that miss events, so the range is read again every poll ` + - `and the Base cursor stays at block ${start} — check the Base RPC`, + `and the Base cursor stays at block ${start}` + + (critical ? ` (for ${Math.round(heldMs / 1000)}s): no later Base event is matched against its burn` : "") + + " — check the Base RPC", ); } return { violation: firstViolation, incomplete: range.gap }; } + logScanHeldSinceMs = undefined; const { approvals, releases } = range; const observedAt = Date.now(); let awaitingSource: string | undefined; From 72c4fa2fc571b71441384a39349734ba57837e74 Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Thu, 22 Oct 2026 14:03:59 +0200 Subject: [PATCH 52/75] feat(monitor): Run the vault balance check on its own while a poll is busy A poll checks the vault balance first, but the next check waited for the rest of the poll: Pendulum reads, a long catch-up. main() now also runs the check on a timer whenever a poll is still busy a poll interval after its last check started. A violation found while a pause attempt is in flight joins that attempt instead of starting another. --- monitor/README.md | 4 +- monitor/src/main.test.ts | 77 ++++++++++++++++++++++++++++++++++ monitor/src/main.ts | 90 +++++++++++++++++++++++++++++----------- 3 files changed, 144 insertions(+), 27 deletions(-) diff --git a/monitor/README.md b/monitor/README.md index ddcb7a05c..f36d87a09 100644 --- a/monitor/README.md +++ b/monitor/README.md @@ -10,7 +10,7 @@ Checks every poll: |---|---|---| | M2 tuple match | Every safe Base `Approved` and `Released` event must match one finalized Pendulum `MigrationInitiated` nonce, recipient, and pallet amount; `Released.tokenAmount` must equal `palletAmount × conversionFactor`. After a violation the Base cursor stays on any block range whose events the source view may still reveal, so a pause caused by a lagging source does not lose those releases. | Alert + auto-pause on any mismatching `Released`, and on approvals of one nonce that disagree with the finalized Pendulum view from two or more distinct attestors, also across polls and restarts. A single attestor's disagreeing approval pages `CRITICAL: MISMATCHED APPROVAL` instead: one approval can never release. A mismatch against a *known* nonce counts immediately. An *unknown* nonce first pages (`UNVERIFIED BASE EVENT`, without delaying the pause decision), then counts as a mismatch as soon as it is provable — the finalized source view passed the event's own timestamp and the nonce still does not exist — or after `UNMATCHED_EVENT_GRACE_SECONDS` if the source view stays behind or cannot be read (a stalled or unreadable source also pages on its own, see below) | | M2a aggregate | `totalReleased <= TotalMigrated × conversionFactor + ACKNOWLEDGED_RELEASE_EXCESS`. This remains independent defence-in-depth around the event reconciliation: it also runs while a Base log range is held, and waits only while a Base event's burn is not in the source view yet. After a real, quantified loss, operators set `ACKNOWLEDGED_RELEASE_EXCESS` to it (RB-3) so the vault can be unpaused; any further excess still fires. | Alert + auto-pause | -| M2b: `balanceOf(vault) + totalReleased + totalSwept >= totalSupply` | Vault-internal conservation. Only a **deficit** alerts: a surplus is a harmless inbound transfer (donation, or a migration whose recipient is the vault) and is ignored, so it cannot false-trigger a pause. It reads Base only and runs first in every poll, so a slow or failing Pendulum source or Base log scan can neither delay nor skip it. | Alert + auto-pause on a deficit | +| M2b: `balanceOf(vault) + totalReleased + totalSwept >= totalSupply` | Vault-internal conservation. Only a **deficit** alerts: a surplus is a harmless inbound transfer (donation, or a migration whose recipient is the vault) and is ignored, so it cannot false-trigger a pause. It reads Base only and runs first in every poll, so a failing Pendulum source or Base log scan cannot skip it. While the rest of a poll takes longer than `POLL_INTERVAL_MS` (a slow source, a long catch-up), it also runs on its own, so that delays it by two poll intervals at most. | Alert + auto-pause on a deficit | | M4: every old migration reaches active approval quorum — and then releases | Liveness of the attestor fleet (`below approval quorum`), and of everything after it (`quorum reached but not released`: a release starved of daily allowance, paused, or made releasable by a threshold cut without a `ReleasePending`, which the releaser cannot see). | One summary page per kind and poll (count and the first nonces; the log lists every nonce), each nonce at most once per `GRACE_SECONDS` | | Source supply | `totalIssuance + TotalMigrated` on Pendulum must never grow past its first-seen anchor. | Alert (`SOURCE SUPPLY GREW`) | | Unreleasable burns | A burn to the zero or vault address can never be approved or released; it is paged once and excluded from the pending count so it cannot page liveness forever or block RB-7. | Alert once | @@ -72,7 +72,7 @@ advance). | `RELAY_ANCHOR_GRACE_SECONDS` | How long the source may go without a relay-anchored head before `CRITICAL: RELAY ANCHOR STALE` pages (default 3600) | | `BASE_RPC_URL` | Base JSON-RPC endpoint — a **single dedicated node**, and a different provider than the attestors use. Load-balanced pools can serve `eth_getLogs` from a lagging replica; each scan range is probed for its end block first and its logs must account for the vault's running totals, which turns a behind-node into a loud replay instead of a silent gap, but a dedicated node removes the hazard entirely | | `VAULT_ADDRESS` | MigrationVault address on Base | -| `POLL_INTERVAL_MS` | Poll cadence (default 60s) | +| `POLL_INTERVAL_MS` | Poll cadence (default 60s); while a poll takes longer, the vault balance check runs on its own at this cadence | | `GRACE_SECONDS` | Liveness alert threshold (default 30 min) | | `UNMATCHED_EVENT_GRACE_SECONDS` | Time a lagging or unreadable Pendulum source view gets to reveal the burn behind a safe Base event before a cannot-verify pause (default 10 min). The first sighting of an unmatched event pages immediately, so this is the window operators have to repair a stalled or failing node. Size it to your node-ops response time | | `SOURCE_CLOCK_SKEW_MARGIN_SECONDS` | Clock-skew allowance for the fabrication proof (default 120s): once the finalized Pendulum view has passed the moment the monitor first observed an unmatched Base event by this margin and the nonce still does not exist, the event is provably fabricated and pauses without waiting out the grace. The anchor is the monitor's own clock, not the Base block timestamp (which trails real time after a sequencer outage); a future-dated source view proves nothing | diff --git a/monitor/src/main.test.ts b/monitor/src/main.test.ts index 3caf91dd4..6f95dde71 100644 --- a/monitor/src/main.test.ts +++ b/monitor/src/main.test.ts @@ -665,6 +665,83 @@ test("a slow Pendulum source does not delay the vault balance check", async (t) } }); +/** Holds every finalized-head read of `pendulum` until the returned function is called. */ +function holdSource(pendulum: Pendulum): () => void { + let respond = () => {}; + pendulum.gate = new Promise((resolve) => (respond = resolve)); + return respond; +} + +test("while a poll waits on a slow source, the vault balance check runs on its own", async (t) => { + const { base, pendulum, api, monitor, subjects } = await setup(t, { env: { POLL_INTERVAL_MS: "20" } }); + const respond = holdSource(pendulum); + const polling = monitor.poll(api); // checks the balance, then waits on the source + try { + await new Promise((resolve) => setTimeout(resolve, 40)); + base.vaultBalance -= 1n; + await monitor.watchVaultBalance(); + assert.ok(subjects().includes("VAULT BALANCE DEFICIT"), JSON.stringify(subjects())); + } finally { + respond(); + await polling; + } +}); + +test("the vault balance check does not run on its own within a poll interval of a poll's, or between polls", async (t) => { + const { base, pendulum, api, monitor, subjects } = await setup(t, { env: { POLL_INTERVAL_MS: "200" } }); + const respond = holdSource(pendulum); + const polling = monitor.poll(api); + try { + await new Promise((resolve) => setTimeout(resolve, 40)); + base.vaultBalance -= 1n; + await monitor.watchVaultBalance(); // the poll checked the balance less than an interval ago + } finally { + respond(); + await polling; + } + await new Promise((resolve) => setTimeout(resolve, 250)); + await monitor.watchVaultBalance(); // no poll is running: the next one checks it + assert.ok(!subjects().includes("VAULT BALANCE DEFICIT"), JSON.stringify(subjects())); +}); + +test("a pause attempt in flight is joined, not started again", async (t) => { + let answerFailedPause = () => {}; + const failedPauseAnswered = new Promise((resolve) => (answerFailedPause = resolve)); + const { base, pendulum, api, monitor, pages } = await setup(t, { + env: { GUARDIAN_PRIVATE_KEY: GUARDIAN_KEY, POLL_INTERVAL_MS: "20" }, + webhook: async ({ subject }) => { + if (subject === "AUTO-PAUSE FAILED") await failedPauseAnswered; // holds that attempt in flight + return 200; + }, + }); + burn(pendulum, 0n, 100n * PEN); + await monitor.poll(api); + const tokenAmount = 100n * PEN * CF; // a release of nonce 0 to another recipient + base.logs.push({ address: VAULT, eventName: "Released", args: { nonce: 0n, recipient: TOKEN, palletAmount: 100n * PEN, tokenAmount }, blockNumber: 1_015n }); + base.totalReleased += tokenAmount; + base.vaultBalance -= tokenAmount; + base.safe = 1_020n; + const respond = holdSource(pendulum); + const polling = monitor.poll(api); + const failedPauses = () => pages.filter((page) => page.subject === "AUTO-PAUSE FAILED").length; + try { + await new Promise((resolve) => setTimeout(resolve, 40)); + base.vaultBalance -= 1n; + const watching = monitor.watchVaultBalance(); // a deficit: the first pause attempt + await until(() => failedPauses() === 1, "the first failed pause page"); + respond(); // the poll goes on to the release, a second violation + await until(() => pages.some((page) => page.subject === "MIGRATION TUPLE VIOLATION"), "the second violation"); + await new Promise((resolve) => setTimeout(resolve, 50)); + answerFailedPause(); + await watching; + } finally { + respond(); + answerFailedPause(); + await polling; + } + assert.equal(failedPauses(), 1, JSON.stringify(pages.map((page) => page.subject))); +}); + test("a source whose chain leaves the blocks ingested so far pages critical and ingests nothing", async (t) => { const { pendulum, api, monitor, restart, pages, subjects, persisted } = await setup(t); burn(pendulum, 0n, 100n * PEN); diff --git a/monitor/src/main.ts b/monitor/src/main.ts index 99a7006b6..c009b7f40 100644 --- a/monitor/src/main.ts +++ b/monitor/src/main.ts @@ -524,7 +524,15 @@ async function waitForBaseFinality( } } -async function pauseVault(): Promise { +/** The pause attempt in flight, which a concurrent violation joins. */ +let pausing: Promise | undefined; + +function pauseVault(): Promise { + pausing ??= pauseOnce().finally(() => (pausing = undefined)); + return pausing; +} + +async function pauseOnce(): Promise { if (!config.guardianPrivateKey) { await alert("AUTO-PAUSE UNAVAILABLE", "no GUARDIAN_PRIVATE_KEY configured; pause manually NOW", "critical"); return false; @@ -1678,6 +1686,55 @@ async function readSource(api: ApiPromise, draft: RuntimeState): Promise { + if (!polling || Date.now() - lastBalanceCheckMs < config.pollIntervalMs) return; + lastBalanceCheckMs = Date.now(); + try { + const confirmed = await publicClient.getBlock({ blockTag: config.baseFinalityTag }); + if (confirmed.number === null) throw new Error(`${config.baseFinalityTag} Base block has no number`); + await checkVaultBalance(confirmed.number); + } catch (balanceError) { + await alert("monitor check failed", `the vault balance check failed: ${balanceError}`); + } +} + /** One pass of the checks; resolves to whether every one of them ran. */ async function check(api: ApiPromise): Promise { const confirmedBase = await publicClient.getBlock({ blockTag: config.baseFinalityTag }); @@ -1700,31 +1757,11 @@ async function check(api: ApiPromise): Promise { await checkAutoPauseReadiness(); // The vault balance check (M2b) reads Base only, so it runs before anything - // that waits on Pendulum or scans Base logs: a slow or failing source, or a - // failing log scan, can neither delay nor skip it. Its pause does not end + // that waits on Pendulum or scans Base logs: a failing source or log scan + // cannot skip it, and while the rest of a poll takes longer than a poll + // interval, `watchVaultBalance` runs it on its own. Its pause does not end // the poll. - const [totalReleased, totalSwept, conversionFactor, tokenAddress] = await Promise.all([ - publicClient.readContract({ address: config.vaultAddress, abi: vaultAbi, functionName: "totalReleased", blockNumber: baseBlock }), - publicClient.readContract({ address: config.vaultAddress, abi: vaultAbi, functionName: "totalSwept", blockNumber: baseBlock }), - publicClient.readContract({ address: config.vaultAddress, abi: vaultAbi, functionName: "conversionFactor", blockNumber: baseBlock }), - publicClient.readContract({ address: config.vaultAddress, abi: vaultAbi, functionName: "token", blockNumber: baseBlock }), - ]); - const [totalSupply, vaultBalance] = await Promise.all([ - publicClient.readContract({ address: tokenAddress, abi: erc20Abi, functionName: "totalSupply", blockNumber: baseBlock }), - publicClient.readContract({ - address: tokenAddress, - abi: erc20Abi, - functionName: "balanceOf", - args: [config.vaultAddress], - blockNumber: baseBlock, - }), - ]); - if (vaultConservationDeficit(vaultBalance, totalReleased, totalSwept, totalSupply)) { - await securityViolation( - "VAULT BALANCE DEFICIT", - `balance ${vaultBalance} + released ${totalReleased} + swept ${totalSwept} < supply ${totalSupply}`, - ); - } + const { totalReleased, totalSwept, conversionFactor, vaultBalance } = await checkVaultBalance(baseBlock); // The admin scan only pages and keeps its own cursor, so nothing else waits // for it, and its failure skips no other check. @@ -1924,6 +1961,7 @@ export async function start(api: ApiPromise, connect?: (url: string) => Promise< /** One poll; a failure pages and the next poll starts afresh. */ export async function poll(api: ApiPromise): Promise { + polling = true; let complete = false; try { complete = await check(api); @@ -1941,11 +1979,13 @@ export async function poll(api: ApiPromise): Promise { } await retryUndeliveredPages(); if (complete) await heartbeat(); + polling = false; } async function main(): Promise { const api = await ApiPromise.create({ provider: new WsProvider(config.pendulumWs) }); await start(api); + setInterval(() => void watchVaultBalance(), config.pollIntervalMs); for (;;) { await poll(api); await new Promise((resolve) => setTimeout(resolve, config.pollIntervalMs)); From 52a43dfcb9e23367857a110abaa2c64927d962fd Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Thu, 22 Oct 2026 14:04:51 +0200 Subject: [PATCH 53/75] fix(monitor): Pause on one disagreeing approval at a threshold of two At a vault threshold of two or less, one more approval releases a nonce, so a single attestor's approval that disagrees with the finalized burn now pauses the vault there. Above that, two distinct attestors are still needed and one pages. The threshold is read before the Base events are reconciled. --- monitor/README.md | 2 +- monitor/src/main.test.ts | 12 ++++++++++++ monitor/src/main.ts | 27 +++++++++++++++------------ 3 files changed, 28 insertions(+), 13 deletions(-) diff --git a/monitor/README.md b/monitor/README.md index f36d87a09..883e16077 100644 --- a/monitor/README.md +++ b/monitor/README.md @@ -8,7 +8,7 @@ Checks every poll: | Check | Meaning | Reaction | |---|---|---| -| M2 tuple match | Every safe Base `Approved` and `Released` event must match one finalized Pendulum `MigrationInitiated` nonce, recipient, and pallet amount; `Released.tokenAmount` must equal `palletAmount × conversionFactor`. After a violation the Base cursor stays on any block range whose events the source view may still reveal, so a pause caused by a lagging source does not lose those releases. | Alert + auto-pause on any mismatching `Released`, and on approvals of one nonce that disagree with the finalized Pendulum view from two or more distinct attestors, also across polls and restarts. A single attestor's disagreeing approval pages `CRITICAL: MISMATCHED APPROVAL` instead: one approval can never release. A mismatch against a *known* nonce counts immediately. An *unknown* nonce first pages (`UNVERIFIED BASE EVENT`, without delaying the pause decision), then counts as a mismatch as soon as it is provable — the finalized source view passed the event's own timestamp and the nonce still does not exist — or after `UNMATCHED_EVENT_GRACE_SECONDS` if the source view stays behind or cannot be read (a stalled or unreadable source also pages on its own, see below) | +| M2 tuple match | Every safe Base `Approved` and `Released` event must match one finalized Pendulum `MigrationInitiated` nonce, recipient, and pallet amount; `Released.tokenAmount` must equal `palletAmount × conversionFactor`. After a violation the Base cursor stays on any block range whose events the source view may still reveal, so a pause caused by a lagging source does not lose those releases. | Alert + auto-pause on any mismatching `Released`, and on approvals of one nonce that disagree with the finalized Pendulum view from two or more distinct attestors, also across polls and restarts (from one at a vault threshold of two or less, where one more approval releases). Otherwise a single attestor's disagreeing approval pages `CRITICAL: MISMATCHED APPROVAL` instead: one approval can never release. A mismatch against a *known* nonce counts immediately. An *unknown* nonce first pages (`UNVERIFIED BASE EVENT`, without delaying the pause decision), then counts as a mismatch as soon as it is provable — the finalized source view passed the event's own timestamp and the nonce still does not exist — or after `UNMATCHED_EVENT_GRACE_SECONDS` if the source view stays behind or cannot be read (a stalled or unreadable source also pages on its own, see below) | | M2a aggregate | `totalReleased <= TotalMigrated × conversionFactor + ACKNOWLEDGED_RELEASE_EXCESS`. This remains independent defence-in-depth around the event reconciliation: it also runs while a Base log range is held, and waits only while a Base event's burn is not in the source view yet. After a real, quantified loss, operators set `ACKNOWLEDGED_RELEASE_EXCESS` to it (RB-3) so the vault can be unpaused; any further excess still fires. | Alert + auto-pause | | M2b: `balanceOf(vault) + totalReleased + totalSwept >= totalSupply` | Vault-internal conservation. Only a **deficit** alerts: a surplus is a harmless inbound transfer (donation, or a migration whose recipient is the vault) and is ignored, so it cannot false-trigger a pause. It reads Base only and runs first in every poll, so a failing Pendulum source or Base log scan cannot skip it. While the rest of a poll takes longer than `POLL_INTERVAL_MS` (a slow source, a long catch-up), it also runs on its own, so that delays it by two poll intervals at most. | Alert + auto-pause on a deficit | | M4: every old migration reaches active approval quorum — and then releases | Liveness of the attestor fleet (`below approval quorum`), and of everything after it (`quorum reached but not released`: a release starved of daily allowance, paused, or made releasable by a threshold cut without a `ReleasePending`, which the releaser cannot see). | One summary page per kind and poll (count and the first nonces; the log lists every nonce), each nonce at most once per `GRACE_SECONDS` | diff --git a/monitor/src/main.test.ts b/monitor/src/main.test.ts index 6f95dde71..6bf6c9c1b 100644 --- a/monitor/src/main.test.ts +++ b/monitor/src/main.test.ts @@ -1325,6 +1325,18 @@ test("one attestor approving against the burn pages critical; a second one pause ); }); +test("at a threshold of two, one attestor approving against the burn pauses", async (t) => { + const { base, pendulum, api, monitor, pages, subjects } = await setup(t, { prepare: (base) => (base.threshold = 2n) }); + burn(pendulum, 0n, 100n * PEN); + approve(base, 0n, TOKEN, ATTESTORS[0], 1_005n); + await monitor.poll(api); + assert.match( + pages.find((page) => page.subject === "MIGRATION TUPLE VIOLATION")?.detail ?? "", + new RegExp(`; attestor ${ATTESTORS[0]} approved tuples for nonce 0`, "i"), + ); + assert.ok(!subjects().includes("CRITICAL: MISMATCHED APPROVAL"), JSON.stringify(subjects())); +}); + test("auto-pause readiness is checked at startup and each problem pages", async (t) => { const { startup } = await setup(t, { env: { GUARDIAN_PRIVATE_KEY: GUARDIAN_KEY }, diff --git a/monitor/src/main.ts b/monitor/src/main.ts index c009b7f40..78256c843 100644 --- a/monitor/src/main.ts +++ b/monitor/src/main.ts @@ -1147,6 +1147,7 @@ async function reconcileBaseEvents( toBlock: bigint, conversionFactor: bigint, sourceFinalizedTsMs: number, + threshold: bigint, ): Promise<{ violation?: string; awaitingSource?: string; incomplete?: string }> { const releasedInScan = new Set(); let firstViolation: string | undefined; @@ -1211,8 +1212,9 @@ async function reconcileBaseEvents( ? " (source view has passed the event's timestamp — fabricated)" : " (source view still behind after the full grace period)"; } - // One attestor's approval cannot release: it pages. Two attestors - // approving tuples that disagree with the burn pause the vault. + // Approvals that disagree with the burn pause the vault once they come + // from two attestors, or from one at a threshold of two or less, where + // one more approval releases. One attestor's approval alone pages. const { nonce, recipient, palletAmount, attestor } = actual; const seen = draft.mismatchedApprovals.some((approval) => approval.nonce === nonce && approval.attestor.toLowerCase() === attestor.toLowerCase() && @@ -1220,9 +1222,10 @@ async function reconcileBaseEvents( ); if (!seen) draft.mismatchedApprovals.push({ nonce, recipient, palletAmount, attestor }); const attestors = disagreeingAttestors(expected, nonce, draft.mismatchedApprovals); - if (attestors.length >= 2) { - firstViolation ??= `Approved at Base block ${event.blockNumber}: ${mismatch}; attestors ` + - `${attestors.join(", ")} approved tuples for nonce ${nonce} that disagree with the finalized Pendulum view`; + if (attestors.length >= (threshold > 2n ? 2 : 1)) { + firstViolation ??= `Approved at Base block ${event.blockNumber}: ${mismatch}; attestor` + + `${attestors.length > 1 ? "s" : ""} ${attestors.join(", ")} approved tuples for nonce ${nonce} that ` + + "disagree with the finalized Pendulum view"; continue; } void pageOnce( @@ -1808,7 +1811,13 @@ async function check(api: ApiPromise): Promise { } } - const reconciliation = await reconcileBaseEvents(draft, baseBlock, conversionFactor, source?.timestampMs ?? 0); + const threshold = await publicClient.readContract({ + address: config.vaultAddress, + abi: vaultAbi, + functionName: "threshold", + blockNumber: baseBlock, + }); + const reconciliation = await reconcileBaseEvents(draft, baseBlock, conversionFactor, source?.timestampMs ?? 0, threshold); if (reconciliation.violation) { if (await securityViolation("MIGRATION TUPLE VIOLATION", reconciliation.violation)) { state = draft; @@ -1852,12 +1861,6 @@ async function check(api: ApiPromise): Promise { } const pending = [...draft.migrations.values()]; - const threshold = await publicClient.readContract({ - address: config.vaultAddress, - abi: vaultAbi, - functionName: "threshold", - blockNumber: baseBlock, - }); const approvals = await activeApprovalCounts(pending, baseBlock); const now = Date.now(); // One page per kind and poll however long the backlog, with the detail From cc2050f6050938bcc4a645016de3fd6fc334090c Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Thu, 22 Oct 2026 14:06:08 +0200 Subject: [PATCH 54/75] feat(monitor): Page critical when the Base admin scan keeps failing While the admin scan fails, vault admin events and scheduled governance operations are not paged, and the canceller's veto window keeps running. Once the scan has failed for longer than UNMATCHED_EVENT_GRACE_SECONDS, the throttled page turns into CRITICAL: BASE ADMIN SCAN FAILED; a scan that reads again ends it. --- monitor/README.md | 4 ++-- monitor/src/main.test.ts | 22 ++++++++++++++++++++++ monitor/src/main.ts | 14 +++++++++++--- 3 files changed, 35 insertions(+), 5 deletions(-) diff --git a/monitor/README.md b/monitor/README.md index 883e16077..a2283c8c0 100644 --- a/monitor/README.md +++ b/monitor/README.md @@ -22,7 +22,7 @@ Checks every poll: | Auto-pause readiness | At startup and every poll: the Base RPC serves `BASE_CHAIN_ID` and, with a guardian key configured, the key parses, is `vault.guardian()` and holds at least `GUARDIAN_MIN_BALANCE_WEI`. A read that fails pages as well and never stops the poll's other checks. | Alert (`AUTO-PAUSE NOT READY`), throttled; `AUTO-PAUSE DISABLED` once at startup without a key | | Pendulum governance | Every pending public proposal, the council's external proposal, every ongoing referendum and every open council and technical-committee motion, with the call decoded. A call that mints, moves or unlocks balances outside normal transfers, or changes the runtime or the migration pallet, pages as critical: minted PEN burned through the migration drains the fixed-supply vault, and only Pendulum governance can stop the call before enactment. A call the monitor cannot read pages as critical too: one over 64 KiB is never fetched, and one that does not decode or nests more than 256 calls or 16 levels deep is unreadable; one whose preimage is not noted yet is re-checked every poll and paged again once it can be read. The watch runs after the poll's conservation checks, so it cannot stop or delay them, and it decodes within fixed bounds, so its cost to the next poll is bounded. | Alert once per item (`PENDULUM GOVERNANCE ITEM`, `CRITICAL: Pendulum governance item …`); a motion proposed again with the same call (a new motion index), or an external proposal made again after it left the queue or became a referendum, pages again; a restart re-pages the items still pending | | Migration pallet admin events | Every finalized `MigrationPauseSet` and `TreasuryDestinationSet`, however fast the motion behind it passed. | Alert once per event (`MIGRATION PALLET ADMIN EVENT`) | -| Vault admin events | Every admin event of the vault (token, pause, unpause, attestors, threshold, caps, guardian, admin transfer, sweep, stale pending cleared). Before the governance handover the admin acts without a delay, so these are the only trace of such a change. This scan (shared with the governance queue) keeps its own persisted cursor: while it fails, every other check still runs, and it resumes where it stopped, also after a restart. | Alert once per event (`VAULT ADMIN EVENT`); the first run also pages the deployment's own events; `BASE ADMIN SCAN FAILED`, throttled, on a failure | +| Vault admin events | Every admin event of the vault (token, pause, unpause, attestors, threshold, caps, guardian, admin transfer, sweep, stale pending cleared). Before the governance handover the admin acts without a delay, so these are the only trace of such a change. This scan (shared with the governance queue) keeps its own persisted cursor: while it fails, every other check still runs, and it resumes where it stopped, also after a restart. | Alert once per event (`VAULT ADMIN EVENT`); the first run also pages the deployment's own events; `BASE ADMIN SCAN FAILED`, throttled, on a failure, and `CRITICAL: BASE ADMIN SCAN FAILED` once it has failed for longer than `UNMATCHED_EVENT_GRACE_SECONDS` | | Governance queue | Every operation the governance Timelock schedules (`CallScheduled`), with its calls decoded and the time it becomes executable. Once the delay has passed anyone may execute a ready operation, so the canceller has exactly the delay to review and veto it. | Alert once per operation and scheduling block (`GOVERNANCE OPERATION SCHEDULED`), so an operation cancelled and scheduled again with the same id pages again | ## Pendulum chain verification @@ -112,7 +112,7 @@ subject that starts with `CRITICAL`; everything else is a `WARNING`. | `CRITICAL: MISMATCHED APPROVAL` | One attestor approved a tuple that disagrees with the finalized Pendulum view; check its key and its Pendulum node. A second attestor doing so for the same nonce pauses the vault | | `LIVENESS: migration below approval quorum`, `LIVENESS: quorum reached but not released` | Stalled migrations, one summary per kind and poll | | `PENDULUM SOURCE VIEW STALLED`, `PENDULUM SOURCE UNREADABLE`, `BASE VIEW STALLED`, `BASE ADMIN SCAN FAILED`, `BASE LOG SCAN INCOMPLETE` | The monitor is going blind on one chain | -| `CRITICAL: BASE LOG SCAN INCOMPLETE` | A Base block range has been held for longer than `UNMATCHED_EVENT_GRACE_SECONDS`: no later Base event is matched against its burn until it reads in full; fix the Base RPC | +| `CRITICAL: BASE LOG SCAN INCOMPLETE`, `CRITICAL: BASE ADMIN SCAN FAILED` | For longer than `UNMATCHED_EVENT_GRACE_SECONDS`, a Base block range has been held, so no later Base event is matched against its burn, or the admin scan has failed, so no vault admin event or scheduled governance operation is paged while its veto window runs; fix the Base RPC | | `CRITICAL: PENDULUM SOURCE CONTRADICTED` | The Pendulum source contradicts the chain the monitor verified; stop trusting that node | | `PENDULUM CROSS-CHECK UNAVAILABLE`, `RELAY ANCHOR UNAVAILABLE` | The second Pendulum node or every relay RPC does not answer, or the only relay RPC that answers records a head that contradicts the source, so the source is used without that check | | `RELAY RPCS DISAGREE` | One relay RPC records a Pendulum head that contradicts the source and the next one that answers does not; the source is used without the anchor until they agree. Check the first one | diff --git a/monitor/src/main.test.ts b/monitor/src/main.test.ts index 6bf6c9c1b..cb08d3e6f 100644 --- a/monitor/src/main.test.ts +++ b/monitor/src/main.test.ts @@ -1416,6 +1416,28 @@ test("a failed admin scan pages, every other check still runs and the scan resum assert.equal(persisted().baseAdminFromBlock, "1011"); }); +test("an admin scan that keeps failing past the grace pages critical", async (t) => { + const { base, api, monitor, pages } = await setup(t, { env: { UNMATCHED_EVENT_GRACE_SECONDS: "0" } }); + const guardianUpdated = toEventSelector("GuardianUpdated(address)"); // only the admin scan asks for it + const failing = (method: string, params: unknown[]) => method === "eth_getLogs" && JSON.stringify(params).includes(guardianUpdated); + base.failing = failing; + await monitor.poll(api); + await new Promise((resolve) => setTimeout(resolve, 5)); + await monitor.poll(api); + const failed = () => pages.filter((page) => page.subject.endsWith("BASE ADMIN SCAN FAILED")); + assert.deepEqual(failed().map((page) => [page.subject, page.severity]), [ + ["BASE ADMIN SCAN FAILED", "WARNING"], + ["CRITICAL: BASE ADMIN SCAN FAILED", "CRITICAL"], + ]); + assert.match(failed()[1].detail, /\(failing for \d+s\)/); + base.failing = undefined; + await monitor.poll(api); // the scan reads again + base.failing = failing; + base.safe = 1_020n; // a new range for the admin scan + await monitor.poll(api); + assert.equal(failed()[2]?.subject, "BASE ADMIN SCAN FAILED"); // a new failure starts as a warning +}); + test("a monitor without a guardian key says so at startup", async (t) => { const { startup } = await setup(t); assert.ok(startup.some((page) => page.subject === "AUTO-PAUSE DISABLED"), JSON.stringify(startup)); diff --git a/monitor/src/main.ts b/monitor/src/main.ts index 78256c843..437590681 100644 --- a/monitor/src/main.ts +++ b/monitor/src/main.ts @@ -438,6 +438,8 @@ let lastRelayAnchorMs = Date.now(); let lastBaseStallAlertMs = 0; let lastReadinessAlertMs = 0; let lastAdminScanFailureAlertMs = 0; +/** Since when the Base admin scan fails, while it does. */ +let adminScanFailingSinceMs: number | undefined; let lastLogScanAlertMs = 0; /** Since when the Base log scan holds a range, while it does. */ let logScanHeldSinceMs: number | undefined; @@ -1771,14 +1773,20 @@ async function check(api: ApiPromise): Promise { let adminScanned = true; try { await watchBaseAdmin(baseBlock); + adminScanFailingSinceMs = undefined; } catch (adminError) { adminScanned = false; + const failingMs = adminScanFailingSinceMs === undefined ? 0 : Date.now() - adminScanFailingSinceMs; + adminScanFailingSinceMs ??= Date.now(); if (Date.now() - lastAdminScanFailureAlertMs > config.unmatchedEventGraceSeconds * 1000) { lastAdminScanFailureAlertMs = Date.now(); + // Meanwhile the veto window of any operation scheduled since runs unseen. + const critical = failingMs > config.unmatchedEventGraceSeconds * 1000; await alert( - "BASE ADMIN SCAN FAILED", - `${adminError}; vault admin events and the governance queue are not paged until the scan reads again, ` + - "and it then resumes where it stopped; every other check still runs", + `${critical ? "CRITICAL: " : ""}BASE ADMIN SCAN FAILED`, + `${adminError}; vault admin events and the governance queue are not paged until the scan reads again` + + (critical ? ` (failing for ${Math.round(failingMs / 1000)}s)` : "") + + ", and it then resumes where it stopped; every other check still runs", ); } } From bfbd2b58e2f28ddc297a415612e38fc05c61eeb5 Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Thu, 22 Oct 2026 14:07:13 +0200 Subject: [PATCH 55/75] fix(monitor): Say that the monitor holds at a treasury violation While the pause after a treasury destination violation is not confirmed, the monitor holds at that block. The hold was paged as an unreadable Pendulum source, which it is not; the violation's page now says that the monitor holds there until the vault is paused, and the hold is logged. --- monitor/README.md | 2 +- monitor/src/main.test.ts | 8 +++++--- monitor/src/main.ts | 12 +++++++++--- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/monitor/README.md b/monitor/README.md index a2283c8c0..3f70a6c19 100644 --- a/monitor/README.md +++ b/monitor/README.md @@ -14,7 +14,7 @@ Checks every poll: | M4: every old migration reaches active approval quorum — and then releases | Liveness of the attestor fleet (`below approval quorum`), and of everything after it (`quorum reached but not released`: a release starved of daily allowance, paused, or made releasable by a threshold cut without a `ReleasePending`, which the releaser cannot see). | One summary page per kind and poll (count and the first nonces; the log lists every nonce), each nonce at most once per `GRACE_SECONDS` | | Source supply | `totalIssuance + TotalMigrated` on Pendulum must never grow past its first-seen anchor. | Alert (`SOURCE SUPPLY GREW`) | | Unreleasable burns | A burn to the zero or vault address can never be approved or released; it is paged once and excluded from the pending count so it cannot page liveness forever or block RB-7. | Alert once | -| Treasury destination | With `EXPECTED_TREASURY_DESTINATION` set, every finalized treasury migration (a `MigrationInitiated` whose `who` is the treasury account, `PalletId(*b"py/trsry")`) must pay that Base address. It is checked when the burn is ingested, before the attestors can release it; while the pause is not confirmed, the monitor does not move past that block. | Alert + auto-pause (`TREASURY DESTINATION VIOLATION`) | +| Treasury destination | With `EXPECTED_TREASURY_DESTINATION` set, every finalized treasury migration (a `MigrationInitiated` whose `who` is the treasury account, `PalletId(*b"py/trsry")`) must pay that Base address. It is checked when the burn is ingested, before the attestors can release it; while the pause is not confirmed, the monitor holds at that block, and the violation pages again every poll saying so. | Alert + auto-pause (`TREASURY DESTINATION VIOLATION`) | | Pendulum source | The finalized Pendulum view must stay fresh and readable. While it cannot be read (an RPC error, a nonce discontinuity, an event-shape change), the Base-side checks — vault balance and its auto-pause, Base view, auto-pause readiness, vault admin events, governance queue — still run, and Base events are reconciled against the last view read in full: an unreadable source counts as a stalled one, so a mismatch on a known nonce counts at once and an unknown nonce after `UNMATCHED_EVENT_GRACE_SECONDS`. M2a and liveness wait for the source. | Alert, throttled: `PENDULUM SOURCE VIEW STALLED` once the view is older than `SOURCE_STALE_ALERT_SECONDS`, `PENDULUM SOURCE UNREADABLE` on a failure | | Pendulum chain | Every finalized block the monitor ingests is read at a hash it verified against the blocks it ingested before and, where configured, a second node and the relay chain; a second node must also agree on the migration totals and on the migration events of each ingested block (see [Pendulum chain verification](#pendulum-chain-verification)). | Alert (`CRITICAL: PENDULUM SOURCE CONTRADICTED`), throttled; nothing from the source is ingested, and it is treated as unreadable (see Pendulum source) until it verifies again. A second node or relay RPC that does not answer only warns, throttled (`PENDULUM CROSS-CHECK UNAVAILABLE`, `RELAY ANCHOR UNAVAILABLE`), and so do relay RPCs that disagree (`RELAY RPCS DISAGREE`); `CRITICAL: RELAY ANCHOR STALE` once no head was anchored for `RELAY_ANCHOR_GRACE_SECONDS` | | Base log completeness | In every scanned block range the `Released` amounts must add up to the growth of `totalReleased`, and the `RemainderSwept` amounts to that of `totalSwept`, between the block before the range and its last block. A range whose logs do not (a backend serving truncated logs) is read once more, then held: the Base cursor stays on it and liveness waits until a read accounts for the totals; M2a still runs. | Alert (`BASE LOG SCAN INCOMPLETE`), throttled; `CRITICAL: BASE LOG SCAN INCOMPLETE` once a range has been held for longer than `UNMATCHED_EVENT_GRACE_SECONDS`, since no later Base event is matched against its burn meanwhile | diff --git a/monitor/src/main.test.ts b/monitor/src/main.test.ts index cb08d3e6f..8c58c3943 100644 --- a/monitor/src/main.test.ts +++ b/monitor/src/main.test.ts @@ -1571,15 +1571,17 @@ test("a treasury migration to any but the expected address pages and pauses", as }); test("while the pause after a treasury violation is not confirmed, the monitor holds at its block", async (t) => { - const { pendulum, api, monitor, subjects, persisted } = await setup(t, { + const { pendulum, api, monitor, pages, subjects, persisted } = await setup(t, { env: { EXPECTED_TREASURY_DESTINATION: TREASURY_DESTINATION }, }); burn(pendulum, 0n, 100n * PEN, 101, { who: TREASURY_ACCOUNT, recipient: TOKEN }); await monitor.poll(api); await monitor.poll(api); - assert.equal(subjects().filter((subject) => subject === "TREASURY DESTINATION VIOLATION").length, 2); + const violations = pages.filter((page) => page.subject === "TREASURY DESTINATION VIOLATION"); + assert.equal(violations.length, 2); + assert.match(violations[1].detail, /; the monitor holds at that block until the vault is paused$/); assert.equal(persisted().lastPendulumBlock, 100); - assert.ok(subjects().includes("PENDULUM SOURCE UNREADABLE"), JSON.stringify(subjects())); + assert.ok(!subjects().includes("PENDULUM SOURCE UNREADABLE"), JSON.stringify(subjects())); // the source reads fine }); test("admin events of the migration pallet page from the ingest", async (t) => { diff --git a/monitor/src/main.ts b/monitor/src/main.ts index 437590681..ad0af79e8 100644 --- a/monitor/src/main.ts +++ b/monitor/src/main.ts @@ -693,6 +693,10 @@ interface BlockRef { * so far, the second node's or the relay chain's. Handled as unreadable. */ class SourceIntegrityError extends Error {} +/** The source holds at a block whose violation must pause the vault first; + * the violation's own page says so. */ +class PauseHold extends Error {} + /** Bound on connecting to, and on each read from, the nodes of the integrity checks. */ const NODE_TIMEOUT_MS = 20_000; @@ -1068,9 +1072,10 @@ async function ingestPendulumEvents( const paused = await securityViolation( "TREASURY DESTINATION VIOLATION", `treasury migration nonce ${nonce} in finalized Pendulum block ${block} pays ${amountValue.toBigInt()} ` + - `pallet units to ${recipient}, not EXPECTED_TREASURY_DESTINATION ${expectedTreasury}`, + `pallet units to ${recipient}, not EXPECTED_TREASURY_DESTINATION ${expectedTreasury}; the monitor ` + + "holds at that block until the vault is paused", ); - if (!paused) throw new Error(`the vault is not confirmed paused after treasury migration nonce ${nonce}`); + if (!paused) throw new PauseHold(`holding at Pendulum block ${block} until the vault is paused`); } if (recipient === ZERO_ADDRESS || recipient.toLowerCase() === config.vaultAddress.toLowerCase()) { // The vault deterministically rejects this tuple (ZeroAddress / @@ -1806,7 +1811,8 @@ async function check(api: ApiPromise): Promise { const consequence = `Base events are checked against the last Pendulum view read in full: one whose nonce ` + `that view lacks pauses the vault ${config.unmatchedEventGraceSeconds}s after its first sighting, and M2a ` + "and liveness are skipped"; - if (contradicted && Date.now() - lastSourceIntegrityAlertMs > config.unmatchedEventGraceSeconds * 1000) { + if (sourceError instanceof PauseHold) log(sourceError.message); + else if (contradicted && Date.now() - lastSourceIntegrityAlertMs > config.unmatchedEventGraceSeconds * 1000) { lastSourceIntegrityAlertMs = Date.now(); await alert( "CRITICAL: PENDULUM SOURCE CONTRADICTED", From e6414ca8c75e3e8dbaa8e44a700f02e8a29f7dee Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Thu, 22 Oct 2026 14:09:11 +0200 Subject: [PATCH 56/75] test(monitor): Cover the re-read of a Base range and a failing totals read A range whose first log read misses an event and whose second accounts for the vault's totals advances the cursor without a page; a totals read that fails at a block where the vault has code fails the poll instead of counting as zero. --- monitor/src/main.test.ts | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/monitor/src/main.test.ts b/monitor/src/main.test.ts index 8c58c3943..771b434a1 100644 --- a/monitor/src/main.test.ts +++ b/monitor/src/main.test.ts @@ -23,6 +23,7 @@ import { keccak256, parseAbi, toEventSelector, + toFunctionSelector, toHex, } from "viem"; import { privateKeyToAccount } from "viem/accounts"; @@ -87,6 +88,9 @@ function baseChain() { logs: [] as Log[], /** Where set, log queries answer as a backend whose head is this block. */ logsHead: undefined as bigint | undefined, + /** Where set and it returns a block, the log query for these topics + * answers as a backend whose head is that block. */ + logsHeadFor: undefined as ((topics: string[]) => bigint | undefined) | undefined, /** Where set, the vault has no code before this block. */ deployedAt: undefined as bigint | undefined, /** Requests to answer with an RPC error. */ @@ -152,10 +156,11 @@ function answer(base: Base, method: string, params: any[]): unknown { case "eth_getLogs": { const [{ address, topics, fromBlock, toBlock }] = params; const wanted = [topics?.[0] ?? []].flat() as string[]; + const head = base.logsHeadFor?.(wanted) ?? base.logsHead; return base.logs.flatMap((log, index) => { if (log.address.toLowerCase() !== String(address).toLowerCase()) return []; if (log.blockNumber < BigInt(fromBlock) || log.blockNumber > BigInt(toBlock)) return []; - if (base.logsHead !== undefined && log.blockNumber > base.logsHead) return []; + if (head !== undefined && log.blockNumber > head) return []; const logTopics = encodeEventTopics({ abi, eventName: log.eventName, args: log.args } as never) as Hex[]; if (wanted.length && !wanted.includes(logTopics[0])) return []; const { inputs } = abi.find((item) => item.type === "event" && item.name === log.eventName) as AbiEvent; @@ -1264,6 +1269,34 @@ test("the conservation check waits while a Base event's burn is not in the sourc assert.deepEqual([...new Set(subjects())], ["UNVERIFIED BASE EVENT"]); }); +test("logs that miss an event once and then account for the totals hold nothing and page nothing", async (t) => { + const { base, pendulum, api, monitor, subjects, persisted } = await setup(t); + burn(pendulum, 0n, 100n * PEN); + release(base, 0n, 100n * PEN, 1_005n); + const released = toEventSelector("Released(uint64,address,uint256,uint256)"); // only reconciliation asks for it + let reads = 0; + base.logsHeadFor = (topics) => (topics.includes(released) && reads++ === 0 ? 1_003n : undefined); + await monitor.poll(api); + assert.equal(reads, 2); // read once more + assert.equal(persisted().baseFromBlock, "1011"); + assert.deepEqual(persisted().migrations, []); + await new Promise((resolve) => setTimeout(resolve, 50)); // an incomplete scan page is sent without waiting + assert.deepEqual(subjects(), []); +}); + +test("a vault total that cannot be read where the vault has code fails the poll, not counting as zero", async (t) => { + const { base, pendulum, api, monitor, subjects } = await setup(t, { prepare: (base) => (base.deployedAt = 900n) }); + burn(pendulum, 0n, 100n * PEN); + release(base, 0n, 100n * PEN, 950n); // before the first Base block the monitor scans + const totalReleased = toFunctionSelector("totalReleased()"); + base.failing = (method, params) => + method === "eth_call" && (params[0] as { data: string }).data === totalReleased && params[1] === toHex(999n); + await monitor.poll(api); + await new Promise((resolve) => setTimeout(resolve, 50)); // an incomplete scan page is sent without waiting + assert.ok(subjects().includes("monitor check failed"), JSON.stringify(subjects())); + assert.ok(!subjects().includes("BASE LOG SCAN INCOMPLETE"), JSON.stringify(subjects())); +}); + test("a release in the first block of a range accounts for the totals", async (t) => { const { base, pendulum, api, monitor, subjects, persisted } = await setup(t); burn(pendulum, 0n, 100n * PEN); From 52be02d69570cffef13c9a9c1d9871656f763627 Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Thu, 22 Oct 2026 14:17:34 +0200 Subject: [PATCH 57/75] test(monitor): Cover each second-node total and a relay head below the walk The second node's nonce and total are now each compared on their own, a relay head below the block ingested last is checked down the source's chain, and the tests of the vault balance timer wait until the poll holds on the source instead of for a fixed time. --- monitor/src/main.test.ts | 75 +++++++++++++++++++++++++++++----------- 1 file changed, 55 insertions(+), 20 deletions(-) diff --git a/monitor/src/main.test.ts b/monitor/src/main.test.ts index 771b434a1..107965e14 100644 --- a/monitor/src/main.test.ts +++ b/monitor/src/main.test.ts @@ -259,6 +259,8 @@ function pendulumState() { failure: undefined as Error | undefined, /** Holds every finalized-head read until it settles, as a slow node would. */ gate: undefined as Promise | undefined, + /** Finalized-head reads so far. */ + finalizedReads: 0, events: new Map(), totalMigrated: 0n, nextNonce: 0n, @@ -349,6 +351,7 @@ function pendulumChain(chain: Pendulum = pendulumState()) { rpc: { chain: { getFinalizedHead: async () => { + chain.finalizedReads++; await chain.gate; if (chain.failure) throw chain.failure; return hash(pendulumHash(chain, chain.finalized)); @@ -670,19 +673,22 @@ test("a slow Pendulum source does not delay the vault balance check", async (t) } }); -/** Holds every finalized-head read of `pendulum` until the returned function is called. */ -function holdSource(pendulum: Pendulum): () => void { +/** Holds every finalized-head read of `pendulum` until `respond` is called; + * `waiting` resolves once a read waits. */ +function holdSource(pendulum: Pendulum) { let respond = () => {}; pendulum.gate = new Promise((resolve) => (respond = resolve)); - return respond; + const reads = pendulum.finalizedReads; + return { respond, waiting: () => until(() => pendulum.finalizedReads > reads, "a poll waiting on the source") }; } test("while a poll waits on a slow source, the vault balance check runs on its own", async (t) => { const { base, pendulum, api, monitor, subjects } = await setup(t, { env: { POLL_INTERVAL_MS: "20" } }); - const respond = holdSource(pendulum); + const { respond, waiting } = holdSource(pendulum); const polling = monitor.poll(api); // checks the balance, then waits on the source try { - await new Promise((resolve) => setTimeout(resolve, 40)); + await waiting(); + await new Promise((resolve) => setTimeout(resolve, 30)); // a poll interval later base.vaultBalance -= 1n; await monitor.watchVaultBalance(); assert.ok(subjects().includes("VAULT BALANCE DEFICIT"), JSON.stringify(subjects())); @@ -693,18 +699,18 @@ test("while a poll waits on a slow source, the vault balance check runs on its o }); test("the vault balance check does not run on its own within a poll interval of a poll's, or between polls", async (t) => { - const { base, pendulum, api, monitor, subjects } = await setup(t, { env: { POLL_INTERVAL_MS: "200" } }); - const respond = holdSource(pendulum); + const { base, pendulum, api, monitor, subjects } = await setup(t, { env: { POLL_INTERVAL_MS: "500" } }); + const { respond, waiting } = holdSource(pendulum); const polling = monitor.poll(api); try { - await new Promise((resolve) => setTimeout(resolve, 40)); + await waiting(); base.vaultBalance -= 1n; await monitor.watchVaultBalance(); // the poll checked the balance less than an interval ago } finally { respond(); await polling; } - await new Promise((resolve) => setTimeout(resolve, 250)); + await new Promise((resolve) => setTimeout(resolve, 550)); await monitor.watchVaultBalance(); // no poll is running: the next one checks it assert.ok(!subjects().includes("VAULT BALANCE DEFICIT"), JSON.stringify(subjects())); }); @@ -726,11 +732,12 @@ test("a pause attempt in flight is joined, not started again", async (t) => { base.totalReleased += tokenAmount; base.vaultBalance -= tokenAmount; base.safe = 1_020n; - const respond = holdSource(pendulum); + const { respond, waiting } = holdSource(pendulum); const polling = monitor.poll(api); const failedPauses = () => pages.filter((page) => page.subject === "AUTO-PAUSE FAILED").length; try { - await new Promise((resolve) => setTimeout(resolve, 40)); + await waiting(); + await new Promise((resolve) => setTimeout(resolve, 30)); // a poll interval later base.vaultBalance -= 1n; const watching = monitor.watchVaultBalance(); // a deficit: the first pause attempt await until(() => failedPauses() === 1, "the first failed pause page"); @@ -860,15 +867,17 @@ test("a second node whose best chain holds another newest block, not yet finaliz assert.equal(persisted().lastPendulumBlock, 101); }); -test("a second node with other migration totals at the finalized head pages critical and nothing is ingested", async (t) => { - const { pendulum, api, monitor, pages, persisted } = await crossCheckSetup(t, { nextNonce: 1n, totalMigrated: 7n * PEN }); - burn(pendulum, 0n, 100n * PEN); - await monitor.poll(api); - assert.match( - pages.find((page) => page.subject === "CRITICAL: PENDULUM SOURCE CONTRADICTED")?.detail ?? "", - /at Pendulum block 101 the second node has nextNonce 1 and totalMigrated 7000000000000, the source 1 and 100000000000000;/, - ); - assert.equal(persisted().lastPendulumBlock, 100); +test("a second node with another nonce or total at the finalized head pages critical and nothing is ingested", async (t) => { + for (const own of [{ nextNonce: 2n }, { totalMigrated: 7n * PEN }]) { + const { pendulum, api, monitor, pages, persisted } = await crossCheckSetup(t, own); + burn(pendulum, 0n, 100n * PEN); + await monitor.poll(api); + assert.match( + pages.find((page) => page.subject === "CRITICAL: PENDULUM SOURCE CONTRADICTED")?.detail ?? "", + /at Pendulum block 101 the second node has nextNonce (2|1) and totalMigrated (100|7)000000000000, the source 1 and 100000000000000;/, + ); + assert.equal(persisted().lastPendulumBlock, 100); + } }); test("a second node with other migration events in a block pages critical and nothing is ingested", async (t) => { @@ -1047,6 +1056,32 @@ test("a relay chain that records another hash at a height it anchored pages crit ); }); +test("a relay record below the block ingested last is checked down the source's chain", async (t) => { + for (const recorded of [pendulumHash({}, 100), pendulumHash({ forkFrom: 0 }, 100)]) { + let relayDown = true; + const { pendulum, api, monitor, pages, persisted } = await relaySetup(t, { + head: () => ({ number: 100, hash: recorded }), + failing: () => relayDown, + }); + burn(pendulum, 0n, 100n * PEN); + await monitor.poll(api); // ingested up to block 101 without the anchor + relayDown = false; + burn(pendulum, 1n, 100n * PEN, 102); + await monitor.poll(api); + if (recorded === pendulumHash({}, 100)) { + assert.deepEqual(pages.map((page) => page.subject), ["RELAY ANCHOR UNAVAILABLE"]); + assert.deepEqual(persisted().relayAnchor, { number: 100, hash: recorded }); + assert.equal(persisted().lastPendulumBlock, 102); + } else { + assert.match( + pages.find((page) => page.subject === "CRITICAL: PENDULUM SOURCE CONTRADICTED")?.detail ?? "", + /relay chain records 0x0+10+64 as Pendulum block 100, the source's chain has 0x0+64;/, + ); + assert.equal(persisted().lastPendulumBlock, 101); + } + } +}); + test("an unreadable relay chain warns, the source is used, and it turns critical after the grace", async (t) => { const { pendulum, api, monitor, pages, persisted } = await relaySetup(t, { failing: () => true }); burn(pendulum, 0n, 100n * PEN); From 34c0c42668c437b41e1d72de98e089bafcd264f5 Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Thu, 22 Oct 2026 14:40:49 +0200 Subject: [PATCH 58/75] test(monitor): Cover range boundaries and header numbers, bound a wait Releases and sweeps in the first and the last block of a scanned range must account for the vault's totals, and a source header that names another block number pages critical. A test that waited for a page without a bound now fails after five seconds instead of hanging. --- monitor/src/main.test.ts | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/monitor/src/main.test.ts b/monitor/src/main.test.ts index 107965e14..e0ab56153 100644 --- a/monitor/src/main.test.ts +++ b/monitor/src/main.test.ts @@ -249,6 +249,8 @@ function pendulumState() { forkFrom: undefined as number | undefined, /** Blocks whose header the node alters, so it no longer hashes to its hash. */ alteredHeaders: new Set(), + /** Blocks whose header names the next block number. */ + renumberedHeaders: new Set(), /** Whether the node names block hashes by number from the chain without the fork. */ canonicalBlockHashes: false, finalized: 100, @@ -361,7 +363,7 @@ function pendulumChain(chain: Pendulum = pendulumState()) { const block = blockOf(requested); const onFork = BigInt(requested) >> 64n !== 0n; return { - number: codec(block), + number: codec(chain.renumberedHeaders.has(block) ? block + 1 : block), parentHash: hash(pendulumHash(onFork ? chain : {}, block - 1)), hash: hash(chain.alteredHeaders.has(block) ? pendulumHash(chain, block + 1_000_000) : requested), }; @@ -781,6 +783,18 @@ test("a source header that does not hash to its block hash pages critical", asyn ); }); +test("a source header that names another block number pages critical", async (t) => { + const { pendulum, api, monitor, pages, persisted } = await setup(t); + burn(pendulum, 0n, 100n * PEN, 102); + pendulum.renumberedHeaders.add(101); + await monitor.poll(api); + assert.match( + pages.find((page) => page.subject === "CRITICAL: PENDULUM SOURCE CONTRADICTED")?.detail ?? "", + /header for Pendulum block 101 does not hash to 0x0+65;/, + ); + assert.equal(persisted().lastPendulumBlock, 100); +}); + const SECOND_NODE = "ws://second-node.test:9944?apikey=secret"; /** A monitor whose second Pendulum node serves the source's chain, except for @@ -1332,13 +1346,22 @@ test("a vault total that cannot be read where the vault has code fails the poll, assert.ok(!subjects().includes("BASE LOG SCAN INCOMPLETE"), JSON.stringify(subjects())); }); -test("a release in the first block of a range accounts for the totals", async (t) => { +test("releases and sweeps in the first and the last block of a range account for the totals", async (t) => { const { base, pendulum, api, monitor, subjects, persisted } = await setup(t); burn(pendulum, 0n, 100n * PEN); + burn(pendulum, 1n, 100n * PEN); release(base, 0n, 100n * PEN, 1_000n); + release(base, 1n, 100n * PEN, 1_010n); + for (const blockNumber of [1_000n, 1_010n]) { + base.logs.push({ address: VAULT, eventName: "RemainderSwept", args: { to: TOKEN, amount: 5n }, blockNumber }); + base.totalSwept += 5n; + base.vaultBalance -= 5n; + } await monitor.poll(api); - assert.deepEqual(subjects(), []); assert.equal(persisted().baseFromBlock, "1011"); + assert.deepEqual(persisted().migrations, []); + await new Promise((resolve) => setTimeout(resolve, 50)); // an incomplete scan page is sent without waiting + assert.deepEqual([...new Set(subjects())], ["VAULT ADMIN EVENT"]); // the sweeps }); test("the totals count as zero before the vault existed", async (t) => { @@ -1728,7 +1751,7 @@ test("first sightings of unmatched events do not delay the pause", async (t) => base.totalReleased += 1n; base.vaultBalance -= 1n; await monitor.poll(api); - while (!events.includes("first sighting answered")) await new Promise((resolve) => setTimeout(resolve, 10)); + await until(() => events.includes("first sighting answered"), "the first sighting's answer", 5_000); assert.deepEqual(events, ["first sighting paged", "violation paged", "first sighting answered"]); }); From e24b668f734ff1dbb8afbd014fb6a5011b0d6b8c Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Thu, 22 Oct 2026 15:29:17 +0200 Subject: [PATCH 59/75] fix(monitor): Ingest only Pendulum blocks the second node has finalized With PENDULUM_WS_CROSSCHECK set, the source's chain is compared with the second node at the second node's finalized head, or at the source's head once the second node has finalized past it, and no block above that height is ingested. The view the monitor checks against (its timestamp, nextNonce, totalMigrated and issuance) is read at the newest ingested block, and the second node's totals are compared there. A second node whose finalized head is older than SOURCE_STALE_ALERT_SECONDS counts as unavailable, like one that cannot be read: it pages a warning and holds nothing back. While the second node is unavailable, the migration events of the ingested blocks are not compared with it. --- monitor/README.md | 19 ++++--- monitor/src/main.test.ts | 73 +++++++++++++++++++++----- monitor/src/main.ts | 107 ++++++++++++++++++++++++++------------- 3 files changed, 143 insertions(+), 56 deletions(-) diff --git a/monitor/README.md b/monitor/README.md index 3f70a6c19..47b9c455c 100644 --- a/monitor/README.md +++ b/monitor/README.md @@ -35,11 +35,14 @@ itself: node and hashed by the monitor. The walk must arrive at the hash of the block ingested last, which the state file keeps, so every ingested block lies on one chain. -- **Second node** (`PENDULUM_WS_CROSSCHECK`). The second node must have the - same hash at the finalized head's height, compared once it has finalized - that height itself, and the same migration totals there (`nextNonce`, - `totalMigrated`). Every ingested block with migration pallet events must - carry the same ones on the second node. +- **Second node** (`PENDULUM_WS_CROSSCHECK`). The source's chain must have + the second node's hash at the height of the second node's finalized head + (at the source's head once the second node has finalized past it), and no + block above that height is ingested. The migration totals (`nextNonce`, + `totalMigrated`) must agree at the block the monitor reads them, and every + ingested block with migration pallet events must carry the same ones on the + second node. A second node whose finalized head is older than + `SOURCE_STALE_ALERT_SECONDS` counts as unavailable. - **Relay-chain anchor** (`POLKADOT_WS`). Each poll reads the Pendulum head that the Polkadot relay chain recorded at its finalized block (`paras.heads(PENDULUM_PARA_ID)`), from the first relay RPC that answers. The @@ -66,7 +69,7 @@ advance). | Variable | Meaning | |---|---| | `PENDULUM_WS` | WebSocket of the monitor's own Pendulum node, run with `--state-pruning archive` (see below) | -| `PENDULUM_WS_CROSSCHECK` | Optional: WebSocket of a second Pendulum node, not the one behind `PENDULUM_WS`; it must agree with the source on the finalized head's hash and migration totals in every poll, and on the migration events of every ingested block that has some. Run it with `--state-pruning archive` too, or those events cannot be compared after a monitor outage longer than its pruning horizon (`PENDULUM CROSS-CHECK UNAVAILABLE`) | +| `PENDULUM_WS_CROSSCHECK` | Optional: WebSocket of a second Pendulum node, not the one behind `PENDULUM_WS`; only blocks on its finalized chain are ingested, and it must agree with the source on the migration totals in every poll and on the migration events of every ingested block that has some. Run it with `--state-pruning archive` too, or those events cannot be compared after a monitor outage longer than its pruning horizon (`PENDULUM CROSS-CHECK UNAVAILABLE`) | | `POLKADOT_WS` | Optional: comma-separated WebSocket RPCs of the Polkadot relay chain, run by third parties and tried in order (for example `wss://rpc.polkadot.io`); turns on the relay-chain anchor. List at least two: a record that contradicts the source needs a second relay RPC's confirmation, unless only one is listed | | `PENDULUM_PARA_ID` | Pendulum's parachain id on the relay chain (default 2094) | | `RELAY_ANCHOR_GRACE_SECONDS` | How long the source may go without a relay-anchored head before `CRITICAL: RELAY ANCHOR STALE` pages (default 3600) | @@ -76,7 +79,7 @@ advance). | `GRACE_SECONDS` | Liveness alert threshold (default 30 min) | | `UNMATCHED_EVENT_GRACE_SECONDS` | Time a lagging or unreadable Pendulum source view gets to reveal the burn behind a safe Base event before a cannot-verify pause (default 10 min). The first sighting of an unmatched event pages immediately, so this is the window operators have to repair a stalled or failing node. Size it to your node-ops response time | | `SOURCE_CLOCK_SKEW_MARGIN_SECONDS` | Clock-skew allowance for the fabrication proof (default 120s): once the finalized Pendulum view has passed the moment the monitor first observed an unmatched Base event by this margin and the nonce still does not exist, the event is provably fabricated and pauses without waiting out the grace. The anchor is the monitor's own clock, not the Base block timestamp (which trails real time after a sequencer outage); a future-dated source view proves nothing | -| `SOURCE_STALE_ALERT_SECONDS` | Age of the finalized Pendulum view (vs. wall clock) past which the monitor pages that it is going blind (default 300s) | +| `SOURCE_STALE_ALERT_SECONDS` | Age of the verified Pendulum view (vs. wall clock) past which the monitor pages that it is going blind (default 300s); a second node whose finalized head is older counts as unavailable | | `BASE_STALE_ALERT_SECONDS` | Age of the safe/finalized Base block (vs. wall clock) past which `BASE VIEW STALLED` pages (default 600s for `safe`, 1800s for `finalized`; Base's safe head normally trails by about a minute, its finalized head by about 20) | | `ISSUANCE_TOLERANCE` | Growth of Pendulum `totalIssuance + TotalMigrated` over its first-seen anchor (12-decimal pallet units) tolerated before `SOURCE SUPPLY GREW` pages (default 0) | | `ACKNOWLEDGED_RELEASE_EXCESS` | Excess of released over migrated, in token units (18 decimals), that operators quantified and accepted after a real loss; M2a fires only beyond it (default 0). Set it at the RB-3 sign-off and restart; the startup log shows it | @@ -114,7 +117,7 @@ subject that starts with `CRITICAL`; everything else is a `WARNING`. | `PENDULUM SOURCE VIEW STALLED`, `PENDULUM SOURCE UNREADABLE`, `BASE VIEW STALLED`, `BASE ADMIN SCAN FAILED`, `BASE LOG SCAN INCOMPLETE` | The monitor is going blind on one chain | | `CRITICAL: BASE LOG SCAN INCOMPLETE`, `CRITICAL: BASE ADMIN SCAN FAILED` | For longer than `UNMATCHED_EVENT_GRACE_SECONDS`, a Base block range has been held, so no later Base event is matched against its burn, or the admin scan has failed, so no vault admin event or scheduled governance operation is paged while its veto window runs; fix the Base RPC | | `CRITICAL: PENDULUM SOURCE CONTRADICTED` | The Pendulum source contradicts the chain the monitor verified; stop trusting that node | -| `PENDULUM CROSS-CHECK UNAVAILABLE`, `RELAY ANCHOR UNAVAILABLE` | The second Pendulum node or every relay RPC does not answer, or the only relay RPC that answers records a head that contradicts the source, so the source is used without that check | +| `PENDULUM CROSS-CHECK UNAVAILABLE`, `RELAY ANCHOR UNAVAILABLE` | The second Pendulum node does not answer or its finalized head is stale, or every relay RPC does not answer, or the only relay RPC that answers records a head that contradicts the source, so the source is used without that check | | `RELAY RPCS DISAGREE` | One relay RPC records a Pendulum head that contradicts the source and the next one that answers does not; the source is used without the anchor until they agree. Check the first one | | `CRITICAL: RELAY ANCHOR STALE` | No Pendulum head was checked against the relay chain for `RELAY_ANCHOR_GRACE_SECONDS` | | `AUTO-PAUSE NOT READY`, `AUTO-PAUSE DISABLED` | The next violation would not pause the vault automatically | diff --git a/monitor/src/main.test.ts b/monitor/src/main.test.ts index e0ab56153..f2ef0cbd1 100644 --- a/monitor/src/main.test.ts +++ b/monitor/src/main.test.ts @@ -307,6 +307,18 @@ function pendulumChain(chain: Pendulum = pendulumState()) { ), }); const optional = (value: T | undefined) => (value === undefined ? { isSome: false } : { isSome: true, unwrap: () => value }); + /** The migration totals at `block`: the current ones less the burns of later blocks. */ + const totalsAt = (block: number) => { + let { nextNonce, totalMigrated } = chain; + for (const [later, events] of chain.events) { + for (const { method, data } of later > block ? events : []) { + if (method !== "MigrationInitiated") continue; + nextNonce--; + totalMigrated -= (data[3] as { toBigInt(): bigint }).toBigInt(); + } + } + return { nextNonce, totalMigrated }; + }; const at = (block: number) => ({ registry: { createType: (_type: string, bytes: Uint8Array) => decodeCall(bytes) }, consts: { democracy: { launchPeriod: codec(36_000) } }, @@ -314,10 +326,10 @@ function pendulumChain(chain: Pendulum = pendulumState()) { timestamp: { now: async () => codec(Date.now() - chain.headAgeMs - (chain.finalized - block) * 12_000) }, system: { events: async () => (chain.events.get(block) ?? []).map((event) => ({ event })) }, tokenMigration: { - totalMigrated: async () => codec(chain.totalMigrated), - nextNonce: async () => codec(chain.nextNonce), + totalMigrated: async () => codec(totalsAt(block).totalMigrated), + nextNonce: async () => codec(totalsAt(block).nextNonce), }, - balances: { totalIssuance: async () => codec(100_000_000n * PEN - chain.totalMigrated) }, + balances: { totalIssuance: async () => codec(100_000_000n * PEN - totalsAt(block).totalMigrated) }, democracy: { publicProps: async () => chain.publicProps.map(([index, bounded]) => [codec(index), bounded, "5Proposer"]), nextExternal: async () => optional(chain.external && [chain.external, "SimpleMajority"]), @@ -864,20 +876,55 @@ test("a node connection that fails with an event says why", async (t) => { assert.match(pages[0]?.detail ?? "", /cannot be read: Error: getaddrinfo ENOTFOUND second-node\.test;/); }); -test("a second node that has not reached the newest block only warns", async (t) => { - const { pendulum, api, monitor, pages, persisted } = await crossCheckSetup(t, { finalized: 100 }); +test("a block the second node has not finalized yet waits for it, without a page", async (t) => { + const { pendulum, api, monitor, pages, persisted, own } = await crossCheckSetup(t, { finalized: 100 }); burn(pendulum, 0n, 100n * PEN); await monitor.poll(api); - assert.deepEqual(pages.map((page) => page.subject), ["PENDULUM CROSS-CHECK UNAVAILABLE"]); - assert.match(pages[0].detail, /has not finalized block 101 yet;/); + assert.deepEqual(pages, []); + assert.equal(persisted().lastPendulumBlock, 100); + own.finalized = 101; + await monitor.poll(api); + assert.deepEqual(pages, []); assert.equal(persisted().lastPendulumBlock, 101); + assert.deepEqual(persisted().migrations.map((migration) => migration.nonce), ["0"]); }); -test("a second node whose best chain holds another newest block, not yet finalized, only warns", async (t) => { +test("a second node whose best chain holds another newest block, not yet finalized, holds that block back", async (t) => { const { pendulum, api, monitor, pages, persisted } = await crossCheckSetup(t, { finalized: 100, best: 101, forkFrom: 101 }); burn(pendulum, 0n, 100n * PEN); await monitor.poll(api); - assert.deepEqual(pages.map((page) => page.subject), ["PENDULUM CROSS-CHECK UNAVAILABLE"]); + assert.deepEqual(pages, []); + assert.equal(persisted().lastPendulumBlock, 100); +}); + +test("a source on another chain one block past the second node's head ingests nothing from that chain", async (t) => { + const { base, pendulum, api, monitor, pages, persisted, own } = await crossCheckSetup(t, { finalized: 101, forkFrom: undefined }); + burn(pendulum, 0n, 100n * PEN); + pendulum.forkFrom = 102; // the source serves another chain from block 102 on... + burn(pendulum, 1n, 999n * PEN, 102); // ...with a fabricated burn, one block past the second node's head + release(base, 1n, 999n * PEN); + await monitor.poll(api); + assert.equal(persisted().lastPendulumBlock, 101); + assert.deepEqual(persisted().migrations.map((migration) => migration.nonce), ["0"]); + assert.equal(persisted().baseFromBlock, "1000"); // the release of the fabricated nonce awaits its burn + assert.deepEqual([...new Set(pages.map((page) => page.subject))], ["UNVERIFIED BASE EVENT"]); + own.finalized = 102; // the second node finalizes its own block 102... + burn(pendulum, 2n, 999n * PEN, 103); // ...and the source stays one block ahead + await monitor.poll(api); + assert.match( + pages.find((page) => page.subject === "CRITICAL: PENDULUM SOURCE CONTRADICTED")?.detail ?? "", + /second Pendulum node has 0x0+66 at block 102, the source 0x0+10+66;/, + ); + assert.equal(persisted().lastPendulumBlock, 101); + assert.deepEqual(persisted().migrations.map((migration) => migration.nonce), ["0"]); +}); + +test("a second node whose finalized head is stale counts as unavailable", async (t) => { + const { pendulum, api, monitor, pages, persisted } = await crossCheckSetup(t, { finalized: 100, headAgeMs: 400_000 }); + burn(pendulum, 0n, 100n * PEN); + await monitor.poll(api); + assert.deepEqual(pages.map((page) => [page.subject, page.severity]), [["PENDULUM CROSS-CHECK UNAVAILABLE", "WARNING"]]); + assert.match(pages[0].detail, /has a finalized head 40\ds old;/); assert.equal(persisted().lastPendulumBlock, 101); }); @@ -911,11 +958,11 @@ test("a second node with other migration events in a block pages critical and no }); test("a second node that cannot read a block's events only warns", async (t) => { - const failing = { - get: () => { + const failing = new (class extends Map { + get(): never { throw new Error("state discarded"); - }, - } as unknown as Pendulum["events"]; + } + })(); const { pendulum, api, monitor, pages, persisted } = await crossCheckSetup(t, { events: failing }); burn(pendulum, 0n, 100n * PEN); await monitor.poll(api); diff --git a/monitor/src/main.ts b/monitor/src/main.ts index ad0af79e8..4772428c1 100644 --- a/monitor/src/main.ts +++ b/monitor/src/main.ts @@ -751,33 +751,55 @@ async function migrationTotals(apiAt: ApiAt): Promise { }; } -/** The second node must have finalized the source's finalized head, with the - * same hash and the same migration totals there. One that cannot answer, or - * has not finalized that block yet, only warns, throttled: it is a check, - * not a source. */ -async function crossCheckSource(finalized: BlockRef, totals: MigrationTotals): Promise { +/** The height up to which the second node confirms the source's chain + * `hashes`, walked down from `top` to `bottom`: the hashes must agree at its + * finalized head, or at `top` once it has finalized past it. Undefined while + * it cannot be read or its finalized head is older than + * SOURCE_STALE_ALERT_SECONDS, which only warns, throttled: it is a check, not + * a source. */ +async function crossCheckSource(hashes: Map, top: BlockRef, bottom: number): Promise { const url = config.pendulumWsCrossCheck; - if (!url) return; - let other: { hash: string; totals?: MigrationTotals } | undefined; + if (!url) return undefined; + let other: BlockRef & { ageMs: number }; try { other = await onNode(url, async (node) => { + const finalizedHash = await node.rpc.chain.getFinalizedHead(); + const finalized = (await node.rpc.chain.getHeader(finalizedHash)).number.toNumber(); + const timestamp = await (await node.at(finalizedHash)).query.timestamp.now(); // Above its finalized head, a node's block at a height may still lose. - const head = await node.rpc.chain.getHeader(await node.rpc.chain.getFinalizedHead()); - if (head.number.toNumber() < finalized.number) return undefined; - const hash = (await node.rpc.chain.getBlockHash(finalized.number)).toHex(); - return { hash, totals: hash === finalized.hash ? await migrationTotals(await node.at(hash)) : undefined }; + const number = Math.min(finalized, top.number); + const hash = number === finalized ? finalizedHash.toHex() : (await node.rpc.chain.getBlockHash(number)).toHex(); + return { number, hash, ageMs: Date.now() - Number(timestamp.toString().replaceAll(",", "")) }; }); } catch (crossCheckError) { - return crossCheckUnavailable(`cannot be read: ${crossCheckError}`); + await crossCheckUnavailable(`cannot be read: ${crossCheckError}`); + return undefined; } - if (other === undefined) return crossCheckUnavailable(`has not finalized block ${finalized.number} yet`); - if (other.hash !== finalized.hash) { - throw new SourceIntegrityError(`the second Pendulum node has ${other.hash} at block ${finalized.number}, the source ${finalized.hash}`); + if (other.ageMs > config.sourceStaleAlertMs) { + await crossCheckUnavailable(`has a finalized head ${Math.round(other.ageMs / 1000)}s old`); + return undefined; } - if (other.totals?.nextNonce !== totals.nextNonce || other.totals.totalMigrated !== totals.totalMigrated) { + if (other.number >= bottom && hashes.get(other.number) !== other.hash) { throw new SourceIntegrityError( - `at Pendulum block ${finalized.number} the second node has nextNonce ${other.totals?.nextNonce} and ` + - `totalMigrated ${other.totals?.totalMigrated}, the source ${totals.nextNonce} and ${totals.totalMigrated}`, + `the second Pendulum node has ${other.hash} at block ${other.number}, the source ${hashes.get(other.number)}`, + ); + } + return other.number; +} + +/** The second node must have the source's migration totals at `blockHash`, a + * block it confirmed. One that cannot answer only warns, throttled. */ +async function crossCheckTotals(block: number, blockHash: string, totals: MigrationTotals): Promise { + let other: MigrationTotals; + try { + other = await onNode(config.pendulumWsCrossCheck as string, async (node) => migrationTotals(await node.at(blockHash))); + } catch (crossCheckError) { + return crossCheckUnavailable(`cannot be read: ${crossCheckError}`); + } + if (other.nextNonce !== totals.nextNonce || other.totalMigrated !== totals.totalMigrated) { + throw new SourceIntegrityError( + `at Pendulum block ${block} the second node has nextNonce ${other.nextNonce} and ` + + `totalMigrated ${other.totalMigrated}, the source ${totals.nextNonce} and ${totals.totalMigrated}`, ); } } @@ -983,8 +1005,14 @@ async function anchorToRelay( /** The hashes of the finalized blocks to ingest, on one chain with every * block ingested before and with the relay-chain record kept from an earlier - * poll; the source is then anchored to the relay chain's newest record. */ -async function sourceHashes(api: ApiPromise, draft: RuntimeState, finalized: BlockRef): Promise> { + * poll, and the newest of them that the second node confirms (`confirmed`, + * at most the source's finalized head; `crossChecked` while the second node + * answers). The source is then anchored to the relay chain's newest record. */ +async function sourceHashes( + api: ApiPromise, + draft: RuntimeState, + finalized: BlockRef, +): Promise<{ hashes: Map; confirmed: number; crossChecked: number | undefined }> { // A kept record past the finalized head binds the source too: the walk starts there. const known = draft.relayAnchor; const top = known && known.number > finalized.number ? known : finalized; @@ -1002,8 +1030,9 @@ async function sourceHashes(api: ApiPromise, draft: RuntimeState, finalized: Blo } } const lowest = Math.min(top.number, draft.lastPendulumBlock); + const crossChecked = await crossCheckSource(hashes, top, lowest); await anchorToRelay(api, draft, hashes, top, { number: lowest, hash: hashes.get(lowest) as string }); - return hashes; + return { hashes, confirmed: Math.min(finalized.number, crossChecked ?? Infinity), crossChecked }; } /** The migration pallet's admin events, each with what to check. */ @@ -1015,22 +1044,24 @@ const PALLET_ADMIN_EVENTS = new Map([ ], ]); -/** Ingests the blocks after the cursor up to `finalizedBlock`, each read at - * its hash in `hashes`: the chain that `sourceHashes` verified. */ +/** Ingests the blocks after the cursor up to `lastBlock`, each read at its + * hash in `hashes`: the chain that `sourceHashes` verified. Their migration + * events are compared with the second node's while it answers. */ async function ingestPendulumEvents( api: ApiPromise, draft: RuntimeState, - finalizedBlock: number, + lastBlock: number, hashes: Map, + crossChecked: boolean, ): Promise { - for (let block = draft.lastPendulumBlock + 1; block <= finalizedBlock; block++) { + for (let block = draft.lastPendulumBlock + 1; block <= lastBlock; block++) { const blockHash = hashes.get(block); if (!blockHash) throw new Error(`no verified hash for Pendulum block ${block}`); const apiAt = await api.at(blockHash); const firstSeenMs = Number((await apiAt.query.timestamp.now()).toString().replaceAll(",", "")); const records = (await apiAt.query.system.events()) as unknown as EventRecords; const migrationEvents = migrationEventText(records); - if (migrationEvents) await crossCheckEvents(block, blockHash, migrationEvents); + if (migrationEvents && crossChecked) await crossCheckEvents(block, blockHash, migrationEvents); for (const [index, record] of records.entries()) { const { section, method, data } = record.event; if (section !== "tokenMigration") continue; @@ -1641,8 +1672,15 @@ async function readSource(api: ApiPromise, draft: RuntimeState): Promise viewBlock ? ` (the source finalized ${finalized.number}, not confirmed yet)` : "") + + `; the monitor cannot verify new Base events against a stalled source — restore the node before ` + `the ${config.unmatchedEventGraceSeconds}s unmatched-event grace forces a pause`, ); } - const finalized = { number: finalizedBlock, hash: finalizedHash.toHex() }; const [totals, totalIssuanceValue] = await Promise.all([migrationTotals(apiAt), apiAt.query.balances.totalIssuance()]); const { nextNonce: chainNextNonce, totalMigrated } = totals; - await crossCheckSource(finalized, totals); - const hashes = await sourceHashes(api, draft, finalized); - await ingestPendulumEvents(api, draft, finalizedBlock, hashes); + if (crossChecked !== undefined && crossChecked >= viewBlock) await crossCheckTotals(viewBlock, viewHash, totals); + await ingestPendulumEvents(api, draft, confirmed, hashes, crossChecked !== undefined); if (chainNextNonce !== draft.nextExpectedNonce) { throw new Error(`finalized nextNonce ${chainNextNonce} != monitor event cursor ${draft.nextExpectedNonce}`); } @@ -1693,7 +1730,7 @@ async function readSource(api: ApiPromise, draft: RuntimeState): Promise Date: Thu, 22 Oct 2026 15:32:28 +0200 Subject: [PATCH 60/75] fix(monitor): Ask the next relay RPC until one anchors the source A relay RPC whose record does not anchor the source (not newer than the kept head, or not imported by the source yet) no longer ends the relay check: the next relay RPC in POLKADOT_WS is asked. A record that contradicts the source is outvoted only by another relay RPC's record that anchors the source at or above the contradicting record's height. That record then anchors the source, and RELAY RPCS DISAGREE asks operators to check both relay RPCs. Any other answer leaves the contradiction unconfirmed and pages RELAY ANCHOR UNAVAILABLE. --- monitor/README.md | 29 ++++++++-------- monitor/src/main.test.ts | 50 +++++++++++++++++++++++++--- monitor/src/main.ts | 72 ++++++++++++++++++++++------------------ 3 files changed, 102 insertions(+), 49 deletions(-) diff --git a/monitor/README.md b/monitor/README.md index 47b9c455c..67d8bf841 100644 --- a/monitor/README.md +++ b/monitor/README.md @@ -16,7 +16,7 @@ Checks every poll: | Unreleasable burns | A burn to the zero or vault address can never be approved or released; it is paged once and excluded from the pending count so it cannot page liveness forever or block RB-7. | Alert once | | Treasury destination | With `EXPECTED_TREASURY_DESTINATION` set, every finalized treasury migration (a `MigrationInitiated` whose `who` is the treasury account, `PalletId(*b"py/trsry")`) must pay that Base address. It is checked when the burn is ingested, before the attestors can release it; while the pause is not confirmed, the monitor holds at that block, and the violation pages again every poll saying so. | Alert + auto-pause (`TREASURY DESTINATION VIOLATION`) | | Pendulum source | The finalized Pendulum view must stay fresh and readable. While it cannot be read (an RPC error, a nonce discontinuity, an event-shape change), the Base-side checks — vault balance and its auto-pause, Base view, auto-pause readiness, vault admin events, governance queue — still run, and Base events are reconciled against the last view read in full: an unreadable source counts as a stalled one, so a mismatch on a known nonce counts at once and an unknown nonce after `UNMATCHED_EVENT_GRACE_SECONDS`. M2a and liveness wait for the source. | Alert, throttled: `PENDULUM SOURCE VIEW STALLED` once the view is older than `SOURCE_STALE_ALERT_SECONDS`, `PENDULUM SOURCE UNREADABLE` on a failure | -| Pendulum chain | Every finalized block the monitor ingests is read at a hash it verified against the blocks it ingested before and, where configured, a second node and the relay chain; a second node must also agree on the migration totals and on the migration events of each ingested block (see [Pendulum chain verification](#pendulum-chain-verification)). | Alert (`CRITICAL: PENDULUM SOURCE CONTRADICTED`), throttled; nothing from the source is ingested, and it is treated as unreadable (see Pendulum source) until it verifies again. A second node or relay RPC that does not answer only warns, throttled (`PENDULUM CROSS-CHECK UNAVAILABLE`, `RELAY ANCHOR UNAVAILABLE`), and so do relay RPCs that disagree (`RELAY RPCS DISAGREE`); `CRITICAL: RELAY ANCHOR STALE` once no head was anchored for `RELAY_ANCHOR_GRACE_SECONDS` | +| Pendulum chain | Every finalized block the monitor ingests is read at a hash it verified against the blocks it ingested before and, where configured, a second node and the relay chain; a second node must also agree on the migration totals and on the migration events of each ingested block (see [Pendulum chain verification](#pendulum-chain-verification)). | Alert (`CRITICAL: PENDULUM SOURCE CONTRADICTED`), throttled; nothing from the source is ingested, and it is treated as unreadable (see Pendulum source) until it verifies again. A second node or relay RPC that does not answer only warns, throttled (`PENDULUM CROSS-CHECK UNAVAILABLE`, `RELAY ANCHOR UNAVAILABLE`), and so does a relay RPC outvoted by another (`RELAY RPCS DISAGREE`); `CRITICAL: RELAY ANCHOR STALE` once no head was anchored for `RELAY_ANCHOR_GRACE_SECONDS` | | Base log completeness | In every scanned block range the `Released` amounts must add up to the growth of `totalReleased`, and the `RemainderSwept` amounts to that of `totalSwept`, between the block before the range and its last block. A range whose logs do not (a backend serving truncated logs) is read once more, then held: the Base cursor stays on it and liveness waits until a read accounts for the totals; M2a still runs. | Alert (`BASE LOG SCAN INCOMPLETE`), throttled; `CRITICAL: BASE LOG SCAN INCOMPLETE` once a range has been held for longer than `UNMATCHED_EVENT_GRACE_SECONDS`, since no later Base event is matched against its burn meanwhile | | Base view | The safe/finalized Base block must not be older than `BASE_STALE_ALERT_SECONDS`: a view that stops advancing hides every later vault event while every check still passes at the old block. | Alert (`BASE VIEW STALLED`), throttled | | Auto-pause readiness | At startup and every poll: the Base RPC serves `BASE_CHAIN_ID` and, with a guardian key configured, the key parses, is `vault.guardian()` and holds at least `GUARDIAN_MIN_BALANCE_WEI`. A read that fails pages as well and never stops the poll's other checks. | Alert (`AUTO-PAUSE NOT READY`), throttled; `AUTO-PAUSE DISABLED` once at startup without a key | @@ -45,16 +45,19 @@ itself: `SOURCE_STALE_ALERT_SECONDS` counts as unavailable. - **Relay-chain anchor** (`POLKADOT_WS`). Each poll reads the Pendulum head that the Polkadot relay chain recorded at its finalized block - (`paras.heads(PENDULUM_PARA_ID)`), from the first relay RPC that answers. The - source's chain must have that hash at that height. A head above the source's - finalized head counts once the source's best chain holds it, and its chain - must then pass through the finalized head. The newest such head is kept in - the state file, and the chain must keep passing through it, also while it - lies above the finalized head. Blocks ingested while no relay RPC answered - are tied to the next head that arrives by the same walk. A head that - contradicts the source counts only when the next relay RPC that answers - contradicts it too, or when `POLKADOT_WS` lists one RPC; otherwise - `RELAY RPCS DISAGREE` pages and the poll goes on without the anchor. + (`paras.heads(PENDULUM_PARA_ID)`), asking the relay RPCs in turn until one + head anchors the source. The source's chain must have that hash at that + height. A head above the source's finalized head counts once the source's + best chain holds it, and its chain must then pass through the finalized + head. The newest such head is kept in the state file, and the chain must + keep passing through it, also while it lies above the finalized head. + Blocks ingested while no relay RPC answered are tied to the next head that + arrives by the same walk. A head that contradicts the source counts only + when another relay RPC's head contradicts it too, or when `POLKADOT_WS` + lists one RPC. A relay RPC whose head anchors the source at or above the + contradicting head's height outvotes it: `RELAY RPCS DISAGREE` pages and + that head anchors the source. Any other answer leaves the contradiction + unconfirmed. A contradiction pages `CRITICAL: PENDULUM SOURCE CONTRADICTED`, nothing from the source is ingested, and the source counts as unreadable until it verifies @@ -117,8 +120,8 @@ subject that starts with `CRITICAL`; everything else is a `WARNING`. | `PENDULUM SOURCE VIEW STALLED`, `PENDULUM SOURCE UNREADABLE`, `BASE VIEW STALLED`, `BASE ADMIN SCAN FAILED`, `BASE LOG SCAN INCOMPLETE` | The monitor is going blind on one chain | | `CRITICAL: BASE LOG SCAN INCOMPLETE`, `CRITICAL: BASE ADMIN SCAN FAILED` | For longer than `UNMATCHED_EVENT_GRACE_SECONDS`, a Base block range has been held, so no later Base event is matched against its burn, or the admin scan has failed, so no vault admin event or scheduled governance operation is paged while its veto window runs; fix the Base RPC | | `CRITICAL: PENDULUM SOURCE CONTRADICTED` | The Pendulum source contradicts the chain the monitor verified; stop trusting that node | -| `PENDULUM CROSS-CHECK UNAVAILABLE`, `RELAY ANCHOR UNAVAILABLE` | The second Pendulum node does not answer or its finalized head is stale, or every relay RPC does not answer, or the only relay RPC that answers records a head that contradicts the source, so the source is used without that check | -| `RELAY RPCS DISAGREE` | One relay RPC records a Pendulum head that contradicts the source and the next one that answers does not; the source is used without the anchor until they agree. Check the first one | +| `PENDULUM CROSS-CHECK UNAVAILABLE`, `RELAY ANCHOR UNAVAILABLE` | The second Pendulum node does not answer or its finalized head is stale, or every relay RPC does not answer, or a relay RPC records a head that contradicts the source and no other relay RPC confirms or outvotes it, so the source is used without that check | +| `RELAY RPCS DISAGREE` | One relay RPC records a Pendulum head that contradicts the source, and another records one at or above that height on the source's chain, which anchors the source. One of them is wrong; check both | | `CRITICAL: RELAY ANCHOR STALE` | No Pendulum head was checked against the relay chain for `RELAY_ANCHOR_GRACE_SECONDS` | | `AUTO-PAUSE NOT READY`, `AUTO-PAUSE DISABLED` | The next violation would not pause the vault automatically | | `SOURCE SUPPLY GREW`, `CRITICAL: unreleasable migration burned on Pendulum` | Supply problems at the source | diff --git a/monitor/src/main.test.ts b/monitor/src/main.test.ts index f2ef0cbd1..17b44ed40 100644 --- a/monitor/src/main.test.ts +++ b/monitor/src/main.test.ts @@ -1202,16 +1202,58 @@ async function twoRelaySetup(t: { after(fn: () => void): void }, heads: Array<(( const honestHead = (pendulum: Pendulum) => ({ number: pendulum.finalized, hash: pendulumHash({}, pendulum.finalized) }); const forkedHead = (pendulum: Pendulum) => ({ number: pendulum.finalized, hash: pendulumHash({ forkFrom: 0 }, pendulum.finalized) }); -test("a relay contradiction the next relay RPC does not confirm only warns, without the anchor", async (t) => { +test("a relay contradiction the next relay RPC outvotes warns, and that RPC's record anchors the source", async (t) => { const { pendulum, api, monitor, pages, persisted } = await twoRelaySetup(t, [forkedHead, honestHead]); burn(pendulum, 0n, 100n * PEN); await monitor.poll(api); assert.deepEqual(pages.map((page) => [page.subject, page.severity]), [["RELAY RPCS DISAGREE", "WARNING"]]); - assert.match(pages[0].detail, /^relay RPC 1 reports that the relay chain records 0x0+10+65 as Pendulum block 101, .*; relay RPC 2 does not;/); - assert.equal(persisted().relayAnchor, undefined); + assert.match( + pages[0].detail, + /^relay RPC 1 reports that the relay chain records 0x0+10+65 as Pendulum block 101, .*; relay RPC 2 records 0x0+65 as Pendulum block 101, on the source's chain, .* check both$/, + ); + assert.deepEqual(persisted().relayAnchor, { number: 101, hash: pendulumHash({}, 101) }); assert.equal(persisted().lastPendulumBlock, 101); }); +test("a relay record below a contradicting one does not outvote it", async (t) => { + const below = (pendulum: Pendulum) => ({ number: pendulum.finalized - 1, hash: pendulumHash({}, pendulum.finalized - 1) }); + const { pendulum, api, monitor, pages, persisted } = await twoRelaySetup(t, [forkedHead, below]); + burn(pendulum, 0n, 100n * PEN, 102); + await monitor.poll(api); + assert.deepEqual(pages.map((page) => page.subject), ["RELAY ANCHOR UNAVAILABLE"]); + assert.match(pages[0].detail, /no other relay RPC confirms or outvotes it \(relay RPC 2: its Pendulum head 101 lies below the contradicting record\);/); + assert.equal(persisted().relayAnchor, undefined); +}); + +/** A relay RPC whose record stays at the source's block 101. */ +const stuckHead = () => ({ number: 101, hash: pendulumHash({}, 101) }); + +test("a relay contradiction that only a stuck relay RPC answers stays unconfirmed", async (t) => { + const { pendulum, api, monitor, pages, persisted } = await twoRelaySetup(t, [honestHead, stuckHead]); + burn(pendulum, 0n, 100n * PEN); + await monitor.poll(api); // relay RPC 1 anchors block 101 + pendulum.forkFrom = 102; // the source moves to another chain... + burn(pendulum, 1n, 999n * PEN, 102); // ...with a fabricated burn, which relay RPC 1 contradicts + await monitor.poll(api); + assert.deepEqual(pages.map((page) => page.subject), ["RELAY ANCHOR UNAVAILABLE"]); + assert.match( + pages[0].detail, + /^relay RPC 1 reports that the relay chain records 0x0+66 as Pendulum block 102, the source's chain has 0x0+10+66, and no other relay RPC confirms or outvotes it \(relay RPC 2: the relay chain's Pendulum head 101 is not newer\);/, + ); + assert.deepEqual(persisted().relayAnchor, { number: 101, hash: pendulumHash({}, 101) }); +}); + +test("a relay RPC whose record does not anchor the source leaves it to the next one", async (t) => { + const { pendulum, api, monitor, pages, persisted } = await twoRelaySetup(t, [stuckHead, honestHead]); + burn(pendulum, 0n, 100n * PEN); + await monitor.poll(api); // relay RPC 1 anchors block 101 + burn(pendulum, 1n, 100n * PEN, 102); + await monitor.poll(api); // relay RPC 1 stays there + assert.deepEqual(pages, []); + assert.deepEqual(persisted().relayAnchor, { number: 102, hash: pendulumHash({}, 102) }); + assert.equal(persisted().lastPendulumBlock, 102); +}); + test("a relay contradiction the next relay RPC confirms pages critical", async (t) => { const { pendulum, api, monitor, pages, persisted } = await twoRelaySetup(t, [forkedHead, forkedHead]); burn(pendulum, 0n, 100n * PEN); @@ -1225,7 +1267,7 @@ test("a relay contradiction no other relay RPC can confirm only warns", async (t burn(pendulum, 0n, 100n * PEN); await monitor.poll(api); assert.deepEqual(pages.map((page) => page.subject), ["RELAY ANCHOR UNAVAILABLE"]); - assert.match(pages[0].detail, /and no other relay RPC answered to confirm it \(relay RPC 2: Error: relay RPC unavailable\)/); + assert.match(pages[0].detail, /and no other relay RPC confirms or outvotes it \(relay RPC 2: Error: relay RPC unavailable\)/); assert.equal(persisted().lastPendulumBlock, 101); }); diff --git a/monitor/src/main.ts b/monitor/src/main.ts index 4772428c1..c8b13103f 100644 --- a/monitor/src/main.ts +++ b/monitor/src/main.ts @@ -928,12 +928,15 @@ async function relayVerdict( } /** Anchors the source to the relay chain's record of the Pendulum head, kept - * as `draft.relayAnchor`, from the first relay RPC in POLKADOT_WS that - * answers. A record that contradicts the source counts only when the next - * relay RPC that answers contradicts it too, or when POLKADOT_WS lists one - * RPC; otherwise the poll goes on without the anchor. Pages while no new head - * is anchored: a warning when the relay RPCs fail or disagree, critical once - * none was for RELAY_ANCHOR_GRACE_SECONDS. All throttled. */ + * as `draft.relayAnchor`, from the first relay RPC in POLKADOT_WS whose + * record anchors it; while a record does not, the next RPC is asked. A + * record that contradicts the source counts only when another relay RPC's + * record contradicts it too, or when POLKADOT_WS lists one RPC. A record that + * anchors the source at or above the contradicting record's height outvotes + * it; any other answer leaves it unconfirmed. Pages while no new head is + * anchored: a warning when the relay RPCs fail or a contradiction stays + * unconfirmed, critical once none was anchored for + * RELAY_ANCHOR_GRACE_SECONDS; and when relay RPCs disagree. All throttled. */ async function anchorToRelay( api: ApiPromise, draft: RuntimeState, @@ -942,47 +945,52 @@ async function anchorToRelay( bottom: BlockRef, ): Promise { if (!config.polkadotWs.length) return; - const failures: string[] = []; - let contradiction: { rpc: number; error: SourceIntegrityError } | undefined; - let reason: string | undefined; + const answers: string[] = []; + let answered = false; + let contradiction: { rpc: number; height: number; error: SourceIntegrityError } | undefined; for (const [index, url] of config.polkadotWs.entries()) { let head: BlockRef; try { head = await relayHead(api, url); } catch (relayError) { - failures.push(`relay RPC ${index + 1}: ${relayError}`); + answers.push(`relay RPC ${index + 1}: ${relayError}`); continue; } + answered = true; + let verdict: string | undefined; try { - reason = await relayVerdict(api, draft, hashes, top, bottom, head); + verdict = await relayVerdict(api, draft, hashes, top, bottom, head); } catch (verdictError) { if (!(verdictError instanceof SourceIntegrityError)) throw verdictError; if (contradiction || config.polkadotWs.length === 1) throw contradiction?.error ?? verdictError; - contradiction = { rpc: index + 1, error: verdictError }; + contradiction = { rpc: index + 1, height: head.number, error: verdictError }; continue; } - if (contradiction) { - reason = `relay RPC ${contradiction.rpc} reports that ${contradiction.error.message}; relay RPC ${index + 1} does not`; - if (Date.now() - lastRelayDisagreementAlertMs > config.graceSeconds * 1000) { - lastRelayDisagreementAlertMs = Date.now(); - await alert( - "RELAY RPCS DISAGREE", - `${reason}; the Pendulum source is used without the relay-chain anchor until they agree — check relay ` + - `RPC ${contradiction.rpc}`, - ); - } - } else if (reason === undefined) { - draft.relayAnchor = head; - lastRelayAnchorMs = Date.now(); - return; + if (verdict === undefined && contradiction && head.number < contradiction.height) { + verdict = `its Pendulum head ${head.number} lies below the contradicting record`; + } + if (verdict !== undefined) { + answers.push(`relay RPC ${index + 1}: ${verdict}`); + continue; + } + draft.relayAnchor = head; + lastRelayAnchorMs = Date.now(); + if (contradiction && Date.now() - lastRelayDisagreementAlertMs > config.graceSeconds * 1000) { + lastRelayDisagreementAlertMs = Date.now(); + await alert( + "RELAY RPCS DISAGREE", + `relay RPC ${contradiction.rpc} reports that ${contradiction.error.message}; relay RPC ${index + 1} records ` + + `${head.hash} as Pendulum block ${head.number}, on the source's chain, and the source is anchored to ` + + "that record — one of the two relay RPCs is wrong, check both", + ); } - break; + return; } - if (reason === undefined) { - reason = contradiction - ? `relay RPC ${contradiction.rpc} reports that ${contradiction.error.message}, and no other relay RPC ` + - `answered to confirm it (${failures.join("; ")})` - : `the relay chain cannot be read: ${failures.join("; ")}`; + const reason = contradiction + ? `relay RPC ${contradiction.rpc} reports that ${contradiction.error.message}, and no other relay RPC confirms ` + + `or outvotes it${answers.length ? ` (${answers.join("; ")})` : ""}` + : `${answered ? "" : "the relay chain cannot be read: "}${answers.join("; ")}`; + if (contradiction || !answered) { if (Date.now() - lastRelayAlertMs > config.graceSeconds * 1000) { lastRelayAlertMs = Date.now(); await alert( From 5c696841328e5d6d3dc174321a07d4eec09cec30 Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Thu, 22 Oct 2026 15:36:18 +0200 Subject: [PATCH 61/75] fix(monitor): Ingest only Pendulum blocks the relay chain anchored While a relay RPC answers, no Pendulum block above the newest relay-anchored head (this poll's, or the kept one) is ingested, and the view the checks use is read at the newest ingested block. Only while no relay RPC answers at all is the source used without the anchor, up to its finalized head. When the relay RPCs answer but none anchors the source's newer blocks (a head that is not newer, one the source has not imported, or a contradiction that stays unconfirmed), RELAY ANCHOR UNAVAILABLE pages at once instead of waiting for RELAY ANCHOR STALE. PENDULUM SOURCE VIEW STALLED now measures the age of the verified view and names the source's newer finalized head. --- monitor/README.md | 31 +++++---- monitor/src/main.test.ts | 133 ++++++++++++++++++++++++++++++++++----- monitor/src/main.ts | 55 +++++++++------- 3 files changed, 169 insertions(+), 50 deletions(-) diff --git a/monitor/README.md b/monitor/README.md index 67d8bf841..f0c27362d 100644 --- a/monitor/README.md +++ b/monitor/README.md @@ -16,7 +16,7 @@ Checks every poll: | Unreleasable burns | A burn to the zero or vault address can never be approved or released; it is paged once and excluded from the pending count so it cannot page liveness forever or block RB-7. | Alert once | | Treasury destination | With `EXPECTED_TREASURY_DESTINATION` set, every finalized treasury migration (a `MigrationInitiated` whose `who` is the treasury account, `PalletId(*b"py/trsry")`) must pay that Base address. It is checked when the burn is ingested, before the attestors can release it; while the pause is not confirmed, the monitor holds at that block, and the violation pages again every poll saying so. | Alert + auto-pause (`TREASURY DESTINATION VIOLATION`) | | Pendulum source | The finalized Pendulum view must stay fresh and readable. While it cannot be read (an RPC error, a nonce discontinuity, an event-shape change), the Base-side checks — vault balance and its auto-pause, Base view, auto-pause readiness, vault admin events, governance queue — still run, and Base events are reconciled against the last view read in full: an unreadable source counts as a stalled one, so a mismatch on a known nonce counts at once and an unknown nonce after `UNMATCHED_EVENT_GRACE_SECONDS`. M2a and liveness wait for the source. | Alert, throttled: `PENDULUM SOURCE VIEW STALLED` once the view is older than `SOURCE_STALE_ALERT_SECONDS`, `PENDULUM SOURCE UNREADABLE` on a failure | -| Pendulum chain | Every finalized block the monitor ingests is read at a hash it verified against the blocks it ingested before and, where configured, a second node and the relay chain; a second node must also agree on the migration totals and on the migration events of each ingested block (see [Pendulum chain verification](#pendulum-chain-verification)). | Alert (`CRITICAL: PENDULUM SOURCE CONTRADICTED`), throttled; nothing from the source is ingested, and it is treated as unreadable (see Pendulum source) until it verifies again. A second node or relay RPC that does not answer only warns, throttled (`PENDULUM CROSS-CHECK UNAVAILABLE`, `RELAY ANCHOR UNAVAILABLE`), and so does a relay RPC outvoted by another (`RELAY RPCS DISAGREE`); `CRITICAL: RELAY ANCHOR STALE` once no head was anchored for `RELAY_ANCHOR_GRACE_SECONDS` | +| Pendulum chain | Every finalized block the monitor ingests is read at a hash it verified against the blocks it ingested before and, where configured, a second node and the relay chain; a second node must also agree on the migration totals and on the migration events of each ingested block (see [Pendulum chain verification](#pendulum-chain-verification)). | Alert (`CRITICAL: PENDULUM SOURCE CONTRADICTED`), throttled; nothing from the source is ingested, and it is treated as unreadable (see Pendulum source) until it verifies again. A second node or relay RPC that does not answer only warns, throttled (`PENDULUM CROSS-CHECK UNAVAILABLE`, `RELAY ANCHOR UNAVAILABLE`), and so does a relay RPC outvoted by another (`RELAY RPCS DISAGREE`); relay RPCs that answer without anchoring the source's newer blocks hold them back (`RELAY ANCHOR UNAVAILABLE`); `CRITICAL: RELAY ANCHOR STALE` once no head was anchored for `RELAY_ANCHOR_GRACE_SECONDS` | | Base log completeness | In every scanned block range the `Released` amounts must add up to the growth of `totalReleased`, and the `RemainderSwept` amounts to that of `totalSwept`, between the block before the range and its last block. A range whose logs do not (a backend serving truncated logs) is read once more, then held: the Base cursor stays on it and liveness waits until a read accounts for the totals; M2a still runs. | Alert (`BASE LOG SCAN INCOMPLETE`), throttled; `CRITICAL: BASE LOG SCAN INCOMPLETE` once a range has been held for longer than `UNMATCHED_EVENT_GRACE_SECONDS`, since no later Base event is matched against its burn meanwhile | | Base view | The safe/finalized Base block must not be older than `BASE_STALE_ALERT_SECONDS`: a view that stops advancing hides every later vault event while every check still passes at the old block. | Alert (`BASE VIEW STALLED`), throttled | | Auto-pause readiness | At startup and every poll: the Base RPC serves `BASE_CHAIN_ID` and, with a guardian key configured, the key parses, is `vault.guardian()` and holds at least `GUARDIAN_MIN_BALANCE_WEI`. A read that fails pages as well and never stops the poll's other checks. | Alert (`AUTO-PAUSE NOT READY`), throttled; `AUTO-PAUSE DISABLED` once at startup without a key | @@ -57,15 +57,21 @@ itself: lists one RPC. A relay RPC whose head anchors the source at or above the contradicting head's height outvotes it: `RELAY RPCS DISAGREE` pages and that head anchors the source. Any other answer leaves the contradiction - unconfirmed. - -A contradiction pages `CRITICAL: PENDULUM SOURCE CONTRADICTED`, nothing from -the source is ingested, and the source counts as unreadable until it verifies -again. A second node or relay RPC that cannot answer only warns, and the -monitor goes on with the source alone; `CRITICAL: RELAY ANCHOR STALE` pages -once no head was anchored for `RELAY_ANCHOR_GRACE_SECONDS`, whatever the reason -(relay RPCs down, the source behind the relay chain, a relay head that does not -advance). + unconfirmed. No block above the newest anchored head is ingested. + +Every block the monitor ingests is confirmed by each checker that answers, +and the values the checks use (the view's timestamp, `nextNonce`, +`totalMigrated` and issuance) are read at the newest ingested block. Relay +RPCs that answer but anchor no newer head, or leave a contradiction +unconfirmed, hold the source's newer blocks back and page +`RELAY ANCHOR UNAVAILABLE`. A contradiction pages +`CRITICAL: PENDULUM SOURCE CONTRADICTED`, nothing from the source is +ingested, and the source counts as unreadable until it verifies again. A +second node or relay RPC that cannot be read only warns and holds nothing +back; with no checker answering, the monitor goes on with the source alone up +to its finalized head. `CRITICAL: RELAY ANCHOR STALE` pages once no head was +anchored for `RELAY_ANCHOR_GRACE_SECONDS`, whatever the reason (relay RPCs +down, the source behind the relay chain, a relay head that does not advance). ## Configuration (environment) @@ -73,7 +79,7 @@ advance). |---|---| | `PENDULUM_WS` | WebSocket of the monitor's own Pendulum node, run with `--state-pruning archive` (see below) | | `PENDULUM_WS_CROSSCHECK` | Optional: WebSocket of a second Pendulum node, not the one behind `PENDULUM_WS`; only blocks on its finalized chain are ingested, and it must agree with the source on the migration totals in every poll and on the migration events of every ingested block that has some. Run it with `--state-pruning archive` too, or those events cannot be compared after a monitor outage longer than its pruning horizon (`PENDULUM CROSS-CHECK UNAVAILABLE`) | -| `POLKADOT_WS` | Optional: comma-separated WebSocket RPCs of the Polkadot relay chain, run by third parties and tried in order (for example `wss://rpc.polkadot.io`); turns on the relay-chain anchor. List at least two: a record that contradicts the source needs a second relay RPC's confirmation, unless only one is listed | +| `POLKADOT_WS` | Optional: comma-separated WebSocket RPCs of the Polkadot relay chain, run by third parties and tried in order (for example `wss://rpc.polkadot.io`); turns on the relay-chain anchor, and while a relay RPC answers, no Pendulum block above the newest anchored head is ingested. List at least two: a record that contradicts the source needs a second relay RPC's confirmation, unless only one is listed | | `PENDULUM_PARA_ID` | Pendulum's parachain id on the relay chain (default 2094) | | `RELAY_ANCHOR_GRACE_SECONDS` | How long the source may go without a relay-anchored head before `CRITICAL: RELAY ANCHOR STALE` pages (default 3600) | | `BASE_RPC_URL` | Base JSON-RPC endpoint — a **single dedicated node**, and a different provider than the attestors use. Load-balanced pools can serve `eth_getLogs` from a lagging replica; each scan range is probed for its end block first and its logs must account for the vault's running totals, which turns a behind-node into a loud replay instead of a silent gap, but a dedicated node removes the hazard entirely | @@ -120,7 +126,8 @@ subject that starts with `CRITICAL`; everything else is a `WARNING`. | `PENDULUM SOURCE VIEW STALLED`, `PENDULUM SOURCE UNREADABLE`, `BASE VIEW STALLED`, `BASE ADMIN SCAN FAILED`, `BASE LOG SCAN INCOMPLETE` | The monitor is going blind on one chain | | `CRITICAL: BASE LOG SCAN INCOMPLETE`, `CRITICAL: BASE ADMIN SCAN FAILED` | For longer than `UNMATCHED_EVENT_GRACE_SECONDS`, a Base block range has been held, so no later Base event is matched against its burn, or the admin scan has failed, so no vault admin event or scheduled governance operation is paged while its veto window runs; fix the Base RPC | | `CRITICAL: PENDULUM SOURCE CONTRADICTED` | The Pendulum source contradicts the chain the monitor verified; stop trusting that node | -| `PENDULUM CROSS-CHECK UNAVAILABLE`, `RELAY ANCHOR UNAVAILABLE` | The second Pendulum node does not answer or its finalized head is stale, or every relay RPC does not answer, or a relay RPC records a head that contradicts the source and no other relay RPC confirms or outvotes it, so the source is used without that check | +| `PENDULUM CROSS-CHECK UNAVAILABLE` | The second Pendulum node does not answer or its finalized head is stale, so the source is used without that check | +| `RELAY ANCHOR UNAVAILABLE` | No relay RPC answers, so the source is used without the anchor; or none anchors the source's newer blocks, or a relay RPC records a head that contradicts the source and no other one confirms or outvotes it, so those blocks wait | | `RELAY RPCS DISAGREE` | One relay RPC records a Pendulum head that contradicts the source, and another records one at or above that height on the source's chain, which anchors the source. One of them is wrong; check both | | `CRITICAL: RELAY ANCHOR STALE` | No Pendulum head was checked against the relay chain for `RELAY_ANCHOR_GRACE_SECONDS` | | `AUTO-PAUSE NOT READY`, `AUTO-PAUSE DISABLED` | The next violation would not pause the vault automatically | diff --git a/monitor/src/main.test.ts b/monitor/src/main.test.ts index 17b44ed40..7bd878aa8 100644 --- a/monitor/src/main.test.ts +++ b/monitor/src/main.test.ts @@ -810,17 +810,23 @@ test("a source header that names another block number pages critical", async (t) const SECOND_NODE = "ws://second-node.test:9944?apikey=secret"; /** A monitor whose second Pendulum node serves the source's chain, except for - * the values in `own`, which the test may change later. */ -async function crossCheckSetup(t: { after(fn: () => void): void }, own: Partial = {}) { + * the values in `own`, which the test may change later; with `relayHead`, + * also against a relay chain recording it. */ +async function crossCheckSetup( + t: { after(fn: () => void): void }, + own: Partial = {}, + relayHead?: () => { number: number; hash: string }, +) { let source: Pendulum | undefined; const chain = new Proxy({} as Pendulum, { get: (_target, key) => (key in own ? own[key as keyof Pendulum] : source![key as keyof Pendulum]), }); const second = pendulumChain(chain); + const relay = relayHead && relayChain(relayHead); const monitor = await setup(t, { - env: { PENDULUM_WS_CROSSCHECK: SECOND_NODE }, + env: { PENDULUM_WS_CROSSCHECK: SECOND_NODE, ...(relay ? { POLKADOT_WS: "wss://relay.test" } : {}) }, prepare: (_base, pendulum) => (source = pendulum), - connect: async (url) => (url === SECOND_NODE ? second.api : assert.fail(url)), + connect: async (url) => (url === SECOND_NODE ? second.api : (url === "wss://relay.test" && relay) || assert.fail(url)), }); return { ...monitor, own }; } @@ -880,11 +886,11 @@ test("a block the second node has not finalized yet waits for it, without a page const { pendulum, api, monitor, pages, persisted, own } = await crossCheckSetup(t, { finalized: 100 }); burn(pendulum, 0n, 100n * PEN); await monitor.poll(api); - assert.deepEqual(pages, []); + assert.equal(pages.length, 0, JSON.stringify(pages)); assert.equal(persisted().lastPendulumBlock, 100); own.finalized = 101; await monitor.poll(api); - assert.deepEqual(pages, []); + assert.equal(pages.length, 0, JSON.stringify(pages)); assert.equal(persisted().lastPendulumBlock, 101); assert.deepEqual(persisted().migrations.map((migration) => migration.nonce), ["0"]); }); @@ -893,7 +899,7 @@ test("a second node whose best chain holds another newest block, not yet finaliz const { pendulum, api, monitor, pages, persisted } = await crossCheckSetup(t, { finalized: 100, best: 101, forkFrom: 101 }); burn(pendulum, 0n, 100n * PEN); await monitor.poll(api); - assert.deepEqual(pages, []); + assert.equal(pages.length, 0, JSON.stringify(pages)); assert.equal(persisted().lastPendulumBlock, 100); }); @@ -919,6 +925,27 @@ test("a source on another chain one block past the second node's head ingests no assert.deepEqual(persisted().migrations.map((migration) => migration.nonce), ["0"]); }); +test("with the second node and the relay chain, a block past both their heads waits, and a fork there is caught", async (t) => { + let recorded = 101; + const { pendulum, api, monitor, pages, persisted, own } = await crossCheckSetup( + t, + { finalized: 101, forkFrom: undefined }, + () => ({ number: recorded, hash: pendulumHash({}, recorded) }), + ); + burn(pendulum, 0n, 100n * PEN); + pendulum.forkFrom = 102; // the source finalizes another block 102... + burn(pendulum, 1n, 999n * PEN, 102); // ...with a fabricated burn + await monitor.poll(api); + assert.equal(pages.length, 0, JSON.stringify(pages)); + assert.equal(persisted().lastPendulumBlock, 101); + assert.deepEqual(persisted().migrations.map((migration) => migration.nonce), ["0"]); + own.finalized = 102; // both checkers reach the real block 102 + recorded = 102; + await monitor.poll(api); + assert.ok(pages.some((page) => page.subject === "CRITICAL: PENDULUM SOURCE CONTRADICTED"), JSON.stringify(pages)); + assert.equal(persisted().lastPendulumBlock, 101); +}); + test("a second node whose finalized head is stale counts as unavailable", async (t) => { const { pendulum, api, monitor, pages, persisted } = await crossCheckSetup(t, { finalized: 100, headAgeMs: 400_000 }); burn(pendulum, 0n, 100n * PEN); @@ -1088,7 +1115,7 @@ test("a relay record past the source's finalized head must descend from that hea assert.equal(persisted().lastPendulumBlock, 100); }); -test("a relay record past the source's finalized head that its best chain does not hold yet only waits", async (t) => { +test("a relay record past the source's finalized head that its best chain does not hold yet holds the source back", async (t) => { const { pendulum, api, monitor, pages, persisted } = await relaySetup(t, { head: () => ({ number: 102, hash: pendulumHash({}, 102) }), }); @@ -1096,11 +1123,82 @@ test("a relay record past the source's finalized head that its best chain does n pendulum.forkFrom = 102; // its best chain holds another block 102 for now pendulum.best = 102; await monitor.poll(api); - assert.equal(pages.length, 0, JSON.stringify(pages)); + assert.deepEqual(pages.map((page) => [page.subject, page.severity]), [["RELAY ANCHOR UNAVAILABLE", "WARNING"]]); + assert.match( + pages[0].detail, + /^relay RPC 1: the source has not imported block 102, the relay chain's Pendulum head, yet; no Pendulum block after 100 is ingested until a relay RPC anchors a newer head,/, + ); assert.equal(persisted().relayAnchor, undefined); + assert.equal(persisted().lastPendulumBlock, 100); +}); + +test("a block past the relay chain's record waits for it, and a fork there is caught once it is recorded", async (t) => { + let recorded = 101; + const { pendulum, api, monitor, pages, persisted } = await relaySetup(t, { + head: () => ({ number: recorded, hash: pendulumHash({}, recorded) }), + }); + burn(pendulum, 0n, 100n * PEN); + pendulum.forkFrom = 102; // the source finalizes another block 102... + burn(pendulum, 1n, 999n * PEN, 102); // ...with a fabricated burn, past the relay chain's record + await monitor.poll(api); + assert.equal(pages.length, 0, JSON.stringify(pages)); + assert.equal(persisted().lastPendulumBlock, 101); + assert.deepEqual(persisted().migrations.map((migration) => migration.nonce), ["0"]); + recorded = 102; // the relay chain records the real block 102 + await monitor.poll(api); + assert.match( + pages.find((page) => page.subject === "CRITICAL: PENDULUM SOURCE CONTRADICTED")?.detail ?? "", + /relay chain records 0x0+66 as Pendulum block 102, the source's chain has 0x0+10+66;/, + ); assert.equal(persisted().lastPendulumBlock, 101); }); +test("a source that finalizes below a relay record it does not hold ingests nothing past the kept record", async (t) => { + let recorded = 101; + const { pendulum, api, monitor, pages, persisted } = await relaySetup(t, { + head: () => ({ number: recorded, hash: pendulumHash({}, recorded) }), + }); + burn(pendulum, 0n, 100n * PEN); + await monitor.poll(api); // block 101 anchored + pendulum.forkFrom = 102; // the source serves another chain from block 102 on... + for (let block = 102; block <= 105; block++) burn(pendulum, BigInt(block - 101), 999n * PEN, block); // ...with fabricated burns + pendulum.best = 106; // ...and holds another block 106 than the relay chain records + recorded = 106; + await monitor.poll(api); + assert.deepEqual(pages.map((page) => page.subject), ["RELAY ANCHOR UNAVAILABLE"]); + assert.match(pages[0].detail, /has not imported block 106, .*; no Pendulum block after 101 is ingested until a relay RPC anchors/); + assert.equal(persisted().lastPendulumBlock, 101); + assert.deepEqual(persisted().migrations.map((migration) => migration.nonce), ["0"]); + assert.deepEqual(persisted().relayAnchor, { number: 101, hash: pendulumHash({}, 101) }); +}); + +test("a Pendulum view the relay chain holds back pages as stalled", async (t) => { + const { pendulum, api, monitor, pages } = await relaySetup(t, { head: () => ({ number: 101, hash: pendulumHash({}, 101) }) }); + burn(pendulum, 0n, 100n * PEN); + pendulum.finalized = 130; // 29 blocks of 12 s past the relay chain's record + await monitor.poll(api); + assert.match( + pages.find((page) => page.subject === "PENDULUM SOURCE VIEW STALLED")?.detail ?? "", + /^verified Pendulum block 101 is 34\ds old \(the source finalized 130, not confirmed yet\);/, + ); +}); + +test("a Base event whose burn the relay chain has not confirmed yet is not proven fabricated by a newer source head", async (t) => { + const { base, pendulum, api, monitor, subjects } = await relaySetup(t, { + head: () => ({ number: 101, hash: pendulumHash({}, 101) }), + env: { SOURCE_CLOCK_SKEW_MARGIN_SECONDS: "0" }, + }); + burn(pendulum, 0n, 100n * PEN); + burn(pendulum, 1n, 100n * PEN, 125); + pendulum.finalized = 130; + release(base, 1n, 100n * PEN); + await monitor.poll(api); + await new Promise((resolve) => setTimeout(resolve, 5)); + await monitor.poll(api); // the source's own head has passed the first sighting, the confirmed view has not + assert.ok(subjects().includes("UNVERIFIED BASE EVENT"), JSON.stringify(subjects())); + assert.ok(!subjects().includes("MIGRATION TUPLE VIOLATION"), JSON.stringify(subjects())); +}); + test("a relay chain that records another hash at a height it anchored pages critical", async (t) => { let recorded = pendulumHash({}, 101); const { pendulum, api, monitor, pages, persisted } = await relaySetup(t, { @@ -1132,7 +1230,7 @@ test("a relay record below the block ingested last is checked down the source's if (recorded === pendulumHash({}, 100)) { assert.deepEqual(pages.map((page) => page.subject), ["RELAY ANCHOR UNAVAILABLE"]); assert.deepEqual(persisted().relayAnchor, { number: 100, hash: recorded }); - assert.equal(persisted().lastPendulumBlock, 102); + assert.equal(persisted().lastPendulumBlock, 101); // block 102 waits for a newer record } else { assert.match( pages.find((page) => page.subject === "CRITICAL: PENDULUM SOURCE CONTRADICTED")?.detail ?? "", @@ -1157,14 +1255,14 @@ test("an unreadable relay chain warns, the source is used, and it turns critical assert.ok(late.subjects().includes("CRITICAL: RELAY ANCHOR STALE"), JSON.stringify(late.subjects())); }); -test("a source behind the relay chain is not anchored, and pages only after the grace", async (t) => { +test("a source behind the relay chain is not anchored, its blocks wait and it pages, critical after the grace", async (t) => { const ahead = (pendulum: Pendulum) => ({ number: pendulum.finalized + 1, hash: pendulumHash(pendulum, pendulum.finalized + 1) }); const { pendulum, api, monitor, pages, persisted } = await relaySetup(t, { head: ahead }); burn(pendulum, 0n, 100n * PEN); await monitor.poll(api); - assert.equal(pages.length, 0, JSON.stringify(pages)); + assert.deepEqual(pages.map((page) => page.subject), ["RELAY ANCHOR UNAVAILABLE"]); assert.equal(persisted().relayAnchor, undefined); - assert.equal(persisted().lastPendulumBlock, 101); + assert.equal(persisted().lastPendulumBlock, 100); const late = await relaySetup(t, { head: ahead, env: { RELAY_ANCHOR_GRACE_SECONDS: "0" } }); await new Promise((resolve) => setTimeout(resolve, 5)); await late.monitor.poll(late.api); @@ -1223,6 +1321,7 @@ test("a relay record below a contradicting one does not outvote it", async (t) = assert.deepEqual(pages.map((page) => page.subject), ["RELAY ANCHOR UNAVAILABLE"]); assert.match(pages[0].detail, /no other relay RPC confirms or outvotes it \(relay RPC 2: its Pendulum head 101 lies below the contradicting record\);/); assert.equal(persisted().relayAnchor, undefined); + assert.equal(persisted().lastPendulumBlock, 100); }); /** A relay RPC whose record stays at the source's block 101. */ @@ -1241,6 +1340,8 @@ test("a relay contradiction that only a stuck relay RPC answers stays unconfirme /^relay RPC 1 reports that the relay chain records 0x0+66 as Pendulum block 102, the source's chain has 0x0+10+66, and no other relay RPC confirms or outvotes it \(relay RPC 2: the relay chain's Pendulum head 101 is not newer\);/, ); assert.deepEqual(persisted().relayAnchor, { number: 101, hash: pendulumHash({}, 101) }); + assert.equal(persisted().lastPendulumBlock, 101); + assert.deepEqual(persisted().migrations.map((migration) => migration.nonce), ["0"]); }); test("a relay RPC whose record does not anchor the source leaves it to the next one", async (t) => { @@ -1249,7 +1350,7 @@ test("a relay RPC whose record does not anchor the source leaves it to the next await monitor.poll(api); // relay RPC 1 anchors block 101 burn(pendulum, 1n, 100n * PEN, 102); await monitor.poll(api); // relay RPC 1 stays there - assert.deepEqual(pages, []); + assert.equal(pages.length, 0, JSON.stringify(pages)); assert.deepEqual(persisted().relayAnchor, { number: 102, hash: pendulumHash({}, 102) }); assert.equal(persisted().lastPendulumBlock, 102); }); @@ -1262,13 +1363,13 @@ test("a relay contradiction the next relay RPC confirms pages critical", async ( assert.equal(persisted().lastPendulumBlock, 100); }); -test("a relay contradiction no other relay RPC can confirm only warns", async (t) => { +test("a relay contradiction no other relay RPC can confirm warns, and the source's blocks wait", async (t) => { const { pendulum, api, monitor, pages, persisted } = await twoRelaySetup(t, [forkedHead, undefined]); burn(pendulum, 0n, 100n * PEN); await monitor.poll(api); assert.deepEqual(pages.map((page) => page.subject), ["RELAY ANCHOR UNAVAILABLE"]); assert.match(pages[0].detail, /and no other relay RPC confirms or outvotes it \(relay RPC 2: Error: relay RPC unavailable\)/); - assert.equal(persisted().lastPendulumBlock, 101); + assert.equal(persisted().lastPendulumBlock, 100); }); test("while Pendulum cannot be read, a Base event with an unknown nonce pauses after the grace", async (t) => { diff --git a/monitor/src/main.ts b/monitor/src/main.ts index c8b13103f..f4980010d 100644 --- a/monitor/src/main.ts +++ b/monitor/src/main.ts @@ -933,18 +933,22 @@ async function relayVerdict( * record that contradicts the source counts only when another relay RPC's * record contradicts it too, or when POLKADOT_WS lists one RPC. A record that * anchors the source at or above the contradicting record's height outvotes - * it; any other answer leaves it unconfirmed. Pages while no new head is - * anchored: a warning when the relay RPCs fail or a contradiction stays - * unconfirmed, critical once none was anchored for - * RELAY_ANCHOR_GRACE_SECONDS; and when relay RPCs disagree. All throttled. */ + * it; any other answer leaves it unconfirmed. Resolves to the height up to + * which the relay chain confirms the source's chain `hashes`, walked down + * from `top` to `bottom`: the newest anchored head, or no higher than + * `bottom` before the first; undefined while no relay RPC answers. Pages + * while no new head is anchored and the source's blocks wait for one, a + * contradiction stays unconfirmed or no relay RPC answers: a warning, + * critical once none was anchored for RELAY_ANCHOR_GRACE_SECONDS; and when + * relay RPCs disagree. All throttled. */ async function anchorToRelay( api: ApiPromise, draft: RuntimeState, hashes: Map, top: BlockRef, bottom: BlockRef, -): Promise { - if (!config.polkadotWs.length) return; +): Promise { + if (!config.polkadotWs.length) return undefined; const answers: string[] = []; let answered = false; let contradiction: { rpc: number; height: number; error: SourceIntegrityError } | undefined; @@ -984,21 +988,25 @@ async function anchorToRelay( "that record — one of the two relay RPCs is wrong, check both", ); } - return; + return head.number; } const reason = contradiction ? `relay RPC ${contradiction.rpc} reports that ${contradiction.error.message}, and no other relay RPC confirms ` + `or outvotes it${answers.length ? ` (${answers.join("; ")})` : ""}` : `${answered ? "" : "the relay chain cannot be read: "}${answers.join("; ")}`; - if (contradiction || !answered) { - if (Date.now() - lastRelayAlertMs > config.graceSeconds * 1000) { - lastRelayAlertMs = Date.now(); - await alert( - "RELAY ANCHOR UNAVAILABLE", - `${reason}; the Pendulum source is used without the relay-chain anchor until a relay RPC answers, ` + - `and this turns critical after ${config.relayAnchorGraceMs / 1000}s`, - ); - } + const confirmed = draft.relayAnchor?.number ?? bottom.number; + const consequence = answered + ? `no Pendulum block after ${Math.max(confirmed, draft.lastPendulumBlock)} is ingested until a relay RPC anchors a newer head` + : "the Pendulum source is used without the relay-chain anchor until a relay RPC answers"; + if ( + (contradiction || !answered || top.number > confirmed) && + Date.now() - lastRelayAlertMs > config.graceSeconds * 1000 + ) { + lastRelayAlertMs = Date.now(); + await alert( + "RELAY ANCHOR UNAVAILABLE", + `${reason}; ${consequence}, and this turns critical after ${config.relayAnchorGraceMs / 1000}s`, + ); } const unanchoredMs = Date.now() - lastRelayAnchorMs; if (unanchoredMs > config.relayAnchorGraceMs && Date.now() - lastRelayStaleAlertMs > config.unmatchedEventGraceSeconds * 1000) { @@ -1006,16 +1014,19 @@ async function anchorToRelay( await alert( "CRITICAL: RELAY ANCHOR STALE", `no Pendulum head was checked against the relay chain for ${Math.round(unanchoredMs / 1000)}s (${reason}); ` + - "the source is trusted without the anchor — restore a relay RPC, or the source node if it lags", + `${consequence} — restore a relay RPC, or the source node if it lags`, ); } + return answered ? confirmed : undefined; } /** The hashes of the finalized blocks to ingest, on one chain with every * block ingested before and with the relay-chain record kept from an earlier - * poll, and the newest of them that the second node confirms (`confirmed`, - * at most the source's finalized head; `crossChecked` while the second node - * answers). The source is then anchored to the relay chain's newest record. */ + * poll, and the newest of them that every checker that answers confirms + * (`confirmed`, at most the source's finalized head): the second node up to + * `crossChecked`, and the relay chain, whose newest record anchors the + * source. A checker that does not answer confirms nothing and holds nothing + * back. */ async function sourceHashes( api: ApiPromise, draft: RuntimeState, @@ -1039,8 +1050,8 @@ async function sourceHashes( } const lowest = Math.min(top.number, draft.lastPendulumBlock); const crossChecked = await crossCheckSource(hashes, top, lowest); - await anchorToRelay(api, draft, hashes, top, { number: lowest, hash: hashes.get(lowest) as string }); - return { hashes, confirmed: Math.min(finalized.number, crossChecked ?? Infinity), crossChecked }; + const anchored = await anchorToRelay(api, draft, hashes, top, { number: lowest, hash: hashes.get(lowest) as string }); + return { hashes, confirmed: Math.min(finalized.number, crossChecked ?? Infinity, anchored ?? Infinity), crossChecked }; } /** The migration pallet's admin events, each with what to check. */ From cb39e15bbbd3eb1f1b43c912be7e81237f653de3 Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Thu, 22 Oct 2026 15:37:26 +0200 Subject: [PATCH 62/75] fix(monitor): Bound the walk down to a relay head below the ingested blocks A relay-chain record of the Pendulum head that lies more than 100 blocks below the block ingested last is not walked down to: it does not anchor the source, and the next relay RPC is asked. --- monitor/README.md | 3 ++- monitor/src/main.test.ts | 13 +++++++++++++ monitor/src/main.ts | 7 +++++++ 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/monitor/README.md b/monitor/README.md index f0c27362d..b8d2633cd 100644 --- a/monitor/README.md +++ b/monitor/README.md @@ -52,7 +52,8 @@ itself: head. The newest such head is kept in the state file, and the chain must keep passing through it, also while it lies above the finalized head. Blocks ingested while no relay RPC answered are tied to the next head that - arrives by the same walk. A head that contradicts the source counts only + arrives by the same walk; a head more than 100 blocks below the block + ingested last is not used. A head that contradicts the source counts only when another relay RPC's head contradicts it too, or when `POLKADOT_WS` lists one RPC. A relay RPC whose head anchors the source at or above the contradicting head's height outvotes it: `RELAY RPCS DISAGREE` pages and diff --git a/monitor/src/main.test.ts b/monitor/src/main.test.ts index 7bd878aa8..b0581a30f 100644 --- a/monitor/src/main.test.ts +++ b/monitor/src/main.test.ts @@ -1241,6 +1241,19 @@ test("a relay record below the block ingested last is checked down the source's } }); +test("a relay record far below the block ingested last is not walked down to", async (t) => { + const { pendulum, api, monitor, pages, persisted } = await relaySetup(t, { + head: () => ({ number: 150, hash: pendulumHash({}, 150) }), + env: { PENDULUM_START_BLOCK: "300" }, + }); + burn(pendulum, 0n, 100n * PEN, 300); + await monitor.poll(api); + assert.deepEqual(pages.map((page) => page.subject), ["RELAY ANCHOR UNAVAILABLE"]); + assert.match(pages[0].detail, /^relay RPC 1: the relay chain's Pendulum head 150 lies more than 100 blocks below block 299;/); + assert.equal(persisted().relayAnchor, undefined); + assert.equal(persisted().lastPendulumBlock, 299); +}); + test("an unreadable relay chain warns, the source is used, and it turns critical after the grace", async (t) => { const { pendulum, api, monitor, pages, persisted } = await relaySetup(t, { failing: () => true }); burn(pendulum, 0n, 100n * PEN); diff --git a/monitor/src/main.ts b/monitor/src/main.ts index f4980010d..40bd677a8 100644 --- a/monitor/src/main.ts +++ b/monitor/src/main.ts @@ -882,6 +882,10 @@ function relayHead(api: ApiPromise, url: string): Promise { }); } +/** How far below the block ingested last the relay chain's record may lie and + * still be walked down to; a deeper one is not used. */ +const RELAY_HEAD_MAX_DEPTH = 100; + /** Why the relay chain's record `head` does not anchor the source this poll, * or undefined when it does. Throws when it contradicts the source's chain: * `hashes`, walked down from `top` to `bottom`. */ @@ -918,6 +922,9 @@ async function relayVerdict( } return undefined; } + if (head.number < bottom.number - RELAY_HEAD_MAX_DEPTH) { + return `the relay chain's Pendulum head ${head.number} lies more than ${RELAY_HEAD_MAX_DEPTH} blocks below block ${bottom.number}`; + } const hash = head.number >= bottom.number ? hashes.get(head.number) : (await walkHeaders(api, bottom, head.number)).get(head.number); From 83235a61f202ca403391c1a742c7472cb111a8cb Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Thu, 22 Oct 2026 15:38:44 +0200 Subject: [PATCH 63/75] fix(monitor): Count a vault paused by hand as paused without a guardian key The pause attempt reads paused() at the confirmed Base block before it looks for GUARDIAN_PRIVATE_KEY. A vault that is already paused counts as paused, so a monitor without a key moves past a treasury violation, and persists the state after a tuple violation, once operators paused the vault by hand. AUTO-PAUSE UNAVAILABLE pages only while the vault is not paused. --- monitor/README.md | 2 +- monitor/src/main.test.ts | 17 +++++++++++++++++ monitor/src/main.ts | 8 ++++---- 3 files changed, 22 insertions(+), 5 deletions(-) diff --git a/monitor/README.md b/monitor/README.md index b8d2633cd..89dd8d2c5 100644 --- a/monitor/README.md +++ b/monitor/README.md @@ -120,7 +120,7 @@ subject that starts with `CRITICAL`; everything else is a `WARNING`. | Subjects | Meaning | |---|---| -| `MIGRATION TUPLE VIOLATION`, `VAULT BALANCE DEFICIT`, `CONSERVATION VIOLATION`, `TREASURY DESTINATION VIOLATION` | A conservation or treasury check failed; the auto-pause follows with `vault auto-pause confirmed`, `vault pause confirmed` (already paused), `AUTO-PAUSE PENDING FINALITY`, `AUTO-PAUSE FAILED` or `AUTO-PAUSE UNAVAILABLE` | +| `MIGRATION TUPLE VIOLATION`, `VAULT BALANCE DEFICIT`, `CONSERVATION VIOLATION`, `TREASURY DESTINATION VIOLATION` | A conservation or treasury check failed; the auto-pause follows with `vault auto-pause confirmed`, `vault pause confirmed` (already paused, also without a guardian key), `AUTO-PAUSE PENDING FINALITY`, `AUTO-PAUSE FAILED` or `AUTO-PAUSE UNAVAILABLE` (no guardian key, and the vault is not paused) | | `UNVERIFIED BASE EVENT` | First sighting of a Base event whose nonce the source view does not know yet | | `CRITICAL: MISMATCHED APPROVAL` | One attestor approved a tuple that disagrees with the finalized Pendulum view; check its key and its Pendulum node. A second attestor doing so for the same nonce pauses the vault | | `LIVENESS: migration below approval quorum`, `LIVENESS: quorum reached but not released` | Stalled migrations, one summary per kind and poll | diff --git a/monitor/src/main.test.ts b/monitor/src/main.test.ts index b0581a30f..9afeba6e9 100644 --- a/monitor/src/main.test.ts +++ b/monitor/src/main.test.ts @@ -1878,6 +1878,23 @@ test("while the pause after a treasury violation is not confirmed, the monitor h assert.ok(!subjects().includes("PENDULUM SOURCE UNREADABLE"), JSON.stringify(subjects())); // the source reads fine }); +test("without a guardian key, a vault paused by hand after a treasury violation lets the monitor move on", async (t) => { + const { base, pendulum, api, monitor, pages, persisted } = await setup(t, { + env: { EXPECTED_TREASURY_DESTINATION: TREASURY_DESTINATION }, + }); + burn(pendulum, 0n, 100n * PEN, 101, { who: TREASURY_ACCOUNT, recipient: TOKEN }); + await monitor.poll(api); + assert.equal(persisted().lastPendulumBlock, 100); + base.paused = true; // operators pause the vault by hand + const before = pages.length; + await monitor.poll(api); + const subjects = pages.slice(before).map((page) => page.subject); + assert.ok(subjects.includes("vault pause confirmed"), JSON.stringify(subjects)); + assert.ok(!subjects.includes("AUTO-PAUSE UNAVAILABLE"), JSON.stringify(subjects)); + assert.equal(persisted().lastPendulumBlock, 101); + assert.deepEqual(persisted().migrations.map((migration) => migration.nonce), ["0"]); +}); + test("admin events of the migration pallet page from the ingest", async (t) => { const { pendulum, api, monitor, pages } = await setup(t); pendulum.finalized = 101; diff --git a/monitor/src/main.ts b/monitor/src/main.ts index 40bd677a8..b8486c8b6 100644 --- a/monitor/src/main.ts +++ b/monitor/src/main.ts @@ -535,10 +535,6 @@ function pauseVault(): Promise { } async function pauseOnce(): Promise { - if (!config.guardianPrivateKey) { - await alert("AUTO-PAUSE UNAVAILABLE", "no GUARDIAN_PRIVATE_KEY configured; pause manually NOW", "critical"); - return false; - } try { const confirmed = await publicClient.getBlock({ blockTag: config.baseFinalityTag }); if (confirmed.number === null) throw new Error(`${config.baseFinalityTag} block has no number`); @@ -561,6 +557,10 @@ async function pauseOnce(): Promise { await alert("AUTO-PAUSE PENDING FINALITY", `pause is mined but not yet ${config.baseFinalityTag}`); return false; } + if (!config.guardianPrivateKey) { + await alert("AUTO-PAUSE UNAVAILABLE", "no GUARDIAN_PRIVATE_KEY configured; pause manually NOW", "critical"); + return false; + } const guardian = privateKeyToAccount(config.guardianPrivateKey); const walletClient = createWalletClient({ account: guardian, chain: baseChain, transport: http(config.baseRpcUrl) }); From 9c4f84ee4cb81a220ff4af4ec055e2416964e8e9 Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Thu, 22 Oct 2026 15:39:58 +0200 Subject: [PATCH 64/75] feat(monitor): Page critical when the treasury destination is not the expected one With EXPECTED_TREASURY_DESTINATION set, a finalized TreasuryDestinationSet to another address pages CRITICAL: MIGRATION PALLET ADMIN EVENT instead of the warning: each treasury migration to it would pause the vault. A destination set to the expected address still pages as a warning. --- monitor/README.md | 6 +++--- monitor/src/main.test.ts | 17 +++++++++++++++++ monitor/src/main.ts | 14 +++++++++++--- 3 files changed, 31 insertions(+), 6 deletions(-) diff --git a/monitor/README.md b/monitor/README.md index 89dd8d2c5..0121b615f 100644 --- a/monitor/README.md +++ b/monitor/README.md @@ -21,7 +21,7 @@ Checks every poll: | Base view | The safe/finalized Base block must not be older than `BASE_STALE_ALERT_SECONDS`: a view that stops advancing hides every later vault event while every check still passes at the old block. | Alert (`BASE VIEW STALLED`), throttled | | Auto-pause readiness | At startup and every poll: the Base RPC serves `BASE_CHAIN_ID` and, with a guardian key configured, the key parses, is `vault.guardian()` and holds at least `GUARDIAN_MIN_BALANCE_WEI`. A read that fails pages as well and never stops the poll's other checks. | Alert (`AUTO-PAUSE NOT READY`), throttled; `AUTO-PAUSE DISABLED` once at startup without a key | | Pendulum governance | Every pending public proposal, the council's external proposal, every ongoing referendum and every open council and technical-committee motion, with the call decoded. A call that mints, moves or unlocks balances outside normal transfers, or changes the runtime or the migration pallet, pages as critical: minted PEN burned through the migration drains the fixed-supply vault, and only Pendulum governance can stop the call before enactment. A call the monitor cannot read pages as critical too: one over 64 KiB is never fetched, and one that does not decode or nests more than 256 calls or 16 levels deep is unreadable; one whose preimage is not noted yet is re-checked every poll and paged again once it can be read. The watch runs after the poll's conservation checks, so it cannot stop or delay them, and it decodes within fixed bounds, so its cost to the next poll is bounded. | Alert once per item (`PENDULUM GOVERNANCE ITEM`, `CRITICAL: Pendulum governance item …`); a motion proposed again with the same call (a new motion index), or an external proposal made again after it left the queue or became a referendum, pages again; a restart re-pages the items still pending | -| Migration pallet admin events | Every finalized `MigrationPauseSet` and `TreasuryDestinationSet`, however fast the motion behind it passed. | Alert once per event (`MIGRATION PALLET ADMIN EVENT`) | +| Migration pallet admin events | Every finalized `MigrationPauseSet` and `TreasuryDestinationSet`, however fast the motion behind it passed. | Alert once per event (`MIGRATION PALLET ADMIN EVENT`); `CRITICAL: MIGRATION PALLET ADMIN EVENT` for a `TreasuryDestinationSet` to another address than `EXPECTED_TREASURY_DESTINATION` | | Vault admin events | Every admin event of the vault (token, pause, unpause, attestors, threshold, caps, guardian, admin transfer, sweep, stale pending cleared). Before the governance handover the admin acts without a delay, so these are the only trace of such a change. This scan (shared with the governance queue) keeps its own persisted cursor: while it fails, every other check still runs, and it resumes where it stopped, also after a restart. | Alert once per event (`VAULT ADMIN EVENT`); the first run also pages the deployment's own events; `BASE ADMIN SCAN FAILED`, throttled, on a failure, and `CRITICAL: BASE ADMIN SCAN FAILED` once it has failed for longer than `UNMATCHED_EVENT_GRACE_SECONDS` | | Governance queue | Every operation the governance Timelock schedules (`CallScheduled`), with its calls decoded and the time it becomes executable. Once the delay has passed anyone may execute a ready operation, so the canceller has exactly the delay to review and veto it. | Alert once per operation and scheduling block (`GOVERNANCE OPERATION SCHEDULED`), so an operation cancelled and scheduled again with the same id pages again | @@ -93,7 +93,7 @@ down, the source behind the relay chain, a relay head that does not advance). | `BASE_STALE_ALERT_SECONDS` | Age of the safe/finalized Base block (vs. wall clock) past which `BASE VIEW STALLED` pages (default 600s for `safe`, 1800s for `finalized`; Base's safe head normally trails by about a minute, its finalized head by about 20) | | `ISSUANCE_TOLERANCE` | Growth of Pendulum `totalIssuance + TotalMigrated` over its first-seen anchor (12-decimal pallet units) tolerated before `SOURCE SUPPLY GREW` pages (default 0) | | `ACKNOWLEDGED_RELEASE_EXCESS` | Excess of released over migrated, in token units (18 decimals), that operators quantified and accepted after a real loss; M2a fires only beyond it (default 0). Set it at the RB-3 sign-off and restart; the startup log shows it | -| `EXPECTED_TREASURY_DESTINATION` | Optional: the Base address every treasury migration must pay, the one set with `tokenMigration.setTreasuryDestination`; any other pauses the vault | +| `EXPECTED_TREASURY_DESTINATION` | Optional: the Base address every treasury migration must pay, the one set with `tokenMigration.setTreasuryDestination`; any other pauses the vault, and setting another one pages critical | | `TIMELOCK_ADDRESS` | Optional: the governance TimelockController. Set it as soon as governance is deployed; the queue is scanned from the monitor's Base cursor onwards, so operations scheduled before the monitor knew the address are not paged | | `ALERT_WEBHOOK_URL` | Slack incoming webhook receiving the pages (see [Pages](#pages)). A page counts as delivered only on a 2xx answer within 10s: once-only pages (governance items, admin events, scheduled operations, first sightings) are retried every poll until delivered, and failed deliveries are logged. Without it the log line is the page | | `HEARTBEAT_URL` | Optional: a dead-man's switch URL (Healthchecks.io style), requested with a `GET` (10s timeout) after every poll that ran all checks; a failed request is logged and never stops the monitor. Have the service page when the requests stop: the monitor is down, stuck, or cannot complete its checks (for example while the Pendulum source cannot be read) | @@ -133,7 +133,7 @@ subject that starts with `CRITICAL`; everything else is a `WARNING`. | `CRITICAL: RELAY ANCHOR STALE` | No Pendulum head was checked against the relay chain for `RELAY_ANCHOR_GRACE_SECONDS` | | `AUTO-PAUSE NOT READY`, `AUTO-PAUSE DISABLED` | The next violation would not pause the vault automatically | | `SOURCE SUPPLY GREW`, `CRITICAL: unreleasable migration burned on Pendulum` | Supply problems at the source | -| `PENDULUM GOVERNANCE ITEM`, `CRITICAL: Pendulum governance item mints, moves or unlocks balances, or changes the runtime`, `CRITICAL: Pendulum governance item cannot be read`, `MIGRATION PALLET ADMIN EVENT` | Pendulum governance | +| `PENDULUM GOVERNANCE ITEM`, `CRITICAL: Pendulum governance item mints, moves or unlocks balances, or changes the runtime`, `CRITICAL: Pendulum governance item cannot be read`, `MIGRATION PALLET ADMIN EVENT`, `CRITICAL: MIGRATION PALLET ADMIN EVENT` | Pendulum governance | | `GOVERNANCE OPERATION SCHEDULED`, `VAULT ADMIN EVENT` | Base governance and vault administration | | `monitor check failed`, `pendulum governance watch failed`, `multicall unavailable, …`, `multicall batch failed, …`, `monitor startup failed` | The monitor itself | diff --git a/monitor/src/main.test.ts b/monitor/src/main.test.ts index 9afeba6e9..ef9685c9e 100644 --- a/monitor/src/main.test.ts +++ b/monitor/src/main.test.ts @@ -1909,6 +1909,23 @@ test("admin events of the migration pallet page from the ingest", async (t) => { assert.match(adminPages[1], /MigrationPauseSet\(false\)/); }); +test("a treasury destination set to any but the expected address pages critical", async (t) => { + const { pendulum, api, monitor, pages } = await setup(t, { env: { EXPECTED_TREASURY_DESTINATION: TREASURY_DESTINATION } }); + pendulum.finalized = 102; + pendulum.events.set(101, [{ section: "tokenMigration", method: "TreasuryDestinationSet", data: [TREASURY_DESTINATION] }]); + pendulum.events.set(102, [{ section: "tokenMigration", method: "TreasuryDestinationSet", data: [TOKEN] }]); + await monitor.poll(api); + const set = pages.filter((page) => page.subject.endsWith("MIGRATION PALLET ADMIN EVENT")); + assert.deepEqual(set.map((page) => [page.subject, page.severity]), [ + ["MIGRATION PALLET ADMIN EVENT", "WARNING"], + ["CRITICAL: MIGRATION PALLET ADMIN EVENT", "CRITICAL"], + ]); + assert.match( + set[1].detail, + new RegExp(`^tokenMigration.TreasuryDestinationSet\\(${TOKEN}\\) in finalized Pendulum block 102: .* not to EXPECTED_TREASURY_DESTINATION ${TREASURY_DESTINATION}, `, "i"), + ); +}); + test("the governance watch runs after the conservation checks", async (t) => { const { base, pendulum, api, monitor, subjects } = await setup(t); base.vaultBalance -= 1n; diff --git a/monitor/src/main.ts b/monitor/src/main.ts index b8486c8b6..cffdd1d5b 100644 --- a/monitor/src/main.ts +++ b/monitor/src/main.ts @@ -1093,11 +1093,19 @@ async function ingestPendulumEvents( if (section !== "tokenMigration") continue; const adminCheck = PALLET_ADMIN_EVENTS.get(method); if (adminCheck) { - // Every admin change, however fast the motion behind it passed. + // Every admin change, however fast the motion behind it passed; a + // treasury destination other than the expected one is critical. + const expected = config.expectedTreasuryDestination; + const unexpected = method === "TreasuryDestinationSet" && expected !== undefined && + String(data[0]).toLowerCase() !== expected.toLowerCase(); await pageOnce( `pallet:${block}:${index}`, - "MIGRATION PALLET ADMIN EVENT", - `tokenMigration.${method}(${data.join(", ")}) in finalized Pendulum block ${block}: ${adminCheck}`, + `${unexpected ? "CRITICAL: " : ""}MIGRATION PALLET ADMIN EVENT`, + `tokenMigration.${method}(${data.join(", ")}) in finalized Pendulum block ${block}: ` + + (unexpected + ? `treasury migrations are released to this Base address from now on, not to ` + + `EXPECTED_TREASURY_DESTINATION ${expected}, and each one pauses the vault — check the motion behind it` + : adminCheck), ); continue; } From 1bfeef3b39d1f0943179aa7b848cb18ff8afa806 Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Thu, 22 Oct 2026 15:40:51 +0200 Subject: [PATCH 65/75] docs(monitor): Correct the vault balance check's timing and an attestor note While a poll is busy, two runs of the vault balance check can be about three poll intervals apart, not two. The helper that collects disagreeing attestors no longer describes the pause rule, which its caller states. --- monitor/README.md | 2 +- monitor/src/checks.ts | 6 ++---- monitor/src/main.ts | 4 ++-- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/monitor/README.md b/monitor/README.md index 0121b615f..e4eccefdb 100644 --- a/monitor/README.md +++ b/monitor/README.md @@ -10,7 +10,7 @@ Checks every poll: |---|---|---| | M2 tuple match | Every safe Base `Approved` and `Released` event must match one finalized Pendulum `MigrationInitiated` nonce, recipient, and pallet amount; `Released.tokenAmount` must equal `palletAmount × conversionFactor`. After a violation the Base cursor stays on any block range whose events the source view may still reveal, so a pause caused by a lagging source does not lose those releases. | Alert + auto-pause on any mismatching `Released`, and on approvals of one nonce that disagree with the finalized Pendulum view from two or more distinct attestors, also across polls and restarts (from one at a vault threshold of two or less, where one more approval releases). Otherwise a single attestor's disagreeing approval pages `CRITICAL: MISMATCHED APPROVAL` instead: one approval can never release. A mismatch against a *known* nonce counts immediately. An *unknown* nonce first pages (`UNVERIFIED BASE EVENT`, without delaying the pause decision), then counts as a mismatch as soon as it is provable — the finalized source view passed the event's own timestamp and the nonce still does not exist — or after `UNMATCHED_EVENT_GRACE_SECONDS` if the source view stays behind or cannot be read (a stalled or unreadable source also pages on its own, see below) | | M2a aggregate | `totalReleased <= TotalMigrated × conversionFactor + ACKNOWLEDGED_RELEASE_EXCESS`. This remains independent defence-in-depth around the event reconciliation: it also runs while a Base log range is held, and waits only while a Base event's burn is not in the source view yet. After a real, quantified loss, operators set `ACKNOWLEDGED_RELEASE_EXCESS` to it (RB-3) so the vault can be unpaused; any further excess still fires. | Alert + auto-pause | -| M2b: `balanceOf(vault) + totalReleased + totalSwept >= totalSupply` | Vault-internal conservation. Only a **deficit** alerts: a surplus is a harmless inbound transfer (donation, or a migration whose recipient is the vault) and is ignored, so it cannot false-trigger a pause. It reads Base only and runs first in every poll, so a failing Pendulum source or Base log scan cannot skip it. While the rest of a poll takes longer than `POLL_INTERVAL_MS` (a slow source, a long catch-up), it also runs on its own, so that delays it by two poll intervals at most. | Alert + auto-pause on a deficit | +| M2b: `balanceOf(vault) + totalReleased + totalSwept >= totalSupply` | Vault-internal conservation. Only a **deficit** alerts: a surplus is a harmless inbound transfer (donation, or a migration whose recipient is the vault) and is ignored, so it cannot false-trigger a pause. It reads Base only and runs first in every poll, so a failing Pendulum source or Base log scan cannot skip it. While the rest of a poll takes longer than `POLL_INTERVAL_MS` (a slow source, a long catch-up), it also runs on its own, so two of its runs are about three poll intervals apart at most. | Alert + auto-pause on a deficit | | M4: every old migration reaches active approval quorum — and then releases | Liveness of the attestor fleet (`below approval quorum`), and of everything after it (`quorum reached but not released`: a release starved of daily allowance, paused, or made releasable by a threshold cut without a `ReleasePending`, which the releaser cannot see). | One summary page per kind and poll (count and the first nonces; the log lists every nonce), each nonce at most once per `GRACE_SECONDS` | | Source supply | `totalIssuance + TotalMigrated` on Pendulum must never grow past its first-seen anchor. | Alert (`SOURCE SUPPLY GREW`) | | Unreleasable burns | A burn to the zero or vault address can never be approved or released; it is paged once and excluded from the pending count so it cannot page liveness forever or block RB-7. | Alert once | diff --git a/monitor/src/checks.ts b/monitor/src/checks.ts index 619f6a619..17c01fa95 100644 --- a/monitor/src/checks.ts +++ b/monitor/src/checks.ts @@ -132,10 +132,8 @@ export function approvalMismatch(expected: MigrationTuple | undefined, actual: M /** * The distinct attestors whose recorded approvals of `nonce` disagree with its - * finalized burn, or with its absence. One attestor's approval can never - * release (the threshold is higher), so one pages; two pause the vault before - * a third could complete a quorum. Approvals that the source view has since - * confirmed no longer count. + * finalized burn, or with its absence. Approvals that the source view has + * since confirmed no longer count. */ export function disagreeingAttestors( expected: MigrationTuple | undefined, diff --git a/monitor/src/main.ts b/monitor/src/main.ts index cffdd1d5b..d874b977e 100644 --- a/monitor/src/main.ts +++ b/monitor/src/main.ts @@ -1802,8 +1802,8 @@ async function checkVaultBalance(blockNumber: bigint) { /** Runs the vault balance check on its own while a poll is still busy a poll * interval after its last one started (a slow Pendulum source, a long - * catch-up). main() calls this every poll interval, so a busy poll delays - * the check by two poll intervals at most. */ + * catch-up). main() calls this every poll interval, so two checks are about + * three poll intervals apart at most. */ export async function watchVaultBalance(): Promise { if (!polling || Date.now() - lastBalanceCheckMs < config.pollIntervalMs) return; lastBalanceCheckMs = Date.now(); From 53e2be589fc2c6ac26ba5ff1e6e7d205e912b6e1 Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Thu, 22 Oct 2026 15:43:00 +0200 Subject: [PATCH 66/75] test(monitor): Cover the checked view's total, a lagging second node and quiet relay polls Pin that the conservation check uses the total of the view the relay chain confirms, that a second node behind the block ingested last holds newer blocks back without a contradiction, that a second node that cannot be read is not asked for the ingested blocks' events, and that a relay record and a source that both stay put page nothing. --- monitor/src/main.test.ts | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/monitor/src/main.test.ts b/monitor/src/main.test.ts index ef9685c9e..877a91bb8 100644 --- a/monitor/src/main.test.ts +++ b/monitor/src/main.test.ts @@ -848,9 +848,11 @@ test("a second node that disagrees on the newest block pages critical and nothin }); test("a second node that cannot answer only warns", async (t) => { + let connections = 0; const { pendulum, api, monitor, pages, persisted, heartbeats } = await setup(t, { env: { PENDULUM_WS_CROSSCHECK: SECOND_NODE }, connect: async (url) => { + connections++; throw new Error(`cannot connect to ${url}`); }, }); @@ -859,6 +861,7 @@ test("a second node that cannot answer only warns", async (t) => { assert.deepEqual(pages.map((page) => [page.subject, page.severity]), [["PENDULUM CROSS-CHECK UNAVAILABLE", "WARNING"]]); assert.match(pages[0].detail, /cannot be read: Error: cannot connect to /); assert.equal(persisted().lastPendulumBlock, 101); + assert.equal(connections, 1); // not asked again for the ingested block's migration events assert.deepEqual(heartbeats, ["GET"]); await monitor.poll(api); assert.equal(pages.length, 1); // throttled @@ -946,6 +949,21 @@ test("with the second node and the relay chain, a block past both their heads wa assert.equal(persisted().lastPendulumBlock, 101); }); +test("a second node behind the block ingested last holds newer blocks back, without a contradiction", async (t) => { + const { pendulum, api, monitor, pages, persisted, own } = await crossCheckSetup(t, { finalized: 100, headAgeMs: 400_000 }); + burn(pendulum, 0n, 100n * PEN); + await monitor.poll(api); // the stale second node is left out: block 101 is ingested + assert.equal(persisted().lastPendulumBlock, 101); + own.headAgeMs = 0; // it answers again, still at block 100 + burn(pendulum, 1n, 100n * PEN, 102); + await monitor.poll(api); + assert.deepEqual(pages.map((page) => page.subject), ["PENDULUM CROSS-CHECK UNAVAILABLE"]); + assert.equal(persisted().lastPendulumBlock, 101); + own.finalized = 102; + await monitor.poll(api); + assert.equal(persisted().lastPendulumBlock, 102); +}); + test("a second node whose finalized head is stale counts as unavailable", async (t) => { const { pendulum, api, monitor, pages, persisted } = await crossCheckSetup(t, { finalized: 100, headAgeMs: 400_000 }); burn(pendulum, 0n, 100n * PEN); @@ -1024,6 +1042,19 @@ test("a relay-chain record of the source's head anchors it and is kept", async ( assert.equal(pages.length, 0, JSON.stringify(pages)); assert.deepEqual(persisted().relayAnchor, { number: 101, hash: pendulumHash({}, 101) }); assert.equal(persisted().lastPendulumBlock, 101); + await monitor.poll(api); // neither the record nor the source moved: nothing waits, nothing pages + assert.equal(pages.length, 0, JSON.stringify(pages)); +}); + +test("the conservation check compares with the total of the view the relay chain confirms", async (t) => { + const { base, pendulum, api, monitor, subjects } = await relaySetup(t, { head: () => ({ number: 101, hash: pendulumHash({}, 101) }) }); + burn(pendulum, 0n, 100n * PEN); + burn(pendulum, 1n, 999n * PEN, 102); // past the relay chain's record + release(base, 0n, 100n * PEN); + base.totalReleased += 50n * PEN * CF; // released without a burn behind it + base.vaultBalance -= 50n * PEN * CF; + await monitor.poll(api); + assert.ok(subjects().includes("CONSERVATION VIOLATION"), JSON.stringify(subjects())); }); test("a source that names another hash than the relay chain's record pages critical", async (t) => { From b87414aa572cedddb5d30004b83bea7b50c8551c Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Thu, 22 Oct 2026 16:05:51 +0200 Subject: [PATCH 67/75] test(monitor): Cover pauses not final yet, unconfirmed tuple pauses and a lagging source Pin that a pause mined by hand but not yet final keeps a treasury hold, that a tuple violation whose pause is not confirmed is found again in the next poll, that a kept relay record above the source's finalized head confirms the blocks below it, and that a source whose finalized head falls behind the block ingested last is read at that block. --- monitor/src/main.test.ts | 60 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 55 insertions(+), 5 deletions(-) diff --git a/monitor/src/main.test.ts b/monitor/src/main.test.ts index 877a91bb8..e94e74010 100644 --- a/monitor/src/main.test.ts +++ b/monitor/src/main.test.ts @@ -76,6 +76,8 @@ function baseChain() { /** How far the confirmed head trails the wall clock. */ safeAgeSeconds: 2, paused: false, + /** Where set, the vault is paused from this block on, and at the latest block. */ + pausedFrom: undefined as bigint | undefined, threshold: 3n, guardian: GUARDIAN as string, totalReleased: 0n, @@ -135,7 +137,7 @@ function answer(base: Base, method: string, params: any[]): unknown { totalSwept: totalAt(base, base.totalSwept, "RemainderSwept", "amount", block), conversionFactor: CF, token: TOKEN, - paused: base.paused, + paused: base.paused || (base.pausedFrom !== undefined && (block === undefined || block >= base.pausedFrom)), threshold: base.threshold, guardian: base.guardian, activeApprovals: base.approvals.get(key) ?? 0n, @@ -784,6 +786,16 @@ test("a source whose chain leaves the blocks ingested so far pages critical and assert.deepEqual(persisted().migrations.map((migration) => migration.nonce), ["0"]); }); +test("a source whose finalized head falls behind the block ingested last is read at that block", async (t) => { + const { pendulum, api, monitor, pages, persisted } = await setup(t); + burn(pendulum, 0n, 100n * PEN); + await monitor.poll(api); + pendulum.finalized = 100; // a node that has not finalized block 101 yet + await monitor.poll(api); + assert.equal(pages.length, 0, JSON.stringify(pages)); + assert.equal(persisted().lastPendulumBlock, 101); +}); + test("a source header that does not hash to its block hash pages critical", async (t) => { const { pendulum, api, monitor, pages } = await setup(t); burn(pendulum, 0n, 100n * PEN, 102); @@ -1130,6 +1142,21 @@ test("a kept relay record past the source's finalized head binds the source whil assert.deepEqual(persisted().migrations.map((migration) => migration.nonce), ["0"]); }); +test("a kept relay record past the source's finalized head confirms the blocks below it while no newer one arrives", async (t) => { + const { pendulum, api, monitor, pages, persisted } = await relaySetup(t, { + head: () => ({ number: 103, hash: pendulumHash({}, 103) }), + }); + burn(pendulum, 0n, 100n * PEN); + pendulum.best = 103; // imported, not yet finalized by the source + await monitor.poll(api); + assert.equal(persisted().lastPendulumBlock, 101); + burn(pendulum, 1n, 100n * PEN, 102); // the source finalizes block 102; the record stays at 103 + await monitor.poll(api); + assert.equal(pages.length, 0, JSON.stringify(pages)); + assert.equal(persisted().lastPendulumBlock, 102); + assert.deepEqual(persisted().relayAnchor, { number: 103, hash: pendulumHash({}, 103) }); +}); + test("a relay record past the source's finalized head must descend from that head", async (t) => { const { pendulum, api, monitor, pages, persisted } = await relaySetup(t, { head: () => ({ number: 102, hash: pendulumHash({}, 102) }), @@ -1444,6 +1471,21 @@ test("while Pendulum cannot be read, a mismatching release of a known nonce paus assert.ok(pages.some((page) => page.subject === "AUTO-PAUSE UNAVAILABLE"), JSON.stringify(pages)); // the pause was asked for }); +test("a tuple violation whose pause is not confirmed is found again in the next poll", async (t) => { + const { base, pendulum, api, monitor, pages, persisted } = await setup(t); + burn(pendulum, 0n, 100n * PEN); + await monitor.poll(api); + const tokenAmount = 100n * PEN * CF; // a release of nonce 0 to another recipient + base.logs.push({ address: VAULT, eventName: "Released", args: { nonce: 0n, recipient: TOKEN, palletAmount: 100n * PEN, tokenAmount }, blockNumber: 1_015n }); + base.totalReleased += tokenAmount; + base.vaultBalance -= tokenAmount; + base.safe = 1_020n; + await monitor.poll(api); + await monitor.poll(api); + assert.equal(pages.filter((page) => page.subject === "MIGRATION TUPLE VIOLATION").length, 2, JSON.stringify(pages)); + assert.equal(persisted().baseFromBlock, "1011"); +}); + test("a release the source never shows pauses after the grace, with no approval in view", async (t) => { const { base, pendulum, api, monitor, pages } = await setup(t, { env: { UNMATCHED_EVENT_GRACE_SECONDS: "0" } }); pendulum.failure = new Error("node unreachable"); @@ -1916,10 +1958,18 @@ test("without a guardian key, a vault paused by hand after a treasury violation burn(pendulum, 0n, 100n * PEN, 101, { who: TREASURY_ACCOUNT, recipient: TOKEN }); await monitor.poll(api); assert.equal(persisted().lastPendulumBlock, 100); - base.paused = true; // operators pause the vault by hand - const before = pages.length; - await monitor.poll(api); - const subjects = pages.slice(before).map((page) => page.subject); + const pollSubjects = async () => { + const before = pages.length; + await monitor.poll(api); + return pages.slice(before).map((page) => page.subject); + }; + base.pausedFrom = 1_015n; // operators pause the vault by hand, after the confirmed Base block + let subjects = await pollSubjects(); + assert.ok(subjects.includes("AUTO-PAUSE PENDING FINALITY"), JSON.stringify(subjects)); + assert.ok(!subjects.includes("AUTO-PAUSE UNAVAILABLE"), JSON.stringify(subjects)); + assert.equal(persisted().lastPendulumBlock, 100); + base.safe = 1_020n; // the pause is confirmed + subjects = await pollSubjects(); assert.ok(subjects.includes("vault pause confirmed"), JSON.stringify(subjects)); assert.ok(!subjects.includes("AUTO-PAUSE UNAVAILABLE"), JSON.stringify(subjects)); assert.equal(persisted().lastPendulumBlock, 101); From 1f70338523101c7f790381ec431c0f4068bfa728 Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Thu, 22 Oct 2026 16:08:29 +0200 Subject: [PATCH 68/75] test(monitor): Cover the auto-pause transaction and its confirmation The Base stand-in accepts the guardian's pause transaction where a test opts in, mined in the confirmed block. A test pins that the auto-pause is reported as confirmed only where paused() is true at the confirmed block after the transaction, and as failed otherwise. --- monitor/src/main.test.ts | 57 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/monitor/src/main.test.ts b/monitor/src/main.test.ts index e94e74010..146b6ca00 100644 --- a/monitor/src/main.test.ts +++ b/monitor/src/main.test.ts @@ -47,6 +47,7 @@ const abi = parseAbi([ "function conversionFactor() view returns (uint256)", "function token() view returns (address)", "function paused() view returns (bool)", + "function pause()", "function threshold() view returns (uint256)", "function guardian() view returns (address)", "function activeApprovals(bytes32 payload) view returns (uint256)", @@ -97,6 +98,11 @@ function baseChain() { deployedAt: undefined as bigint | undefined, /** Requests to answer with an RPC error. */ failing: undefined as ((method: string, params: unknown[]) => boolean) | undefined, + /** Where set, the guardian's pause transaction is accepted and mined in the + * confirmed block; it pauses the vault there only where `true`. */ + pauseTakesEffect: undefined as boolean | undefined, + /** Hashes of the transactions sent. */ + sent: [] as string[], }; } @@ -112,6 +118,39 @@ type Base = ReturnType; function answer(base: Base, method: string, params: any[]): unknown { if (base.failing?.(method, params)) throw new Error("rate limited"); switch (method) { + case "eth_getTransactionCount": + return "0x0"; + case "eth_gasPrice": + return "0x1"; + case "eth_estimateGas": + return "0x5208"; + case "eth_blockNumber": + return toHex(base.safe); + case "eth_sendRawTransaction": { + if (base.pauseTakesEffect === undefined) break; + base.sent.push(keccak256(params[0])); + if (base.pauseTakesEffect) base.pausedFrom ??= base.safe; + return keccak256(params[0]); + } + case "eth_getTransactionReceipt": { + if (!base.sent.includes(params[0])) return null; + return { + transactionHash: params[0], + transactionIndex: "0x0", + blockHash: toHex(base.safe, { size: 32 }), + blockNumber: toHex(base.safe), + from: GUARDIAN, + to: VAULT, + cumulativeGasUsed: "0x5208", + gasUsed: "0x5208", + effectiveGasPrice: "0x1", + contractAddress: null, + logs: [], + logsBloom: `0x${"00".repeat(256)}`, + status: "0x1", + type: "0x0", + }; + } case "eth_chainId": return toHex(base.chainId); case "eth_getCode": { @@ -128,6 +167,10 @@ function answer(base: Base, method: string, params: any[]): unknown { } case "eth_call": { const { functionName, args } = decodeFunctionData({ abi, data: params[0].data }); + if (functionName === "pause") { + if (base.pauseTakesEffect === undefined) throw new Error("unmocked pause"); + return "0x"; + } const key = (args as readonly unknown[] | undefined)?.[0] as string; const block = String(params[1]).startsWith("0x") ? BigInt(params[1]) : undefined; const undeployed = base.deployedAt !== undefined && block !== undefined && block < base.deployedAt; @@ -634,6 +677,20 @@ test("a failing heartbeat never stops the monitor", async (t) => { assert.deepEqual([...subjects(), ...unreachable.subjects()], []); }); +test("an auto-pause counts as confirmed only where the vault is paused at the confirmed block", async (t) => { + for (const pauseTakesEffect of [true, false]) { + const { base, api, monitor, subjects } = await setup(t, { + env: { GUARDIAN_PRIVATE_KEY: GUARDIAN_KEY, BASE_FINALITY_POLL_MS: "10" }, + prepare: (base) => (base.pauseTakesEffect = pauseTakesEffect), + }); + base.vaultBalance -= 1n; + await monitor.poll(api); + assert.equal(base.sent.length, 1); + const outcome = subjects().filter((subject) => subject.includes("PAUSE") || subject.includes("pause")); + assert.deepEqual(outcome, [pauseTakesEffect ? "vault auto-pause confirmed" : "AUTO-PAUSE FAILED"]); + } +}); + test("a vault balance deficit pages and asks for the auto-pause", async (t) => { const { base, api, monitor, subjects } = await setup(t); base.vaultBalance -= 1n; From 1bd1425fbf4a5968ee03744b96b95cf5ed1568b6 Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Thu, 22 Oct 2026 18:53:36 +0200 Subject: [PATCH 69/75] fix(monitor): Count a relay RPC with a stale relay chain view as not answering --- monitor/README.md | 40 +++++++++++++++------------- monitor/src/main.test.ts | 57 +++++++++++++++++++++++++++------------- monitor/src/main.ts | 48 +++++++++++++++++++-------------- 3 files changed, 90 insertions(+), 55 deletions(-) diff --git a/monitor/README.md b/monitor/README.md index e4eccefdb..8fff820b2 100644 --- a/monitor/README.md +++ b/monitor/README.md @@ -46,19 +46,21 @@ itself: - **Relay-chain anchor** (`POLKADOT_WS`). Each poll reads the Pendulum head that the Polkadot relay chain recorded at its finalized block (`paras.heads(PENDULUM_PARA_ID)`), asking the relay RPCs in turn until one - head anchors the source. The source's chain must have that hash at that - height. A head above the source's finalized head counts once the source's - best chain holds it, and its chain must then pass through the finalized - head. The newest such head is kept in the state file, and the chain must - keep passing through it, also while it lies above the finalized head. - Blocks ingested while no relay RPC answered are tied to the next head that - arrives by the same walk; a head more than 100 blocks below the block - ingested last is not used. A head that contradicts the source counts only - when another relay RPC's head contradicts it too, or when `POLKADOT_WS` - lists one RPC. A relay RPC whose head anchors the source at or above the - contradicting head's height outvotes it: `RELAY RPCS DISAGREE` pages and - that head anchors the source. Any other answer leaves the contradiction - unconfirmed. No block above the newest anchored head is ingested. + head anchors the source. A relay RPC whose finalized relay block is older + than `SOURCE_STALE_ALERT_SECONDS` counts as not answering. The source's + chain must have that hash at that height. A head above the source's + finalized head counts once the source's best chain holds it, and its chain + must then pass through the finalized head. The newest such head is kept in + the state file, and the chain must keep passing through it, also while it + lies above the finalized head. Blocks ingested while no relay RPC answered + are tied to the next head that arrives by the same walk; a head more than + 100 blocks below the block ingested last is not used. A head that + contradicts the source counts only when another relay RPC's head + contradicts it too, or when `POLKADOT_WS` lists one RPC. A relay RPC whose + head anchors the source at or above the contradicting head's height + outvotes it: `RELAY RPCS DISAGREE` pages and that head anchors the source. + Any other answer leaves the contradiction unconfirmed. No block above the + newest anchored head is ingested. Every block the monitor ingests is confirmed by each checker that answers, and the values the checks use (the view's timestamp, `nextNonce`, @@ -68,9 +70,11 @@ unconfirmed, hold the source's newer blocks back and page `RELAY ANCHOR UNAVAILABLE`. A contradiction pages `CRITICAL: PENDULUM SOURCE CONTRADICTED`, nothing from the source is ingested, and the source counts as unreadable until it verifies again. A -second node or relay RPC that cannot be read only warns and holds nothing -back; with no checker answering, the monitor goes on with the source alone up -to its finalized head. `CRITICAL: RELAY ANCHOR STALE` pages once no head was +second node or relay RPC that cannot be read, or whose finalized head (for a +relay RPC, its finalized relay block) is older than +`SOURCE_STALE_ALERT_SECONDS`, only warns and holds nothing back; with no +checker answering, the monitor goes on with the source alone up to its +finalized head. `CRITICAL: RELAY ANCHOR STALE` pages once no head was anchored for `RELAY_ANCHOR_GRACE_SECONDS`, whatever the reason (relay RPCs down, the source behind the relay chain, a relay head that does not advance). @@ -89,7 +93,7 @@ down, the source behind the relay chain, a relay head that does not advance). | `GRACE_SECONDS` | Liveness alert threshold (default 30 min) | | `UNMATCHED_EVENT_GRACE_SECONDS` | Time a lagging or unreadable Pendulum source view gets to reveal the burn behind a safe Base event before a cannot-verify pause (default 10 min). The first sighting of an unmatched event pages immediately, so this is the window operators have to repair a stalled or failing node. Size it to your node-ops response time | | `SOURCE_CLOCK_SKEW_MARGIN_SECONDS` | Clock-skew allowance for the fabrication proof (default 120s): once the finalized Pendulum view has passed the moment the monitor first observed an unmatched Base event by this margin and the nonce still does not exist, the event is provably fabricated and pauses without waiting out the grace. The anchor is the monitor's own clock, not the Base block timestamp (which trails real time after a sequencer outage); a future-dated source view proves nothing | -| `SOURCE_STALE_ALERT_SECONDS` | Age of the verified Pendulum view (vs. wall clock) past which the monitor pages that it is going blind (default 300s); a second node whose finalized head is older counts as unavailable | +| `SOURCE_STALE_ALERT_SECONDS` | Age of the verified Pendulum view (vs. wall clock) past which the monitor pages that it is going blind (default 300s); a second node whose finalized head, or a relay RPC whose finalized relay block, is older counts as not answering | | `BASE_STALE_ALERT_SECONDS` | Age of the safe/finalized Base block (vs. wall clock) past which `BASE VIEW STALLED` pages (default 600s for `safe`, 1800s for `finalized`; Base's safe head normally trails by about a minute, its finalized head by about 20) | | `ISSUANCE_TOLERANCE` | Growth of Pendulum `totalIssuance + TotalMigrated` over its first-seen anchor (12-decimal pallet units) tolerated before `SOURCE SUPPLY GREW` pages (default 0) | | `ACKNOWLEDGED_RELEASE_EXCESS` | Excess of released over migrated, in token units (18 decimals), that operators quantified and accepted after a real loss; M2a fires only beyond it (default 0). Set it at the RB-3 sign-off and restart; the startup log shows it | @@ -128,7 +132,7 @@ subject that starts with `CRITICAL`; everything else is a `WARNING`. | `CRITICAL: BASE LOG SCAN INCOMPLETE`, `CRITICAL: BASE ADMIN SCAN FAILED` | For longer than `UNMATCHED_EVENT_GRACE_SECONDS`, a Base block range has been held, so no later Base event is matched against its burn, or the admin scan has failed, so no vault admin event or scheduled governance operation is paged while its veto window runs; fix the Base RPC | | `CRITICAL: PENDULUM SOURCE CONTRADICTED` | The Pendulum source contradicts the chain the monitor verified; stop trusting that node | | `PENDULUM CROSS-CHECK UNAVAILABLE` | The second Pendulum node does not answer or its finalized head is stale, so the source is used without that check | -| `RELAY ANCHOR UNAVAILABLE` | No relay RPC answers, so the source is used without the anchor; or none anchors the source's newer blocks, or a relay RPC records a head that contradicts the source and no other one confirms or outvotes it, so those blocks wait | +| `RELAY ANCHOR UNAVAILABLE` | No relay RPC answers from a finalized relay block younger than `SOURCE_STALE_ALERT_SECONDS`, so the source is used without the anchor; or none anchors the source's newer blocks, or a relay RPC records a head that contradicts the source and no other one confirms or outvotes it, so those blocks wait | | `RELAY RPCS DISAGREE` | One relay RPC records a Pendulum head that contradicts the source, and another records one at or above that height on the source's chain, which anchors the source. One of them is wrong; check both | | `CRITICAL: RELAY ANCHOR STALE` | No Pendulum head was checked against the relay chain for `RELAY_ANCHOR_GRACE_SECONDS` | | `AUTO-PAUSE NOT READY`, `AUTO-PAUSE DISABLED` | The next violation would not pause the vault automatically | diff --git a/monitor/src/main.test.ts b/monitor/src/main.test.ts index 146b6ca00..787de2c3e 100644 --- a/monitor/src/main.test.ts +++ b/monitor/src/main.test.ts @@ -443,9 +443,9 @@ function pendulumChain(chain: Pendulum = pendulumState()) { return { chain, api: api as unknown as ApiPromise }; } -/** A relay chain whose finalized block records `head()` as the Pendulum head; - * every read fails while `failing()` holds. */ -function relayChain(head: () => { number: number; hash: string }, failing = () => false): ApiPromise { +/** A relay chain whose finalized block, `ageMs()` old, records `head()` as the + * Pendulum head; every read fails while `failing()` holds. */ +function relayChain(head: () => { number: number; hash: string }, failing = () => false, ageMs = () => 0): ApiPromise { const check = () => { if (failing()) throw new Error("relay RPC unavailable"); }; @@ -453,6 +453,7 @@ function relayChain(head: () => { number: number; hash: string }, failing = () = rpc: { chain: { getFinalizedHead: async () => (check(), hash(`0x${"77".repeat(32)}`)) } }, at: async () => ({ query: { + timestamp: { now: async () => codec(Date.now() - ageMs()) }, paras: { heads: async (paraId: number) => { check(); @@ -879,23 +880,19 @@ test("a source header that names another block number pages critical", async (t) const SECOND_NODE = "ws://second-node.test:9944?apikey=secret"; /** A monitor whose second Pendulum node serves the source's chain, except for - * the values in `own`, which the test may change later; with `relayHead`, - * also against a relay chain recording it. */ -async function crossCheckSetup( - t: { after(fn: () => void): void }, - own: Partial = {}, - relayHead?: () => { number: number; hash: string }, -) { + * the values in `own`, which the test may change later; with `relays`, also + * against these relay RPCs, in this order. */ +async function crossCheckSetup(t: { after(fn: () => void): void }, own: Partial = {}, relays: ApiPromise[] = []) { let source: Pendulum | undefined; const chain = new Proxy({} as Pendulum, { get: (_target, key) => (key in own ? own[key as keyof Pendulum] : source![key as keyof Pendulum]), }); const second = pendulumChain(chain); - const relay = relayHead && relayChain(relayHead); + const urls = relays.map((_relay, index) => `wss://relay-${index + 1}.test`); const monitor = await setup(t, { - env: { PENDULUM_WS_CROSSCHECK: SECOND_NODE, ...(relay ? { POLKADOT_WS: "wss://relay.test" } : {}) }, + env: { PENDULUM_WS_CROSSCHECK: SECOND_NODE, ...(urls.length ? { POLKADOT_WS: urls.join(",") } : {}) }, prepare: (_base, pendulum) => (source = pendulum), - connect: async (url) => (url === SECOND_NODE ? second.api : (url === "wss://relay.test" && relay) || assert.fail(url)), + connect: async (url) => (url === SECOND_NODE ? second.api : relays[urls.indexOf(url)] ?? assert.fail(url)), }); return { ...monitor, own }; } @@ -999,11 +996,9 @@ test("a source on another chain one block past the second node's head ingests no test("with the second node and the relay chain, a block past both their heads waits, and a fork there is caught", async (t) => { let recorded = 101; - const { pendulum, api, monitor, pages, persisted, own } = await crossCheckSetup( - t, - { finalized: 101, forkFrom: undefined }, - () => ({ number: recorded, hash: pendulumHash({}, recorded) }), - ); + const { pendulum, api, monitor, pages, persisted, own } = await crossCheckSetup(t, { finalized: 101, forkFrom: undefined }, [ + relayChain(() => ({ number: recorded, hash: pendulumHash({}, recorded) })), + ]); burn(pendulum, 0n, 100n * PEN); pendulum.forkFrom = 102; // the source finalizes another block 102... burn(pendulum, 1n, 999n * PEN, 102); // ...with a fabricated burn @@ -1500,6 +1495,32 @@ test("a relay contradiction no other relay RPC can confirm warns, and the source assert.equal(persisted().lastPendulumBlock, 100); }); +test("a relay RPC whose finalized relay block is stale counts as not answering, and the second node still caps the source", async (t) => { + let relayAgeMs = 0; + const { base, pendulum, api, monitor, pages, persisted, own } = await crossCheckSetup(t, { finalized: 101 }, [ + relayChain(stuckHead, () => false, () => relayAgeMs), + relayChain(stuckHead, () => true), + ]); + burn(pendulum, 0n, 100n * PEN); + await monitor.poll(api); // relay RPC 1 anchors block 101 + assert.deepEqual(persisted().relayAnchor, { number: 101, hash: pendulumHash({}, 101) }); + relayAgeMs = 400_000; // relay RPC 1 stops following the relay chain at that record; relay RPC 2 is down + burn(pendulum, 1n, 100n * PEN, 103); + pendulum.finalized = 104; + own.finalized = 103; + release(base, 1n, 100n * PEN, 1_015n); + base.safe = 1_020n; + await monitor.poll(api); + assert.deepEqual(pages.map((page) => [page.subject, page.severity]), [["RELAY ANCHOR UNAVAILABLE", "WARNING"]]); + assert.match( + pages[0].detail, + /^the relay chain cannot be read: relay RPC 1: its finalized relay block is 400s old; relay RPC 2: Error: relay RPC unavailable; the Pendulum source is used without the relay-chain anchor/, + ); + assert.equal(persisted().lastPendulumBlock, 103); + assert.deepEqual(persisted().migrations.map((migration) => migration.nonce), ["0"]); + assert.equal(persisted().baseFromBlock, "1021"); +}); + test("while Pendulum cannot be read, a Base event with an unknown nonce pauses after the grace", async (t) => { const { base, pendulum, api, monitor, subjects } = await setup(t, { env: { UNMATCHED_EVENT_GRACE_SECONDS: "0" } }); pendulum.failure = new Error("node unreachable"); diff --git a/monitor/src/main.ts b/monitor/src/main.ts index d874b977e..20d70abf1 100644 --- a/monitor/src/main.ts +++ b/monitor/src/main.ts @@ -864,11 +864,12 @@ async function walkHeaders(api: ApiPromise, top: BlockRef, bottom: number): Prom } /** The Pendulum head that the relay chain behind the RPC at `url` recorded - * at its finalized block. */ -function relayHead(api: ApiPromise, url: string): Promise { + * at its finalized block, and the age of that block. */ +function relayHead(api: ApiPromise, url: string): Promise { return onNode(url, async (relay) => { - const relayFinalized = await relay.rpc.chain.getFinalizedHead(); - const head = (await (await relay.at(relayFinalized)).query.paras.heads(config.pendulumParaId)) as unknown as { + const relayAt = await relay.at(await relay.rpc.chain.getFinalizedHead()); + const timestamp = await relayAt.query.timestamp.now(); + const head = (await relayAt.query.paras.heads(config.pendulumParaId)) as unknown as { isSome: boolean; unwrap(): { toU8a(bare: boolean): Uint8Array }; }; @@ -878,7 +879,11 @@ function relayHead(api: ApiPromise, url: string): Promise { number: { toNumber(): number }; hash: { toHex(): string }; }; - return { number: header.number.toNumber(), hash: header.hash.toHex() }; + return { + number: header.number.toNumber(), + hash: header.hash.toHex(), + ageMs: Date.now() - Number(timestamp.toString().replaceAll(",", "")), + }; }); } @@ -936,18 +941,19 @@ async function relayVerdict( /** Anchors the source to the relay chain's record of the Pendulum head, kept * as `draft.relayAnchor`, from the first relay RPC in POLKADOT_WS whose - * record anchors it; while a record does not, the next RPC is asked. A - * record that contradicts the source counts only when another relay RPC's - * record contradicts it too, or when POLKADOT_WS lists one RPC. A record that - * anchors the source at or above the contradicting record's height outvotes - * it; any other answer leaves it unconfirmed. Resolves to the height up to - * which the relay chain confirms the source's chain `hashes`, walked down - * from `top` to `bottom`: the newest anchored head, or no higher than - * `bottom` before the first; undefined while no relay RPC answers. Pages - * while no new head is anchored and the source's blocks wait for one, a - * contradiction stays unconfirmed or no relay RPC answers: a warning, - * critical once none was anchored for RELAY_ANCHOR_GRACE_SECONDS; and when - * relay RPCs disagree. All throttled. */ + * record anchors it; while a record does not, the next RPC is asked. A relay + * RPC whose finalized relay block is older than SOURCE_STALE_ALERT_SECONDS + * counts as not answering. A record that contradicts the source counts only + * when another relay RPC's record contradicts it too, or when POLKADOT_WS + * lists one RPC. A record that anchors the source at or above the + * contradicting record's height outvotes it; any other answer leaves it + * unconfirmed. Resolves to the height up to which the relay chain confirms + * the source's chain `hashes`, walked down from `top` to `bottom`: the newest + * anchored head, or no higher than `bottom` before the first; undefined while + * no relay RPC answers. Pages while no new head is anchored and the source's + * blocks wait for one, a contradiction stays unconfirmed or no relay RPC + * answers: a warning, critical once none was anchored for + * RELAY_ANCHOR_GRACE_SECONDS; and when relay RPCs disagree. All throttled. */ async function anchorToRelay( api: ApiPromise, draft: RuntimeState, @@ -960,13 +966,17 @@ async function anchorToRelay( let answered = false; let contradiction: { rpc: number; height: number; error: SourceIntegrityError } | undefined; for (const [index, url] of config.polkadotWs.entries()) { - let head: BlockRef; + let head: BlockRef & { ageMs: number }; try { head = await relayHead(api, url); } catch (relayError) { answers.push(`relay RPC ${index + 1}: ${relayError}`); continue; } + if (head.ageMs > config.sourceStaleAlertMs) { + answers.push(`relay RPC ${index + 1}: its finalized relay block is ${Math.round(head.ageMs / 1000)}s old`); + continue; + } answered = true; let verdict: string | undefined; try { @@ -984,7 +994,7 @@ async function anchorToRelay( answers.push(`relay RPC ${index + 1}: ${verdict}`); continue; } - draft.relayAnchor = head; + draft.relayAnchor = { number: head.number, hash: head.hash }; lastRelayAnchorMs = Date.now(); if (contradiction && Date.now() - lastRelayDisagreementAlertMs > config.graceSeconds * 1000) { lastRelayDisagreementAlertMs = Date.now(); From 1d1627d7ee45407a4b30326bb02a98bff3e1f839 Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Thu, 22 Oct 2026 18:54:27 +0200 Subject: [PATCH 70/75] fix(monitor): Anchor the highest relay record while the anchored head lags the source --- monitor/README.md | 28 +++++++++-------- monitor/src/main.test.ts | 38 ++++++++++++++++++----- monitor/src/main.ts | 66 +++++++++++++++++++++------------------- 3 files changed, 80 insertions(+), 52 deletions(-) diff --git a/monitor/README.md b/monitor/README.md index 8fff820b2..b198f0bcc 100644 --- a/monitor/README.md +++ b/monitor/README.md @@ -45,22 +45,24 @@ itself: `SOURCE_STALE_ALERT_SECONDS` counts as unavailable. - **Relay-chain anchor** (`POLKADOT_WS`). Each poll reads the Pendulum head that the Polkadot relay chain recorded at its finalized block - (`paras.heads(PENDULUM_PARA_ID)`), asking the relay RPCs in turn until one - head anchors the source. A relay RPC whose finalized relay block is older - than `SOURCE_STALE_ALERT_SECONDS` counts as not answering. The source's - chain must have that hash at that height. A head above the source's - finalized head counts once the source's best chain holds it, and its chain - must then pass through the finalized head. The newest such head is kept in - the state file, and the chain must keep passing through it, also while it - lies above the finalized head. Blocks ingested while no relay RPC answered - are tied to the next head that arrives by the same walk; a head more than - 100 blocks below the block ingested last is not used. A head that - contradicts the source counts only when another relay RPC's head + (`paras.heads(PENDULUM_PARA_ID)`), asking the relay RPCs in turn until a + head at or above the source's finalized head anchors it; the highest head + that anchors the source is used. A relay RPC whose finalized relay block is + older than `SOURCE_STALE_ALERT_SECONDS` counts as not answering. The + source's chain must have that hash at that height. A head above the + source's finalized head counts once the source's best chain holds it, and + its chain must then pass through the finalized head. The newest such head + is kept in the state file, and the chain must keep passing through it, + also while it lies above the finalized head. Blocks ingested while no relay + RPC answered are tied to the next head that arrives by the same walk; a + head more than 100 blocks below the block ingested last is not used. A + head that contradicts the source counts only when another relay RPC's head contradicts it too, or when `POLKADOT_WS` lists one RPC. A relay RPC whose head anchors the source at or above the contradicting head's height outvotes it: `RELAY RPCS DISAGREE` pages and that head anchors the source. - Any other answer leaves the contradiction unconfirmed. No block above the - newest anchored head is ingested. + Any other answer leaves the contradiction unconfirmed; a lower head still + anchors the source up to its own height. No block above the newest + anchored head is ingested. Every block the monitor ingests is confirmed by each checker that answers, and the values the checks use (the view's timestamp, `nextNonce`, diff --git a/monitor/src/main.test.ts b/monitor/src/main.test.ts index 787de2c3e..66bb55ff7 100644 --- a/monitor/src/main.test.ts +++ b/monitor/src/main.test.ts @@ -1436,15 +1436,23 @@ test("a relay contradiction the next relay RPC outvotes warns, and that RPC's re assert.equal(persisted().lastPendulumBlock, 101); }); -test("a relay record below a contradicting one does not outvote it", async (t) => { +test("a relay record below a contradicting one does not outvote it, and anchors the source up to its own height", async (t) => { const below = (pendulum: Pendulum) => ({ number: pendulum.finalized - 1, hash: pendulumHash({}, pendulum.finalized - 1) }); - const { pendulum, api, monitor, pages, persisted } = await twoRelaySetup(t, [forkedHead, below]); - burn(pendulum, 0n, 100n * PEN, 102); - await monitor.poll(api); - assert.deepEqual(pages.map((page) => page.subject), ["RELAY ANCHOR UNAVAILABLE"]); - assert.match(pages[0].detail, /no other relay RPC confirms or outvotes it \(relay RPC 2: its Pendulum head 101 lies below the contradicting record\);/); - assert.equal(persisted().relayAnchor, undefined); - assert.equal(persisted().lastPendulumBlock, 100); + for (const [heads, contradicting, lower] of [[[forkedHead, below], 1, 2], [[below, forkedHead], 2, 1]] as const) { + const { pendulum, api, monitor, pages, persisted } = await twoRelaySetup(t, [...heads]); + burn(pendulum, 0n, 100n * PEN, 102); + await monitor.poll(api); + assert.deepEqual(pages.map((page) => page.subject), ["RELAY ANCHOR UNAVAILABLE"]); + assert.match( + pages[0].detail, + new RegExp( + `^relay RPC ${contradicting} reports that .* no other relay RPC confirms or outvotes it \\(relay RPC ${lower}: ` + + "its Pendulum head 101 lies below the contradicting record\\); no Pendulum block after 101 is ingested", + ), + ); + assert.deepEqual(persisted().relayAnchor, { number: 101, hash: pendulumHash({}, 101) }); + assert.equal(persisted().lastPendulumBlock, 101); + } }); /** A relay RPC whose record stays at the source's block 101. */ @@ -1478,6 +1486,20 @@ test("a relay RPC whose record does not anchor the source leaves it to the next assert.equal(persisted().lastPendulumBlock, 102); }); +test("a relay RPC whose record lags the source's head leaves the newer blocks to the next one", async (t) => { + const lagging = (pendulum: Pendulum) => ({ number: pendulum.finalized - 70, hash: pendulumHash({}, pendulum.finalized - 70) }); + const { base, pendulum, api, monitor, pages, persisted } = await twoRelaySetup(t, [lagging, honestHead]); + burn(pendulum, 0n, 100n * PEN, 190); + pendulum.finalized = 200; + release(base, 0n, 100n * PEN); + await monitor.poll(api); + assert.equal(pages.length, 0, JSON.stringify(pages)); + assert.deepEqual(persisted().relayAnchor, { number: 200, hash: pendulumHash({}, 200) }); + assert.equal(persisted().lastPendulumBlock, 200); + assert.deepEqual(persisted().migrations, []); + assert.equal(persisted().baseFromBlock, "1011"); +}); + test("a relay contradiction the next relay RPC confirms pages critical", async (t) => { const { pendulum, api, monitor, pages, persisted } = await twoRelaySetup(t, [forkedHead, forkedHead]); burn(pendulum, 0n, 100n * PEN); diff --git a/monitor/src/main.ts b/monitor/src/main.ts index 20d70abf1..b9cb39feb 100644 --- a/monitor/src/main.ts +++ b/monitor/src/main.ts @@ -940,20 +940,21 @@ async function relayVerdict( } /** Anchors the source to the relay chain's record of the Pendulum head, kept - * as `draft.relayAnchor`, from the first relay RPC in POLKADOT_WS whose - * record anchors it; while a record does not, the next RPC is asked. A relay - * RPC whose finalized relay block is older than SOURCE_STALE_ALERT_SECONDS - * counts as not answering. A record that contradicts the source counts only - * when another relay RPC's record contradicts it too, or when POLKADOT_WS - * lists one RPC. A record that anchors the source at or above the - * contradicting record's height outvotes it; any other answer leaves it - * unconfirmed. Resolves to the height up to which the relay chain confirms - * the source's chain `hashes`, walked down from `top` to `bottom`: the newest - * anchored head, or no higher than `bottom` before the first; undefined while - * no relay RPC answers. Pages while no new head is anchored and the source's - * blocks wait for one, a contradiction stays unconfirmed or no relay RPC - * answers: a warning, critical once none was anchored for - * RELAY_ANCHOR_GRACE_SECONDS; and when relay RPCs disagree. All throttled. */ + * as `draft.relayAnchor`: the highest record that anchors it, from the relay + * RPCs in POLKADOT_WS, asked in turn until a record anchors the source's + * head `top`. A relay RPC whose finalized relay block is older than + * SOURCE_STALE_ALERT_SECONDS counts as not answering. A record that + * contradicts the source counts only when another relay RPC's record + * contradicts it too, or when POLKADOT_WS lists one RPC. A record that + * anchors the source at or above the contradicting record's height outvotes + * it; otherwise it stays unconfirmed. Resolves to the height up to which the + * relay chain confirms the source's chain `hashes`, walked down from `top` to + * `bottom`: the newest anchored head, or no higher than `bottom` before the + * first; undefined while no relay RPC answers. Pages while no new head is + * anchored and the source's blocks wait for one, a contradiction stays + * unconfirmed or no relay RPC answers: a warning, critical once none was + * anchored for RELAY_ANCHOR_GRACE_SECONDS; and when relay RPCs disagree. All + * throttled. */ async function anchorToRelay( api: ApiPromise, draft: RuntimeState, @@ -964,8 +965,10 @@ async function anchorToRelay( if (!config.polkadotWs.length) return undefined; const answers: string[] = []; let answered = false; + let anchor: (BlockRef & { rpc: number }) | undefined; let contradiction: { rpc: number; height: number; error: SourceIntegrityError } | undefined; for (const [index, url] of config.polkadotWs.entries()) { + if (anchor && anchor.number >= Math.max(top.number, contradiction?.height ?? 0)) break; let head: BlockRef & { ageMs: number }; try { head = await relayHead(api, url); @@ -987,25 +990,26 @@ async function anchorToRelay( contradiction = { rpc: index + 1, height: head.number, error: verdictError }; continue; } - if (verdict === undefined && contradiction && head.number < contradiction.height) { - verdict = `its Pendulum head ${head.number} lies below the contradicting record`; - } - if (verdict !== undefined) { - answers.push(`relay RPC ${index + 1}: ${verdict}`); - continue; - } - draft.relayAnchor = { number: head.number, hash: head.hash }; + if (verdict !== undefined) answers.push(`relay RPC ${index + 1}: ${verdict}`); + else if (!anchor || head.number > anchor.number) anchor = { number: head.number, hash: head.hash, rpc: index + 1 }; + } + if (anchor) { + draft.relayAnchor = { number: anchor.number, hash: anchor.hash }; lastRelayAnchorMs = Date.now(); - if (contradiction && Date.now() - lastRelayDisagreementAlertMs > config.graceSeconds * 1000) { - lastRelayDisagreementAlertMs = Date.now(); - await alert( - "RELAY RPCS DISAGREE", - `relay RPC ${contradiction.rpc} reports that ${contradiction.error.message}; relay RPC ${index + 1} records ` + - `${head.hash} as Pendulum block ${head.number}, on the source's chain, and the source is anchored to ` + - "that record — one of the two relay RPCs is wrong, check both", - ); + if (!contradiction) return anchor.number; + if (anchor.number >= contradiction.height) { + if (Date.now() - lastRelayDisagreementAlertMs > config.graceSeconds * 1000) { + lastRelayDisagreementAlertMs = Date.now(); + await alert( + "RELAY RPCS DISAGREE", + `relay RPC ${contradiction.rpc} reports that ${contradiction.error.message}; relay RPC ${anchor.rpc} records ` + + `${anchor.hash} as Pendulum block ${anchor.number}, on the source's chain, and the source is anchored to ` + + "that record — one of the two relay RPCs is wrong, check both", + ); + } + return anchor.number; } - return head.number; + answers.push(`relay RPC ${anchor.rpc}: its Pendulum head ${anchor.number} lies below the contradicting record`); } const reason = contradiction ? `relay RPC ${contradiction.rpc} reports that ${contradiction.error.message}, and no other relay RPC confirms ` + From 492513a07e57cf398a98efec805980dadc59abb5 Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Thu, 22 Oct 2026 18:55:24 +0200 Subject: [PATCH 71/75] fix(monitor): Name the checker that holds a stalled Pendulum view back --- monitor/README.md | 5 +++-- monitor/src/main.test.ts | 24 +++++++++++++++--------- monitor/src/main.ts | 27 +++++++++++++++++---------- 3 files changed, 35 insertions(+), 21 deletions(-) diff --git a/monitor/README.md b/monitor/README.md index b198f0bcc..993a7c54a 100644 --- a/monitor/README.md +++ b/monitor/README.md @@ -15,7 +15,7 @@ Checks every poll: | Source supply | `totalIssuance + TotalMigrated` on Pendulum must never grow past its first-seen anchor. | Alert (`SOURCE SUPPLY GREW`) | | Unreleasable burns | A burn to the zero or vault address can never be approved or released; it is paged once and excluded from the pending count so it cannot page liveness forever or block RB-7. | Alert once | | Treasury destination | With `EXPECTED_TREASURY_DESTINATION` set, every finalized treasury migration (a `MigrationInitiated` whose `who` is the treasury account, `PalletId(*b"py/trsry")`) must pay that Base address. It is checked when the burn is ingested, before the attestors can release it; while the pause is not confirmed, the monitor holds at that block, and the violation pages again every poll saying so. | Alert + auto-pause (`TREASURY DESTINATION VIOLATION`) | -| Pendulum source | The finalized Pendulum view must stay fresh and readable. While it cannot be read (an RPC error, a nonce discontinuity, an event-shape change), the Base-side checks — vault balance and its auto-pause, Base view, auto-pause readiness, vault admin events, governance queue — still run, and Base events are reconciled against the last view read in full: an unreadable source counts as a stalled one, so a mismatch on a known nonce counts at once and an unknown nonce after `UNMATCHED_EVENT_GRACE_SECONDS`. M2a and liveness wait for the source. | Alert, throttled: `PENDULUM SOURCE VIEW STALLED` once the view is older than `SOURCE_STALE_ALERT_SECONDS`, `PENDULUM SOURCE UNREADABLE` on a failure | +| Pendulum source | The finalized Pendulum view must stay fresh and readable. While it cannot be read (an RPC error, a nonce discontinuity, an event-shape change), the Base-side checks — vault balance and its auto-pause, Base view, auto-pause readiness, vault admin events, governance queue — still run, and Base events are reconciled against the last view read in full: an unreadable source counts as a stalled one, so a mismatch on a known nonce counts at once and an unknown nonce after `UNMATCHED_EVENT_GRACE_SECONDS`. M2a and liveness wait for the source. | Alert, throttled: `PENDULUM SOURCE VIEW STALLED` once the view is older than `SOURCE_STALE_ALERT_SECONDS` (naming the second node or the relay RPCs when they hold the view back), `PENDULUM SOURCE UNREADABLE` on a failure | | Pendulum chain | Every finalized block the monitor ingests is read at a hash it verified against the blocks it ingested before and, where configured, a second node and the relay chain; a second node must also agree on the migration totals and on the migration events of each ingested block (see [Pendulum chain verification](#pendulum-chain-verification)). | Alert (`CRITICAL: PENDULUM SOURCE CONTRADICTED`), throttled; nothing from the source is ingested, and it is treated as unreadable (see Pendulum source) until it verifies again. A second node or relay RPC that does not answer only warns, throttled (`PENDULUM CROSS-CHECK UNAVAILABLE`, `RELAY ANCHOR UNAVAILABLE`), and so does a relay RPC outvoted by another (`RELAY RPCS DISAGREE`); relay RPCs that answer without anchoring the source's newer blocks hold them back (`RELAY ANCHOR UNAVAILABLE`); `CRITICAL: RELAY ANCHOR STALE` once no head was anchored for `RELAY_ANCHOR_GRACE_SECONDS` | | Base log completeness | In every scanned block range the `Released` amounts must add up to the growth of `totalReleased`, and the `RemainderSwept` amounts to that of `totalSwept`, between the block before the range and its last block. A range whose logs do not (a backend serving truncated logs) is read once more, then held: the Base cursor stays on it and liveness waits until a read accounts for the totals; M2a still runs. | Alert (`BASE LOG SCAN INCOMPLETE`), throttled; `CRITICAL: BASE LOG SCAN INCOMPLETE` once a range has been held for longer than `UNMATCHED_EVENT_GRACE_SECONDS`, since no later Base event is matched against its burn meanwhile | | Base view | The safe/finalized Base block must not be older than `BASE_STALE_ALERT_SECONDS`: a view that stops advancing hides every later vault event while every check still passes at the old block. | Alert (`BASE VIEW STALLED`), throttled | @@ -166,7 +166,8 @@ the monitor then pages `PENDULUM SOURCE UNREADABLE` on restart. Recovery is pointing `PENDULUM_WS` at an archive node, never editing the state file. A monitor whose *running* source view stalls pages `PENDULUM SOURCE VIEW STALLED` well before the unmatched-event grace can force a cannot-verify -pause; treat that page as "restore the node now". The Base-side checks keep +pause; treat that page as "restore the node now", or the second node or +relay RPCs it names as holding the view back. The Base-side checks keep running while the source cannot be read, and an unreadable source counts as a stalled one: once a Base event it cannot verify appears, the monitor pages `UNVERIFIED BASE EVENT` and pauses when the grace runs out, so treat diff --git a/monitor/src/main.test.ts b/monitor/src/main.test.ts index 66bb55ff7..3d6798f41 100644 --- a/monitor/src/main.test.ts +++ b/monitor/src/main.test.ts @@ -1282,15 +1282,21 @@ test("a source that finalizes below a relay record it does not hold ingests noth assert.deepEqual(persisted().relayAnchor, { number: 101, hash: pendulumHash({}, 101) }); }); -test("a Pendulum view the relay chain holds back pages as stalled", async (t) => { - const { pendulum, api, monitor, pages } = await relaySetup(t, { head: () => ({ number: 101, hash: pendulumHash({}, 101) }) }); - burn(pendulum, 0n, 100n * PEN); - pendulum.finalized = 130; // 29 blocks of 12 s past the relay chain's record - await monitor.poll(api); - assert.match( - pages.find((page) => page.subject === "PENDULUM SOURCE VIEW STALLED")?.detail ?? "", - /^verified Pendulum block 101 is 34\ds old \(the source finalized 130, not confirmed yet\);/, - ); +test("a Pendulum view a checker holds back pages as stalled, naming that checker", async (t) => { + const relayed = await relaySetup(t, { head: () => ({ number: 101, hash: pendulumHash({}, 101) }) }); + const crossChecked = await crossCheckSetup(t, { finalized: 101 }); + for (const [{ pendulum, api, monitor, pages }, checker] of [ + [relayed, "the relay RPCs \\(POLKADOT_WS\\)"], + [crossChecked, "the second node \\(PENDULUM_WS_CROSSCHECK\\)"], + ] as const) { + burn(pendulum, 0n, 100n * PEN); + pendulum.finalized = 130; // 29 blocks of 12 s past the checker's head + await monitor.poll(api); + assert.match( + pages.find((page) => page.subject === "PENDULUM SOURCE VIEW STALLED")?.detail ?? "", + new RegExp(`^verified Pendulum block 101 is 34\\ds old \\(the source finalized 130, not confirmed yet by ${checker}\\);.* — check ${checker} before`), + ); + } }); test("a Base event whose burn the relay chain has not confirmed yet is not proven fabricated by a newer source head", async (t) => { diff --git a/monitor/src/main.ts b/monitor/src/main.ts index b9cb39feb..30c336c04 100644 --- a/monitor/src/main.ts +++ b/monitor/src/main.ts @@ -1045,14 +1045,14 @@ async function anchorToRelay( * block ingested before and with the relay-chain record kept from an earlier * poll, and the newest of them that every checker that answers confirms * (`confirmed`, at most the source's finalized head): the second node up to - * `crossChecked`, and the relay chain, whose newest record anchors the - * source. A checker that does not answer confirms nothing and holds nothing - * back. */ + * `crossChecked`, and the relay chain up to `anchored`, its newest record + * that anchors the source. A checker that does not answer confirms nothing + * and holds nothing back. */ async function sourceHashes( api: ApiPromise, draft: RuntimeState, finalized: BlockRef, -): Promise<{ hashes: Map; confirmed: number; crossChecked: number | undefined }> { +): Promise<{ hashes: Map; confirmed: number; crossChecked: number | undefined; anchored: number | undefined }> { // A kept record past the finalized head binds the source too: the walk starts there. const known = draft.relayAnchor; const top = known && known.number > finalized.number ? known : finalized; @@ -1072,7 +1072,7 @@ async function sourceHashes( const lowest = Math.min(top.number, draft.lastPendulumBlock); const crossChecked = await crossCheckSource(hashes, top, lowest); const anchored = await anchorToRelay(api, draft, hashes, top, { number: lowest, hash: hashes.get(lowest) as string }); - return { hashes, confirmed: Math.min(finalized.number, crossChecked ?? Infinity, anchored ?? Infinity), crossChecked }; + return { hashes, confirmed: Math.min(finalized.number, crossChecked ?? Infinity, anchored ?? Infinity), crossChecked, anchored }; } /** The migration pallet's admin events, each with what to check. */ @@ -1721,7 +1721,7 @@ async function readSource(api: ApiPromise, draft: RuntimeState): Promise config.sourceStaleAlertMs && Date.now() - lastSourceStallAlertMs > config.unmatchedEventGraceSeconds * 1000 ) { lastSourceStallAlertMs = Date.now(); + // The checkers that confirm less than the source finalized hold the view back. + const holding = finalized.number <= viewBlock ? "" : [ + crossChecked !== undefined && crossChecked < finalized.number && "the second node (PENDULUM_WS_CROSSCHECK)", + anchored !== undefined && anchored < finalized.number && "the relay RPCs (POLKADOT_WS)", + ].filter(Boolean).join(" and "); await alert( "PENDULUM SOURCE VIEW STALLED", `verified Pendulum block ${viewBlock} is ${Math.round((Date.now() - sourceFinalizedTsMs) / 1000)}s old` + - (finalized.number > viewBlock ? ` (the source finalized ${finalized.number}, not confirmed yet)` : "") + - `; the monitor cannot verify new Base events against a stalled source — restore the node before ` + - `the ${config.unmatchedEventGraceSeconds}s unmatched-event grace forces a pause`, + (holding ? ` (the source finalized ${finalized.number}, not confirmed yet by ${holding})` : "") + + `; the monitor cannot verify new Base events against a stalled view — ` + + `${holding ? `check ${holding}` : "restore the node"} before the ${config.unmatchedEventGraceSeconds}s ` + + "unmatched-event grace forces a pause", ); } const [totals, totalIssuanceValue] = await Promise.all([migrationTotals(apiAt), apiAt.query.balances.totalIssuance()]); From 6c7ce15f22f717f758f8c4bb26eac8af25285acf Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Thu, 22 Oct 2026 18:55:53 +0200 Subject: [PATCH 72/75] test(monitor): Cover a second node that disagrees on the block ingested last --- monitor/src/main.test.ts | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/monitor/src/main.test.ts b/monitor/src/main.test.ts index 3d6798f41..005345fd3 100644 --- a/monitor/src/main.test.ts +++ b/monitor/src/main.test.ts @@ -913,6 +913,21 @@ test("a second node that disagrees on the newest block pages critical and nothin assert.equal(persisted().lastPendulumBlock, 101); }); +test("a second node that disagrees on the block ingested last, its finalized head, pages critical", async (t) => { + const { pendulum, api, monitor, pages, persisted, own } = await crossCheckSetup(t, { finalized: 101 }); + burn(pendulum, 0n, 100n * PEN); + await monitor.poll(api); + assert.equal(persisted().lastPendulumBlock, 101); + own.forkFrom = 101; // the second node now has another block 101 + burn(pendulum, 1n, 100n * PEN, 102); + await monitor.poll(api); + assert.match( + pages.find((page) => page.subject === "CRITICAL: PENDULUM SOURCE CONTRADICTED")?.detail ?? "", + /second Pendulum node has 0x0+10+65 at block 101, the source 0x0+65;/, + ); + assert.equal(persisted().lastPendulumBlock, 101); +}); + test("a second node that cannot answer only warns", async (t) => { let connections = 0; const { pendulum, api, monitor, pages, persisted, heartbeats } = await setup(t, { From 3b70784e07f595506f66260739807126babcdb93 Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Thu, 22 Oct 2026 19:06:27 +0200 Subject: [PATCH 73/75] test(monitor): Cover a Base event the source view provably lacks --- monitor/src/main.test.ts | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/monitor/src/main.test.ts b/monitor/src/main.test.ts index 005345fd3..b21f75fc4 100644 --- a/monitor/src/main.test.ts +++ b/monitor/src/main.test.ts @@ -1564,6 +1564,23 @@ test("a relay RPC whose finalized relay block is stale counts as not answering, assert.equal(persisted().baseFromBlock, "1021"); }); +test("a Base event whose nonce the source view passed without pauses before the grace, as fabricated", async (t) => { + const { base, pendulum, api, monitor, pages, subjects } = await setup(t, { env: { SOURCE_CLOCK_SKEW_MARGIN_SECONDS: "0" } }); + burn(pendulum, 0n, 100n * PEN); + pendulum.headAgeMs = 1_000; // the view trails the first sighting + release(base, 7n, 5_000n * PEN); // no burn behind it + await monitor.poll(api); + assert.ok(subjects().includes("UNVERIFIED BASE EVENT"), JSON.stringify(subjects())); + assert.ok(!subjects().includes("MIGRATION TUPLE VIOLATION"), JSON.stringify(subjects())); + pendulum.headAgeMs = 0; + await new Promise((resolve) => setTimeout(resolve, 5)); + await monitor.poll(api); // the view has passed the first sighting, still without nonce 7 + assert.match( + pages.find((page) => page.subject === "MIGRATION TUPLE VIOLATION")?.detail ?? "", + /nonce 7 .*source view has passed the event's timestamp — fabricated/, + ); +}); + test("while Pendulum cannot be read, a Base event with an unknown nonce pauses after the grace", async (t) => { const { base, pendulum, api, monitor, subjects } = await setup(t, { env: { UNMATCHED_EVENT_GRACE_SECONDS: "0" } }); pendulum.failure = new Error("node unreachable"); From fc5f5b0ef49d00c0edee18498b49a9ccefd9128f Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Thu, 22 Oct 2026 19:06:52 +0200 Subject: [PATCH 74/75] test(monitor): Cover a relay contradiction confirmed past a lower relay record --- monitor/src/main.test.ts | 36 ++++++++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/monitor/src/main.test.ts b/monitor/src/main.test.ts index b21f75fc4..2ef2ed682 100644 --- a/monitor/src/main.test.ts +++ b/monitor/src/main.test.ts @@ -1430,10 +1430,10 @@ test("the relay RPCs are tried in order", async (t) => { assert.equal(persisted().relayAnchor?.number, 101); }); -/** A monitor with two relay RPCs; each records `head` or, without one, is down. */ -async function twoRelaySetup(t: { after(fn: () => void): void }, heads: Array<((pendulum: Pendulum) => Head) | undefined>) { +/** A monitor with relay RPCs, in this order; each records `head` or, without one, is down. */ +async function relayRpcsSetup(t: { after(fn: () => void): void }, heads: Array<((pendulum: Pendulum) => Head) | undefined>) { let chain: Pendulum | undefined; - const urls = ["wss://first.test", "wss://second.test"]; + const urls = heads.map((_head, index) => `wss://relay-${index + 1}.test`); const relays = heads.map((head) => relayChain(() => head!(chain!), () => !head)); return setup(t, { env: { POLKADOT_WS: urls.join(",") }, @@ -1445,7 +1445,7 @@ const honestHead = (pendulum: Pendulum) => ({ number: pendulum.finalized, hash: const forkedHead = (pendulum: Pendulum) => ({ number: pendulum.finalized, hash: pendulumHash({ forkFrom: 0 }, pendulum.finalized) }); test("a relay contradiction the next relay RPC outvotes warns, and that RPC's record anchors the source", async (t) => { - const { pendulum, api, monitor, pages, persisted } = await twoRelaySetup(t, [forkedHead, honestHead]); + const { pendulum, api, monitor, pages, persisted } = await relayRpcsSetup(t, [forkedHead, honestHead]); burn(pendulum, 0n, 100n * PEN); await monitor.poll(api); assert.deepEqual(pages.map((page) => [page.subject, page.severity]), [["RELAY RPCS DISAGREE", "WARNING"]]); @@ -1460,7 +1460,7 @@ test("a relay contradiction the next relay RPC outvotes warns, and that RPC's re test("a relay record below a contradicting one does not outvote it, and anchors the source up to its own height", async (t) => { const below = (pendulum: Pendulum) => ({ number: pendulum.finalized - 1, hash: pendulumHash({}, pendulum.finalized - 1) }); for (const [heads, contradicting, lower] of [[[forkedHead, below], 1, 2], [[below, forkedHead], 2, 1]] as const) { - const { pendulum, api, monitor, pages, persisted } = await twoRelaySetup(t, [...heads]); + const { pendulum, api, monitor, pages, persisted } = await relayRpcsSetup(t, [...heads]); burn(pendulum, 0n, 100n * PEN, 102); await monitor.poll(api); assert.deepEqual(pages.map((page) => page.subject), ["RELAY ANCHOR UNAVAILABLE"]); @@ -1480,7 +1480,7 @@ test("a relay record below a contradicting one does not outvote it, and anchors const stuckHead = () => ({ number: 101, hash: pendulumHash({}, 101) }); test("a relay contradiction that only a stuck relay RPC answers stays unconfirmed", async (t) => { - const { pendulum, api, monitor, pages, persisted } = await twoRelaySetup(t, [honestHead, stuckHead]); + const { pendulum, api, monitor, pages, persisted } = await relayRpcsSetup(t, [honestHead, stuckHead]); burn(pendulum, 0n, 100n * PEN); await monitor.poll(api); // relay RPC 1 anchors block 101 pendulum.forkFrom = 102; // the source moves to another chain... @@ -1497,7 +1497,7 @@ test("a relay contradiction that only a stuck relay RPC answers stays unconfirme }); test("a relay RPC whose record does not anchor the source leaves it to the next one", async (t) => { - const { pendulum, api, monitor, pages, persisted } = await twoRelaySetup(t, [stuckHead, honestHead]); + const { pendulum, api, monitor, pages, persisted } = await relayRpcsSetup(t, [stuckHead, honestHead]); burn(pendulum, 0n, 100n * PEN); await monitor.poll(api); // relay RPC 1 anchors block 101 burn(pendulum, 1n, 100n * PEN, 102); @@ -1509,7 +1509,7 @@ test("a relay RPC whose record does not anchor the source leaves it to the next test("a relay RPC whose record lags the source's head leaves the newer blocks to the next one", async (t) => { const lagging = (pendulum: Pendulum) => ({ number: pendulum.finalized - 70, hash: pendulumHash({}, pendulum.finalized - 70) }); - const { base, pendulum, api, monitor, pages, persisted } = await twoRelaySetup(t, [lagging, honestHead]); + const { base, pendulum, api, monitor, pages, persisted } = await relayRpcsSetup(t, [lagging, honestHead]); burn(pendulum, 0n, 100n * PEN, 190); pendulum.finalized = 200; release(base, 0n, 100n * PEN); @@ -1522,15 +1522,31 @@ test("a relay RPC whose record lags the source's head leaves the newer blocks to }); test("a relay contradiction the next relay RPC confirms pages critical", async (t) => { - const { pendulum, api, monitor, pages, persisted } = await twoRelaySetup(t, [forkedHead, forkedHead]); + const { pendulum, api, monitor, pages, persisted } = await relayRpcsSetup(t, [forkedHead, forkedHead]); burn(pendulum, 0n, 100n * PEN); await monitor.poll(api); assert.ok(pages.some((page) => page.subject === "CRITICAL: PENDULUM SOURCE CONTRADICTED"), JSON.stringify(pages)); assert.equal(persisted().lastPendulumBlock, 100); }); +test("a relay contradiction above the source's head is confirmed by a later relay RPC, though a record between anchors", async (t) => { + const above = () => ({ number: 102, hash: pendulumHash({}, 102) }); + const onSourceChain = (pendulum: Pendulum) => ({ number: 101, hash: pendulumHash(pendulum, 101) }); + const { pendulum, api, monitor, pages, persisted } = await relayRpcsSetup(t, [above, onSourceChain, above]); + burn(pendulum, 0n, 100n * PEN); + pendulum.forkFrom = 101; // its finalized block 101 is on another chain... + pendulum.canonicalBlockHashes = true; // ...though its best chain holds the relay chain's block 102 + pendulum.best = 102; + await monitor.poll(api); + assert.match( + pages.find((page) => page.subject === "CRITICAL: PENDULUM SOURCE CONTRADICTED")?.detail ?? "", + /records 0x0+66 as Pendulum block 102, whose chain has 0x0+65 at block 101, where the source has 0x0+10+65;/, + ); + assert.equal(persisted().lastPendulumBlock, 100); +}); + test("a relay contradiction no other relay RPC can confirm warns, and the source's blocks wait", async (t) => { - const { pendulum, api, monitor, pages, persisted } = await twoRelaySetup(t, [forkedHead, undefined]); + const { pendulum, api, monitor, pages, persisted } = await relayRpcsSetup(t, [forkedHead, undefined]); burn(pendulum, 0n, 100n * PEN); await monitor.poll(api); assert.deepEqual(pages.map((page) => page.subject), ["RELAY ANCHOR UNAVAILABLE"]); From 480cc50441c9f7458b7b68524b565e83002d6b77 Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Thu, 22 Oct 2026 19:32:32 +0200 Subject: [PATCH 75/75] test(monitor): Tidy the batched call test --- monitor/src/checks.test.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/monitor/src/checks.test.ts b/monitor/src/checks.test.ts index e5cd838e4..24a33fbd5 100644 --- a/monitor/src/checks.test.ts +++ b/monitor/src/checks.test.ts @@ -192,10 +192,9 @@ test("a page is Slack text with its control characters escaped and a long detail test("a governance call tree is walked through batches and minting calls are flagged", () => { const setBalance = { section: "tokens", method: "setBalance", args: ["who", "Native", 4_000_000n] }; - // The recurring attack proposal: utility.batch of tokens.setBalance to the proposer. - const attack = toCallNode({ section: "utility", method: "batch", args: [[setBalance, setBalance]] }); - assert.equal(describeCall(attack), "utility.batch(tokens.setBalance, tokens.setBalance)"); - assert.deepEqual(riskyCalls(attack), ["tokens.setBalance"]); + const batch = toCallNode({ section: "utility", method: "batch", args: [[setBalance, setBalance]] }); + assert.equal(describeCall(batch), "utility.batch(tokens.setBalance, tokens.setBalance)"); + assert.deepEqual(riskyCalls(batch), ["tokens.setBalance"]); const nested = toCallNode({ section: "utility", method: "dispatchAs", args: ["Root", { section: "tokenMigration", method: "setPaused", args: [false] }] }); assert.deepEqual(riskyCalls(nested), ["tokenMigration.setPaused"]); assert.deepEqual(riskyCalls(toCallNode({ section: "system", method: "remark", args: ["0x00"] })), []);