diff --git a/docs/pen-migration-security.md b/docs/pen-migration-security.md index d8b5d630b..e4ab6b42c 100644 --- a/docs/pen-migration-security.md +++ b/docs/pen-migration-security.md @@ -37,7 +37,8 @@ team, and the safeguards below limit what a compromise can do. never unpause; unpausing requires the vault admin. - **Admin changes.** Admin transfers are two-step. After the governance handover every admin action passes a 48-hour timelock, during which a - designated canceller can veto a queued operation. + designated canceller can veto a queued operation. The monitor pages every + operation the timelock schedules. - **Sweep floor.** The unmigrated remainder cannot be moved before 2027-03-01 00:00 UTC (immutable), and only through a governance decision. - **Separation of duties.** The guardian and the monitor are operated by diff --git a/monitor/README.md b/monitor/README.md index 59e9c237b..993a7c54a 100644 --- a/monitor/README.md +++ b/monitor/README.md @@ -8,36 +8,141 @@ Checks every poll: | Check | Meaning | Reaction | |---|---|---| -| M2 tuple match | Every safe Base `Approved` and `Released` event must match one finalized Pendulum `MigrationInitiated` nonce, recipient, and pallet amount; `Released.tokenAmount` must equal `palletAmount × conversionFactor`. | Alert + auto-pause. A mismatch against a *known* nonce pauses immediately. An *unknown* nonce first pages (`UNVERIFIED BASE EVENT`), then pauses as soon as it is provable — the finalized source view passed the event's own timestamp and the nonce still does not exist — or after `UNMATCHED_EVENT_GRACE_SECONDS` if the source view stays behind (a stalled source also pages on its own, see below) | -| M2a aggregate | `totalReleased <= TotalMigrated × conversionFactor`. This remains independent defence-in-depth around the event reconciliation. | Alert + auto-pause | -| M2b: `balanceOf(vault) + totalReleased + totalSwept >= totalSupply` | Vault-internal conservation. Only a **deficit** alerts: a surplus is a harmless inbound transfer (donation, or a migration whose recipient is the vault) and is ignored, so it cannot false-trigger a pause. | Alert + auto-pause on a deficit | -| M4: every old migration reaches active approval quorum — and then releases | Liveness of the attestor fleet (`below approval quorum`), and of everything after it (`quorum reached but not released`: a release starved of daily allowance, paused, or made releasable by a threshold cut without a `ReleasePending`, which the releaser cannot see). | Alert, throttled per nonce | +| M2 tuple match | Every safe Base `Approved` and `Released` event must match one finalized Pendulum `MigrationInitiated` nonce, recipient, and pallet amount; `Released.tokenAmount` must equal `palletAmount × conversionFactor`. After a violation the Base cursor stays on any block range whose events the source view may still reveal, so a pause caused by a lagging source does not lose those releases. | Alert + auto-pause on any mismatching `Released`, and on approvals of one nonce that disagree with the finalized Pendulum view from two or more distinct attestors, also across polls and restarts (from one at a vault threshold of two or less, where one more approval releases). Otherwise a single attestor's disagreeing approval pages `CRITICAL: MISMATCHED APPROVAL` instead: one approval can never release. A mismatch against a *known* nonce counts immediately. An *unknown* nonce first pages (`UNVERIFIED BASE EVENT`, without delaying the pause decision), then counts as a mismatch as soon as it is provable — the finalized source view passed the event's own timestamp and the nonce still does not exist — or after `UNMATCHED_EVENT_GRACE_SECONDS` if the source view stays behind or cannot be read (a stalled or unreadable source also pages on its own, see below) | +| M2a aggregate | `totalReleased <= TotalMigrated × conversionFactor + ACKNOWLEDGED_RELEASE_EXCESS`. This remains independent defence-in-depth around the event reconciliation: it also runs while a Base log range is held, and waits only while a Base event's burn is not in the source view yet. After a real, quantified loss, operators set `ACKNOWLEDGED_RELEASE_EXCESS` to it (RB-3) so the vault can be unpaused; any further excess still fires. | Alert + auto-pause | +| M2b: `balanceOf(vault) + totalReleased + totalSwept >= totalSupply` | Vault-internal conservation. Only a **deficit** alerts: a surplus is a harmless inbound transfer (donation, or a migration whose recipient is the vault) and is ignored, so it cannot false-trigger a pause. It reads Base only and runs first in every poll, so a failing Pendulum source or Base log scan cannot skip it. While the rest of a poll takes longer than `POLL_INTERVAL_MS` (a slow source, a long catch-up), it also runs on its own, so two of its runs are about three poll intervals apart at most. | Alert + auto-pause on a deficit | +| M4: every old migration reaches active approval quorum — and then releases | Liveness of the attestor fleet (`below approval quorum`), and of everything after it (`quorum reached but not released`: a release starved of daily allowance, paused, or made releasable by a threshold cut without a `ReleasePending`, which the releaser cannot see). | One summary page per kind and poll (count and the first nonces; the log lists every nonce), each nonce at most once per `GRACE_SECONDS` | | Source supply | `totalIssuance + TotalMigrated` on Pendulum must never grow past its first-seen anchor. | Alert (`SOURCE SUPPLY GREW`) | | Unreleasable burns | A burn to the zero or vault address can never be approved or released; it is paged once and excluded from the pending count so it cannot page liveness forever or block RB-7. | Alert once | +| Treasury destination | With `EXPECTED_TREASURY_DESTINATION` set, every finalized treasury migration (a `MigrationInitiated` whose `who` is the treasury account, `PalletId(*b"py/trsry")`) must pay that Base address. It is checked when the burn is ingested, before the attestors can release it; while the pause is not confirmed, the monitor holds at that block, and the violation pages again every poll saying so. | Alert + auto-pause (`TREASURY DESTINATION VIOLATION`) | +| Pendulum source | The finalized Pendulum view must stay fresh and readable. While it cannot be read (an RPC error, a nonce discontinuity, an event-shape change), the Base-side checks — vault balance and its auto-pause, Base view, auto-pause readiness, vault admin events, governance queue — still run, and Base events are reconciled against the last view read in full: an unreadable source counts as a stalled one, so a mismatch on a known nonce counts at once and an unknown nonce after `UNMATCHED_EVENT_GRACE_SECONDS`. M2a and liveness wait for the source. | Alert, throttled: `PENDULUM SOURCE VIEW STALLED` once the view is older than `SOURCE_STALE_ALERT_SECONDS` (naming the second node or the relay RPCs when they hold the view back), `PENDULUM SOURCE UNREADABLE` on a failure | +| Pendulum chain | Every finalized block the monitor ingests is read at a hash it verified against the blocks it ingested before and, where configured, a second node and the relay chain; a second node must also agree on the migration totals and on the migration events of each ingested block (see [Pendulum chain verification](#pendulum-chain-verification)). | Alert (`CRITICAL: PENDULUM SOURCE CONTRADICTED`), throttled; nothing from the source is ingested, and it is treated as unreadable (see Pendulum source) until it verifies again. A second node or relay RPC that does not answer only warns, throttled (`PENDULUM CROSS-CHECK UNAVAILABLE`, `RELAY ANCHOR UNAVAILABLE`), and so does a relay RPC outvoted by another (`RELAY RPCS DISAGREE`); relay RPCs that answer without anchoring the source's newer blocks hold them back (`RELAY ANCHOR UNAVAILABLE`); `CRITICAL: RELAY ANCHOR STALE` once no head was anchored for `RELAY_ANCHOR_GRACE_SECONDS` | +| Base log completeness | In every scanned block range the `Released` amounts must add up to the growth of `totalReleased`, and the `RemainderSwept` amounts to that of `totalSwept`, between the block before the range and its last block. A range whose logs do not (a backend serving truncated logs) is read once more, then held: the Base cursor stays on it and liveness waits until a read accounts for the totals; M2a still runs. | Alert (`BASE LOG SCAN INCOMPLETE`), throttled; `CRITICAL: BASE LOG SCAN INCOMPLETE` once a range has been held for longer than `UNMATCHED_EVENT_GRACE_SECONDS`, since no later Base event is matched against its burn meanwhile | +| Base view | The safe/finalized Base block must not be older than `BASE_STALE_ALERT_SECONDS`: a view that stops advancing hides every later vault event while every check still passes at the old block. | Alert (`BASE VIEW STALLED`), throttled | +| Auto-pause readiness | At startup and every poll: the Base RPC serves `BASE_CHAIN_ID` and, with a guardian key configured, the key parses, is `vault.guardian()` and holds at least `GUARDIAN_MIN_BALANCE_WEI`. A read that fails pages as well and never stops the poll's other checks. | Alert (`AUTO-PAUSE NOT READY`), throttled; `AUTO-PAUSE DISABLED` once at startup without a key | +| Pendulum governance | Every pending public proposal, the council's external proposal, every ongoing referendum and every open council and technical-committee motion, with the call decoded. A call that mints, moves or unlocks balances outside normal transfers, or changes the runtime or the migration pallet, pages as critical: minted PEN burned through the migration drains the fixed-supply vault, and only Pendulum governance can stop the call before enactment. A call the monitor cannot read pages as critical too: one over 64 KiB is never fetched, and one that does not decode or nests more than 256 calls or 16 levels deep is unreadable; one whose preimage is not noted yet is re-checked every poll and paged again once it can be read. The watch runs after the poll's conservation checks, so it cannot stop or delay them, and it decodes within fixed bounds, so its cost to the next poll is bounded. | Alert once per item (`PENDULUM GOVERNANCE ITEM`, `CRITICAL: Pendulum governance item …`); a motion proposed again with the same call (a new motion index), or an external proposal made again after it left the queue or became a referendum, pages again; a restart re-pages the items still pending | +| Migration pallet admin events | Every finalized `MigrationPauseSet` and `TreasuryDestinationSet`, however fast the motion behind it passed. | Alert once per event (`MIGRATION PALLET ADMIN EVENT`); `CRITICAL: MIGRATION PALLET ADMIN EVENT` for a `TreasuryDestinationSet` to another address than `EXPECTED_TREASURY_DESTINATION` | +| Vault admin events | Every admin event of the vault (token, pause, unpause, attestors, threshold, caps, guardian, admin transfer, sweep, stale pending cleared). Before the governance handover the admin acts without a delay, so these are the only trace of such a change. This scan (shared with the governance queue) keeps its own persisted cursor: while it fails, every other check still runs, and it resumes where it stopped, also after a restart. | Alert once per event (`VAULT ADMIN EVENT`); the first run also pages the deployment's own events; `BASE ADMIN SCAN FAILED`, throttled, on a failure, and `CRITICAL: BASE ADMIN SCAN FAILED` once it has failed for longer than `UNMATCHED_EVENT_GRACE_SECONDS` | +| Governance queue | Every operation the governance Timelock schedules (`CallScheduled`), with its calls decoded and the time it becomes executable. Once the delay has passed anyone may execute a ready operation, so the canceller has exactly the delay to review and veto it. | Alert once per operation and scheduling block (`GOVERNANCE OPERATION SCHEDULED`), so an operation cancelled and scheduled again with the same id pages again | + +## Pendulum chain verification + +The monitor reads every new finalized Pendulum block at a hash it verified +itself: + +- **Parent-hash chain.** Each poll walks the parent hashes down from the + finalized head to the block ingested last, through headers fetched from the + node and hashed by the monitor. The walk must arrive at the hash of the block + ingested last, which the state file keeps, so every ingested block lies on + one chain. +- **Second node** (`PENDULUM_WS_CROSSCHECK`). The source's chain must have + the second node's hash at the height of the second node's finalized head + (at the source's head once the second node has finalized past it), and no + block above that height is ingested. The migration totals (`nextNonce`, + `totalMigrated`) must agree at the block the monitor reads them, and every + ingested block with migration pallet events must carry the same ones on the + second node. A second node whose finalized head is older than + `SOURCE_STALE_ALERT_SECONDS` counts as unavailable. +- **Relay-chain anchor** (`POLKADOT_WS`). Each poll reads the Pendulum head + that the Polkadot relay chain recorded at its finalized block + (`paras.heads(PENDULUM_PARA_ID)`), asking the relay RPCs in turn until a + head at or above the source's finalized head anchors it; the highest head + that anchors the source is used. A relay RPC whose finalized relay block is + older than `SOURCE_STALE_ALERT_SECONDS` counts as not answering. The + source's chain must have that hash at that height. A head above the + source's finalized head counts once the source's best chain holds it, and + its chain must then pass through the finalized head. The newest such head + is kept in the state file, and the chain must keep passing through it, + also while it lies above the finalized head. Blocks ingested while no relay + RPC answered are tied to the next head that arrives by the same walk; a + head more than 100 blocks below the block ingested last is not used. A + head that contradicts the source counts only when another relay RPC's head + contradicts it too, or when `POLKADOT_WS` lists one RPC. A relay RPC whose + head anchors the source at or above the contradicting head's height + outvotes it: `RELAY RPCS DISAGREE` pages and that head anchors the source. + Any other answer leaves the contradiction unconfirmed; a lower head still + anchors the source up to its own height. No block above the newest + anchored head is ingested. + +Every block the monitor ingests is confirmed by each checker that answers, +and the values the checks use (the view's timestamp, `nextNonce`, +`totalMigrated` and issuance) are read at the newest ingested block. Relay +RPCs that answer but anchor no newer head, or leave a contradiction +unconfirmed, hold the source's newer blocks back and page +`RELAY ANCHOR UNAVAILABLE`. A contradiction pages +`CRITICAL: PENDULUM SOURCE CONTRADICTED`, nothing from the source is +ingested, and the source counts as unreadable until it verifies again. A +second node or relay RPC that cannot be read, or whose finalized head (for a +relay RPC, its finalized relay block) is older than +`SOURCE_STALE_ALERT_SECONDS`, only warns and holds nothing back; with no +checker answering, the monitor goes on with the source alone up to its +finalized head. `CRITICAL: RELAY ANCHOR STALE` pages once no head was +anchored for `RELAY_ANCHOR_GRACE_SECONDS`, whatever the reason (relay RPCs +down, the source behind the relay chain, a relay head that does not advance). ## Configuration (environment) | Variable | Meaning | |---|---| | `PENDULUM_WS` | WebSocket of the monitor's own Pendulum node, run with `--state-pruning archive` (see below) | -| `BASE_RPC_URL` | Base JSON-RPC endpoint — a **single dedicated node**, and a different provider than the attestors use. Load-balanced pools can serve `eth_getLogs` from a lagging replica; each scan range is probed for its end block first, which turns a behind-node into a loud replay instead of a silent gap, but a dedicated node removes the hazard entirely | +| `PENDULUM_WS_CROSSCHECK` | Optional: WebSocket of a second Pendulum node, not the one behind `PENDULUM_WS`; only blocks on its finalized chain are ingested, and it must agree with the source on the migration totals in every poll and on the migration events of every ingested block that has some. Run it with `--state-pruning archive` too, or those events cannot be compared after a monitor outage longer than its pruning horizon (`PENDULUM CROSS-CHECK UNAVAILABLE`) | +| `POLKADOT_WS` | Optional: comma-separated WebSocket RPCs of the Polkadot relay chain, run by third parties and tried in order (for example `wss://rpc.polkadot.io`); turns on the relay-chain anchor, and while a relay RPC answers, no Pendulum block above the newest anchored head is ingested. List at least two: a record that contradicts the source needs a second relay RPC's confirmation, unless only one is listed | +| `PENDULUM_PARA_ID` | Pendulum's parachain id on the relay chain (default 2094) | +| `RELAY_ANCHOR_GRACE_SECONDS` | How long the source may go without a relay-anchored head before `CRITICAL: RELAY ANCHOR STALE` pages (default 3600) | +| `BASE_RPC_URL` | Base JSON-RPC endpoint — a **single dedicated node**, and a different provider than the attestors use. Load-balanced pools can serve `eth_getLogs` from a lagging replica; each scan range is probed for its end block first and its logs must account for the vault's running totals, which turns a behind-node into a loud replay instead of a silent gap, but a dedicated node removes the hazard entirely | | `VAULT_ADDRESS` | MigrationVault address on Base | -| `POLL_INTERVAL_MS` | Poll cadence (default 60s) | +| `POLL_INTERVAL_MS` | Poll cadence (default 60s); while a poll takes longer, the vault balance check runs on its own at this cadence | | `GRACE_SECONDS` | Liveness alert threshold (default 30 min) | -| `UNMATCHED_EVENT_GRACE_SECONDS` | Time a lagging Pendulum source view gets to reveal the burn behind a safe Base event before a cannot-verify pause (default 10 min). The first sighting of an unmatched event pages immediately, so this is the window operators have to repair a stalled node. Size it to your node-ops response time | +| `UNMATCHED_EVENT_GRACE_SECONDS` | Time a lagging or unreadable Pendulum source view gets to reveal the burn behind a safe Base event before a cannot-verify pause (default 10 min). The first sighting of an unmatched event pages immediately, so this is the window operators have to repair a stalled or failing node. Size it to your node-ops response time | | `SOURCE_CLOCK_SKEW_MARGIN_SECONDS` | Clock-skew allowance for the fabrication proof (default 120s): once the finalized Pendulum view has passed the moment the monitor first observed an unmatched Base event by this margin and the nonce still does not exist, the event is provably fabricated and pauses without waiting out the grace. The anchor is the monitor's own clock, not the Base block timestamp (which trails real time after a sequencer outage); a future-dated source view proves nothing | -| `SOURCE_STALE_ALERT_SECONDS` | Age of the finalized Pendulum view (vs. wall clock) past which the monitor pages that it is going blind (default 300s) | +| `SOURCE_STALE_ALERT_SECONDS` | Age of the verified Pendulum view (vs. wall clock) past which the monitor pages that it is going blind (default 300s); a second node whose finalized head, or a relay RPC whose finalized relay block, is older counts as not answering | +| `BASE_STALE_ALERT_SECONDS` | Age of the safe/finalized Base block (vs. wall clock) past which `BASE VIEW STALLED` pages (default 600s for `safe`, 1800s for `finalized`; Base's safe head normally trails by about a minute, its finalized head by about 20) | | `ISSUANCE_TOLERANCE` | Growth of Pendulum `totalIssuance + TotalMigrated` over its first-seen anchor (12-decimal pallet units) tolerated before `SOURCE SUPPLY GREW` pages (default 0) | -| `ALERT_WEBHOOK_URL` | Webhook receiving JSON alerts — wire this to paging | -| `GUARDIAN_PRIVATE_KEY` | Optional: a guardian key enabling automatic `pause()` on conservation violations | -| `BASE_CHAIN_ID` | Default 8453 | +| `ACKNOWLEDGED_RELEASE_EXCESS` | Excess of released over migrated, in token units (18 decimals), that operators quantified and accepted after a real loss; M2a fires only beyond it (default 0). Set it at the RB-3 sign-off and restart; the startup log shows it | +| `EXPECTED_TREASURY_DESTINATION` | Optional: the Base address every treasury migration must pay, the one set with `tokenMigration.setTreasuryDestination`; any other pauses the vault, and setting another one pages critical | +| `TIMELOCK_ADDRESS` | Optional: the governance TimelockController. Set it as soon as governance is deployed; the queue is scanned from the monitor's Base cursor onwards, so operations scheduled before the monitor knew the address are not paged | +| `ALERT_WEBHOOK_URL` | Slack incoming webhook receiving the pages (see [Pages](#pages)). A page counts as delivered only on a 2xx answer within 10s: once-only pages (governance items, admin events, scheduled operations, first sightings) are retried every poll until delivered, and failed deliveries are logged. Without it the log line is the page | +| `HEARTBEAT_URL` | Optional: a dead-man's switch URL (Healthchecks.io style), requested with a `GET` (10s timeout) after every poll that ran all checks; a failed request is logged and never stops the monitor. Have the service page when the requests stop: the monitor is down, stuck, or cannot complete its checks (for example while the Pendulum source cannot be read) | +| `GUARDIAN_PRIVATE_KEY` | Optional: the key of `vault.guardian()`, enabling automatic `pause()` on conservation violations; its readiness is checked at startup and every poll, and any other key pages `AUTO-PAUSE NOT READY` (the vault admin may pause too, but its key does not belong on the monitor) | +| `GUARDIAN_MIN_BALANCE_WEI` | Guardian ETH balance below which `AUTO-PAUSE NOT READY` pages (default 10^15 wei = 0.001 ETH) | +| `BASE_CHAIN_ID` | Default 8453; the Base RPC must serve it | | `PENDULUM_START_BLOCK` | **Required on first run:** first block to scan, normally the pallet activation block or the next finalized block before unpausing | | `PENDULUM_START_NONCE` | Nonce expected at that block (default 0; set explicitly when starting after any migrations) | | `BASE_START_BLOCK` | **Required on first run:** vault deployment block or an earlier block | -| `STATE_FILE` | Durable two-chain cursor, pending tuples, and liveness timestamps (default `./monitor-state.json`) | +| `STATE_FILE` | Durable cursors (Pendulum with the hash of its last block, Base, Base admin scan), the newest relay-anchored Pendulum head, pending tuples, and liveness timestamps (default `./monitor-state.json`) | | `BASE_FINALITY_TAG` | `safe` (default) or `finalized`; only events inside this boundary are trusted | | `BASE_MAX_BLOCK_RANGE` | Maximum `eth_getLogs` range (default 10,000 blocks) | +## Pages + +Each page is a Slack incoming-webhook message, a JSON `POST` of `{ "text": … }` +to `ALERT_WEBHOOK_URL`. The text is `[CRITICAL] pen-monitor: SUBJECT` or +`[WARNING] pen-monitor: SUBJECT`, then the detail on the next line. Slack's +control characters `&`, `<` and `>` are escaped, so no detail can mention +anyone or hide a link, and a detail longer than 3,000 characters is cut short +(the monitor's log line keeps the full text). `CRITICAL` marks the violations +that pause the vault, `AUTO-PAUSE FAILED`, `AUTO-PAUSE UNAVAILABLE` and every +subject that starts with `CRITICAL`; everything else is a `WARNING`. + +| Subjects | Meaning | +|---|---| +| `MIGRATION TUPLE VIOLATION`, `VAULT BALANCE DEFICIT`, `CONSERVATION VIOLATION`, `TREASURY DESTINATION VIOLATION` | A conservation or treasury check failed; the auto-pause follows with `vault auto-pause confirmed`, `vault pause confirmed` (already paused, also without a guardian key), `AUTO-PAUSE PENDING FINALITY`, `AUTO-PAUSE FAILED` or `AUTO-PAUSE UNAVAILABLE` (no guardian key, and the vault is not paused) | +| `UNVERIFIED BASE EVENT` | First sighting of a Base event whose nonce the source view does not know yet | +| `CRITICAL: MISMATCHED APPROVAL` | One attestor approved a tuple that disagrees with the finalized Pendulum view; check its key and its Pendulum node. A second attestor doing so for the same nonce pauses the vault | +| `LIVENESS: migration below approval quorum`, `LIVENESS: quorum reached but not released` | Stalled migrations, one summary per kind and poll | +| `PENDULUM SOURCE VIEW STALLED`, `PENDULUM SOURCE UNREADABLE`, `BASE VIEW STALLED`, `BASE ADMIN SCAN FAILED`, `BASE LOG SCAN INCOMPLETE` | The monitor is going blind on one chain | +| `CRITICAL: BASE LOG SCAN INCOMPLETE`, `CRITICAL: BASE ADMIN SCAN FAILED` | For longer than `UNMATCHED_EVENT_GRACE_SECONDS`, a Base block range has been held, so no later Base event is matched against its burn, or the admin scan has failed, so no vault admin event or scheduled governance operation is paged while its veto window runs; fix the Base RPC | +| `CRITICAL: PENDULUM SOURCE CONTRADICTED` | The Pendulum source contradicts the chain the monitor verified; stop trusting that node | +| `PENDULUM CROSS-CHECK UNAVAILABLE` | The second Pendulum node does not answer or its finalized head is stale, so the source is used without that check | +| `RELAY ANCHOR UNAVAILABLE` | No relay RPC answers from a finalized relay block younger than `SOURCE_STALE_ALERT_SECONDS`, so the source is used without the anchor; or none anchors the source's newer blocks, or a relay RPC records a head that contradicts the source and no other one confirms or outvotes it, so those blocks wait | +| `RELAY RPCS DISAGREE` | One relay RPC records a Pendulum head that contradicts the source, and another records one at or above that height on the source's chain, which anchors the source. One of them is wrong; check both | +| `CRITICAL: RELAY ANCHOR STALE` | No Pendulum head was checked against the relay chain for `RELAY_ANCHOR_GRACE_SECONDS` | +| `AUTO-PAUSE NOT READY`, `AUTO-PAUSE DISABLED` | The next violation would not pause the vault automatically | +| `SOURCE SUPPLY GREW`, `CRITICAL: unreleasable migration burned on Pendulum` | Supply problems at the source | +| `PENDULUM GOVERNANCE ITEM`, `CRITICAL: Pendulum governance item mints, moves or unlocks balances, or changes the runtime`, `CRITICAL: Pendulum governance item cannot be read`, `MIGRATION PALLET ADMIN EVENT`, `CRITICAL: MIGRATION PALLET ADMIN EVENT` | Pendulum governance | +| `GOVERNANCE OPERATION SCHEDULED`, `VAULT ADMIN EVENT` | Base governance and vault administration | +| `monitor check failed`, `pendulum governance watch failed`, `multicall unavailable, …`, `multicall batch failed, …`, `monitor startup failed` | The monitor itself | + ## Run ```sh @@ -47,9 +152,9 @@ npm start ``` Run it under a process manager and treat "monitor down" itself as a paging -condition: an unwatched migration is the risk model failing silently. The -two-chain cursors, unresolved canonical tuples, their original finalized block -timestamps, and alert throttles are atomically persisted. A malformed or +condition (`HEARTBEAT_URL`): an unwatched migration is the risk model failing +silently. The two-chain cursors, unresolved canonical tuples, their original +finalized block timestamps, and alert throttles are atomically persisted. A malformed or Pendulum-genesis/Base-chain/vault-mismatched state file is fatal rather than silently resetting the security history. @@ -57,11 +162,19 @@ silently resetting the security history. archive` (or `archive-canonical`): the monitor catches up block by block through historical state, and a default-pruned node (256 blocks ≈ 51 min) cannot serve that after any monitor outage longer than the pruning horizon — -the monitor then wedges loudly on restart. Recovery is pointing `PENDULUM_WS` -at an archive node, never editing the state file. A monitor whose *running* -source view stalls pages `PENDULUM SOURCE VIEW STALLED` well before the -unmatched-event grace can force a cannot-verify pause; treat that page as -"restore the node now". +the monitor then pages `PENDULUM SOURCE UNREADABLE` on restart. Recovery is +pointing `PENDULUM_WS` at an archive node, never editing the state file. A +monitor whose *running* source view stalls pages `PENDULUM SOURCE VIEW +STALLED` well before the unmatched-event grace can force a cannot-verify +pause; treat that page as "restore the node now", or the second node or +relay RPCs it names as holding the view back. The Base-side checks keep +running while the source cannot be read, and an unreadable source counts as +a stalled one: once a Base event it cannot verify appears, the monitor pages +`UNVERIFIED BASE EVENT` and pauses when the grace runs out, so treat +`PENDULUM SOURCE UNREADABLE` the same way. The Base node must answer +`eth_call` at the block before each scanned range and at its last block (the +log completeness check); after a monitor outage those are old blocks, so a +node that keeps only recent state stops the scan until it is an archive node. An automatic pause is reported as successful only after the transaction is canonical inside the configured Base confirmation boundary and `paused()` is diff --git a/monitor/src/checks.test.ts b/monitor/src/checks.test.ts index c4803013b..24a33fbd5 100644 --- a/monitor/src/checks.test.ts +++ b/monitor/src/checks.test.ts @@ -1,9 +1,9 @@ /** * Unit tests for the monitor's conservation/liveness predicates. * - * Run with `npm test` (compiles, then `node --test`). These lock in the - * round-5 fixes: the M2b check must ignore a token surplus (donation / vault - * recipient) and only fire on a real deficit. + * Run with `npm test` (compiles, then `node --test`). Among others, these lock + * in that the M2b check ignores a token surplus (donation / vault recipient) + * and only fires on a real deficit. */ import assert from "node:assert/strict"; @@ -12,11 +12,17 @@ import { approvalMismatch, blockRanges, chunks, + describeCall, + describeTimelockOperation, + disagreeingAttestors, isStale, provablyUnsourced, releaseMismatch, releasedExceedsMigrated, + riskyCalls, shouldAwaitSource, + slackText, + toCallNode, vaultConservationDeficit, } from "./checks.js"; @@ -32,6 +38,11 @@ test("M2a: released exceeding migrated fires", () => { assert.equal(releasedExceedsMigrated(6n * CF, 5n, CF), true); }); +test("M2a: an acknowledged loss is tolerated, and nothing beyond it", () => { + assert.equal(releasedExceedsMigrated(6n * CF, 5n, CF, CF), false); + assert.equal(releasedExceedsMigrated(6n * CF + 1n, 5n, CF, CF), true); +}); + test("M2b: exact conservation is not a deficit", () => { // balance + released + swept == totalSupply assert.equal(vaultConservationDeficit(SUPPLY - 30n, 20n, 10n, SUPPLY), false); @@ -113,6 +124,20 @@ test("M2 tuple matching rejects missing, wrong-recipient and wrong-amount approv assert.equal(approvalMismatch(expected, { ...expected, recipient: expected.recipient.toUpperCase() }), undefined); }); +test("approvals against the burn count once per attestor and only while they disagree", () => { + const burn = { nonce: 7n, recipient: "0x1111111111111111111111111111111111111111", palletAmount: 100n }; + const [a, b] = ["0xA1a1a1a1a1A1a1a1a1a1a1A1a1a1a1a1a1a1A1a1", "0xa2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2"]; + const approvals = [ + { ...burn, palletAmount: 99n, attestor: a }, + { ...burn, palletAmount: 98n, attestor: a.toLowerCase() }, + { ...burn, attestor: b }, // agrees + { ...burn, nonce: 8n, palletAmount: 99n, attestor: b }, // another nonce + ]; + assert.deepEqual(disagreeingAttestors(burn, 7n, approvals), [a.toLowerCase()]); + assert.deepEqual(disagreeingAttestors(undefined, 7n, approvals), [a.toLowerCase(), b]); // no burn known + assert.deepEqual(disagreeingAttestors({ ...burn, palletAmount: 99n }, 7n, approvals), [a.toLowerCase(), b]); +}); + test("M2 release matching includes the one-and-only decimal conversion", () => { const expected = { nonce: 7n, recipient: "0x1111111111111111111111111111111111111111", palletAmount: 100n }; assert.equal(releaseMismatch(expected, { ...expected, tokenAmount: 100n * CF }, CF), undefined); @@ -124,3 +149,64 @@ test("RPC work is split into inclusive block ranges and bounded batches", () => assert.deepEqual(blockRanges(11n, 10n, 4n), []); assert.deepEqual(chunks([1, 2, 3, 4, 5], 2), [[1, 2], [3, 4], [5]]); }); + +test("a scheduled governance operation is described with names, targets and the veto deadline", () => { + const vault = "0x2222222222222222222222222222222222222222"; + const labels = { [vault]: "vault" }; + const functions = { "0x0e18b681": "acceptAdmin()", "0x212bf316": "setCaps(uint256,uint256)" }; + const calls = [ + { index: 1n, target: vault.toUpperCase().replace("0X", "0x"), value: 0n, data: "0x212bf316" + "00".repeat(64) }, + { index: 0n, target: vault, value: 0n, data: "0x0e18b681" }, + ]; + const now = Date.UTC(2026, 9, 21, 10, 0, 0); + const text = describeTimelockOperation("0xabc", calls, BigInt((now + (48 * 60 + 5) * 60_000) / 1000), now, labels, functions); + assert.match(text, /^operation 0xabc: #0 vault\.acceptAdmin\(\) data=0x0e18b681; #1 vault\.setCaps\(uint256,uint256\)/); + assert.match(text, /executable by anyone from 2026-10-23T10:05:00\.000Z \(in 48h 5m\)/); + assert.match(text, /canceller must cancel it before then$/); +}); + +test("a scheduled operation with unknown parts or a settled timestamp still pages readably", () => { + const other = "0x3333333333333333333333333333333333333333"; + const call = { index: 0n, target: other, value: 5n, data: "0xdeadbeef" }; + assert.match(describeTimelockOperation("0x1", [call], 0n, 0, {}, {}), new RegExp(`#0 ${other}\\.unknown function 0xdeadbeef value=5 .*cancelled\\)$`)); + assert.match(describeTimelockOperation("0x1", [call], 1n, 0, {}, {}), /already executed$/); + assert.match(describeTimelockOperation("0x1", [call], 100n, 200_000, {}, {}), /\(ready now\)/); +}); + +test("a scheduled operation pages whatever its delay or its event carries", () => { + const call = { index: 0n, target: "0x3333333333333333333333333333333333333333", value: 0n, data: "0x8456cb59" }; + assert.match(describeTimelockOperation("0x1", [call], 10n ** 20n, 0, {}, {}), /from timestamp 100000000000000000000 \(in /); + const undecoded = { index: undefined, target: undefined, value: undefined, data: undefined } as unknown as typeof call; + assert.match(describeTimelockOperation("0x1", [undecoded, call], 5n, 0, {}, {}), /^operation 0x1: #undefined undefined\.unknown function undefined/); +}); + +test("a page is Slack text with its control characters escaped and a long detail cut short", () => { + assert.equal( + slackText("pen-monitor", "critical", "SUBJECT", "to & "), + "[CRITICAL] pen-monitor: SUBJECT\nto <!channel> & <https://x.example|click>", + ); + const text = slackText("pen-monitor", "warning", "SUBJECT", "a".repeat(10_000)); + assert.ok(text.startsWith(`[WARNING] pen-monitor: SUBJECT\n${"a".repeat(3_000)}… (cut short`), text.slice(0, 80)); + assert.ok(text.length < 3_200, String(text.length)); +}); + +test("a governance call tree is walked through batches and minting calls are flagged", () => { + const setBalance = { section: "tokens", method: "setBalance", args: ["who", "Native", 4_000_000n] }; + const batch = toCallNode({ section: "utility", method: "batch", args: [[setBalance, setBalance]] }); + assert.equal(describeCall(batch), "utility.batch(tokens.setBalance, tokens.setBalance)"); + assert.deepEqual(riskyCalls(batch), ["tokens.setBalance"]); + const nested = toCallNode({ section: "utility", method: "dispatchAs", args: ["Root", { section: "tokenMigration", method: "setPaused", args: [false] }] }); + assert.deepEqual(riskyCalls(nested), ["tokenMigration.setPaused"]); + assert.deepEqual(riskyCalls(toCallNode({ section: "system", method: "remark", args: ["0x00"] })), []); +}); + +test("the call-tree walk is bounded in size and depth", () => { + const remark = { section: "system", method: "remark", args: [] }; + const batch = (calls: unknown[]) => ({ section: "utility", method: "batch", args: [calls] }); + assert.equal(toCallNode(batch(Array(255).fill(remark))).calls.length, 255); + assert.throws(() => toCallNode(batch(Array(256).fill(remark))), /more than 256 calls/); + let nested: { section: string; method: string; args: unknown[] } = remark; + for (let depth = 1; depth < 16; depth++) nested = batch([nested]); + assert.equal(describeCall(toCallNode(nested)).split("(").length, 16); + assert.throws(() => toCallNode(batch([nested])), /more than 16 levels/); +}); diff --git a/monitor/src/checks.ts b/monitor/src/checks.ts index 92082992b..17c01fa95 100644 --- a/monitor/src/checks.ts +++ b/monitor/src/checks.ts @@ -2,8 +2,7 @@ * Pure conservation/liveness predicates for the invariant monitor. * * These are isolated from the chain-client plumbing in `main.ts` so they can be - * unit-tested exhaustively (see checks.test.ts). Every subtlety that has bitten - * a review round lives here. + * unit-tested exhaustively (see checks.test.ts). */ /** @@ -13,13 +12,18 @@ * units. It lags `totalMigrated` only via finality delay, never the other way, * because a release requires attestations of an already-finalized burn — so any * excess is the strongest possible signal of attestor compromise. + * + * `acknowledgedExcess` is a loss operators have quantified and accepted: it + * stays in the cumulative totals for good, and without it the check would + * re-pause the vault after every recovery unpause. */ export function releasedExceedsMigrated( totalReleased: bigint, totalMigrated: bigint, conversionFactor: bigint, + acknowledgedExcess = 0n, ): boolean { - return totalReleased > totalMigrated * conversionFactor; + return totalReleased > totalMigrated * conversionFactor + acknowledgedExcess; } /** @@ -90,7 +94,7 @@ export function shouldAwaitSource( * timestamp: OP-stack L2 block timestamps trail real time by the length of any * sequencer outage while it catches up, and an artificially old event * timestamp would let a merely-lagging source "prove" a legitimate event - * fabricated (review round 9). A lagging source never satisfies this predicate + * fabricated. A lagging source never satisfies this predicate * (its head timestamp trails the observation), and a future-dated source view * (a collator clock ahead of real time) proves nothing either and is excluded, * so neither can turn node lag into a false pause. @@ -126,6 +130,22 @@ export function approvalMismatch(expected: MigrationTuple | undefined, actual: M return undefined; } +/** + * The distinct attestors whose recorded approvals of `nonce` disagree with its + * finalized burn, or with its absence. Approvals that the source view has + * since confirmed no longer count. + */ +export function disagreeingAttestors( + expected: MigrationTuple | undefined, + nonce: bigint, + approvals: Array, +): string[] { + const attestors = approvals + .filter((approval) => approval.nonce === nonce && approvalMismatch(expected, approval)) + .map((approval) => approval.attestor.toLowerCase()); + return [...new Set(attestors)]; +} + export function releaseMismatch( expected: MigrationTuple | undefined, actual: MigrationTuple & { tokenAmount: bigint }, @@ -156,3 +176,142 @@ export function chunks(items: T[], size: number): T[][] { for (let start = 0; start < items.length; start += size) result.push(items.slice(start, start + size)); return result; } + +/** One call of a governance Timelock operation (one `CallScheduled` event). */ +export interface ScheduledCall { + index: bigint; + target: string; + value: bigint; + data: string; +} + +/** + * The page for one operation the governance Timelock scheduled. Every such + * operation pages: once its delay has passed, anyone may execute it, so the + * canceller has exactly the delay to review and veto it. + * + * `readyAtSeconds` is the Timelock's `getTimestamp(id)`, whose sentinels are + * 0 (unknown or cancelled) and 1 (executed). `labels` maps lowercased + * addresses and `functions` maps selectors to readable names. Never throws, + * whatever the event carries: one operation must not hide the next. + */ +export function describeTimelockOperation( + id: string, + calls: ScheduledCall[], + readyAtSeconds: bigint, + nowMs: number, + labels: Record, + functions: Record, +): string { + const callText = [...calls] + .sort((a, b) => (a.index < b.index ? -1 : a.index > b.index ? 1 : 0)) + .map((call) => { + const target = labels[String(call.target).toLowerCase()] ?? call.target; + const selector = String(call.data).slice(0, 10).toLowerCase(); + const name = functions[selector] ?? `unknown function ${selector}`; + return `#${call.index} ${target}.${name}${call.value ? ` value=${call.value}` : ""} data=${call.data}`; + }) + .join("; "); + let status: string; + if (readyAtSeconds === 0n) status = "no longer pending (cancelled)"; + else if (readyAtSeconds === 1n) status = "already executed"; + else { + const readyMs = Number(readyAtSeconds) * 1000; + const leftMinutes = Math.floor((readyMs - nowMs) / 60_000); + // A Date holds at most 8.64e15 ms; a later time is shown in seconds. + const readyText = readyMs <= 8.64e15 ? new Date(readyMs).toISOString() : `timestamp ${readyAtSeconds}`; + status = + `executable by anyone from ${readyText}` + + (leftMinutes > 0 ? ` (in ${Math.floor(leftMinutes / 60)}h ${leftMinutes % 60}m)` : " (ready now)") + + "; if it is not intended, the canceller must cancel it before then"; + } + return `operation ${id}: ${callText} — ${status}`; +} + +export type Severity = "critical" | "warning"; + +/** Longest page detail sent; the monitor's log line keeps the full text. */ +const MAX_PAGE_DETAIL = 3_000; + +/** + * A page as the `text` of a Slack incoming-webhook message: severity, subject + * and detail, with Slack's control characters `&`, `<` and `>` escaped, so + * that no detail can mention anyone or hide a link. + */ +export function slackText(service: string, severity: Severity, subject: string, detail: string): string { + const body = detail.length > MAX_PAGE_DETAIL + ? `${detail.slice(0, MAX_PAGE_DETAIL)}… (cut short; the monitor log has the full text)` + : detail; + const escapes: Record = { "&": "&", "<": "<", ">": ">" }; + return `[${severity.toUpperCase()}] ${service}: ${subject}\n${body}`.replace(/[&<>]/g, (char) => escapes[char]); +} + +/** A decoded Pendulum call: `section.method` plus any calls nested in its arguments. */ +export interface CallNode { + name: string; + calls: CallNode[]; +} + +/** The duck type of a polkadot.js `Call`, enough to walk nested calls. */ +interface CallLike { + section: string; + method: string; + args: unknown[]; +} + +function isCallLike(value: unknown): value is CallLike { + const call = value as CallLike | null; + return typeof call?.section === "string" && typeof call.method === "string" && Array.isArray(call.args); +} + +/** Bounds on a walked call tree. A governance call is untrusted data; + * legitimate ones stay far below both, and a bigger tree is unreadable. */ +const MAX_CALL_NODES = 256; +const MAX_CALL_DEPTH = 16; + +/** Walks a decoded call, descending into arguments that are calls or lists of + * calls (batch, dispatch_as, ...). Throws past the node or depth bound. */ +export function toCallNode(call: CallLike): CallNode { + let nodes = 0; + const walk = (node: CallLike, depth: number): CallNode => { + if (++nodes > MAX_CALL_NODES) throw new Error(`call tree has more than ${MAX_CALL_NODES} calls`); + if (depth > MAX_CALL_DEPTH) throw new Error(`call tree is nested more than ${MAX_CALL_DEPTH} levels deep`); + const calls: CallNode[] = []; + for (const arg of node.args) { + if (isCallLike(arg)) calls.push(walk(arg, depth + 1)); + else if (Array.isArray(arg)) for (const item of arg) if (isCallLike(item)) calls.push(walk(item, depth + 1)); + } + return { name: `${node.section}.${node.method}`, calls }; + }; + return walk(call, 1); +} + +/** Calls that mint, move or unlock balances outside normal transfers, or change + * the runtime or the migration pallet. Minted PEN that is burned through the + * migration drains the fixed-supply vault on Base, and only Pendulum governance + * can stop such a call before it is enacted. */ +const RISKY_CALLS = new Set([ + "balances.forceSetBalance", "balances.setBalance", "balances.setBalanceDeprecated", "balances.forceTransfer", + "balances.forceUnreserve", "balances.forceAdjustTotalIssuance", "tokens.setBalance", "tokens.forceTransfer", + "currencies.updateBalance", "vesting.forceVestedTransfer", "vestingManager.removeVestingSchedule", + "system.setCode", "system.setCodeWithoutChecks", "system.setStorage", "system.killStorage", "system.killPrefix", + "system.authorizeUpgrade", "system.authorizeUpgradeWithoutChecks", "parachainSystem.authorizeUpgrade", + "parachainSystem.enactAuthorizedUpgrade", +]); +const RISKY_SECTIONS = new Set(["sudo", "tokenMigration"]); + +/** The risky calls anywhere in a call tree, deduplicated. */ +export function riskyCalls(call: CallNode): string[] { + const found = new Set(); + const walk = (node: CallNode) => { + if (RISKY_CALLS.has(node.name) || RISKY_SECTIONS.has(node.name.split(".")[0])) found.add(node.name); + node.calls.forEach(walk); + }; + walk(call); + return [...found]; +} + +/** Compact one-line form: `utility.batch(tokens.setBalance, tokens.setBalance)`. */ +export function describeCall(call: CallNode): string { + return call.calls.length ? `${call.name}(${call.calls.map(describeCall).join(", ")})` : call.name; +} diff --git a/monitor/src/main.test.ts b/monitor/src/main.test.ts new file mode 100644 index 000000000..2ef2ed682 --- /dev/null +++ b/monitor/src/main.test.ts @@ -0,0 +1,2310 @@ +/** + * Call-site tests for the poll loop in main.ts. Each test loads a fresh copy + * of the module (it reads its configuration at import) against stand-in + * chains: a JSON-RPC server for Base and a plain object shaped like the + * polkadot.js API for Pendulum. The same server receives every page. + */ + +import assert from "node:assert/strict"; +import { mkdtempSync, readFileSync } from "node:fs"; +import { createServer, type IncomingMessage } from "node:http"; +import type { AddressInfo } from "node:net"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { test } from "node:test"; +import type { ApiPromise } from "@polkadot/api"; +import { + type AbiEvent, + type AbiFunction, + decodeFunctionData, + encodeAbiParameters, + encodeEventTopics, + type Hex, + keccak256, + parseAbi, + toEventSelector, + toFunctionSelector, + toHex, +} from "viem"; +import { privateKeyToAccount } from "viem/accounts"; + +(globalThis as { penMonitorUnderTest?: boolean }).penMonitorUnderTest = true; + +const VAULT = "0x1111111111111111111111111111111111111111"; +const TOKEN = "0x2222222222222222222222222222222222222222"; +const TIMELOCK = "0x3333333333333333333333333333333333333333"; +const RECIPIENT = "0x4444444444444444444444444444444444444444"; +const ATTESTORS = ["0xa1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1", "0xa2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2", "0xa3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3a3"]; +const CF = 1_000_000n; +const SUPPLY = 150_000_000n * 10n ** 18n; +const PEN = 10n ** 12n; +const GUARDIAN_KEY = `0x${"11".repeat(32)}` as const; +const GUARDIAN = privateKeyToAccount(GUARDIAN_KEY).address; + +const abi = parseAbi([ + "function totalReleased() view returns (uint256)", + "function totalSwept() view returns (uint256)", + "function conversionFactor() view returns (uint256)", + "function token() view returns (address)", + "function paused() view returns (bool)", + "function pause()", + "function threshold() view returns (uint256)", + "function guardian() view returns (address)", + "function activeApprovals(bytes32 payload) view returns (uint256)", + "function totalSupply() view returns (uint256)", + "function balanceOf(address owner) view returns (uint256)", + "function getTimestamp(bytes32 id) view returns (uint256)", + "event Approved(uint64 indexed nonce, address indexed recipient, uint256 palletAmount, address indexed attestor)", + "event Released(uint64 indexed nonce, address indexed recipient, uint256 palletAmount, uint256 tokenAmount)", + "event RemainderSwept(address indexed to, uint256 amount)", + "event GuardianUpdated(address indexed guardian)", + "event CapsUpdated(uint256 perReleaseCap, uint256 dailyCap)", + "event CallScheduled(bytes32 indexed id, uint256 indexed index, address target, uint256 value, bytes data, bytes32 predecessor, uint256 delay)", +]); + +interface Log { + address: string; + eventName: string; + args: Record; + blockNumber: bigint; +} + +/** The Base side: the vault, its token and the governance timelock. */ +function baseChain() { + return { + chainId: 8453, + safe: 1_010n, + /** How far the confirmed head trails the wall clock. */ + safeAgeSeconds: 2, + paused: false, + /** Where set, the vault is paused from this block on, and at the latest block. */ + pausedFrom: undefined as bigint | undefined, + threshold: 3n, + guardian: GUARDIAN as string, + totalReleased: 0n, + totalSwept: 0n, + vaultBalance: SUPPLY, + approvals: new Map(), + readyAt: new Map(), + /** Ether balances by lowercased address. */ + balances: new Map([[GUARDIAN.toLowerCase(), 10n ** 18n]]), + logs: [] as Log[], + /** Where set, log queries answer as a backend whose head is this block. */ + logsHead: undefined as bigint | undefined, + /** Where set and it returns a block, the log query for these topics + * answers as a backend whose head is that block. */ + logsHeadFor: undefined as ((topics: string[]) => bigint | undefined) | undefined, + /** Where set, the vault has no code before this block. */ + deployedAt: undefined as bigint | undefined, + /** Requests to answer with an RPC error. */ + failing: undefined as ((method: string, params: unknown[]) => boolean) | undefined, + /** Where set, the guardian's pause transaction is accepted and mined in the + * confirmed block; it pauses the vault there only where `true`. */ + pauseTakesEffect: undefined as boolean | undefined, + /** Hashes of the transactions sent. */ + sent: [] as string[], + }; +} + +/** A vault total at `block`: its current value less what later events added + * (`sign` -1 for the balance, which those events lowered). */ +function totalAt(base: Base, total: bigint, eventName: string, field: string, block: bigint | undefined, sign = 1n): bigint { + const later = base.logs.filter((log) => log.eventName === eventName && block !== undefined && log.blockNumber > block); + return later.reduce((sum, log) => sum - sign * (log.args[field] as bigint), total); +} +type Base = ReturnType; + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +function answer(base: Base, method: string, params: any[]): unknown { + if (base.failing?.(method, params)) throw new Error("rate limited"); + switch (method) { + case "eth_getTransactionCount": + return "0x0"; + case "eth_gasPrice": + return "0x1"; + case "eth_estimateGas": + return "0x5208"; + case "eth_blockNumber": + return toHex(base.safe); + case "eth_sendRawTransaction": { + if (base.pauseTakesEffect === undefined) break; + base.sent.push(keccak256(params[0])); + if (base.pauseTakesEffect) base.pausedFrom ??= base.safe; + return keccak256(params[0]); + } + case "eth_getTransactionReceipt": { + if (!base.sent.includes(params[0])) return null; + return { + transactionHash: params[0], + transactionIndex: "0x0", + blockHash: toHex(base.safe, { size: 32 }), + blockNumber: toHex(base.safe), + from: GUARDIAN, + to: VAULT, + cumulativeGasUsed: "0x5208", + gasUsed: "0x5208", + effectiveGasPrice: "0x1", + contractAddress: null, + logs: [], + logsBloom: `0x${"00".repeat(256)}`, + status: "0x1", + type: "0x0", + }; + } + case "eth_chainId": + return toHex(base.chainId); + case "eth_getCode": { + const block = String(params[1]).startsWith("0x") ? BigInt(params[1]) : undefined; + const deployed = base.deployedAt !== undefined && (block === undefined || block >= base.deployedAt); + return String(params[0]).toLowerCase() === VAULT && deployed ? "0x6001" : "0x"; + } + case "eth_getBalance": + return toHex(base.balances.get(String(params[0]).toLowerCase()) ?? 0n); + case "eth_getBlockByNumber": { + const number = String(params[0]).startsWith("0x") ? BigInt(params[0]) : base.safe; + const timestamp = Math.floor(Date.now() / 1000) - base.safeAgeSeconds; + return { number: toHex(number), hash: toHex(number, { size: 32 }), timestamp: toHex(timestamp), transactions: [] }; + } + case "eth_call": { + const { functionName, args } = decodeFunctionData({ abi, data: params[0].data }); + if (functionName === "pause") { + if (base.pauseTakesEffect === undefined) throw new Error("unmocked pause"); + return "0x"; + } + const key = (args as readonly unknown[] | undefined)?.[0] as string; + const block = String(params[1]).startsWith("0x") ? BigInt(params[1]) : undefined; + const undeployed = base.deployedAt !== undefined && block !== undefined && block < base.deployedAt; + if (params[0].to.toLowerCase() === VAULT && undeployed) return "0x"; + const result = { + totalReleased: totalAt(base, base.totalReleased, "Released", "tokenAmount", block), + totalSwept: totalAt(base, base.totalSwept, "RemainderSwept", "amount", block), + conversionFactor: CF, + token: TOKEN, + paused: base.paused || (base.pausedFrom !== undefined && (block === undefined || block >= base.pausedFrom)), + threshold: base.threshold, + guardian: base.guardian, + activeApprovals: base.approvals.get(key) ?? 0n, + totalSupply: SUPPLY, + balanceOf: totalAt( + base, + totalAt(base, base.vaultBalance, "Released", "tokenAmount", block, -1n), + "RemainderSwept", + "amount", + block, + -1n, + ), + getTimestamp: base.readyAt.get(key) ?? 0n, + }[functionName]; + const { outputs } = abi.find((item) => item.type === "function" && item.name === functionName) as AbiFunction; + return encodeAbiParameters(outputs, [result]); + } + case "eth_getLogs": { + const [{ address, topics, fromBlock, toBlock }] = params; + const wanted = [topics?.[0] ?? []].flat() as string[]; + const head = base.logsHeadFor?.(wanted) ?? base.logsHead; + return base.logs.flatMap((log, index) => { + if (log.address.toLowerCase() !== String(address).toLowerCase()) return []; + if (log.blockNumber < BigInt(fromBlock) || log.blockNumber > BigInt(toBlock)) return []; + if (head !== undefined && log.blockNumber > head) return []; + const logTopics = encodeEventTopics({ abi, eventName: log.eventName, args: log.args } as never) as Hex[]; + if (wanted.length && !wanted.includes(logTopics[0])) return []; + const { inputs } = abi.find((item) => item.type === "event" && item.name === log.eventName) as AbiEvent; + const data = inputs.filter((input) => !input.indexed); + return [{ + address: log.address, + topics: logTopics, + data: encodeAbiParameters(data, data.map((input) => log.args[input.name!])), + blockNumber: toHex(log.blockNumber), + blockHash: toHex(log.blockNumber, { size: 32 }), + transactionHash: toHex(index + 1, { size: 32 }), + transactionIndex: "0x0", + logIndex: toHex(index), + removed: false, + }]; + }); + } + } + throw new Error(`unmocked ${method}`); +} + +/** A migration approved by three attestors and released on Base. */ +function release(base: Base, nonce: bigint, palletAmount: bigint, blockNumber = 1_005n): void { + for (const attestor of ATTESTORS) { + base.logs.push({ address: VAULT, eventName: "Approved", args: { nonce, recipient: RECIPIENT, palletAmount, attestor }, blockNumber }); + } + const tokenAmount = palletAmount * CF; + base.logs.push({ address: VAULT, eventName: "Released", args: { nonce, recipient: RECIPIENT, palletAmount, tokenAmount }, blockNumber }); + base.totalReleased += tokenAmount; + base.vaultBalance -= tokenAmount; +} + +interface Call { + section: string; + method: string; + args: unknown[]; +} + +/** Calls the stand-in registry decodes; a call's bytes are its name. */ +const CALLS: Record = { + remark: { section: "system", method: "remark", args: ["0x"] }, + mint: { section: "utility", method: "batch", args: [[{ section: "tokens", method: "setBalance", args: [] }]] }, + setTreasuryDestination: { section: "tokenMigration", method: "setTreasuryDestination", args: ["0x55"] }, +}; +const encodeCall = (name: string) => new TextEncoder().encode(name); +function decodeCall(bytes: Uint8Array): Call { + const call = CALLS[new TextDecoder().decode(bytes)]; + if (!call) throw new Error("Unable to find Call"); + return call; +} +const codec = (value: number | bigint) => ({ + toString: () => String(value), + toNumber: () => Number(value), + toBigInt: () => BigInt(value), +}); +const hash = (hex: string) => ({ toHex: () => hex, toString: () => hex, isEmpty: /^0x0*$/.test(hex) }); +const inline = (name: string) => ({ isInline: true, asInline: encodeCall(name), isLookup: false, isLegacy: false, toString: () => `inline ${name}` }); +const lookup = (hex: string, len: number) => ({ + isInline: false, + isLookup: true, + asLookup: { hash_: hash(hex), len: codec(len) }, + isLegacy: false, + toString: () => `lookup ${hex}`, +}); +const legacy = (hex: string) => ({ + isInline: false, + isLookup: false, + isLegacy: true, + asLegacy: { hash_: hash(hex) }, + toString: () => `legacy ${hex}`, +}); +type Bounded = ReturnType | ReturnType | ReturnType; + +interface PendulumEvent { + section: string; + method: string; + data: unknown[]; +} + +/** The Pendulum side: finalized blocks with their events, the migration + * pallet's totals and the governance queues. A block hash encodes the block + * number, and whether it is on the fork the node serves from `forkFrom` on. */ +function pendulumState() { + return { + /** Where set, the node serves another chain from this block on. */ + forkFrom: undefined as number | undefined, + /** Blocks whose header the node alters, so it no longer hashes to its hash. */ + alteredHeaders: new Set(), + /** Blocks whose header names the next block number. */ + renumberedHeaders: new Set(), + /** Whether the node names block hashes by number from the chain without the fork. */ + canonicalBlockHashes: false, + finalized: 100, + /** The newest block the node imported, where it is past the finalized one. */ + best: undefined as number | undefined, + /** Age of the finalized head; earlier blocks are 12 s apart. */ + headAgeMs: 0, + failure: undefined as Error | undefined, + /** Holds every finalized-head read until it settles, as a slow node would. */ + gate: undefined as Promise | undefined, + /** Finalized-head reads so far. */ + finalizedReads: 0, + events: new Map(), + totalMigrated: 0n, + nextNonce: 0n, + publicProps: [] as Array<[number, Bounded]>, + external: undefined as Bounded | undefined, + referenda: new Map(), + /** Noted preimages by `${hash}:${len}`, and their lengths by hash (old: + * noted before the pallet's status migration). */ + preimages: new Map(), + notedLengths: new Map(), + oldNotedLengths: new Map(), + preimageReads: 0, + /** Preimage reads still to fail, as an RPC error would. */ + preimageFailures: 0, + /** Open motions by hash, their index where it is not 0, the stored size + * where it is not the call's, and the motions whose call was fetched. */ + motions: { council: new Map(), technicalCommittee: new Map() }, + motionIndices: new Map(), + motionSizes: new Map(), + motionReads: new Set(), + }; +} +type Pendulum = ReturnType; + +/** A node serving `chain`, as much of the polkadot.js API as the monitor uses. */ +function pendulumChain(chain: Pendulum = pendulumState()) { + const collective = (motions: Map) => ({ + proposals: async () => [...motions.keys()].map(hash), + voting: async (motion: { toHex(): string }) => ({ + toJSON: () => (motions.has(motion.toHex()) ? { index: chain.motionIndices.get(motion.toHex()) ?? 0 } : null), + }), + proposalOf: Object.assign( + async (motion: { toHex(): string }) => { + chain.motionReads.add(motion.toHex()); + const name = motions.get(motion.toHex()); + return optional(name === undefined ? undefined : decodeCall(encodeCall(name))); + }, + { + size: async (motion: { toHex(): string }) => + codec(chain.motionSizes.get(motion.toHex()) ?? encodeCall(motions.get(motion.toHex()) ?? "").length), + }, + ), + }); + const optional = (value: T | undefined) => (value === undefined ? { isSome: false } : { isSome: true, unwrap: () => value }); + /** The migration totals at `block`: the current ones less the burns of later blocks. */ + const totalsAt = (block: number) => { + let { nextNonce, totalMigrated } = chain; + for (const [later, events] of chain.events) { + for (const { method, data } of later > block ? events : []) { + if (method !== "MigrationInitiated") continue; + nextNonce--; + totalMigrated -= (data[3] as { toBigInt(): bigint }).toBigInt(); + } + } + return { nextNonce, totalMigrated }; + }; + const at = (block: number) => ({ + registry: { createType: (_type: string, bytes: Uint8Array) => decodeCall(bytes) }, + consts: { democracy: { launchPeriod: codec(36_000) } }, + query: { + timestamp: { now: async () => codec(Date.now() - chain.headAgeMs - (chain.finalized - block) * 12_000) }, + system: { events: async () => (chain.events.get(block) ?? []).map((event) => ({ event })) }, + tokenMigration: { + totalMigrated: async () => codec(totalsAt(block).totalMigrated), + nextNonce: async () => codec(totalsAt(block).nextNonce), + }, + balances: { totalIssuance: async () => codec(100_000_000n * PEN - totalsAt(block).totalMigrated) }, + democracy: { + publicProps: async () => chain.publicProps.map(([index, bounded]) => [codec(index), bounded, "5Proposer"]), + nextExternal: async () => optional(chain.external && [chain.external, "SimpleMajority"]), + referendumCount: async () => codec(chain.referenda.size ? Math.max(...chain.referenda.keys()) + 1 : 0), + lowestUnbaked: async () => codec(chain.referenda.size ? Math.min(...chain.referenda.keys()) : 0), + referendumInfoOf: async (index: number) => { + const proposal = chain.referenda.get(index); + return optional(proposal && { isOngoing: true, asOngoing: { proposal, end: codec(9_000_000) } }); + }, + }, + preimage: { + preimageFor: async ([preimage, len]: [{ toHex(): string }, unknown]) => { + chain.preimageReads++; + if (chain.preimageFailures > 0 && chain.preimageFailures--) throw new Error("request timed out"); + const name = chain.preimages.get(`${preimage.toHex()}:${len}`); + return optional(name === undefined ? undefined : encodeCall(name)); + }, + requestStatusFor: async (preimage: { toHex(): string }) => { + const len = chain.notedLengths.get(preimage.toHex()); + return { toJSON: () => (len === undefined ? null : { unrequested: { ticket: ["5Noter", 1], len } }) }; + }, + statusFor: async (preimage: { toHex(): string }) => { + const len = chain.oldNotedLengths.get(preimage.toHex()); + return { toJSON: () => (len === undefined ? null : { unrequested: { deposit: ["5Noter", 1], len } }) }; + }, + }, + council: collective(chain.motions.council), + technicalCommittee: collective(chain.motions.technicalCommittee), + }, + }); + const api = { + genesisHash: hash(`0x${"5d".repeat(32)}`), + rpc: { + chain: { + getFinalizedHead: async () => { + chain.finalizedReads++; + await chain.gate; + if (chain.failure) throw chain.failure; + return hash(pendulumHash(chain, chain.finalized)); + }, + getHeader: async (value: string | { toHex(): string }) => { + const requested = typeof value === "string" ? value : value.toHex(); + const block = blockOf(requested); + const onFork = BigInt(requested) >> 64n !== 0n; + return { + number: codec(chain.renumberedHeaders.has(block) ? block + 1 : block), + parentHash: hash(pendulumHash(onFork ? chain : {}, block - 1)), + hash: hash(chain.alteredHeaders.has(block) ? pendulumHash(chain, block + 1_000_000) : requested), + }; + }, + getBlockHash: async (block: number) => + hash( + block > (chain.best ?? chain.finalized) + ? toHex(0, { size: 32 }) + : pendulumHash(chain.canonicalBlockHashes ? {} : chain, block), + ), + }, + }, + /** Decodes the head data of the stand-in relay chain. */ + createType: (_type: string, bytes: Uint8Array) => { + const header = JSON.parse(new TextDecoder().decode(bytes)) as { number: number; hash: string }; + return { number: codec(header.number), hash: hash(header.hash) }; + }, + at: async (value: string | { toHex(): string }) => at(blockOf(typeof value === "string" ? value : value.toHex())), + }; + return { chain, api: api as unknown as ApiPromise }; +} + +/** A relay chain whose finalized block, `ageMs()` old, records `head()` as the + * Pendulum head; every read fails while `failing()` holds. */ +function relayChain(head: () => { number: number; hash: string }, failing = () => false, ageMs = () => 0): ApiPromise { + const check = () => { + if (failing()) throw new Error("relay RPC unavailable"); + }; + const relay = { + rpc: { chain: { getFinalizedHead: async () => (check(), hash(`0x${"77".repeat(32)}`)) } }, + at: async () => ({ + query: { + timestamp: { now: async () => codec(Date.now() - ageMs()) }, + paras: { + heads: async (paraId: number) => { + check(); + const data = new TextEncoder().encode(JSON.stringify(head())); + return paraId === 2094 ? { isSome: true, unwrap: () => ({ toU8a: () => data }) } : { isSome: false }; + }, + }, + }, + }), + }; + return relay as unknown as ApiPromise; +} + +/** The hash of `block` on the chain the node serves. */ +function pendulumHash(chain: { forkFrom?: number }, block: number): string { + const fork = chain.forkFrom !== undefined && block >= chain.forkFrom ? 1n << 64n : 0n; + return toHex(fork + BigInt(block), { size: 32 }); +} +const blockOf = (blockHash: string) => Number(BigInt(blockHash) & 0xffff_ffff_ffff_ffffn); + +const MIGRATOR = `0x${"ab".repeat(32)}`; +const TREASURY_ACCOUNT = `0x${Buffer.from("modlpy/trsry").toString("hex")}${"00".repeat(20)}`; + +/** A finalized burn on Pendulum by `who` for `recipient`, in block 101 unless stated. */ +function burn( + pendulum: Pendulum, + nonce: bigint, + palletAmount: bigint, + block = 101, + { who = MIGRATOR, recipient = RECIPIENT } = {}, +): void { + pendulum.finalized = Math.max(pendulum.finalized, block); + const events = pendulum.events.get(block) ?? []; + events.push({ + section: "tokenMigration", + method: "MigrationInitiated", + data: [codec(nonce), hash(who), hash(recipient), codec(palletAmount)], + }); + pendulum.events.set(block, events); + pendulum.totalMigrated += palletAmount; + pendulum.nextNonce = nonce + 1n; +} + +interface Page { + subject: string; + detail: string; + severity: string; + /** The Slack message as sent. */ + message: Record; + status?: number; +} + +/** A page from the Slack message the monitor sent, with Slack's escapes undone. */ +function readPage(message: Record): Page { + const [, severity, subject, detail] = /^\[(CRITICAL|WARNING)\] pen-monitor: (.*)\n([\s\S]*)$/.exec(String(message.text)) ?? []; + assert.ok(subject, `not a monitor page: ${JSON.stringify(message)}`); + const unescaped = detail.replaceAll("<", "<").replaceAll(">", ">").replaceAll("&", "&"); + return { subject, detail: unescaped, severity, message }; +} + +async function body(request: IncomingMessage): Promise { + let text = ""; + for await (const chunk of request) text += chunk; + return text; +} + +let instances = 0; +let monitorEnv: string[] = []; + +/** A started monitor against fresh stand-ins, which `prepare` can change + * before startup. `webhook` answers each page with an HTTP status; `pages` + * lists them in arrival order from after startup, `startup` the ones before. + * `heartbeats` lists the heartbeat requests, which `heartbeat` answers. */ +async function setup( + t: { after(fn: () => void): void }, + { + env = {}, + webhook = () => 200, + heartbeat = () => 200, + prepare = () => {}, + connect, + }: { + env?: Record; + webhook?: (page: Page) => number | Promise; + heartbeat?: () => number; + prepare?: (base: Base, pendulum: Pendulum) => void; + /** Reaches the other nodes: the second Pendulum node and relay RPCs. */ + connect?: (url: string) => Promise; + } = {}, +) { + const base = baseChain(); + const { chain: pendulum, api } = pendulumChain(); + prepare(base, pendulum); + const pages: Page[] = []; + const heartbeats: string[] = []; + const server = createServer(async (request, response) => { + const text = await body(request); + if (request.url === "/heartbeat") { + heartbeats.push(request.method ?? ""); + response.writeHead(heartbeat()).end(); + return; + } + if (request.url === "/alert") { + const page = readPage(JSON.parse(text)); + pages.push(page); + page.status = await webhook(page); + response.writeHead(page.status).end(); + return; + } + const { id, method, params } = JSON.parse(text); + let reply: object; + try { + reply = { result: answer(base, method, params) }; + } catch (error) { + reply = { error: { code: -32603, message: String(error) } }; + } + response.setHeader("content-type", "application/json"); + response.end(JSON.stringify({ jsonrpc: "2.0", id, ...reply })); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + t.after(() => { + server.closeAllConnections(); + server.close(); + }); + const url = `http://127.0.0.1:${(server.address() as AddressInfo).port}`; + const stateFile = join(mkdtempSync(join(tmpdir(), "pen-monitor-test-")), "state.json"); + const variables: Record = { + PENDULUM_WS: "ws://127.0.0.1:1", + BASE_RPC_URL: url, + VAULT_ADDRESS: VAULT, + BASE_START_BLOCK: "1000", + PENDULUM_START_BLOCK: "101", + STATE_FILE: stateFile, + ALERT_WEBHOOK_URL: `${url}/alert`, + HEARTBEAT_URL: `${url}/heartbeat`, + ...env, + }; + for (const name of monitorEnv) delete process.env[name]; + monitorEnv = Object.keys(variables); + Object.assign(process.env, variables); + const launch = async () => { + const monitor: typeof import("./main.js") = await import(`./main.js?instance=${++instances}`); + await monitor.start(api, connect ?? (async (url) => assert.fail(`unexpected connection to ${url}`))); + return monitor; + }; + const monitor = await launch(); + const startup = pages.splice(0); + return { + base, + pendulum, + api, + pages, + startup, + heartbeats, + monitor, + /** A new monitor process on the same state file and stand-ins. */ + restart: launch, + subjects: () => pages.map((page) => page.subject), + persisted: () => JSON.parse(readFileSync(stateFile, "utf8")) as { + lastPendulumBlock: number; + lastPendulumHash: string; + relayAnchor?: { number: number; hash: string }; + baseFromBlock: string; + baseAdminFromBlock: string; + migrations: Array<{ nonce: string }>; + mismatchedApprovals: Array<{ nonce: string; attestor: string }>; + }, + }; +} + +test("a released migration reconciles and the poll pages nothing", async (t) => { + const { base, pendulum, api, monitor, subjects, persisted } = await setup(t); + burn(pendulum, 0n, 100n * PEN); + release(base, 0n, 100n * PEN); + await monitor.poll(api); + assert.deepEqual(subjects(), []); + assert.deepEqual(persisted().migrations, []); + assert.equal(persisted().baseFromBlock, "1011"); +}); + +/** Pages sent concurrently can arrive in either order. */ +const sorted = (subjects: string[]) => [...subjects].sort(); + +test("pages are Slack messages carrying severity, subject and detail in their text only", async (t) => { + const { base, api, monitor, pages } = await setup(t); + base.vaultBalance -= 1n; + base.safeAgeSeconds = 601; + await monitor.poll(api); + assert.ok(pages.length >= 3, JSON.stringify(pages)); + for (const page of pages) assert.deepEqual(Object.keys(page.message), ["text"]); + const severity = (subject: string) => pages.find((page) => page.subject === subject)?.severity; + assert.equal(severity("VAULT BALANCE DEFICIT"), "CRITICAL"); + assert.equal(severity("AUTO-PAUSE UNAVAILABLE"), "CRITICAL"); + assert.equal(severity("BASE VIEW STALLED"), "WARNING"); + assert.match(pages.find((page) => page.subject === "VAULT BALANCE DEFICIT")!.detail, /^balance \d+ \+ released 0 \+ swept 0 < supply/); +}); + +test("only a poll that runs every check requests the heartbeat", async (t) => { + const { base, pendulum, api, monitor, heartbeats } = await setup(t); + await monitor.poll(api); + assert.deepEqual(heartbeats, ["GET"]); + pendulum.failure = new Error("node unreachable"); + await monitor.poll(api); + pendulum.failure = undefined; + const guardianUpdated = toEventSelector("GuardianUpdated(address)"); // only the admin scan asks for it + base.failing = (method, params) => method === "eth_getLogs" && JSON.stringify(params).includes(guardianUpdated); + base.safe = 1_020n; // a new range for the admin scan + await monitor.poll(api); + assert.deepEqual(heartbeats, ["GET"]); + base.failing = undefined; + await monitor.poll(api); + assert.deepEqual(heartbeats, ["GET", "GET"]); +}); + +test("a failing heartbeat never stops the monitor", async (t) => { + const { api, monitor, heartbeats, subjects } = await setup(t, { heartbeat: () => 500 }); + await monitor.poll(api); + await monitor.poll(api); + assert.deepEqual(heartbeats, ["GET", "GET"]); + const unreachable = await setup(t, { env: { HEARTBEAT_URL: "http://127.0.0.1:1/heartbeat" } }); + await unreachable.monitor.poll(unreachable.api); + assert.deepEqual([...subjects(), ...unreachable.subjects()], []); +}); + +test("an auto-pause counts as confirmed only where the vault is paused at the confirmed block", async (t) => { + for (const pauseTakesEffect of [true, false]) { + const { base, api, monitor, subjects } = await setup(t, { + env: { GUARDIAN_PRIVATE_KEY: GUARDIAN_KEY, BASE_FINALITY_POLL_MS: "10" }, + prepare: (base) => (base.pauseTakesEffect = pauseTakesEffect), + }); + base.vaultBalance -= 1n; + await monitor.poll(api); + assert.equal(base.sent.length, 1); + const outcome = subjects().filter((subject) => subject.includes("PAUSE") || subject.includes("pause")); + assert.deepEqual(outcome, [pauseTakesEffect ? "vault auto-pause confirmed" : "AUTO-PAUSE FAILED"]); + } +}); + +test("a vault balance deficit pages and asks for the auto-pause", async (t) => { + const { base, api, monitor, subjects } = await setup(t); + base.vaultBalance -= 1n; + await monitor.poll(api); + assert.deepEqual(sorted(subjects()), ["AUTO-PAUSE UNAVAILABLE", "VAULT BALANCE DEFICIT"]); +}); + +test("a Pendulum failure pages on its own and the vault balance check still runs", async (t) => { + const { base, pendulum, api, monitor, subjects } = await setup(t); + pendulum.failure = new Error("node unreachable"); + base.vaultBalance -= 1n; + await monitor.poll(api); + assert.deepEqual(sorted(subjects()), [ + "AUTO-PAUSE UNAVAILABLE", + "PENDULUM SOURCE UNREADABLE", + "VAULT BALANCE DEFICIT", + "pendulum governance watch failed", + ]); +}); + +test("a failing Base log read does not skip the vault balance check", async (t) => { + const { base, pendulum, api, monitor, subjects } = await setup(t, { env: { GUARDIAN_PRIVATE_KEY: GUARDIAN_KEY } }); + pendulum.failure = new Error("node unreachable"); + const released = toEventSelector("Released(uint64,address,uint256,uint256)"); // only reconciliation asks for it + base.failing = (method, params) => method === "eth_getLogs" && JSON.stringify(params).includes(released); + base.vaultBalance -= 1n; + base.paused = true; // so the auto-pause only confirms + await monitor.poll(api); + assert.ok(subjects().includes("VAULT BALANCE DEFICIT"), JSON.stringify(subjects())); + assert.ok(subjects().includes("vault pause confirmed"), JSON.stringify(subjects())); +}); + +/** Waits until `predicate` holds, failing after `ms`. */ +async function until(predicate: () => boolean, label: string, ms = 2_000): Promise { + const deadline = Date.now() + ms; + while (!predicate()) { + if (Date.now() > deadline) assert.fail(`timed out waiting for ${label}`); + await new Promise((resolve) => setTimeout(resolve, 10)); + } +} + +test("a slow Pendulum source does not delay the vault balance check", async (t) => { + const { base, pendulum, api, monitor, subjects } = await setup(t); + let respond = () => {}; + pendulum.gate = new Promise((resolve) => (respond = resolve)); + base.vaultBalance -= 1n; + const polling = monitor.poll(api); + try { + await until(() => subjects().includes("VAULT BALANCE DEFICIT"), "the deficit page while Pendulum is still answering"); + } finally { + respond(); + await polling; + } +}); + +/** Holds every finalized-head read of `pendulum` until `respond` is called; + * `waiting` resolves once a read waits. */ +function holdSource(pendulum: Pendulum) { + let respond = () => {}; + pendulum.gate = new Promise((resolve) => (respond = resolve)); + const reads = pendulum.finalizedReads; + return { respond, waiting: () => until(() => pendulum.finalizedReads > reads, "a poll waiting on the source") }; +} + +test("while a poll waits on a slow source, the vault balance check runs on its own", async (t) => { + const { base, pendulum, api, monitor, subjects } = await setup(t, { env: { POLL_INTERVAL_MS: "20" } }); + const { respond, waiting } = holdSource(pendulum); + const polling = monitor.poll(api); // checks the balance, then waits on the source + try { + await waiting(); + await new Promise((resolve) => setTimeout(resolve, 30)); // a poll interval later + base.vaultBalance -= 1n; + await monitor.watchVaultBalance(); + assert.ok(subjects().includes("VAULT BALANCE DEFICIT"), JSON.stringify(subjects())); + } finally { + respond(); + await polling; + } +}); + +test("the vault balance check does not run on its own within a poll interval of a poll's, or between polls", async (t) => { + const { base, pendulum, api, monitor, subjects } = await setup(t, { env: { POLL_INTERVAL_MS: "500" } }); + const { respond, waiting } = holdSource(pendulum); + const polling = monitor.poll(api); + try { + await waiting(); + base.vaultBalance -= 1n; + await monitor.watchVaultBalance(); // the poll checked the balance less than an interval ago + } finally { + respond(); + await polling; + } + await new Promise((resolve) => setTimeout(resolve, 550)); + await monitor.watchVaultBalance(); // no poll is running: the next one checks it + assert.ok(!subjects().includes("VAULT BALANCE DEFICIT"), JSON.stringify(subjects())); +}); + +test("a pause attempt in flight is joined, not started again", async (t) => { + let answerFailedPause = () => {}; + const failedPauseAnswered = new Promise((resolve) => (answerFailedPause = resolve)); + const { base, pendulum, api, monitor, pages } = await setup(t, { + env: { GUARDIAN_PRIVATE_KEY: GUARDIAN_KEY, POLL_INTERVAL_MS: "20" }, + webhook: async ({ subject }) => { + if (subject === "AUTO-PAUSE FAILED") await failedPauseAnswered; // holds that attempt in flight + return 200; + }, + }); + burn(pendulum, 0n, 100n * PEN); + await monitor.poll(api); + const tokenAmount = 100n * PEN * CF; // a release of nonce 0 to another recipient + base.logs.push({ address: VAULT, eventName: "Released", args: { nonce: 0n, recipient: TOKEN, palletAmount: 100n * PEN, tokenAmount }, blockNumber: 1_015n }); + base.totalReleased += tokenAmount; + base.vaultBalance -= tokenAmount; + base.safe = 1_020n; + const { respond, waiting } = holdSource(pendulum); + const polling = monitor.poll(api); + const failedPauses = () => pages.filter((page) => page.subject === "AUTO-PAUSE FAILED").length; + try { + await waiting(); + await new Promise((resolve) => setTimeout(resolve, 30)); // a poll interval later + base.vaultBalance -= 1n; + const watching = monitor.watchVaultBalance(); // a deficit: the first pause attempt + await until(() => failedPauses() === 1, "the first failed pause page"); + respond(); // the poll goes on to the release, a second violation + await until(() => pages.some((page) => page.subject === "MIGRATION TUPLE VIOLATION"), "the second violation"); + await new Promise((resolve) => setTimeout(resolve, 50)); + answerFailedPause(); + await watching; + } finally { + respond(); + answerFailedPause(); + await polling; + } + assert.equal(failedPauses(), 1, JSON.stringify(pages.map((page) => page.subject))); +}); + +test("a source whose chain leaves the blocks ingested so far pages critical and ingests nothing", async (t) => { + const { pendulum, api, monitor, restart, pages, subjects, persisted } = await setup(t); + burn(pendulum, 0n, 100n * PEN); + await monitor.poll(api); + const restarted = await restart(); + pendulum.forkFrom = 101; // the node now serves another chain from block 101 on + burn(pendulum, 1n, 100n * PEN, 102); + await restarted.poll(api); + const page = pages.find((page) => page.subject === "CRITICAL: PENDULUM SOURCE CONTRADICTED"); + assert.equal(page?.severity, "CRITICAL", JSON.stringify(pages)); + assert.match(page.detail, /chain has 0x0+10+65 at Pendulum block 101, where the monitor ingested 0x0+65;/); + assert.ok(!subjects().includes("PENDULUM SOURCE UNREADABLE"), JSON.stringify(subjects())); + assert.equal(persisted().lastPendulumBlock, 101); + assert.deepEqual(persisted().migrations.map((migration) => migration.nonce), ["0"]); +}); + +test("a source whose finalized head falls behind the block ingested last is read at that block", async (t) => { + const { pendulum, api, monitor, pages, persisted } = await setup(t); + burn(pendulum, 0n, 100n * PEN); + await monitor.poll(api); + pendulum.finalized = 100; // a node that has not finalized block 101 yet + await monitor.poll(api); + assert.equal(pages.length, 0, JSON.stringify(pages)); + assert.equal(persisted().lastPendulumBlock, 101); +}); + +test("a source header that does not hash to its block hash pages critical", async (t) => { + const { pendulum, api, monitor, pages } = await setup(t); + burn(pendulum, 0n, 100n * PEN, 102); + pendulum.alteredHeaders.add(102); + await monitor.poll(api); + assert.match( + pages.find((page) => page.subject === "CRITICAL: PENDULUM SOURCE CONTRADICTED")?.detail ?? "", + /header for Pendulum block 102 does not hash to 0x0+66;/, + ); +}); + +test("a source header that names another block number pages critical", async (t) => { + const { pendulum, api, monitor, pages, persisted } = await setup(t); + burn(pendulum, 0n, 100n * PEN, 102); + pendulum.renumberedHeaders.add(101); + await monitor.poll(api); + assert.match( + pages.find((page) => page.subject === "CRITICAL: PENDULUM SOURCE CONTRADICTED")?.detail ?? "", + /header for Pendulum block 101 does not hash to 0x0+65;/, + ); + assert.equal(persisted().lastPendulumBlock, 100); +}); + +const SECOND_NODE = "ws://second-node.test:9944?apikey=secret"; + +/** A monitor whose second Pendulum node serves the source's chain, except for + * the values in `own`, which the test may change later; with `relays`, also + * against these relay RPCs, in this order. */ +async function crossCheckSetup(t: { after(fn: () => void): void }, own: Partial = {}, relays: ApiPromise[] = []) { + let source: Pendulum | undefined; + const chain = new Proxy({} as Pendulum, { + get: (_target, key) => (key in own ? own[key as keyof Pendulum] : source![key as keyof Pendulum]), + }); + const second = pendulumChain(chain); + const urls = relays.map((_relay, index) => `wss://relay-${index + 1}.test`); + const monitor = await setup(t, { + env: { PENDULUM_WS_CROSSCHECK: SECOND_NODE, ...(urls.length ? { POLKADOT_WS: urls.join(",") } : {}) }, + prepare: (_base, pendulum) => (source = pendulum), + connect: async (url) => (url === SECOND_NODE ? second.api : relays[urls.indexOf(url)] ?? assert.fail(url)), + }); + return { ...monitor, own }; +} + +test("a second node that disagrees on the newest block pages critical and nothing is ingested", async (t) => { + const { pendulum, api, monitor, pages, persisted, own } = await crossCheckSetup(t, { finalized: 200 }); + burn(pendulum, 0n, 100n * PEN); + await monitor.poll(api); + assert.equal(pages.length, 0, JSON.stringify(pages)); + assert.equal(persisted().lastPendulumBlock, 101); + own.forkFrom = 102; + burn(pendulum, 1n, 100n * PEN, 102); + await monitor.poll(api); + assert.match( + pages.find((page) => page.subject === "CRITICAL: PENDULUM SOURCE CONTRADICTED")?.detail ?? "", + /second Pendulum node has 0x0+10+66 at block 102, the source 0x0+66;/, + ); + assert.equal(persisted().lastPendulumBlock, 101); +}); + +test("a second node that disagrees on the block ingested last, its finalized head, pages critical", async (t) => { + const { pendulum, api, monitor, pages, persisted, own } = await crossCheckSetup(t, { finalized: 101 }); + burn(pendulum, 0n, 100n * PEN); + await monitor.poll(api); + assert.equal(persisted().lastPendulumBlock, 101); + own.forkFrom = 101; // the second node now has another block 101 + burn(pendulum, 1n, 100n * PEN, 102); + await monitor.poll(api); + assert.match( + pages.find((page) => page.subject === "CRITICAL: PENDULUM SOURCE CONTRADICTED")?.detail ?? "", + /second Pendulum node has 0x0+10+65 at block 101, the source 0x0+65;/, + ); + assert.equal(persisted().lastPendulumBlock, 101); +}); + +test("a second node that cannot answer only warns", async (t) => { + let connections = 0; + const { pendulum, api, monitor, pages, persisted, heartbeats } = await setup(t, { + env: { PENDULUM_WS_CROSSCHECK: SECOND_NODE }, + connect: async (url) => { + connections++; + throw new Error(`cannot connect to ${url}`); + }, + }); + burn(pendulum, 0n, 100n * PEN); + await monitor.poll(api); + assert.deepEqual(pages.map((page) => [page.subject, page.severity]), [["PENDULUM CROSS-CHECK UNAVAILABLE", "WARNING"]]); + assert.match(pages[0].detail, /cannot be read: Error: cannot connect to /); + assert.equal(persisted().lastPendulumBlock, 101); + assert.equal(connections, 1); // not asked again for the ingested block's migration events + assert.deepEqual(heartbeats, ["GET"]); + await monitor.poll(api); + assert.equal(pages.length, 1); // throttled +}); + +test("the URLs of the checking nodes are validated at startup", async (t) => { + await assert.rejects(setup(t, { env: { POLKADOT_WS: "wss://relay.test, https://relay.test" } }), /invalid POLKADOT_WS/); + await assert.rejects( + setup(t, { env: { PENDULUM_WS_CROSSCHECK: "wss://a.test,wss://b.test" } }), + /invalid PENDULUM_WS_CROSSCHECK: expected one ws/, + ); +}); + +test("a node connection that fails with an event says why", async (t) => { + const { pendulum, api, monitor, pages } = await setup(t, { + env: { PENDULUM_WS_CROSSCHECK: SECOND_NODE }, + connect: () => Promise.reject({ type: "error", message: "getaddrinfo ENOTFOUND second-node.test" }), + }); + burn(pendulum, 0n, 100n * PEN); + await monitor.poll(api); + assert.match(pages[0]?.detail ?? "", /cannot be read: Error: getaddrinfo ENOTFOUND second-node\.test;/); +}); + +test("a block the second node has not finalized yet waits for it, without a page", async (t) => { + const { pendulum, api, monitor, pages, persisted, own } = await crossCheckSetup(t, { finalized: 100 }); + burn(pendulum, 0n, 100n * PEN); + await monitor.poll(api); + assert.equal(pages.length, 0, JSON.stringify(pages)); + assert.equal(persisted().lastPendulumBlock, 100); + own.finalized = 101; + await monitor.poll(api); + assert.equal(pages.length, 0, JSON.stringify(pages)); + assert.equal(persisted().lastPendulumBlock, 101); + assert.deepEqual(persisted().migrations.map((migration) => migration.nonce), ["0"]); +}); + +test("a second node whose best chain holds another newest block, not yet finalized, holds that block back", async (t) => { + const { pendulum, api, monitor, pages, persisted } = await crossCheckSetup(t, { finalized: 100, best: 101, forkFrom: 101 }); + burn(pendulum, 0n, 100n * PEN); + await monitor.poll(api); + assert.equal(pages.length, 0, JSON.stringify(pages)); + assert.equal(persisted().lastPendulumBlock, 100); +}); + +test("a source on another chain one block past the second node's head ingests nothing from that chain", async (t) => { + const { base, pendulum, api, monitor, pages, persisted, own } = await crossCheckSetup(t, { finalized: 101, forkFrom: undefined }); + burn(pendulum, 0n, 100n * PEN); + pendulum.forkFrom = 102; // the source serves another chain from block 102 on... + burn(pendulum, 1n, 999n * PEN, 102); // ...with a fabricated burn, one block past the second node's head + release(base, 1n, 999n * PEN); + await monitor.poll(api); + assert.equal(persisted().lastPendulumBlock, 101); + assert.deepEqual(persisted().migrations.map((migration) => migration.nonce), ["0"]); + assert.equal(persisted().baseFromBlock, "1000"); // the release of the fabricated nonce awaits its burn + assert.deepEqual([...new Set(pages.map((page) => page.subject))], ["UNVERIFIED BASE EVENT"]); + own.finalized = 102; // the second node finalizes its own block 102... + burn(pendulum, 2n, 999n * PEN, 103); // ...and the source stays one block ahead + await monitor.poll(api); + assert.match( + pages.find((page) => page.subject === "CRITICAL: PENDULUM SOURCE CONTRADICTED")?.detail ?? "", + /second Pendulum node has 0x0+66 at block 102, the source 0x0+10+66;/, + ); + assert.equal(persisted().lastPendulumBlock, 101); + assert.deepEqual(persisted().migrations.map((migration) => migration.nonce), ["0"]); +}); + +test("with the second node and the relay chain, a block past both their heads waits, and a fork there is caught", async (t) => { + let recorded = 101; + const { pendulum, api, monitor, pages, persisted, own } = await crossCheckSetup(t, { finalized: 101, forkFrom: undefined }, [ + relayChain(() => ({ number: recorded, hash: pendulumHash({}, recorded) })), + ]); + burn(pendulum, 0n, 100n * PEN); + pendulum.forkFrom = 102; // the source finalizes another block 102... + burn(pendulum, 1n, 999n * PEN, 102); // ...with a fabricated burn + await monitor.poll(api); + assert.equal(pages.length, 0, JSON.stringify(pages)); + assert.equal(persisted().lastPendulumBlock, 101); + assert.deepEqual(persisted().migrations.map((migration) => migration.nonce), ["0"]); + own.finalized = 102; // both checkers reach the real block 102 + recorded = 102; + await monitor.poll(api); + assert.ok(pages.some((page) => page.subject === "CRITICAL: PENDULUM SOURCE CONTRADICTED"), JSON.stringify(pages)); + assert.equal(persisted().lastPendulumBlock, 101); +}); + +test("a second node behind the block ingested last holds newer blocks back, without a contradiction", async (t) => { + const { pendulum, api, monitor, pages, persisted, own } = await crossCheckSetup(t, { finalized: 100, headAgeMs: 400_000 }); + burn(pendulum, 0n, 100n * PEN); + await monitor.poll(api); // the stale second node is left out: block 101 is ingested + assert.equal(persisted().lastPendulumBlock, 101); + own.headAgeMs = 0; // it answers again, still at block 100 + burn(pendulum, 1n, 100n * PEN, 102); + await monitor.poll(api); + assert.deepEqual(pages.map((page) => page.subject), ["PENDULUM CROSS-CHECK UNAVAILABLE"]); + assert.equal(persisted().lastPendulumBlock, 101); + own.finalized = 102; + await monitor.poll(api); + assert.equal(persisted().lastPendulumBlock, 102); +}); + +test("a second node whose finalized head is stale counts as unavailable", async (t) => { + const { pendulum, api, monitor, pages, persisted } = await crossCheckSetup(t, { finalized: 100, headAgeMs: 400_000 }); + burn(pendulum, 0n, 100n * PEN); + await monitor.poll(api); + assert.deepEqual(pages.map((page) => [page.subject, page.severity]), [["PENDULUM CROSS-CHECK UNAVAILABLE", "WARNING"]]); + assert.match(pages[0].detail, /has a finalized head 40\ds old;/); + assert.equal(persisted().lastPendulumBlock, 101); +}); + +test("a second node with another nonce or total at the finalized head pages critical and nothing is ingested", async (t) => { + for (const own of [{ nextNonce: 2n }, { totalMigrated: 7n * PEN }]) { + const { pendulum, api, monitor, pages, persisted } = await crossCheckSetup(t, own); + burn(pendulum, 0n, 100n * PEN); + await monitor.poll(api); + assert.match( + pages.find((page) => page.subject === "CRITICAL: PENDULUM SOURCE CONTRADICTED")?.detail ?? "", + /at Pendulum block 101 the second node has nextNonce (2|1) and totalMigrated (100|7)000000000000, the source 1 and 100000000000000;/, + ); + assert.equal(persisted().lastPendulumBlock, 100); + } +}); + +test("a second node with other migration events in a block pages critical and nothing is ingested", async (t) => { + const { pendulum, api, monitor, pages, persisted, own } = await crossCheckSetup(t); + burn(pendulum, 0n, 100n * PEN); + const events = new Map(pendulum.events); + own.events = events; + burn(pendulum, 1n, 100n * PEN, 102); + events.set(102, [{ ...pendulum.events.get(102)![0], data: [codec(1n), hash(MIGRATOR), hash(TOKEN), codec(100n * PEN)] }]); + await monitor.poll(api); + assert.match( + pages.find((page) => page.subject === "CRITICAL: PENDULUM SOURCE CONTRADICTED")?.detail ?? "", + new RegExp(`in Pendulum block 102 the second node reports the migration events MigrationInitiated\\(1, ${MIGRATOR}, ${TOKEN}, `, "i"), + ); + assert.equal(persisted().lastPendulumBlock, 100); + assert.deepEqual(persisted().migrations, []); +}); + +test("a second node that cannot read a block's events only warns", async (t) => { + const failing = new (class extends Map { + get(): never { + throw new Error("state discarded"); + } + })(); + const { pendulum, api, monitor, pages, persisted } = await crossCheckSetup(t, { events: failing }); + burn(pendulum, 0n, 100n * PEN); + await monitor.poll(api); + assert.deepEqual(pages.map((page) => page.subject), ["PENDULUM CROSS-CHECK UNAVAILABLE"]); + assert.match(pages[0].detail, /cannot be read: .*state discarded/); + assert.equal(persisted().lastPendulumBlock, 101); +}); + +const RELAY = "wss://relay.test"; +type Head = { number: number; hash: string }; + +/** A monitor with the relay-chain anchor, against a relay chain recording + * `head` (by default the source's finalized head, on the chain without a fork). */ +async function relaySetup( + t: { after(fn: () => void): void }, + { head, failing, env = {} }: { head?: (pendulum: Pendulum) => Head; failing?: () => boolean; env?: Record } = {}, +) { + let chain: Pendulum | undefined; + const honest = (pendulum: Pendulum) => ({ number: pendulum.finalized, hash: pendulumHash({}, pendulum.finalized) }); + const relay = relayChain(() => (head ?? honest)(chain!), failing); + return setup(t, { + env: { POLKADOT_WS: RELAY, ...env }, + prepare: (_base, pendulum) => (chain = pendulum), + connect: async (url) => (url === RELAY ? relay : assert.fail(url)), + }); +} + +test("a relay-chain record of the source's head anchors it and is kept", async (t) => { + const { pendulum, api, monitor, pages, persisted } = await relaySetup(t); + burn(pendulum, 0n, 100n * PEN); + await monitor.poll(api); + assert.equal(pages.length, 0, JSON.stringify(pages)); + assert.deepEqual(persisted().relayAnchor, { number: 101, hash: pendulumHash({}, 101) }); + assert.equal(persisted().lastPendulumBlock, 101); + await monitor.poll(api); // neither the record nor the source moved: nothing waits, nothing pages + assert.equal(pages.length, 0, JSON.stringify(pages)); +}); + +test("the conservation check compares with the total of the view the relay chain confirms", async (t) => { + const { base, pendulum, api, monitor, subjects } = await relaySetup(t, { head: () => ({ number: 101, hash: pendulumHash({}, 101) }) }); + burn(pendulum, 0n, 100n * PEN); + burn(pendulum, 1n, 999n * PEN, 102); // past the relay chain's record + release(base, 0n, 100n * PEN); + base.totalReleased += 50n * PEN * CF; // released without a burn behind it + base.vaultBalance -= 50n * PEN * CF; + await monitor.poll(api); + assert.ok(subjects().includes("CONSERVATION VIOLATION"), JSON.stringify(subjects())); +}); + +test("a source that names another hash than the relay chain's record pages critical", async (t) => { + const { pendulum, api, monitor, pages, persisted } = await relaySetup(t, { + head: (pendulum) => ({ number: pendulum.finalized, hash: pendulumHash({ forkFrom: 0 }, pendulum.finalized) }), + }); + burn(pendulum, 0n, 100n * PEN); + await monitor.poll(api); + assert.match( + pages.find((page) => page.subject === "CRITICAL: PENDULUM SOURCE CONTRADICTED")?.detail ?? "", + /relay chain records 0x0+10+65 as Pendulum block 101, the source's chain has 0x0+65;/, + ); + assert.equal(persisted().lastPendulumBlock, 100); +}); + +test("a source whose chain avoids the relay chain's record pages critical, whatever hash it names", async (t) => { + const { pendulum, api, monitor, pages } = await relaySetup(t, { + head: () => ({ number: 101, hash: pendulumHash({}, 101) }), + }); + pendulum.forkFrom = 101; + pendulum.canonicalBlockHashes = true; // it names the relay chain's hash for block 101 + burn(pendulum, 0n, 100n * PEN, 102); + await monitor.poll(api); + assert.match( + pages.find((page) => page.subject === "CRITICAL: PENDULUM SOURCE CONTRADICTED")?.detail ?? "", + /relay chain records 0x0+65 as Pendulum block 101, the source's chain has 0x0+10+65;/, + ); +}); + +test("a relay record past the source's finalized head anchors it, and its chain must keep passing through it", async (t) => { + let relayDown = false; + const { pendulum, api, monitor, pages, persisted } = await relaySetup(t, { + head: (pendulum) => ({ number: 102, hash: pendulumHash(pendulum, 102) }), + failing: () => relayDown, + }); + burn(pendulum, 0n, 100n * PEN); + pendulum.best = 102; // imported, not yet finalized by the source + await monitor.poll(api); + assert.equal(pages.length, 0, JSON.stringify(pages)); + assert.deepEqual(persisted().relayAnchor, { number: 102, hash: pendulumHash({}, 102) }); + assert.equal(persisted().lastPendulumBlock, 101); + relayDown = true; + pendulum.forkFrom = 102; // the source moves to another chain above its last ingested block + burn(pendulum, 1n, 100n * PEN, 103); + await monitor.poll(api); + assert.match( + pages.find((page) => page.subject === "CRITICAL: PENDULUM SOURCE CONTRADICTED")?.detail ?? "", + /chain has 0x0+10+66 at Pendulum block 102, where the relay chain recorded 0x0+66;/, + ); +}); + +test("a kept relay record past the source's finalized head binds the source while no newer one arrives", async (t) => { + let relayDown = false; + const { pendulum, api, monitor, pages, persisted } = await relaySetup(t, { + head: () => ({ number: 103, hash: pendulumHash({}, 103) }), + failing: () => relayDown, + }); + burn(pendulum, 0n, 100n * PEN); + pendulum.best = 103; // imported, not yet finalized by the source + await monitor.poll(api); + assert.equal(pages.length, 0, JSON.stringify(pages)); + assert.deepEqual(persisted().relayAnchor, { number: 103, hash: pendulumHash({}, 103) }); + assert.equal(persisted().lastPendulumBlock, 101); + relayDown = true; + pendulum.forkFrom = 102; // the source finalizes another chain below the kept record... + burn(pendulum, 1n, 999n * PEN, 102); // ...with a burn on it + await monitor.poll(api); + assert.match( + pages.find((page) => page.subject === "CRITICAL: PENDULUM SOURCE CONTRADICTED")?.detail ?? "", + /chain has 0x0+66 at Pendulum block 102, where the source's finalized head is 0x0+10+66;/, + ); + assert.equal(persisted().lastPendulumBlock, 101); + assert.deepEqual(persisted().migrations.map((migration) => migration.nonce), ["0"]); +}); + +test("a kept relay record past the source's finalized head confirms the blocks below it while no newer one arrives", async (t) => { + const { pendulum, api, monitor, pages, persisted } = await relaySetup(t, { + head: () => ({ number: 103, hash: pendulumHash({}, 103) }), + }); + burn(pendulum, 0n, 100n * PEN); + pendulum.best = 103; // imported, not yet finalized by the source + await monitor.poll(api); + assert.equal(persisted().lastPendulumBlock, 101); + burn(pendulum, 1n, 100n * PEN, 102); // the source finalizes block 102; the record stays at 103 + await monitor.poll(api); + assert.equal(pages.length, 0, JSON.stringify(pages)); + assert.equal(persisted().lastPendulumBlock, 102); + assert.deepEqual(persisted().relayAnchor, { number: 103, hash: pendulumHash({}, 103) }); +}); + +test("a relay record past the source's finalized head must descend from that head", async (t) => { + const { pendulum, api, monitor, pages, persisted } = await relaySetup(t, { + head: () => ({ number: 102, hash: pendulumHash({}, 102) }), + }); + burn(pendulum, 0n, 100n * PEN); + pendulum.forkFrom = 101; // its finalized block 101 is on another chain... + pendulum.canonicalBlockHashes = true; // ...though it names the relay chain's hash for block 102 + pendulum.best = 102; + await monitor.poll(api); + assert.match( + pages.find((page) => page.subject === "CRITICAL: PENDULUM SOURCE CONTRADICTED")?.detail ?? "", + /records 0x0+66 as Pendulum block 102, whose chain has 0x0+65 at block 101, where the source has 0x0+10+65;/, + ); + assert.equal(persisted().lastPendulumBlock, 100); +}); + +test("a relay record past the source's finalized head that its best chain does not hold yet holds the source back", async (t) => { + const { pendulum, api, monitor, pages, persisted } = await relaySetup(t, { + head: () => ({ number: 102, hash: pendulumHash({}, 102) }), + }); + burn(pendulum, 0n, 100n * PEN); + pendulum.forkFrom = 102; // its best chain holds another block 102 for now + pendulum.best = 102; + await monitor.poll(api); + assert.deepEqual(pages.map((page) => [page.subject, page.severity]), [["RELAY ANCHOR UNAVAILABLE", "WARNING"]]); + assert.match( + pages[0].detail, + /^relay RPC 1: the source has not imported block 102, the relay chain's Pendulum head, yet; no Pendulum block after 100 is ingested until a relay RPC anchors a newer head,/, + ); + assert.equal(persisted().relayAnchor, undefined); + assert.equal(persisted().lastPendulumBlock, 100); +}); + +test("a block past the relay chain's record waits for it, and a fork there is caught once it is recorded", async (t) => { + let recorded = 101; + const { pendulum, api, monitor, pages, persisted } = await relaySetup(t, { + head: () => ({ number: recorded, hash: pendulumHash({}, recorded) }), + }); + burn(pendulum, 0n, 100n * PEN); + pendulum.forkFrom = 102; // the source finalizes another block 102... + burn(pendulum, 1n, 999n * PEN, 102); // ...with a fabricated burn, past the relay chain's record + await monitor.poll(api); + assert.equal(pages.length, 0, JSON.stringify(pages)); + assert.equal(persisted().lastPendulumBlock, 101); + assert.deepEqual(persisted().migrations.map((migration) => migration.nonce), ["0"]); + recorded = 102; // the relay chain records the real block 102 + await monitor.poll(api); + assert.match( + pages.find((page) => page.subject === "CRITICAL: PENDULUM SOURCE CONTRADICTED")?.detail ?? "", + /relay chain records 0x0+66 as Pendulum block 102, the source's chain has 0x0+10+66;/, + ); + assert.equal(persisted().lastPendulumBlock, 101); +}); + +test("a source that finalizes below a relay record it does not hold ingests nothing past the kept record", async (t) => { + let recorded = 101; + const { pendulum, api, monitor, pages, persisted } = await relaySetup(t, { + head: () => ({ number: recorded, hash: pendulumHash({}, recorded) }), + }); + burn(pendulum, 0n, 100n * PEN); + await monitor.poll(api); // block 101 anchored + pendulum.forkFrom = 102; // the source serves another chain from block 102 on... + for (let block = 102; block <= 105; block++) burn(pendulum, BigInt(block - 101), 999n * PEN, block); // ...with fabricated burns + pendulum.best = 106; // ...and holds another block 106 than the relay chain records + recorded = 106; + await monitor.poll(api); + assert.deepEqual(pages.map((page) => page.subject), ["RELAY ANCHOR UNAVAILABLE"]); + assert.match(pages[0].detail, /has not imported block 106, .*; no Pendulum block after 101 is ingested until a relay RPC anchors/); + assert.equal(persisted().lastPendulumBlock, 101); + assert.deepEqual(persisted().migrations.map((migration) => migration.nonce), ["0"]); + assert.deepEqual(persisted().relayAnchor, { number: 101, hash: pendulumHash({}, 101) }); +}); + +test("a Pendulum view a checker holds back pages as stalled, naming that checker", async (t) => { + const relayed = await relaySetup(t, { head: () => ({ number: 101, hash: pendulumHash({}, 101) }) }); + const crossChecked = await crossCheckSetup(t, { finalized: 101 }); + for (const [{ pendulum, api, monitor, pages }, checker] of [ + [relayed, "the relay RPCs \\(POLKADOT_WS\\)"], + [crossChecked, "the second node \\(PENDULUM_WS_CROSSCHECK\\)"], + ] as const) { + burn(pendulum, 0n, 100n * PEN); + pendulum.finalized = 130; // 29 blocks of 12 s past the checker's head + await monitor.poll(api); + assert.match( + pages.find((page) => page.subject === "PENDULUM SOURCE VIEW STALLED")?.detail ?? "", + new RegExp(`^verified Pendulum block 101 is 34\\ds old \\(the source finalized 130, not confirmed yet by ${checker}\\);.* — check ${checker} before`), + ); + } +}); + +test("a Base event whose burn the relay chain has not confirmed yet is not proven fabricated by a newer source head", async (t) => { + const { base, pendulum, api, monitor, subjects } = await relaySetup(t, { + head: () => ({ number: 101, hash: pendulumHash({}, 101) }), + env: { SOURCE_CLOCK_SKEW_MARGIN_SECONDS: "0" }, + }); + burn(pendulum, 0n, 100n * PEN); + burn(pendulum, 1n, 100n * PEN, 125); + pendulum.finalized = 130; + release(base, 1n, 100n * PEN); + await monitor.poll(api); + await new Promise((resolve) => setTimeout(resolve, 5)); + await monitor.poll(api); // the source's own head has passed the first sighting, the confirmed view has not + assert.ok(subjects().includes("UNVERIFIED BASE EVENT"), JSON.stringify(subjects())); + assert.ok(!subjects().includes("MIGRATION TUPLE VIOLATION"), JSON.stringify(subjects())); +}); + +test("a relay chain that records another hash at a height it anchored pages critical", async (t) => { + let recorded = pendulumHash({}, 101); + const { pendulum, api, monitor, pages, persisted } = await relaySetup(t, { + head: () => ({ number: 101, hash: recorded }), + }); + burn(pendulum, 0n, 100n * PEN); + await monitor.poll(api); + assert.deepEqual(persisted().relayAnchor, { number: 101, hash: pendulumHash({}, 101) }); + recorded = pendulumHash({ forkFrom: 0 }, 101); + await monitor.poll(api); + assert.match( + pages.find((page) => page.subject === "CRITICAL: PENDULUM SOURCE CONTRADICTED")?.detail ?? "", + /relay chain now records 0x0+10+65 as Pendulum block 101, where it recorded 0x0+65;/, + ); +}); + +test("a relay record below the block ingested last is checked down the source's chain", async (t) => { + for (const recorded of [pendulumHash({}, 100), pendulumHash({ forkFrom: 0 }, 100)]) { + let relayDown = true; + const { pendulum, api, monitor, pages, persisted } = await relaySetup(t, { + head: () => ({ number: 100, hash: recorded }), + failing: () => relayDown, + }); + burn(pendulum, 0n, 100n * PEN); + await monitor.poll(api); // ingested up to block 101 without the anchor + relayDown = false; + burn(pendulum, 1n, 100n * PEN, 102); + await monitor.poll(api); + if (recorded === pendulumHash({}, 100)) { + assert.deepEqual(pages.map((page) => page.subject), ["RELAY ANCHOR UNAVAILABLE"]); + assert.deepEqual(persisted().relayAnchor, { number: 100, hash: recorded }); + assert.equal(persisted().lastPendulumBlock, 101); // block 102 waits for a newer record + } else { + assert.match( + pages.find((page) => page.subject === "CRITICAL: PENDULUM SOURCE CONTRADICTED")?.detail ?? "", + /relay chain records 0x0+10+64 as Pendulum block 100, the source's chain has 0x0+64;/, + ); + assert.equal(persisted().lastPendulumBlock, 101); + } + } +}); + +test("a relay record far below the block ingested last is not walked down to", async (t) => { + const { pendulum, api, monitor, pages, persisted } = await relaySetup(t, { + head: () => ({ number: 150, hash: pendulumHash({}, 150) }), + env: { PENDULUM_START_BLOCK: "300" }, + }); + burn(pendulum, 0n, 100n * PEN, 300); + await monitor.poll(api); + assert.deepEqual(pages.map((page) => page.subject), ["RELAY ANCHOR UNAVAILABLE"]); + assert.match(pages[0].detail, /^relay RPC 1: the relay chain's Pendulum head 150 lies more than 100 blocks below block 299;/); + assert.equal(persisted().relayAnchor, undefined); + assert.equal(persisted().lastPendulumBlock, 299); +}); + +test("an unreadable relay chain warns, the source is used, and it turns critical after the grace", async (t) => { + const { pendulum, api, monitor, pages, persisted } = await relaySetup(t, { failing: () => true }); + burn(pendulum, 0n, 100n * PEN); + await monitor.poll(api); + assert.deepEqual(pages.map((page) => [page.subject, page.severity]), [["RELAY ANCHOR UNAVAILABLE", "WARNING"]]); + assert.match(pages[0].detail, /relay RPC 1: Error: relay RPC unavailable;/); + assert.equal(persisted().lastPendulumBlock, 101); + const late = await relaySetup(t, { failing: () => true, env: { RELAY_ANCHOR_GRACE_SECONDS: "0" } }); + burn(late.pendulum, 0n, 100n * PEN); + await new Promise((resolve) => setTimeout(resolve, 5)); + await late.monitor.poll(late.api); + assert.ok(late.subjects().includes("CRITICAL: RELAY ANCHOR STALE"), JSON.stringify(late.subjects())); +}); + +test("a source behind the relay chain is not anchored, its blocks wait and it pages, critical after the grace", async (t) => { + const ahead = (pendulum: Pendulum) => ({ number: pendulum.finalized + 1, hash: pendulumHash(pendulum, pendulum.finalized + 1) }); + const { pendulum, api, monitor, pages, persisted } = await relaySetup(t, { head: ahead }); + burn(pendulum, 0n, 100n * PEN); + await monitor.poll(api); + assert.deepEqual(pages.map((page) => page.subject), ["RELAY ANCHOR UNAVAILABLE"]); + assert.equal(persisted().relayAnchor, undefined); + assert.equal(persisted().lastPendulumBlock, 100); + const late = await relaySetup(t, { head: ahead, env: { RELAY_ANCHOR_GRACE_SECONDS: "0" } }); + await new Promise((resolve) => setTimeout(resolve, 5)); + await late.monitor.poll(late.api); + assert.match( + late.pages.find((page) => page.subject === "CRITICAL: RELAY ANCHOR STALE")?.detail ?? "", + /the source has not imported block 101, the relay chain's Pendulum head, yet/, + ); +}); + +test("the relay RPCs are tried in order", async (t) => { + let chain: Pendulum | undefined; + const relay = relayChain(() => ({ number: chain!.finalized, hash: pendulumHash({}, chain!.finalized) })); + const { pendulum, api, monitor, pages, persisted } = await setup(t, { + env: { POLKADOT_WS: "wss://down.test, wss://up.test" }, + prepare: (_base, pendulum) => (chain = pendulum), + connect: async (url) => (url === "wss://up.test" ? relay : assert.fail(`cannot reach ${url}`)), + }); + burn(pendulum, 0n, 100n * PEN); + await monitor.poll(api); + assert.equal(pages.length, 0, JSON.stringify(pages)); + assert.equal(persisted().relayAnchor?.number, 101); +}); + +/** A monitor with relay RPCs, in this order; each records `head` or, without one, is down. */ +async function relayRpcsSetup(t: { after(fn: () => void): void }, heads: Array<((pendulum: Pendulum) => Head) | undefined>) { + let chain: Pendulum | undefined; + const urls = heads.map((_head, index) => `wss://relay-${index + 1}.test`); + const relays = heads.map((head) => relayChain(() => head!(chain!), () => !head)); + return setup(t, { + env: { POLKADOT_WS: urls.join(",") }, + prepare: (_base, pendulum) => (chain = pendulum), + connect: async (url) => relays[urls.indexOf(url)] ?? assert.fail(url), + }); +} +const honestHead = (pendulum: Pendulum) => ({ number: pendulum.finalized, hash: pendulumHash({}, pendulum.finalized) }); +const forkedHead = (pendulum: Pendulum) => ({ number: pendulum.finalized, hash: pendulumHash({ forkFrom: 0 }, pendulum.finalized) }); + +test("a relay contradiction the next relay RPC outvotes warns, and that RPC's record anchors the source", async (t) => { + const { pendulum, api, monitor, pages, persisted } = await relayRpcsSetup(t, [forkedHead, honestHead]); + burn(pendulum, 0n, 100n * PEN); + await monitor.poll(api); + assert.deepEqual(pages.map((page) => [page.subject, page.severity]), [["RELAY RPCS DISAGREE", "WARNING"]]); + assert.match( + pages[0].detail, + /^relay RPC 1 reports that the relay chain records 0x0+10+65 as Pendulum block 101, .*; relay RPC 2 records 0x0+65 as Pendulum block 101, on the source's chain, .* check both$/, + ); + assert.deepEqual(persisted().relayAnchor, { number: 101, hash: pendulumHash({}, 101) }); + assert.equal(persisted().lastPendulumBlock, 101); +}); + +test("a relay record below a contradicting one does not outvote it, and anchors the source up to its own height", async (t) => { + const below = (pendulum: Pendulum) => ({ number: pendulum.finalized - 1, hash: pendulumHash({}, pendulum.finalized - 1) }); + for (const [heads, contradicting, lower] of [[[forkedHead, below], 1, 2], [[below, forkedHead], 2, 1]] as const) { + const { pendulum, api, monitor, pages, persisted } = await relayRpcsSetup(t, [...heads]); + burn(pendulum, 0n, 100n * PEN, 102); + await monitor.poll(api); + assert.deepEqual(pages.map((page) => page.subject), ["RELAY ANCHOR UNAVAILABLE"]); + assert.match( + pages[0].detail, + new RegExp( + `^relay RPC ${contradicting} reports that .* no other relay RPC confirms or outvotes it \\(relay RPC ${lower}: ` + + "its Pendulum head 101 lies below the contradicting record\\); no Pendulum block after 101 is ingested", + ), + ); + assert.deepEqual(persisted().relayAnchor, { number: 101, hash: pendulumHash({}, 101) }); + assert.equal(persisted().lastPendulumBlock, 101); + } +}); + +/** A relay RPC whose record stays at the source's block 101. */ +const stuckHead = () => ({ number: 101, hash: pendulumHash({}, 101) }); + +test("a relay contradiction that only a stuck relay RPC answers stays unconfirmed", async (t) => { + const { pendulum, api, monitor, pages, persisted } = await relayRpcsSetup(t, [honestHead, stuckHead]); + burn(pendulum, 0n, 100n * PEN); + await monitor.poll(api); // relay RPC 1 anchors block 101 + pendulum.forkFrom = 102; // the source moves to another chain... + burn(pendulum, 1n, 999n * PEN, 102); // ...with a fabricated burn, which relay RPC 1 contradicts + await monitor.poll(api); + assert.deepEqual(pages.map((page) => page.subject), ["RELAY ANCHOR UNAVAILABLE"]); + assert.match( + pages[0].detail, + /^relay RPC 1 reports that the relay chain records 0x0+66 as Pendulum block 102, the source's chain has 0x0+10+66, and no other relay RPC confirms or outvotes it \(relay RPC 2: the relay chain's Pendulum head 101 is not newer\);/, + ); + assert.deepEqual(persisted().relayAnchor, { number: 101, hash: pendulumHash({}, 101) }); + assert.equal(persisted().lastPendulumBlock, 101); + assert.deepEqual(persisted().migrations.map((migration) => migration.nonce), ["0"]); +}); + +test("a relay RPC whose record does not anchor the source leaves it to the next one", async (t) => { + const { pendulum, api, monitor, pages, persisted } = await relayRpcsSetup(t, [stuckHead, honestHead]); + burn(pendulum, 0n, 100n * PEN); + await monitor.poll(api); // relay RPC 1 anchors block 101 + burn(pendulum, 1n, 100n * PEN, 102); + await monitor.poll(api); // relay RPC 1 stays there + assert.equal(pages.length, 0, JSON.stringify(pages)); + assert.deepEqual(persisted().relayAnchor, { number: 102, hash: pendulumHash({}, 102) }); + assert.equal(persisted().lastPendulumBlock, 102); +}); + +test("a relay RPC whose record lags the source's head leaves the newer blocks to the next one", async (t) => { + const lagging = (pendulum: Pendulum) => ({ number: pendulum.finalized - 70, hash: pendulumHash({}, pendulum.finalized - 70) }); + const { base, pendulum, api, monitor, pages, persisted } = await relayRpcsSetup(t, [lagging, honestHead]); + burn(pendulum, 0n, 100n * PEN, 190); + pendulum.finalized = 200; + release(base, 0n, 100n * PEN); + await monitor.poll(api); + assert.equal(pages.length, 0, JSON.stringify(pages)); + assert.deepEqual(persisted().relayAnchor, { number: 200, hash: pendulumHash({}, 200) }); + assert.equal(persisted().lastPendulumBlock, 200); + assert.deepEqual(persisted().migrations, []); + assert.equal(persisted().baseFromBlock, "1011"); +}); + +test("a relay contradiction the next relay RPC confirms pages critical", async (t) => { + const { pendulum, api, monitor, pages, persisted } = await relayRpcsSetup(t, [forkedHead, forkedHead]); + burn(pendulum, 0n, 100n * PEN); + await monitor.poll(api); + assert.ok(pages.some((page) => page.subject === "CRITICAL: PENDULUM SOURCE CONTRADICTED"), JSON.stringify(pages)); + assert.equal(persisted().lastPendulumBlock, 100); +}); + +test("a relay contradiction above the source's head is confirmed by a later relay RPC, though a record between anchors", async (t) => { + const above = () => ({ number: 102, hash: pendulumHash({}, 102) }); + const onSourceChain = (pendulum: Pendulum) => ({ number: 101, hash: pendulumHash(pendulum, 101) }); + const { pendulum, api, monitor, pages, persisted } = await relayRpcsSetup(t, [above, onSourceChain, above]); + burn(pendulum, 0n, 100n * PEN); + pendulum.forkFrom = 101; // its finalized block 101 is on another chain... + pendulum.canonicalBlockHashes = true; // ...though its best chain holds the relay chain's block 102 + pendulum.best = 102; + await monitor.poll(api); + assert.match( + pages.find((page) => page.subject === "CRITICAL: PENDULUM SOURCE CONTRADICTED")?.detail ?? "", + /records 0x0+66 as Pendulum block 102, whose chain has 0x0+65 at block 101, where the source has 0x0+10+65;/, + ); + assert.equal(persisted().lastPendulumBlock, 100); +}); + +test("a relay contradiction no other relay RPC can confirm warns, and the source's blocks wait", async (t) => { + const { pendulum, api, monitor, pages, persisted } = await relayRpcsSetup(t, [forkedHead, undefined]); + burn(pendulum, 0n, 100n * PEN); + await monitor.poll(api); + assert.deepEqual(pages.map((page) => page.subject), ["RELAY ANCHOR UNAVAILABLE"]); + assert.match(pages[0].detail, /and no other relay RPC confirms or outvotes it \(relay RPC 2: Error: relay RPC unavailable\)/); + assert.equal(persisted().lastPendulumBlock, 100); +}); + +test("a relay RPC whose finalized relay block is stale counts as not answering, and the second node still caps the source", async (t) => { + let relayAgeMs = 0; + const { base, pendulum, api, monitor, pages, persisted, own } = await crossCheckSetup(t, { finalized: 101 }, [ + relayChain(stuckHead, () => false, () => relayAgeMs), + relayChain(stuckHead, () => true), + ]); + burn(pendulum, 0n, 100n * PEN); + await monitor.poll(api); // relay RPC 1 anchors block 101 + assert.deepEqual(persisted().relayAnchor, { number: 101, hash: pendulumHash({}, 101) }); + relayAgeMs = 400_000; // relay RPC 1 stops following the relay chain at that record; relay RPC 2 is down + burn(pendulum, 1n, 100n * PEN, 103); + pendulum.finalized = 104; + own.finalized = 103; + release(base, 1n, 100n * PEN, 1_015n); + base.safe = 1_020n; + await monitor.poll(api); + assert.deepEqual(pages.map((page) => [page.subject, page.severity]), [["RELAY ANCHOR UNAVAILABLE", "WARNING"]]); + assert.match( + pages[0].detail, + /^the relay chain cannot be read: relay RPC 1: its finalized relay block is 400s old; relay RPC 2: Error: relay RPC unavailable; the Pendulum source is used without the relay-chain anchor/, + ); + assert.equal(persisted().lastPendulumBlock, 103); + assert.deepEqual(persisted().migrations.map((migration) => migration.nonce), ["0"]); + assert.equal(persisted().baseFromBlock, "1021"); +}); + +test("a Base event whose nonce the source view passed without pauses before the grace, as fabricated", async (t) => { + const { base, pendulum, api, monitor, pages, subjects } = await setup(t, { env: { SOURCE_CLOCK_SKEW_MARGIN_SECONDS: "0" } }); + burn(pendulum, 0n, 100n * PEN); + pendulum.headAgeMs = 1_000; // the view trails the first sighting + release(base, 7n, 5_000n * PEN); // no burn behind it + await monitor.poll(api); + assert.ok(subjects().includes("UNVERIFIED BASE EVENT"), JSON.stringify(subjects())); + assert.ok(!subjects().includes("MIGRATION TUPLE VIOLATION"), JSON.stringify(subjects())); + pendulum.headAgeMs = 0; + await new Promise((resolve) => setTimeout(resolve, 5)); + await monitor.poll(api); // the view has passed the first sighting, still without nonce 7 + assert.match( + pages.find((page) => page.subject === "MIGRATION TUPLE VIOLATION")?.detail ?? "", + /nonce 7 .*source view has passed the event's timestamp — fabricated/, + ); +}); + +test("while Pendulum cannot be read, a Base event with an unknown nonce pauses after the grace", async (t) => { + const { base, pendulum, api, monitor, subjects } = await setup(t, { env: { UNMATCHED_EVENT_GRACE_SECONDS: "0" } }); + pendulum.failure = new Error("node unreachable"); + release(base, 7n, 5_000n * PEN); // no burn behind it + await monitor.poll(api); + assert.ok(subjects().includes("UNVERIFIED BASE EVENT"), JSON.stringify(subjects())); + assert.ok(!subjects().includes("MIGRATION TUPLE VIOLATION"), JSON.stringify(subjects())); + await new Promise((resolve) => setTimeout(resolve, 5)); + await monitor.poll(api); // the grace has run out + assert.ok(subjects().includes("MIGRATION TUPLE VIOLATION"), JSON.stringify(subjects())); +}); + +test("while Pendulum cannot be read, a mismatching release of a known nonce pauses at once", async (t) => { + const { base, pendulum, api, monitor, pages } = await setup(t); + burn(pendulum, 0n, 100n * PEN); + await monitor.poll(api); + pendulum.failure = new Error("node unreachable"); + const tokenAmount = 100n * PEN * CF; + const released = { nonce: 0n, recipient: TOKEN, palletAmount: 100n * PEN, tokenAmount }; + base.logs.push({ address: VAULT, eventName: "Released", args: released, blockNumber: 1_012n }); + base.totalReleased += tokenAmount; + base.vaultBalance -= tokenAmount; + base.safe = 1_020n; + await monitor.poll(api); + assert.match(pages.find((page) => page.subject === "MIGRATION TUPLE VIOLATION")?.detail ?? "", /recipient/); + assert.ok(pages.some((page) => page.subject === "AUTO-PAUSE UNAVAILABLE"), JSON.stringify(pages)); // the pause was asked for +}); + +test("a tuple violation whose pause is not confirmed is found again in the next poll", async (t) => { + const { base, pendulum, api, monitor, pages, persisted } = await setup(t); + burn(pendulum, 0n, 100n * PEN); + await monitor.poll(api); + const tokenAmount = 100n * PEN * CF; // a release of nonce 0 to another recipient + base.logs.push({ address: VAULT, eventName: "Released", args: { nonce: 0n, recipient: TOKEN, palletAmount: 100n * PEN, tokenAmount }, blockNumber: 1_015n }); + base.totalReleased += tokenAmount; + base.vaultBalance -= tokenAmount; + base.safe = 1_020n; + await monitor.poll(api); + await monitor.poll(api); + assert.equal(pages.filter((page) => page.subject === "MIGRATION TUPLE VIOLATION").length, 2, JSON.stringify(pages)); + assert.equal(persisted().baseFromBlock, "1011"); +}); + +test("a release the source never shows pauses after the grace, with no approval in view", async (t) => { + const { base, pendulum, api, monitor, pages } = await setup(t, { env: { UNMATCHED_EVENT_GRACE_SECONDS: "0" } }); + pendulum.failure = new Error("node unreachable"); + const tokenAmount = 5n * PEN * CF; + base.logs.push({ address: VAULT, eventName: "Released", args: { nonce: 7n, recipient: RECIPIENT, palletAmount: 5n * PEN, tokenAmount }, blockNumber: 1_005n }); + base.totalReleased += tokenAmount; + base.vaultBalance -= tokenAmount; + await monitor.poll(api); + await new Promise((resolve) => setTimeout(resolve, 5)); + await monitor.poll(api); // the grace has run out + assert.match(pages.find((page) => page.subject === "MIGRATION TUPLE VIOLATION")?.detail ?? "", /^Released at Base block 1005: nonce 7 has no finalized/); +}); + +test("a second release of one nonce is a violation", async (t) => { + const { base, pendulum, api, monitor, pages } = await setup(t); + burn(pendulum, 0n, 100n * PEN); + release(base, 0n, 100n * PEN, 1_004n); + release(base, 0n, 100n * PEN, 1_006n); + await monitor.poll(api); + assert.match(pages.find((page) => page.subject === "MIGRATION TUPLE VIOLATION")?.detail ?? "", /duplicate Released event for nonce 0/); +}); + +test("vault events past the confirmed Base block do not trip the balance check", async (t) => { + const { base, pendulum, api, monitor, pages } = await setup(t); + burn(pendulum, 0n, 100n * PEN); + release(base, 0n, 100n * PEN, 1_015n); // after the safe block 1010 + await monitor.poll(api); + assert.equal(pages.length, 0, JSON.stringify(pages)); +}); + +test("logs that miss a release hold the Base cursor and page until a read accounts for the totals", async (t) => { + const { base, pendulum, api, monitor, pages, persisted } = await setup(t); + burn(pendulum, 0n, 100n * PEN); + release(base, 0n, 100n * PEN, 1_005n); + base.logsHead = 1_003n; // logs from a backend that has not reached the release + await monitor.poll(api); + await until(() => pages.some((page) => page.subject === "BASE LOG SCAN INCOMPLETE"), "the incomplete scan page"); + assert.match( + pages.find((page) => page.subject === "BASE LOG SCAN INCOMPLETE")!.detail, + /in Base blocks 1000-1010 the Released events add up to 0 .* totalReleased grew by 100000000000000000000/, + ); + assert.equal(persisted().baseFromBlock, "1000"); + assert.deepEqual(persisted().migrations.map((migration) => migration.nonce), ["0"]); + base.logsHead = undefined; + await monitor.poll(api); + assert.equal(persisted().baseFromBlock, "1011"); + assert.deepEqual(persisted().migrations, []); +}); + +test("logs that miss a sweep hold the Base cursor too", async (t) => { + const { base, api, monitor, pages, persisted } = await setup(t); + base.logs.push({ address: VAULT, eventName: "RemainderSwept", args: { to: TOKEN, amount: 5n }, blockNumber: 1_005n }); + base.totalSwept += 5n; + base.vaultBalance -= 5n; + base.logsHead = 1_003n; + await monitor.poll(api); + await until(() => pages.some((page) => page.subject === "BASE LOG SCAN INCOMPLETE"), "the incomplete scan page"); + assert.match(pages.find((page) => page.subject === "BASE LOG SCAN INCOMPLETE")!.detail, /RemainderSwept events to 0, .* totalSwept by 5/); + assert.equal(persisted().baseFromBlock, "1000"); + base.logsHead = undefined; + await monitor.poll(api); + assert.equal(persisted().baseFromBlock, "1011"); +}); + +test("a Base log range held past the grace pages critical", async (t) => { + const { base, pendulum, api, monitor, pages } = await setup(t, { env: { UNMATCHED_EVENT_GRACE_SECONDS: "0" } }); + burn(pendulum, 0n, 100n * PEN); + release(base, 0n, 100n * PEN, 1_005n); + base.logsHead = 1_003n; + await monitor.poll(api); + await new Promise((resolve) => setTimeout(resolve, 5)); + await monitor.poll(api); + const held = () => pages.filter((page) => page.subject.endsWith("BASE LOG SCAN INCOMPLETE")); + await until(() => held().length === 2, "two incomplete scan pages"); + assert.deepEqual(held().map((page) => [page.subject, page.severity]), [ + ["BASE LOG SCAN INCOMPLETE", "WARNING"], + ["CRITICAL: BASE LOG SCAN INCOMPLETE", "CRITICAL"], + ]); + assert.match(held()[1].detail, /stays at block 1000 \(for \d+s\): no later Base event is matched against its burn/); + base.logsHead = undefined; + await monitor.poll(api); // the range reads in full + burn(pendulum, 1n, 100n * PEN, 102); + release(base, 1n, 100n * PEN, 1_015n); + base.logsHead = 1_012n; + base.safe = 1_020n; + await monitor.poll(api); // a new hold starts as a warning + await until(() => held().length === 3, "the next incomplete scan page"); + assert.equal(held()[2].subject, "BASE LOG SCAN INCOMPLETE"); +}); + +test("the conservation check still runs while a Base log range is held", async (t) => { + const { base, pendulum, api, monitor, subjects } = await setup(t); + burn(pendulum, 0n, 100n * PEN); + release(base, 0n, 100n * PEN, 1_005n); + release(base, 0n, 100n * PEN, 1_007n); // a second release lifts totalReleased past TotalMigrated... + base.logsHead = 1_006n; // ...while the log backend drops its block + await monitor.poll(api); + assert.ok(subjects().includes("CONSERVATION VIOLATION"), JSON.stringify(subjects())); + assert.ok(subjects().includes("AUTO-PAUSE UNAVAILABLE"), JSON.stringify(subjects())); // the pause was asked for +}); + +test("the conservation check waits while a Base event's burn is not in the source view yet", async (t) => { + const { base, api, monitor, subjects } = await setup(t); + release(base, 0n, 100n * PEN); // the burn is not in the lagging source view yet + await monitor.poll(api); + assert.deepEqual([...new Set(subjects())], ["UNVERIFIED BASE EVENT"]); +}); + +test("logs that miss an event once and then account for the totals hold nothing and page nothing", async (t) => { + const { base, pendulum, api, monitor, subjects, persisted } = await setup(t); + burn(pendulum, 0n, 100n * PEN); + release(base, 0n, 100n * PEN, 1_005n); + const released = toEventSelector("Released(uint64,address,uint256,uint256)"); // only reconciliation asks for it + let reads = 0; + base.logsHeadFor = (topics) => (topics.includes(released) && reads++ === 0 ? 1_003n : undefined); + await monitor.poll(api); + assert.equal(reads, 2); // read once more + assert.equal(persisted().baseFromBlock, "1011"); + assert.deepEqual(persisted().migrations, []); + await new Promise((resolve) => setTimeout(resolve, 50)); // an incomplete scan page is sent without waiting + assert.deepEqual(subjects(), []); +}); + +test("a vault total that cannot be read where the vault has code fails the poll, not counting as zero", async (t) => { + const { base, pendulum, api, monitor, subjects } = await setup(t, { prepare: (base) => (base.deployedAt = 900n) }); + burn(pendulum, 0n, 100n * PEN); + release(base, 0n, 100n * PEN, 950n); // before the first Base block the monitor scans + const totalReleased = toFunctionSelector("totalReleased()"); + base.failing = (method, params) => + method === "eth_call" && (params[0] as { data: string }).data === totalReleased && params[1] === toHex(999n); + await monitor.poll(api); + await new Promise((resolve) => setTimeout(resolve, 50)); // an incomplete scan page is sent without waiting + assert.ok(subjects().includes("monitor check failed"), JSON.stringify(subjects())); + assert.ok(!subjects().includes("BASE LOG SCAN INCOMPLETE"), JSON.stringify(subjects())); +}); + +test("releases and sweeps in the first and the last block of a range account for the totals", async (t) => { + const { base, pendulum, api, monitor, subjects, persisted } = await setup(t); + burn(pendulum, 0n, 100n * PEN); + burn(pendulum, 1n, 100n * PEN); + release(base, 0n, 100n * PEN, 1_000n); + release(base, 1n, 100n * PEN, 1_010n); + for (const blockNumber of [1_000n, 1_010n]) { + base.logs.push({ address: VAULT, eventName: "RemainderSwept", args: { to: TOKEN, amount: 5n }, blockNumber }); + base.totalSwept += 5n; + base.vaultBalance -= 5n; + } + await monitor.poll(api); + assert.equal(persisted().baseFromBlock, "1011"); + assert.deepEqual(persisted().migrations, []); + await new Promise((resolve) => setTimeout(resolve, 50)); // an incomplete scan page is sent without waiting + assert.deepEqual([...new Set(subjects())], ["VAULT ADMIN EVENT"]); // the sweeps +}); + +test("the totals count as zero before the vault existed", async (t) => { + const { base, pendulum, api, monitor, subjects, persisted } = await setup(t, { prepare: (base) => (base.deployedAt = 1_000n) }); + burn(pendulum, 0n, 100n * PEN); + release(base, 0n, 100n * PEN, 1_005n); + await monitor.poll(api); + assert.deepEqual(subjects(), []); + assert.equal(persisted().baseFromBlock, "1011"); +}); + +/** An approval of `nonce` for `recipient` from attestor `attestor`. */ +function approve(base: Base, nonce: bigint, recipient: string, attestor: string, blockNumber: bigint): void { + base.logs.push({ address: VAULT, eventName: "Approved", args: { nonce, recipient, palletAmount: 100n * PEN, attestor }, blockNumber }); +} + +test("the release of a nonce forgets the approvals of it that disagreed", async (t) => { + const { base, pendulum, api, monitor, persisted } = await setup(t); + burn(pendulum, 0n, 100n * PEN); + approve(base, 0n, TOKEN, ATTESTORS[0], 1_004n); + await monitor.poll(api); + assert.equal(persisted().mismatchedApprovals.length, 1); + release(base, 0n, 100n * PEN, 1_012n); + base.safe = 1_020n; + await monitor.poll(api); + assert.deepEqual(persisted().mismatchedApprovals, []); +}); + +test("one attestor approving against the burn pages critical; a second one pauses, also polls later", async (t) => { + const { base, pendulum, api, monitor, restart, pages, subjects } = await setup(t); + burn(pendulum, 0n, 100n * PEN); + await monitor.poll(api); + approve(base, 0n, TOKEN, ATTESTORS[0], 1_012n); + base.safe = 1_020n; + await monitor.poll(api); + const page = pages.find((page) => page.subject === "CRITICAL: MISMATCHED APPROVAL"); + assert.equal(page?.severity, "CRITICAL", JSON.stringify(pages)); + assert.match(page.detail, new RegExp(`by attestor ${ATTESTORS[0]}: nonce 0 recipient ${TOKEN}`, "i")); + assert.ok(!subjects().includes("MIGRATION TUPLE VIOLATION"), JSON.stringify(subjects())); + approve(base, 0n, RECIPIENT, ATTESTORS[1], 1_022n); // agrees with the burn + approve(base, 0n, "0x5555555555555555555555555555555555555555", ATTESTORS[0], 1_023n); // the same attestor again + base.safe = 1_030n; + await monitor.poll(api); + assert.ok(!subjects().includes("MIGRATION TUPLE VIOLATION"), JSON.stringify(subjects())); + const restarted = await restart(); + approve(base, 0n, TOKEN, ATTESTORS[2], 1_032n); + base.safe = 1_040n; + await restarted.poll(api); + assert.match( + pages.find((page) => page.subject === "MIGRATION TUPLE VIOLATION")?.detail ?? "", + new RegExp(`attestors ${ATTESTORS[0]}, ${ATTESTORS[2]} approved tuples for nonce 0`, "i"), + ); +}); + +test("at a threshold of two, one attestor approving against the burn pauses", async (t) => { + const { base, pendulum, api, monitor, pages, subjects } = await setup(t, { prepare: (base) => (base.threshold = 2n) }); + burn(pendulum, 0n, 100n * PEN); + approve(base, 0n, TOKEN, ATTESTORS[0], 1_005n); + await monitor.poll(api); + assert.match( + pages.find((page) => page.subject === "MIGRATION TUPLE VIOLATION")?.detail ?? "", + new RegExp(`; attestor ${ATTESTORS[0]} approved tuples for nonce 0`, "i"), + ); + assert.ok(!subjects().includes("CRITICAL: MISMATCHED APPROVAL"), JSON.stringify(subjects())); +}); + +test("auto-pause readiness is checked at startup and each problem pages", async (t) => { + const { startup } = await setup(t, { + env: { GUARDIAN_PRIVATE_KEY: GUARDIAN_KEY }, + prepare: (base) => { + base.chainId = 84532; + base.guardian = "0x5555555555555555555555555555555555555555"; + base.balances.clear(); + }, + }); + const detail = startup.find((page) => page.subject === "AUTO-PAUSE NOT READY")?.detail ?? ""; + assert.match(detail, /serves chain 84532, not BASE_CHAIN_ID 8453/); + assert.match(detail, new RegExp(`${GUARDIAN} is not the vault guardian 0x5{40}`)); + assert.match(detail, /holds 0 wei/); +}); + +test("the guardian balance floor rejects a negative value and keeps its default when empty", async (t) => { + await assert.rejects(setup(t, { env: { GUARDIAN_MIN_BALANCE_WEI: "-1" } }), /invalid GUARDIAN_MIN_BALANCE_WEI: -1/); + const { startup } = await setup(t, { + env: { GUARDIAN_PRIVATE_KEY: GUARDIAN_KEY, GUARDIAN_MIN_BALANCE_WEI: "" }, + prepare: (base) => base.balances.clear(), + }); + const detail = startup.find((page) => page.subject === "AUTO-PAUSE NOT READY")?.detail ?? ""; + assert.match(detail, /holds 0 wei, less than GUARDIAN_MIN_BALANCE_WEI 1000000000000000/); +}); + +test("auto-pause readiness is re-checked every poll", async (t) => { + const { base, api, monitor, startup, subjects } = await setup(t, { env: { GUARDIAN_PRIVATE_KEY: GUARDIAN_KEY } }); + assert.ok(!startup.some((page) => page.subject.startsWith("AUTO-PAUSE")), JSON.stringify(startup)); + base.guardian = "0x5555555555555555555555555555555555555555"; + await monitor.poll(api); + assert.deepEqual(subjects(), ["AUTO-PAUSE NOT READY"]); +}); + +test("a failed readiness read pages and the vault balance check still runs", async (t) => { + const { base, api, monitor, pages, subjects } = await setup(t, { env: { GUARDIAN_PRIVATE_KEY: GUARDIAN_KEY } }); + base.failing = (method) => method === "eth_getBalance"; + base.vaultBalance -= 1n; + base.paused = true; // so the auto-pause only confirms + await monitor.poll(api); + assert.deepEqual(sorted(subjects()), ["AUTO-PAUSE NOT READY", "VAULT BALANCE DEFICIT", "vault pause confirmed"]); + assert.match(pages.find((page) => page.subject === "AUTO-PAUSE NOT READY")?.detail ?? "", /readiness reads failed.*rate limited/s); +}); + +test("vault admin events page once each from the Base scan", async (t) => { + const { base, api, monitor, pages } = await setup(t); + const guardian = "0x5555555555555555555555555555555555555555"; + base.logs.push({ address: VAULT, eventName: "GuardianUpdated", args: { guardian }, blockNumber: 1_004n }); + base.logs.push({ address: VAULT, eventName: "CapsUpdated", args: { perReleaseCap: 5n, dailyCap: 7n }, blockNumber: 1_012n }); + await monitor.poll(api); + base.safe = 1_020n; + await monitor.poll(api); + const adminPages = pages.filter((page) => page.subject === "VAULT ADMIN EVENT").map((page) => page.detail); + assert.equal(adminPages.length, 2, JSON.stringify(pages)); + assert.match(adminPages[0], /^GuardianUpdated\(guardian=0x5{40}\) at Base block 1004/i); + assert.match(adminPages[1], /^CapsUpdated\(perReleaseCap=5, dailyCap=7\) at Base block 1012/); +}); + +test("a failed admin scan pages, every other check still runs and the scan resumes where it stopped", async (t) => { + const { base, pendulum, api, monitor, restart, subjects, persisted } = await setup(t, { + env: { GUARDIAN_PRIVATE_KEY: GUARDIAN_KEY }, + }); + burn(pendulum, 0n, 100n * PEN); + release(base, 0n, 100n * PEN); + base.logs.push({ address: VAULT, eventName: "GuardianUpdated", args: { guardian: GUARDIAN }, blockNumber: 1_004n }); + const guardianUpdated = toEventSelector("GuardianUpdated(address)"); // only the admin scan asks for it + base.failing = (method, params) => method === "eth_getLogs" && JSON.stringify(params).includes(guardianUpdated); + base.vaultBalance -= 1n; + base.paused = true; // so the auto-pause only confirms and the poll persists its state + await monitor.poll(api); + assert.deepEqual(sorted(subjects()), ["BASE ADMIN SCAN FAILED", "VAULT BALANCE DEFICIT", "vault pause confirmed"]); + assert.equal(persisted().baseFromBlock, "1011"); // reconciled although the admin scan failed + assert.equal(persisted().baseAdminFromBlock, "1000"); + base.failing = undefined; + base.vaultBalance += 1n; + await (await restart()).poll(api); + assert.ok(subjects().includes("VAULT ADMIN EVENT"), JSON.stringify(subjects())); + assert.equal(persisted().baseAdminFromBlock, "1011"); +}); + +test("an admin scan that keeps failing past the grace pages critical", async (t) => { + const { base, api, monitor, pages } = await setup(t, { env: { UNMATCHED_EVENT_GRACE_SECONDS: "0" } }); + const guardianUpdated = toEventSelector("GuardianUpdated(address)"); // only the admin scan asks for it + const failing = (method: string, params: unknown[]) => method === "eth_getLogs" && JSON.stringify(params).includes(guardianUpdated); + base.failing = failing; + await monitor.poll(api); + await new Promise((resolve) => setTimeout(resolve, 5)); + await monitor.poll(api); + const failed = () => pages.filter((page) => page.subject.endsWith("BASE ADMIN SCAN FAILED")); + assert.deepEqual(failed().map((page) => [page.subject, page.severity]), [ + ["BASE ADMIN SCAN FAILED", "WARNING"], + ["CRITICAL: BASE ADMIN SCAN FAILED", "CRITICAL"], + ]); + assert.match(failed()[1].detail, /\(failing for \d+s\)/); + base.failing = undefined; + await monitor.poll(api); // the scan reads again + base.failing = failing; + base.safe = 1_020n; // a new range for the admin scan + await monitor.poll(api); + assert.equal(failed()[2]?.subject, "BASE ADMIN SCAN FAILED"); // a new failure starts as a warning +}); + +test("a monitor without a guardian key says so at startup", async (t) => { + const { startup } = await setup(t); + assert.ok(startup.some((page) => page.subject === "AUTO-PAUSE DISABLED"), JSON.stringify(startup)); +}); + +test("a Base view that stops advancing pages", async (t) => { + const { base, api, monitor, pages } = await setup(t); + base.safeAgeSeconds = 601; + await monitor.poll(api); + assert.match(pages.find((page) => page.subject === "BASE VIEW STALLED")?.detail ?? "", /^safe block 1010 is 60\ds old/); +}); + +const UNREADABLE = "CRITICAL: Pendulum governance item cannot be read"; +const RISKY = "CRITICAL: Pendulum governance item mints, moves or unlocks balances, or changes the runtime"; +const pageFor = (pages: Page[], subject: string, label: string) => + pages.find((page) => page.subject === subject && page.detail.startsWith(label)); + +test("an undecodable governance item pages as unreadable and hides nothing after it", async (t) => { + const { pendulum, api, monitor, pages } = await setup(t); + pendulum.publicProps.push([5, inline("garbage")]); + pendulum.referenda.set(9, inline("mint")); + await monitor.poll(api); + assert.ok(pageFor(pages, UNREADABLE, "public proposal #5"), JSON.stringify(pages)); + assert.ok(pageFor(pages, RISKY, "referendum #9"), JSON.stringify(pages)); +}); + +test("an oversized governance call pages as unreadable without being fetched", async (t) => { + const { pendulum, api, monitor, pages } = await setup(t); + pendulum.publicProps.push([5, lookup(`0x${"ab".repeat(32)}`, 3_900_000)]); + pendulum.preimages.set(`0x${"ab".repeat(32)}:3900000`, "mint"); + await monitor.poll(api); + assert.match(pageFor(pages, UNREADABLE, "public proposal #5")?.detail ?? "", /3900000 bytes/); + assert.equal(pendulum.preimageReads, 0); +}); + +test("a governance call that cannot be read yet is re-checked and pages critical once readable", async (t) => { + const { pendulum, api, monitor, pages } = await setup(t); + const preimage = `0x${"cd".repeat(32)}`; + pendulum.referenda.set(9, lookup(preimage, 4)); + await monitor.poll(api); + assert.match(pageFor(pages, UNREADABLE, "referendum #9")?.detail ?? "", /not noted yet/); + pendulum.preimages.set(`${preimage}:4`, "mint"); + await monitor.poll(api); + assert.ok(pageFor(pages, RISKY, "referendum #9"), JSON.stringify(pages)); +}); + +test("a governance item whose read fails is retried and hides nothing after it", async (t) => { + const { pendulum, api, monitor, pages } = await setup(t); + const preimage = `0x${"aa".repeat(32)}`; + pendulum.publicProps.push([5, lookup(preimage, 4)]); + pendulum.preimages.set(`${preimage}:4`, "mint"); + pendulum.referenda.set(9, inline("mint")); + pendulum.preimageFailures = 1; + await monitor.poll(api); + assert.match(pageFor(pages, UNREADABLE, "public proposal #5")?.detail ?? "", /reading the call failed/); + assert.ok(pageFor(pages, RISKY, "referendum #9"), JSON.stringify(pages)); + await monitor.poll(api); + assert.ok(pageFor(pages, RISKY, "public proposal #5"), JSON.stringify(pages)); +}); + +test("a legacy call reference is read through the noted preimage's length", async (t) => { + const { pendulum, api, monitor, pages } = await setup(t); + const preimage = `0x${"ef".repeat(32)}`; + pendulum.referenda.set(9, legacy(preimage)); + pendulum.notedLengths.set(preimage, 4); + pendulum.preimages.set(`${preimage}:4`, "mint"); + await monitor.poll(api); + assert.ok(pageFor(pages, RISKY, "referendum #9"), JSON.stringify(pages)); +}); + +test("a legacy call reference to a preimage noted before the status migration is read too", async (t) => { + const { pendulum, api, monitor, pages } = await setup(t); + const preimage = `0x${"fe".repeat(32)}`; + pendulum.referenda.set(9, legacy(preimage)); + pendulum.oldNotedLengths.set(preimage, 4); + pendulum.preimages.set(`${preimage}:4`, "mint"); + await monitor.poll(api); + assert.ok(pageFor(pages, RISKY, "referendum #9"), JSON.stringify(pages)); +}); + +test("council and technical-committee motions are decoded with the same risk rules", async (t) => { + const { pendulum, api, monitor, pages } = await setup(t); + const [anchor, remark, huge] = ["0x01", "0x02", "0x03"].map((prefix) => prefix + "00".repeat(31)); + pendulum.motions.council.set(anchor, "setTreasuryDestination"); + pendulum.motions.technicalCommittee.set(remark, "remark"); + pendulum.motions.council.set(huge, "mint"); + pendulum.motionSizes.set(huge, 100_000); + await monitor.poll(api); + assert.ok(pageFor(pages, RISKY, `council motion ${anchor}`), JSON.stringify(pages)); + assert.ok(pageFor(pages, "PENDULUM GOVERNANCE ITEM", `technical committee motion ${remark}`), JSON.stringify(pages)); + assert.match(pageFor(pages, UNREADABLE, `council motion ${huge}`)?.detail ?? "", /100000 bytes/); + assert.ok(!pendulum.motionReads.has(huge)); +}); + +test("a motion or external proposal made again with the same call pages again", async (t) => { + const { pendulum, api, monitor, pages } = await setup(t); + const motion = `0x04${"00".repeat(31)}`; + pendulum.motions.council.set(motion, "remark"); + pendulum.motionIndices.set(motion, 3); + pendulum.external = inline("remark"); + await monitor.poll(api); + pendulum.motionIndices.set(motion, 4); // closed and proposed again between two polls + pendulum.referenda.set(9, inline("remark")); // the external proposal tabled, and proposed again between two polls + await monitor.poll(api); + pendulum.external = undefined; // tabled or vetoed + await monitor.poll(api); + pendulum.external = inline("remark"); // proposed again + await monitor.poll(api); + assert.equal(pages.filter((page) => page.detail.startsWith(`council motion ${motion}`)).length, 2, JSON.stringify(pages)); + assert.equal(pages.filter((page) => page.detail.startsWith("the council's external proposal")).length, 3, JSON.stringify(pages)); +}); + +const TREASURY_DESTINATION = "0x5555555555555555555555555555555555555555"; + +test("a treasury migration to any but the expected address pages and pauses", async (t) => { + const { base, pendulum, api, monitor, pages, persisted } = await setup(t, { + env: { EXPECTED_TREASURY_DESTINATION: TREASURY_DESTINATION, GUARDIAN_PRIVATE_KEY: GUARDIAN_KEY }, + }); + base.paused = true; // so the auto-pause only confirms + burn(pendulum, 0n, 100n * PEN, 101, { who: TREASURY_ACCOUNT, recipient: TREASURY_DESTINATION }); + burn(pendulum, 1n, 100n * PEN, 101, { recipient: TOKEN }); // anyone else may migrate anywhere + await monitor.poll(api); + assert.equal(pages.length, 0, JSON.stringify(pages)); + burn(pendulum, 2n, 7n * PEN, 102, { who: TREASURY_ACCOUNT, recipient: TOKEN }); + await monitor.poll(api); + const page = pages.find((page) => page.subject === "TREASURY DESTINATION VIOLATION"); + assert.equal(page?.severity, "CRITICAL", JSON.stringify(pages)); + assert.match(page.detail, new RegExp(`nonce 2 in finalized Pendulum block 102 pays 7000000000000 pallet units to ${TOKEN}`, "i")); + assert.ok(pages.some((page) => page.subject === "vault pause confirmed"), JSON.stringify(pages)); + assert.deepEqual(persisted().migrations.map((migration) => migration.nonce), ["0", "1", "2"]); +}); + +test("while the pause after a treasury violation is not confirmed, the monitor holds at its block", async (t) => { + const { pendulum, api, monitor, pages, subjects, persisted } = await setup(t, { + env: { EXPECTED_TREASURY_DESTINATION: TREASURY_DESTINATION }, + }); + burn(pendulum, 0n, 100n * PEN, 101, { who: TREASURY_ACCOUNT, recipient: TOKEN }); + await monitor.poll(api); + await monitor.poll(api); + const violations = pages.filter((page) => page.subject === "TREASURY DESTINATION VIOLATION"); + assert.equal(violations.length, 2); + assert.match(violations[1].detail, /; the monitor holds at that block until the vault is paused$/); + assert.equal(persisted().lastPendulumBlock, 100); + assert.ok(!subjects().includes("PENDULUM SOURCE UNREADABLE"), JSON.stringify(subjects())); // the source reads fine +}); + +test("without a guardian key, a vault paused by hand after a treasury violation lets the monitor move on", async (t) => { + const { base, pendulum, api, monitor, pages, persisted } = await setup(t, { + env: { EXPECTED_TREASURY_DESTINATION: TREASURY_DESTINATION }, + }); + burn(pendulum, 0n, 100n * PEN, 101, { who: TREASURY_ACCOUNT, recipient: TOKEN }); + await monitor.poll(api); + assert.equal(persisted().lastPendulumBlock, 100); + const pollSubjects = async () => { + const before = pages.length; + await monitor.poll(api); + return pages.slice(before).map((page) => page.subject); + }; + base.pausedFrom = 1_015n; // operators pause the vault by hand, after the confirmed Base block + let subjects = await pollSubjects(); + assert.ok(subjects.includes("AUTO-PAUSE PENDING FINALITY"), JSON.stringify(subjects)); + assert.ok(!subjects.includes("AUTO-PAUSE UNAVAILABLE"), JSON.stringify(subjects)); + assert.equal(persisted().lastPendulumBlock, 100); + base.safe = 1_020n; // the pause is confirmed + subjects = await pollSubjects(); + assert.ok(subjects.includes("vault pause confirmed"), JSON.stringify(subjects)); + assert.ok(!subjects.includes("AUTO-PAUSE UNAVAILABLE"), JSON.stringify(subjects)); + assert.equal(persisted().lastPendulumBlock, 101); + assert.deepEqual(persisted().migrations.map((migration) => migration.nonce), ["0"]); +}); + +test("admin events of the migration pallet page from the ingest", async (t) => { + const { pendulum, api, monitor, pages } = await setup(t); + pendulum.finalized = 101; + pendulum.events.set(101, [ + { section: "tokenMigration", method: "TreasuryDestinationSet", data: ["0x5555555555555555555555555555555555555555"] }, + { section: "tokenMigration", method: "MigrationPauseSet", data: [false] }, + ]); + await monitor.poll(api); + const adminPages = pages.filter((page) => page.subject === "MIGRATION PALLET ADMIN EVENT").map((page) => page.detail); + assert.equal(adminPages.length, 2, JSON.stringify(pages)); + assert.match(adminPages[0], /TreasuryDestinationSet\(0x5{40}\) in finalized Pendulum block 101/); + assert.match(adminPages[1], /MigrationPauseSet\(false\)/); +}); + +test("a treasury destination set to any but the expected address pages critical", async (t) => { + const { pendulum, api, monitor, pages } = await setup(t, { env: { EXPECTED_TREASURY_DESTINATION: TREASURY_DESTINATION } }); + pendulum.finalized = 102; + pendulum.events.set(101, [{ section: "tokenMigration", method: "TreasuryDestinationSet", data: [TREASURY_DESTINATION] }]); + pendulum.events.set(102, [{ section: "tokenMigration", method: "TreasuryDestinationSet", data: [TOKEN] }]); + await monitor.poll(api); + const set = pages.filter((page) => page.subject.endsWith("MIGRATION PALLET ADMIN EVENT")); + assert.deepEqual(set.map((page) => [page.subject, page.severity]), [ + ["MIGRATION PALLET ADMIN EVENT", "WARNING"], + ["CRITICAL: MIGRATION PALLET ADMIN EVENT", "CRITICAL"], + ]); + assert.match( + set[1].detail, + new RegExp(`^tokenMigration.TreasuryDestinationSet\\(${TOKEN}\\) in finalized Pendulum block 102: .* not to EXPECTED_TREASURY_DESTINATION ${TREASURY_DESTINATION}, `, "i"), + ); +}); + +test("the governance watch runs after the conservation checks", async (t) => { + const { base, pendulum, api, monitor, subjects } = await setup(t); + base.vaultBalance -= 1n; + pendulum.publicProps.push([5, inline("remark")]); + await monitor.poll(api); + assert.deepEqual(subjects().slice(2), ["PENDULUM GOVERNANCE ITEM"]); + assert.deepEqual(sorted(subjects().slice(0, 2)), ["AUTO-PAUSE UNAVAILABLE", "VAULT BALANCE DEFICIT"]); +}); + +test("stale migrations page one liveness summary per kind and poll", async (t) => { + const { base, pendulum, api, monitor, pages } = await setup(t, { env: { GRACE_SECONDS: "30" } }); + pendulum.headAgeMs = 60_000; + for (let nonce = 0n; nonce < 27n; nonce++) burn(pendulum, nonce, 100n * PEN); + for (const nonce of [25n, 26n]) { + const payload = keccak256(encodeAbiParameters( + [{ type: "uint64" }, { type: "address" }, { type: "uint256" }], + [nonce, RECIPIENT, 100n * PEN], + )); + base.approvals.set(payload, 3n); + } + await monitor.poll(api); + await monitor.poll(api); + const liveness = pages.filter((page) => page.subject.startsWith("LIVENESS")); + assert.equal(liveness.length, 2, JSON.stringify(liveness)); + assert.match(pageFor(liveness, "LIVENESS: migration below approval quorum", "25 ")!.detail, /nonce 0, 1, .*, 9 and 15 more\)/); + assert.match(pageFor(liveness, "LIVENESS: quorum reached but not released", "2 ")!.detail, /\(nonce 25, 26\)/); +}); + +test("first sightings of unmatched events do not delay the pause", async (t) => { + const events: string[] = []; + let violationArrived = () => {}; + const arrived = new Promise((resolve) => (violationArrived = resolve)); + const { base, pendulum, api, monitor } = await setup(t, { + webhook: async ({ subject }) => { + if (subject === "MIGRATION TUPLE VIOLATION") { + events.push("violation paged"); + violationArrived(); + } + if (subject === "UNVERIFIED BASE EVENT") { + events.push("first sighting paged"); + await Promise.race([arrived, new Promise((resolve) => setTimeout(resolve, 3_000).unref())]); + events.push("first sighting answered"); + } + return 200; + }, + }); + burn(pendulum, 0n, 100n * PEN); + base.logs.push({ + address: VAULT, + eventName: "Approved", + args: { nonce: 5n, recipient: RECIPIENT, palletAmount: 100n * PEN, attestor: ATTESTORS[0] }, + blockNumber: 1_005n, + }); + base.logs.push({ + address: VAULT, + eventName: "Released", + args: { nonce: 0n, recipient: RECIPIENT, palletAmount: 100n * PEN, tokenAmount: 1n }, + blockNumber: 1_006n, + }); + base.totalReleased += 1n; + base.vaultBalance -= 1n; + await monitor.poll(api); + await until(() => events.includes("first sighting answered"), "the first sighting's answer", 5_000); + assert.deepEqual(events, ["first sighting paged", "violation paged", "first sighting answered"]); +}); + +test("after a violation the Base cursor stays on events still awaiting their source", async (t) => { + const { base, pendulum, api, monitor, subjects, persisted } = await setup(t, { + env: { UNMATCHED_EVENT_GRACE_SECONDS: "0", GUARDIAN_PRIVATE_KEY: GUARDIAN_KEY }, + }); + base.paused = true; // so the pause is confirmed and the poll persists its state + release(base, 0n, 100n * PEN); // the burn is not in the lagging source view yet + await monitor.poll(api); + await new Promise((resolve) => setTimeout(resolve, 5)); + await monitor.poll(api); // the grace has run out: a cannot-verify violation + assert.ok(subjects().includes("MIGRATION TUPLE VIOLATION"), JSON.stringify(subjects())); + assert.equal(persisted().baseFromBlock, "1000"); + burn(pendulum, 0n, 100n * PEN); // the source view catches up + await monitor.poll(api); + assert.deepEqual(persisted().migrations, []); + assert.equal(persisted().baseFromBlock, "1011"); +}); + +test("the conservation check fires only beyond an acknowledged loss", async (t) => { + const { base, api, monitor, subjects } = await setup(t, { env: { ACKNOWLEDGED_RELEASE_EXCESS: "1000" } }); + base.totalReleased = 1_000n; // released without a burn, quantified and accepted + base.vaultBalance = SUPPLY - 1_000n; + await monitor.poll(api); + assert.deepEqual(subjects(), []); + base.totalReleased += 1n; + base.vaultBalance -= 1n; + await monitor.poll(api); + assert.ok(subjects().includes("CONSERVATION VIOLATION"), JSON.stringify(subjects())); +}); + +/** A timelock operation scheduling the vault's `pause()`, ready in 48 h. */ +function schedule(base: Base, id: Hex, blockNumber = 1_005n): void { + const call = { target: VAULT, value: 0n, data: "0x8456cb59", predecessor: toHex(0, { size: 32 }), delay: 172_800n }; + base.logs.push({ address: TIMELOCK, eventName: "CallScheduled", args: { id, index: 0n, ...call }, blockNumber }); + base.readyAt.set(id, BigInt(Math.floor(Date.now() / 1000) + 172_800)); +} + +test("a scheduled operation with an extreme delay pages and hides nothing after it", async (t) => { + const { base, api, monitor, pages } = await setup(t, { env: { TIMELOCK_ADDRESS: TIMELOCK } }); + const [extreme, ordinary] = [toHex(1, { size: 32 }), toHex(2, { size: 32 })]; + schedule(base, extreme, 1_004n); + base.readyAt.set(extreme, 10n ** 20n); + schedule(base, ordinary, 1_006n); + await monitor.poll(api); + const scheduled = pages.filter((page) => page.subject === "GOVERNANCE OPERATION SCHEDULED").map((page) => page.detail); + assert.equal(scheduled.length, 2, JSON.stringify(pages)); + assert.ok(scheduled.some((detail) => detail.startsWith(`operation ${ordinary}`)), JSON.stringify(scheduled)); + assert.ok(!pages.some((page) => page.subject === "BASE ADMIN SCAN FAILED"), JSON.stringify(pages)); +}); + +test("an operation cancelled and scheduled again with the same id pages again", async (t) => { + const { base, api, monitor, pages } = await setup(t, { env: { TIMELOCK_ADDRESS: TIMELOCK } }); + const id = toHex(7, { size: 32 }); + schedule(base, id, 1_005n); + await monitor.poll(api); + base.readyAt.set(id, 0n); // cancelled + schedule(base, id, 1_015n); + base.safe = 1_020n; + await monitor.poll(api); + const scheduled = pages.filter((page) => page.subject === "GOVERNANCE OPERATION SCHEDULED"); + assert.equal(scheduled.length, 2, JSON.stringify(pages)); +}); + +test("once-only pages the webhook rejects are sent again on the next poll", async (t) => { + const failedOnce = new Set(); + const { base, pendulum, api, monitor, pages } = await setup(t, { + env: { TIMELOCK_ADDRESS: TIMELOCK }, + webhook: ({ subject }) => { + if (failedOnce.has(subject)) return 200; + failedOnce.add(subject); + return 500; + }, + }); + schedule(base, toHex(1, { size: 32 })); + pendulum.publicProps.push([7, inline("remark")]); + await monitor.poll(api); + await monitor.poll(api); + const delivered = pages.filter((page) => page.status === 200).map((page) => page.subject); + assert.ok(delivered.includes("GOVERNANCE OPERATION SCHEDULED"), `delivered: ${delivered}`); + assert.ok(delivered.includes("PENDULUM GOVERNANCE ITEM"), `delivered: ${delivered}`); + assert.ok(delivered.includes("AUTO-PAUSE DISABLED"), `delivered: ${delivered}`); // first sent at startup +}); diff --git a/monitor/src/main.ts b/monitor/src/main.ts index 533620efb..30c336c04 100644 --- a/monitor/src/main.ts +++ b/monitor/src/main.ts @@ -15,17 +15,28 @@ import { encodeAbiParameters, http, keccak256, + parseAbi, + toFunctionSelector, } from "viem"; import { privateKeyToAccount } from "viem/accounts"; import { approvalMismatch, blockRanges, + type CallNode, chunks, + describeCall, + describeTimelockOperation, + disagreeingAttestors, isStale, provablyUnsourced, releaseMismatch, releasedExceedsMigrated, + riskyCalls, + type ScheduledCall, + type Severity, shouldAwaitSource, + slackText, + toCallNode, type MigrationTuple, vaultConservationDeficit, } from "./checks.js"; @@ -45,6 +56,7 @@ const vaultAbi = [ { type: "function", name: "token", stateMutability: "view", inputs: [], outputs: [{ type: "address" }] }, { type: "function", name: "paused", stateMutability: "view", inputs: [], outputs: [{ type: "bool" }] }, { type: "function", name: "threshold", stateMutability: "view", inputs: [], outputs: [{ type: "uint256" }] }, + { type: "function", name: "guardian", stateMutability: "view", inputs: [], outputs: [{ type: "address" }] }, { type: "function", name: "activeApprovals", @@ -73,8 +85,32 @@ const vaultAbi = [ { name: "tokenAmount", type: "uint256", indexed: false }, ], }, + { + type: "event", + name: "RemainderSwept", + inputs: [ + { name: "to", type: "address", indexed: true }, + { name: "amount", type: "uint256", indexed: false }, + ], + }, ] as const; +/** The vault events of its admin (and of the guardian's pause). */ +const vaultAdminEventsAbi = parseAbi([ + "event TokenSet(address indexed token)", + "event Paused(address indexed by)", + "event Unpaused(address indexed by)", + "event AttestorAdded(address indexed attestor)", + "event AttestorRemoved(address indexed attestor)", + "event ThresholdUpdated(uint256 threshold)", + "event CapsUpdated(uint256 perReleaseCap, uint256 dailyCap)", + "event GuardianUpdated(address indexed guardian)", + "event AdminTransferStarted(address indexed pendingAdmin)", + "event AdminTransferred(address indexed newAdmin)", + "event RemainderSwept(address indexed to, uint256 amount)", + "event StalePendingCleared(bytes32 indexed payload, uint256 tokenAmount)", +]); + const erc20Abi = [ { type: "function", name: "totalSupply", stateMutability: "view", inputs: [], outputs: [{ type: "uint256" }] }, { @@ -86,6 +122,44 @@ const erc20Abi = [ }, ] as const; +const timelockAbi = [ + { + type: "function", + name: "getTimestamp", + stateMutability: "view", + inputs: [{ name: "id", type: "bytes32" }], + outputs: [{ type: "uint256" }], + }, + { + type: "event", + name: "CallScheduled", + inputs: [ + { name: "id", type: "bytes32", indexed: true }, + { name: "index", type: "uint256", indexed: true }, + { name: "target", type: "address", indexed: false }, + { name: "value", type: "uint256", indexed: false }, + { name: "data", type: "bytes", indexed: false }, + { name: "predecessor", type: "bytes32", indexed: false }, + { name: "delay", type: "uint256", indexed: false }, + ], + }, +] as const; + +/** Calls governance can be expected to schedule, named in the page. Anything + * else still pages, with its raw selector. */ +const KNOWN_FUNCTIONS = Object.fromEntries([ + // MigrationVault + "acceptAdmin()", "transferAdmin(address)", "setCaps(uint256,uint256)", "setGuardian(address)", + "setThreshold(uint256)", "addAttestor(address)", "removeAttestor(address)", "pause()", "unpause()", + "sweepRemainder(address,uint256)", "clearStalePending(uint64,address,uint256)", "setToken(address)", + // TimelockController and Governor self-administration + "updateDelay(uint256)", "grantRole(bytes32,address)", "revokeRole(bytes32,address)", + "setVotingDelay(uint48)", "setVotingPeriod(uint32)", "setProposalThreshold(uint256)", + "updateQuorumNumerator(uint256)", "updateTimelock(address)", "relay(address,uint256,bytes)", + // Treasury spends + "transfer(address,uint256)", "approve(address,uint256)", +].map((signature) => [toFunctionSelector(signature), signature])); + function required(name: string): string { const value = process.env[name]; if (!value) throw new Error(`Missing required environment variable ${name}`); @@ -106,6 +180,30 @@ function envNumber(name: string, fallback: number | undefined): number { return value; } +/** A non-negative integer amount from the environment, validated at startup. */ +function envAmount(name: string, fallback = "0"): bigint { + const raw = process.env[name] || fallback; + if (!/^\d+$/.test(raw)) throw new Error(`invalid ${name}: ${raw}`); + return BigInt(raw); +} + +function optionalAddress(name: string): `0x${string}` | undefined { + const value = process.env[name]; + if (!value) return undefined; + if (!/^0x[0-9a-fA-F]{40}$/.test(value)) throw new Error(`invalid ${name}: ${value}`); + return value as `0x${string}`; +} + +/** Comma-separated WebSocket URLs from the environment, at most `most`, + * validated at startup. */ +function wsUrls(name: string, most = Infinity): string[] { + const urls = (process.env[name] ?? "").split(",").map((url) => url.trim()).filter(Boolean); + if (urls.length > most || urls.some((url) => !/^wss?:\/\//i.test(url))) { + throw new Error(`invalid ${name}: expected ${most === 1 ? "one ws:// or wss:// URL" : "ws:// or wss:// URLs"}`); + } + return urls; +} + function finalityTag(): "safe" | "finalized" { const value = process.env.BASE_FINALITY_TAG ?? "safe"; if (value !== "safe" && value !== "finalized") throw new Error(`invalid BASE_FINALITY_TAG ${value}`); @@ -116,12 +214,22 @@ const baseFinalityTag = finalityTag(); const config = { pendulumWs: required("PENDULUM_WS"), + /** Optional: a second Pendulum node whose block hashes must agree. */ + pendulumWsCrossCheck: wsUrls("PENDULUM_WS_CROSSCHECK", 1)[0] as string | undefined, + /** Optional: third-party Polkadot RPCs, tried in order, whose record of + * the Pendulum head anchors the source. */ + polkadotWs: wsUrls("POLKADOT_WS"), + pendulumParaId: envNumber("PENDULUM_PARA_ID", 2094), + /** How long the source may go without a relay-anchored head before that pages critical. */ + relayAnchorGraceMs: envNumber("RELAY_ANCHOR_GRACE_SECONDS", 3600) * 1000, baseRpcUrl: required("BASE_RPC_URL"), vaultAddress: required("VAULT_ADDRESS") as `0x${string}`, pollIntervalMs: envNumber("POLL_INTERVAL_MS", 60_000), graceSeconds: envNumber("GRACE_SECONDS", 1800), unmatchedEventGraceSeconds: envNumber("UNMATCHED_EVENT_GRACE_SECONDS", 600), alertWebhookUrl: process.env.ALERT_WEBHOOK_URL, + /** Optional: requested after every poll that ran all checks. */ + heartbeatUrl: process.env.HEARTBEAT_URL, guardianPrivateKey: process.env.GUARDIAN_PRIVATE_KEY as `0x${string}` | undefined, baseChainId: envNumber("BASE_CHAIN_ID", 8453), baseFinalityTag, @@ -150,9 +258,26 @@ const config = { /** Growth of (Pendulum totalIssuance + TotalMigrated) over its first-seen * anchor, in 12-decimal pallet units, tolerated before paging. */ issuanceToleranceRaw: BigInt(process.env.ISSUANCE_TOLERANCE ?? "0"), + /** Optional: the governance TimelockController; every operation it + * schedules pages, because the canceller must veto inside the delay. */ + timelockAddress: optionalAddress("TIMELOCK_ADDRESS"), + /** Optional: the only Base address a treasury migration may pay. */ + expectedTreasuryDestination: optionalAddress("EXPECTED_TREASURY_DESTINATION"), + /** Age of the confirmed Base block (vs. wall clock) past which the monitor + * pages that its Base view stopped advancing. `safe` normally trails by + * about a minute, `finalized` by about 20 minutes. */ + baseStaleAlertMs: envNumber("BASE_STALE_ALERT_SECONDS", baseFinalityTag === "finalized" ? 1800 : 600) * 1000, + /** Guardian balance (wei) below which the auto-pause counts as not ready. */ + guardianMinBalanceWei: envAmount("GUARDIAN_MIN_BALANCE_WEI", "1000000000000000"), + /** Excess of released over migrated, in token units, that operators have + * quantified and accepted after a real loss (RB-3); M2a fires beyond it. */ + acknowledgedReleaseExcess: envAmount("ACKNOWLEDGED_RELEASE_EXCESS"), }; const ZERO_ADDRESS = "0x0000000000000000000000000000000000000000"; +/** The runtime's treasury account, `PalletId(*b"py/trsry")` as an account + * ("modl" + id, zero-padded): the `who` of every treasury migration. */ +const PENDULUM_TREASURY_ACCOUNT = `0x6d6f646c70792f7472737279${"00".repeat(20)}`; const baseChain = defineChain({ id: config.baseChainId, @@ -173,11 +298,21 @@ interface RuntimeState { vaultAddress: `0x${string}`; pendulumGenesisHash: `0x${string}` | undefined; lastPendulumBlock: number; + /** The hash of `lastPendulumBlock`: the next blocks must descend from it. */ + lastPendulumHash: string | undefined; + /** The newest Pendulum head the relay chain recorded that the source was + * verified against; the source's chain must keep passing through it. */ + relayAnchor: { number: number; hash: string } | undefined; nextExpectedNonce: bigint; baseFromBlock: bigint; + /** The admin scan's own cursor: it only pages, so nothing waits for it. */ + baseAdminFromBlock: bigint; migrations: Map; livenessAlertedAt: Map; unmatchedBaseFirstSeenAt: Map; + /** Approvals that disagreed with the finalized burn, or found none, kept + * until their nonce is released (see `disagreeingAttestors`). */ + mismatchedApprovals: Array; /** Pendulum totalIssuance + TotalMigrated at first observation. Under every * legitimate flow this sum can only fall (teleport-out) and later recover * to at most its original value (teleport-in); growth means PEN was minted @@ -194,11 +329,15 @@ function hydrateState(): RuntimeState { vaultAddress: config.vaultAddress, pendulumGenesisHash: undefined, lastPendulumBlock: config.pendulumStartBlock - 1, + lastPendulumHash: undefined, + relayAnchor: undefined, nextExpectedNonce: config.pendulumStartNonce, baseFromBlock: config.baseStartBlock, + baseAdminFromBlock: config.baseStartBlock, migrations: new Map(), livenessAlertedAt: new Map(), unmatchedBaseFirstSeenAt: new Map(), + mismatchedApprovals: [], issuanceAnchor: undefined, }; } @@ -207,8 +346,11 @@ function hydrateState(): RuntimeState { vaultAddress: persisted.vaultAddress, pendulumGenesisHash: persisted.pendulumGenesisHash, lastPendulumBlock: persisted.lastPendulumBlock, + lastPendulumHash: persisted.lastPendulumHash, + relayAnchor: persisted.relayAnchor, nextExpectedNonce: BigInt(persisted.nextExpectedNonce), baseFromBlock: BigInt(persisted.baseFromBlock), + baseAdminFromBlock: BigInt(persisted.baseAdminFromBlock ?? persisted.baseFromBlock), migrations: new Map(persisted.migrations.map((migration) => [ BigInt(migration.nonce), { @@ -221,6 +363,11 @@ function hydrateState(): RuntimeState { ])), livenessAlertedAt: new Map(persisted.livenessAlertedAt.map(([nonce, at]) => [BigInt(nonce), at])), unmatchedBaseFirstSeenAt: new Map(persisted.unmatchedBaseFirstSeenAt), + mismatchedApprovals: (persisted.mismatchedApprovals ?? []).map((approval) => ({ + ...approval, + nonce: BigInt(approval.nonce), + palletAmount: BigInt(approval.palletAmount), + })), issuanceAnchor: persisted.issuanceAnchor === undefined ? undefined : BigInt(persisted.issuanceAnchor), }; } @@ -231,11 +378,15 @@ function cloneState(source: RuntimeState): RuntimeState { vaultAddress: source.vaultAddress, pendulumGenesisHash: source.pendulumGenesisHash, lastPendulumBlock: source.lastPendulumBlock, + lastPendulumHash: source.lastPendulumHash, + relayAnchor: source.relayAnchor, nextExpectedNonce: source.nextExpectedNonce, baseFromBlock: source.baseFromBlock, + baseAdminFromBlock: source.baseAdminFromBlock, migrations: new Map(source.migrations), livenessAlertedAt: new Map(source.livenessAlertedAt), unmatchedBaseFirstSeenAt: new Map(source.unmatchedBaseFirstSeenAt), + mismatchedApprovals: [...source.mismatchedApprovals], issuanceAnchor: source.issuanceAnchor, }; } @@ -248,8 +399,11 @@ function persistState(state: RuntimeState): void { vaultAddress: state.vaultAddress, pendulumGenesisHash: state.pendulumGenesisHash, lastPendulumBlock: state.lastPendulumBlock, + lastPendulumHash: state.lastPendulumHash as `0x${string}` | undefined, + relayAnchor: state.relayAnchor as { number: number; hash: `0x${string}` } | undefined, nextExpectedNonce: state.nextExpectedNonce.toString(), baseFromBlock: state.baseFromBlock.toString(), + baseAdminFromBlock: state.baseAdminFromBlock.toString(), migrations: [...state.migrations.values()].map((migration) => ({ nonce: migration.nonce.toString(), recipient: migration.recipient as `0x${string}`, @@ -259,6 +413,12 @@ function persistState(state: RuntimeState): void { })), livenessAlertedAt: [...state.livenessAlertedAt].map(([nonce, at]) => [nonce.toString(), at]), unmatchedBaseFirstSeenAt: [...state.unmatchedBaseFirstSeenAt], + mismatchedApprovals: state.mismatchedApprovals.map((approval) => ({ + nonce: approval.nonce.toString(), + recipient: approval.recipient as `0x${string}`, + palletAmount: approval.palletAmount.toString(), + attestor: approval.attestor as `0x${string}`, + })), issuanceAnchor: state.issuanceAnchor?.toString(), }; saveMonitorState(config.stateFile, persisted); @@ -267,33 +427,81 @@ function persistState(state: RuntimeState): void { let state: RuntimeState; let multicallUnavailable = false; let lastSourceStallAlertMs = 0; +let lastSourceFailureAlertMs = 0; +let lastSourceIntegrityAlertMs = 0; +let lastCrossCheckAlertMs = 0; +let lastRelayAlertMs = 0; +let lastRelayDisagreementAlertMs = 0; +let lastRelayStaleAlertMs = 0; +/** When the source last verified against a new relay-chain record (or the start). */ +let lastRelayAnchorMs = Date.now(); +let lastBaseStallAlertMs = 0; +let lastReadinessAlertMs = 0; +let lastAdminScanFailureAlertMs = 0; +/** Since when the Base admin scan fails, while it does. */ +let adminScanFailingSinceMs: number | undefined; +let lastLogScanAlertMs = 0; +/** Since when the Base log scan holds a range, while it does. */ +let logScanHeldSinceMs: number | undefined; let lastIssuanceAlertMs = 0; +/** Once-only pages by key, each held until the webhook accepts it. */ +// ponytail: in memory; a restart re-pages what it finds again (Base admin events after the persisted cursor, pending governance items) and drops undelivered pages it does not find again. +const onceOnlyPages = new Map }>(); +let lastGovernanceWatchFailureMs = 0; function log(message: string): void { console.log(`${new Date().toISOString()} ${message}`); } /** Strip URLs before anything leaves the process: RPC endpoints commonly embed - * API keys, and viem error texts quote the endpoint verbatim. */ + * API keys, and viem and polkadot.js error texts quote the endpoint verbatim. */ function redact(text: string): string { - return text.replace(/https?:\/\/[^\s"')]+/gi, ""); + return text.replace(/(?:https?|wss?):\/\/[^\s"')]+/gi, ""); } -async function alert(subject: string, detail: string): Promise { +/** Sends one page, as a Slack incoming-webhook message, and resolves to + * whether it was delivered: the webhook must answer 2xx within the timeout. + * Without a webhook the log line is the page. */ +async function alert( + subject: string, + detail: string, + severity: Severity = subject.startsWith("CRITICAL") ? "critical" : "warning", +): Promise { console.error(`${new Date().toISOString()} ALERT: ${subject} — ${detail}`); - if (!config.alertWebhookUrl) return; + if (!config.alertWebhookUrl) return true; try { - await fetch(config.alertWebhookUrl, { + const response = await fetch(config.alertWebhookUrl, { method: "POST", headers: { "content-type": "application/json" }, - body: JSON.stringify({ service: "pen-monitor", subject, detail: redact(detail) }), + body: JSON.stringify({ text: slackText("pen-monitor", severity, subject, redact(detail)) }), // The webhook is an external dependency; it must never stall the // check loop (or the pause that follows a violation) indefinitely. signal: AbortSignal.timeout(10_000), }); + if (response.ok) return true; + console.error(`alert webhook answered ${response.status} for: ${subject}`); } catch (webhookError) { - console.error("alert webhook failed", webhookError); + console.error(`alert webhook failed for: ${subject}: ${redact(`${webhookError}`)}`); } + return false; +} + +/** Pages `key` once. A page the webhook does not accept stays queued, and + * every poll retries it until it is delivered. */ +function pageOnce(key: string, subject: string, detail: string): Promise { + const queued = onceOnlyPages.get(key); + if (queued) return queued.delivery; + const delivery = alert(subject, detail); + onceOnlyPages.set(key, { subject, detail, delivery }); + return delivery; +} + +async function retryUndeliveredPages(): Promise { + await Promise.all([...onceOnlyPages.values()].map(async (page) => { + if (await page.delivery) return; + page.delivery = alert(page.subject, page.detail); + await page.delivery; + })); } async function waitForBaseFinality( @@ -318,11 +526,15 @@ async function waitForBaseFinality( } } -async function pauseVault(): Promise { - if (!config.guardianPrivateKey) { - await alert("AUTO-PAUSE UNAVAILABLE", "no GUARDIAN_PRIVATE_KEY configured; pause manually NOW"); - return false; - } +/** The pause attempt in flight, which a concurrent violation joins. */ +let pausing: Promise | undefined; + +function pauseVault(): Promise { + pausing ??= pauseOnce().finally(() => (pausing = undefined)); + return pausing; +} + +async function pauseOnce(): Promise { try { const confirmed = await publicClient.getBlock({ blockTag: config.baseFinalityTag }); if (confirmed.number === null) throw new Error(`${config.baseFinalityTag} block has no number`); @@ -345,6 +557,10 @@ async function pauseVault(): Promise { await alert("AUTO-PAUSE PENDING FINALITY", `pause is mined but not yet ${config.baseFinalityTag}`); return false; } + if (!config.guardianPrivateKey) { + await alert("AUTO-PAUSE UNAVAILABLE", "no GUARDIAN_PRIVATE_KEY configured; pause manually NOW", "critical"); + return false; + } const guardian = privateKeyToAccount(config.guardianPrivateKey); const walletClient = createWalletClient({ account: guardian, chain: baseChain, transport: http(config.baseRpcUrl) }); @@ -368,15 +584,56 @@ async function pauseVault(): Promise { await alert("vault auto-pause confirmed", `tx=${txHash} ${config.baseFinalityTag}Block=${safeBlock}`); return true; } catch (pauseError) { - await alert("AUTO-PAUSE FAILED", `${pauseError}; pause manually NOW`); + await alert("AUTO-PAUSE FAILED", `${pauseError}; pause manually NOW`, "critical"); return false; } } +/** What would make an auto-pause fail now: an RPC on another chain, or a key + * that does not parse, is not the vault's guardian or cannot pay for the + * pause. Pages, throttled, so it surfaces before a violation needs it. A read + * that fails is a problem too: it must not stop the poll's other checks. */ +async function checkAutoPauseReadiness(): Promise { + const problems: string[] = []; + try { + const chainId = await publicClient.getChainId(); + if (chainId !== config.baseChainId) { + problems.push(`the Base RPC serves chain ${chainId}, not BASE_CHAIN_ID ${config.baseChainId}`); + } + if (config.guardianPrivateKey) { + let guardian: `0x${string}` | undefined; + try { + guardian = privateKeyToAccount(config.guardianPrivateKey).address; + } catch { + problems.push("GUARDIAN_PRIVATE_KEY is not a valid private key"); + } + if (guardian) { + const [vaultGuardian, balance] = await Promise.all([ + publicClient.readContract({ address: config.vaultAddress, abi: vaultAbi, functionName: "guardian" }), + publicClient.getBalance({ address: guardian }), + ]); + if (vaultGuardian.toLowerCase() !== guardian.toLowerCase()) { + problems.push(`the key's address ${guardian} is not the vault guardian ${vaultGuardian}`); + } + if (balance < config.guardianMinBalanceWei) { + problems.push(`guardian ${guardian} holds ${balance} wei, less than GUARDIAN_MIN_BALANCE_WEI ${config.guardianMinBalanceWei}`); + } + } + } + } catch (readError) { + problems.push(`the readiness reads failed: ${readError}`); + } + if (problems.length && Date.now() - lastReadinessAlertMs > config.graceSeconds * 1000) { + lastReadinessAlertMs = Date.now(); + await alert("AUTO-PAUSE NOT READY", `${problems.join("; ")} — fix it before a violation needs the pause`); + } + return problems; +} + async function securityViolation(subject: string, detail: string): Promise { // Pause FIRST. The alert webhook is an external dependency with its own // latency; nothing may sit between detecting a violation and the pause. - const notified = alert(subject, detail); + const notified = alert(subject, detail, "critical"); const paused = await pauseVault(); await notified; return paused; @@ -398,9 +655,10 @@ async function securityViolation(subject: string, detail: string): Promise { +): "proven" | "await" | "expired" { const alreadyKnown = draft.unmatchedBaseFirstSeenAt.has(key); const firstSeen = draft.unmatchedBaseFirstSeenAt.get(key) ?? observedAt; draft.unmatchedBaseFirstSeenAt.set(key, firstSeen); if (!alreadyKnown) { - await alert( + void pageOnce( + `unverified:${key}`, "UNVERIFIED BASE EVENT", `${kind} nonce ${nonce} at Base block ${eventBlockNumber} has no finalized Pendulum source yet; ` + `waiting up to ${config.unmatchedEventGraceSeconds}s for the source view to reveal its burn`, @@ -425,21 +684,450 @@ async function unmatchedDisposition( : "expired"; } -async function ingestPendulumEvents(api: ApiPromise, draft: RuntimeState, finalizedBlock: number): Promise { - for (let block = draft.lastPendulumBlock + 1; block <= finalizedBlock; block++) { - const blockHash = await api.rpc.chain.getBlockHash(block); +interface BlockRef { + number: number; + hash: string; +} + +/** The Pendulum source contradicts the chain it must be on: the one ingested + * so far, the second node's or the relay chain's. Handled as unreadable. */ +class SourceIntegrityError extends Error {} + +/** The source holds at a block whose violation must pause the vault first; + * the violation's own page says so. */ +class PauseHold extends Error {} + +/** Bound on connecting to, and on each read from, the nodes of the integrity checks. */ +const NODE_TIMEOUT_MS = 20_000; + +function withTimeout(promise: Promise, ms: number): Promise { + let timer: NodeJS.Timeout | undefined; + const timeout = new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error(`no answer within ${ms / 1000}s`)), ms); + }); + return Promise.race([promise, timeout]).finally(() => clearTimeout(timer)); +} + +/** Connects to a node of the integrity checks; tests replace it. */ +let connectNode = async (url: string): Promise => { + const provider = new WsProvider(url, 2_500, {}, NODE_TIMEOUT_MS); + try { + return await withTimeout(ApiPromise.create({ provider, throwOnConnect: true, noInitWarn: true }), NODE_TIMEOUT_MS); + } catch (connectError) { + await provider.disconnect().catch(() => {}); + throw connectError; + } +}; +const nodeConnections = new Map>(); + +/** Runs `read` against the node at `url` over a kept connection, bounded in + * time; a failure drops the connection, so the next poll connects afresh. */ +async function onNode(url: string, read: (node: ApiPromise) => Promise): Promise { + let connection = nodeConnections.get(url); + if (!connection) nodeConnections.set(url, (connection = connectNode(url))); + try { + return await withTimeout(connection.then(read), NODE_TIMEOUT_MS); + } catch (nodeError) { + nodeConnections.delete(url); + connection.then((node) => node.disconnect()).catch(() => {}); + // A failed WebSocket connection rejects with an event, not an Error. + throw nodeError instanceof Error ? nodeError : new Error(String((nodeError as { message?: unknown })?.message ?? nodeError)); + } +} + +interface MigrationTotals { + nextNonce: bigint; + totalMigrated: bigint; +} + +async function migrationTotals(apiAt: ApiAt): Promise { + const [nextNonce, totalMigrated] = await Promise.all([ + apiAt.query.tokenMigration.nextNonce(), + apiAt.query.tokenMigration.totalMigrated(), + ]); + return { + nextNonce: BigInt(nextNonce.toString().replaceAll(",", "")), + totalMigrated: BigInt(totalMigrated.toString().replaceAll(",", "")), + }; +} + +/** The height up to which the second node confirms the source's chain + * `hashes`, walked down from `top` to `bottom`: the hashes must agree at its + * finalized head, or at `top` once it has finalized past it. Undefined while + * it cannot be read or its finalized head is older than + * SOURCE_STALE_ALERT_SECONDS, which only warns, throttled: it is a check, not + * a source. */ +async function crossCheckSource(hashes: Map, top: BlockRef, bottom: number): Promise { + const url = config.pendulumWsCrossCheck; + if (!url) return undefined; + let other: BlockRef & { ageMs: number }; + try { + other = await onNode(url, async (node) => { + const finalizedHash = await node.rpc.chain.getFinalizedHead(); + const finalized = (await node.rpc.chain.getHeader(finalizedHash)).number.toNumber(); + const timestamp = await (await node.at(finalizedHash)).query.timestamp.now(); + // Above its finalized head, a node's block at a height may still lose. + const number = Math.min(finalized, top.number); + const hash = number === finalized ? finalizedHash.toHex() : (await node.rpc.chain.getBlockHash(number)).toHex(); + return { number, hash, ageMs: Date.now() - Number(timestamp.toString().replaceAll(",", "")) }; + }); + } catch (crossCheckError) { + await crossCheckUnavailable(`cannot be read: ${crossCheckError}`); + return undefined; + } + if (other.ageMs > config.sourceStaleAlertMs) { + await crossCheckUnavailable(`has a finalized head ${Math.round(other.ageMs / 1000)}s old`); + return undefined; + } + if (other.number >= bottom && hashes.get(other.number) !== other.hash) { + throw new SourceIntegrityError( + `the second Pendulum node has ${other.hash} at block ${other.number}, the source ${hashes.get(other.number)}`, + ); + } + return other.number; +} + +/** The second node must have the source's migration totals at `blockHash`, a + * block it confirmed. One that cannot answer only warns, throttled. */ +async function crossCheckTotals(block: number, blockHash: string, totals: MigrationTotals): Promise { + let other: MigrationTotals; + try { + other = await onNode(config.pendulumWsCrossCheck as string, async (node) => migrationTotals(await node.at(blockHash))); + } catch (crossCheckError) { + return crossCheckUnavailable(`cannot be read: ${crossCheckError}`); + } + if (other.nextNonce !== totals.nextNonce || other.totalMigrated !== totals.totalMigrated) { + throw new SourceIntegrityError( + `at Pendulum block ${block} the second node has nextNonce ${other.nextNonce} and ` + + `totalMigrated ${other.totalMigrated}, the source ${totals.nextNonce} and ${totals.totalMigrated}`, + ); + } +} + +type EventRecords = Array<{ event: { section: string; method: string; data: unknown[] } }>; + +/** The migration pallet's events in a block, as one text; empty without any. */ +function migrationEventText(records: EventRecords): string { + return records + .filter(({ event }) => event.section === "tokenMigration") + .map(({ event }) => `${event.method}(${event.data.join(", ")})`) + .join("; "); +} + +/** The second node must report the same migration events in the block at + * `blockHash`. One that cannot answer only warns, throttled. */ +async function crossCheckEvents(block: number, blockHash: string, events: string): Promise { + const url = config.pendulumWsCrossCheck; + if (!url) return; + let other: string; + try { + other = await onNode(url, async (node) => + migrationEventText((await (await node.at(blockHash)).query.system.events()) as unknown as EventRecords) + ); + } catch (crossCheckError) { + return crossCheckUnavailable(`cannot be read: ${crossCheckError}`); + } + if (other !== events) { + throw new SourceIntegrityError( + `in Pendulum block ${block} the second node reports the migration events ${other || "none"}, the source ${events}`, + ); + } +} + +/** Pages, throttled, that the source cannot be compared with the second node. */ +async function crossCheckUnavailable(reason: string): Promise { + if (Date.now() - lastCrossCheckAlertMs <= config.graceSeconds * 1000) return; + lastCrossCheckAlertMs = Date.now(); + await alert( + "PENDULUM CROSS-CHECK UNAVAILABLE", + `the second Pendulum node (PENDULUM_WS_CROSSCHECK) ${reason}; the source's blocks are not compared ` + + "with it until it answers — restore it", + ); +} + +/** Walks parent hashes down from `top` to `bottom` through headers fetched + * from the node, recomputing each hash here, so that the result is one + * chain the node cannot fake: block hashes by height, `bottom` included. */ +async function walkHeaders(api: ApiPromise, top: BlockRef, bottom: number): Promise> { + // ponytail: one pass, held in memory and started over after a failed read; make it resumable if catch-ups grow to weeks. + const hashes = new Map([[top.number, top.hash]]); + let hash = top.hash; + for (let height = top.number; height > bottom; height--) { + const header = await api.rpc.chain.getHeader(hash); + if (header.hash.toHex() !== hash || header.number.toNumber() !== height) { + throw new SourceIntegrityError(`the source's header for Pendulum block ${height} does not hash to ${hash}`); + } + hash = header.parentHash.toHex(); + hashes.set(height - 1, hash); + } + return hashes; +} + +/** The Pendulum head that the relay chain behind the RPC at `url` recorded + * at its finalized block, and the age of that block. */ +function relayHead(api: ApiPromise, url: string): Promise { + return onNode(url, async (relay) => { + const relayAt = await relay.at(await relay.rpc.chain.getFinalizedHead()); + const timestamp = await relayAt.query.timestamp.now(); + const head = (await relayAt.query.paras.heads(config.pendulumParaId)) as unknown as { + isSome: boolean; + unwrap(): { toU8a(bare: boolean): Uint8Array }; + }; + if (!head.isSome) throw new Error(`no head recorded for para ${config.pendulumParaId}`); + // The head data is the Pendulum header; its hash is computed here. + const header = api.createType("Header", head.unwrap().toU8a(true)) as unknown as { + number: { toNumber(): number }; + hash: { toHex(): string }; + }; + return { + number: header.number.toNumber(), + hash: header.hash.toHex(), + ageMs: Date.now() - Number(timestamp.toString().replaceAll(",", "")), + }; + }); +} + +/** How far below the block ingested last the relay chain's record may lie and + * still be walked down to; a deeper one is not used. */ +const RELAY_HEAD_MAX_DEPTH = 100; + +/** Why the relay chain's record `head` does not anchor the source this poll, + * or undefined when it does. Throws when it contradicts the source's chain: + * `hashes`, walked down from `top` to `bottom`. */ +async function relayVerdict( + api: ApiPromise, + draft: RuntimeState, + hashes: Map, + top: BlockRef, + bottom: BlockRef, + head: BlockRef, +): Promise { + const known = draft.relayAnchor; + if (known && head.number <= known.number) { + if (head.number === known.number && head.hash !== known.hash) { + throw new SourceIntegrityError( + `the relay chain now records ${head.hash} as Pendulum block ${head.number}, where it recorded ${known.hash}`, + ); + } + return `the relay chain's Pendulum head ${head.number} is not newer`; + } + if (head.number > top.number) { + // Past the finalized head, the source's best chain may still hold another + // block at that height. The record anchors the source once it holds the + // record, whose chain must then pass through `top`. + if ((await api.rpc.chain.getBlockHash(head.number)).toHex() !== head.hash) { + return `the source has not imported block ${head.number}, the relay chain's Pendulum head, yet`; + } + const hash = (await walkHeaders(api, head, top.number)).get(top.number); + if (hash !== top.hash) { + throw new SourceIntegrityError( + `the relay chain records ${head.hash} as Pendulum block ${head.number}, whose chain has ${hash} at block ` + + `${top.number}, where the source has ${top.hash}`, + ); + } + return undefined; + } + if (head.number < bottom.number - RELAY_HEAD_MAX_DEPTH) { + return `the relay chain's Pendulum head ${head.number} lies more than ${RELAY_HEAD_MAX_DEPTH} blocks below block ${bottom.number}`; + } + const hash = head.number >= bottom.number + ? hashes.get(head.number) + : (await walkHeaders(api, bottom, head.number)).get(head.number); + if (hash !== head.hash) { + throw new SourceIntegrityError(`the relay chain records ${head.hash} as Pendulum block ${head.number}, the source's chain has ${hash}`); + } + return undefined; +} + +/** Anchors the source to the relay chain's record of the Pendulum head, kept + * as `draft.relayAnchor`: the highest record that anchors it, from the relay + * RPCs in POLKADOT_WS, asked in turn until a record anchors the source's + * head `top`. A relay RPC whose finalized relay block is older than + * SOURCE_STALE_ALERT_SECONDS counts as not answering. A record that + * contradicts the source counts only when another relay RPC's record + * contradicts it too, or when POLKADOT_WS lists one RPC. A record that + * anchors the source at or above the contradicting record's height outvotes + * it; otherwise it stays unconfirmed. Resolves to the height up to which the + * relay chain confirms the source's chain `hashes`, walked down from `top` to + * `bottom`: the newest anchored head, or no higher than `bottom` before the + * first; undefined while no relay RPC answers. Pages while no new head is + * anchored and the source's blocks wait for one, a contradiction stays + * unconfirmed or no relay RPC answers: a warning, critical once none was + * anchored for RELAY_ANCHOR_GRACE_SECONDS; and when relay RPCs disagree. All + * throttled. */ +async function anchorToRelay( + api: ApiPromise, + draft: RuntimeState, + hashes: Map, + top: BlockRef, + bottom: BlockRef, +): Promise { + if (!config.polkadotWs.length) return undefined; + const answers: string[] = []; + let answered = false; + let anchor: (BlockRef & { rpc: number }) | undefined; + let contradiction: { rpc: number; height: number; error: SourceIntegrityError } | undefined; + for (const [index, url] of config.polkadotWs.entries()) { + if (anchor && anchor.number >= Math.max(top.number, contradiction?.height ?? 0)) break; + let head: BlockRef & { ageMs: number }; + try { + head = await relayHead(api, url); + } catch (relayError) { + answers.push(`relay RPC ${index + 1}: ${relayError}`); + continue; + } + if (head.ageMs > config.sourceStaleAlertMs) { + answers.push(`relay RPC ${index + 1}: its finalized relay block is ${Math.round(head.ageMs / 1000)}s old`); + continue; + } + answered = true; + let verdict: string | undefined; + try { + verdict = await relayVerdict(api, draft, hashes, top, bottom, head); + } catch (verdictError) { + if (!(verdictError instanceof SourceIntegrityError)) throw verdictError; + if (contradiction || config.polkadotWs.length === 1) throw contradiction?.error ?? verdictError; + contradiction = { rpc: index + 1, height: head.number, error: verdictError }; + continue; + } + if (verdict !== undefined) answers.push(`relay RPC ${index + 1}: ${verdict}`); + else if (!anchor || head.number > anchor.number) anchor = { number: head.number, hash: head.hash, rpc: index + 1 }; + } + if (anchor) { + draft.relayAnchor = { number: anchor.number, hash: anchor.hash }; + lastRelayAnchorMs = Date.now(); + if (!contradiction) return anchor.number; + if (anchor.number >= contradiction.height) { + if (Date.now() - lastRelayDisagreementAlertMs > config.graceSeconds * 1000) { + lastRelayDisagreementAlertMs = Date.now(); + await alert( + "RELAY RPCS DISAGREE", + `relay RPC ${contradiction.rpc} reports that ${contradiction.error.message}; relay RPC ${anchor.rpc} records ` + + `${anchor.hash} as Pendulum block ${anchor.number}, on the source's chain, and the source is anchored to ` + + "that record — one of the two relay RPCs is wrong, check both", + ); + } + return anchor.number; + } + answers.push(`relay RPC ${anchor.rpc}: its Pendulum head ${anchor.number} lies below the contradicting record`); + } + const reason = contradiction + ? `relay RPC ${contradiction.rpc} reports that ${contradiction.error.message}, and no other relay RPC confirms ` + + `or outvotes it${answers.length ? ` (${answers.join("; ")})` : ""}` + : `${answered ? "" : "the relay chain cannot be read: "}${answers.join("; ")}`; + const confirmed = draft.relayAnchor?.number ?? bottom.number; + const consequence = answered + ? `no Pendulum block after ${Math.max(confirmed, draft.lastPendulumBlock)} is ingested until a relay RPC anchors a newer head` + : "the Pendulum source is used without the relay-chain anchor until a relay RPC answers"; + if ( + (contradiction || !answered || top.number > confirmed) && + Date.now() - lastRelayAlertMs > config.graceSeconds * 1000 + ) { + lastRelayAlertMs = Date.now(); + await alert( + "RELAY ANCHOR UNAVAILABLE", + `${reason}; ${consequence}, and this turns critical after ${config.relayAnchorGraceMs / 1000}s`, + ); + } + const unanchoredMs = Date.now() - lastRelayAnchorMs; + if (unanchoredMs > config.relayAnchorGraceMs && Date.now() - lastRelayStaleAlertMs > config.unmatchedEventGraceSeconds * 1000) { + lastRelayStaleAlertMs = Date.now(); + await alert( + "CRITICAL: RELAY ANCHOR STALE", + `no Pendulum head was checked against the relay chain for ${Math.round(unanchoredMs / 1000)}s (${reason}); ` + + `${consequence} — restore a relay RPC, or the source node if it lags`, + ); + } + return answered ? confirmed : undefined; +} + +/** The hashes of the finalized blocks to ingest, on one chain with every + * block ingested before and with the relay-chain record kept from an earlier + * poll, and the newest of them that every checker that answers confirms + * (`confirmed`, at most the source's finalized head): the second node up to + * `crossChecked`, and the relay chain up to `anchored`, its newest record + * that anchors the source. A checker that does not answer confirms nothing + * and holds nothing back. */ +async function sourceHashes( + api: ApiPromise, + draft: RuntimeState, + finalized: BlockRef, +): Promise<{ hashes: Map; confirmed: number; crossChecked: number | undefined; anchored: number | undefined }> { + // A kept record past the finalized head binds the source too: the walk starts there. + const known = draft.relayAnchor; + const top = known && known.number > finalized.number ? known : finalized; + const hashes = await walkHeaders(api, top, draft.lastPendulumBlock); + const ingested = draft.lastPendulumHash ? { number: draft.lastPendulumBlock, hash: draft.lastPendulumHash } : undefined; + const expected: Array<[BlockRef | undefined, string]> = [ + [finalized, "the source's finalized head is"], + [ingested, "the monitor ingested"], + [known, "the relay chain recorded"], + ]; + for (const [block, what] of expected) { + const hash = block && hashes.get(block.number); + if (block && hash !== undefined && hash !== block.hash) { + throw new SourceIntegrityError(`the source's chain has ${hash} at Pendulum block ${block.number}, where ${what} ${block.hash}`); + } + } + const lowest = Math.min(top.number, draft.lastPendulumBlock); + const crossChecked = await crossCheckSource(hashes, top, lowest); + const anchored = await anchorToRelay(api, draft, hashes, top, { number: lowest, hash: hashes.get(lowest) as string }); + return { hashes, confirmed: Math.min(finalized.number, crossChecked ?? Infinity, anchored ?? Infinity), crossChecked, anchored }; +} + +/** The migration pallet's admin events, each with what to check. */ +const PALLET_ADMIN_EVENTS = new Map([ + ["MigrationPauseSet", "migrations were paused or resumed; check that it was intended"], + [ + "TreasuryDestinationSet", + "treasury migrations are released to this Base address from now on; check that it is the governance timelock", + ], +]); + +/** Ingests the blocks after the cursor up to `lastBlock`, each read at its + * hash in `hashes`: the chain that `sourceHashes` verified. Their migration + * events are compared with the second node's while it answers. */ +async function ingestPendulumEvents( + api: ApiPromise, + draft: RuntimeState, + lastBlock: number, + hashes: Map, + crossChecked: boolean, +): Promise { + for (let block = draft.lastPendulumBlock + 1; block <= lastBlock; block++) { + const blockHash = hashes.get(block); + if (!blockHash) throw new Error(`no verified hash for Pendulum block ${block}`); const apiAt = await api.at(blockHash); const firstSeenMs = Number((await apiAt.query.timestamp.now()).toString().replaceAll(",", "")); - const records = (await apiAt.query.system.events()) as unknown as { - event: { section: string; method: string; data: unknown[] }; - }[]; - for (const record of records) { + const records = (await apiAt.query.system.events()) as unknown as EventRecords; + const migrationEvents = migrationEventText(records); + if (migrationEvents && crossChecked) await crossCheckEvents(block, blockHash, migrationEvents); + for (const [index, record] of records.entries()) { const { section, method, data } = record.event; - if (section !== "tokenMigration" || method !== "MigrationInitiated") continue; + if (section !== "tokenMigration") continue; + const adminCheck = PALLET_ADMIN_EVENTS.get(method); + if (adminCheck) { + // Every admin change, however fast the motion behind it passed; a + // treasury destination other than the expected one is critical. + const expected = config.expectedTreasuryDestination; + const unexpected = method === "TreasuryDestinationSet" && expected !== undefined && + String(data[0]).toLowerCase() !== expected.toLowerCase(); + await pageOnce( + `pallet:${block}:${index}`, + `${unexpected ? "CRITICAL: " : ""}MIGRATION PALLET ADMIN EVENT`, + `tokenMigration.${method}(${data.join(", ")}) in finalized Pendulum block ${block}: ` + + (unexpected + ? `treasury migrations are released to this Base address from now on, not to ` + + `EXPECTED_TREASURY_DESTINATION ${expected}, and each one pauses the vault — check the motion behind it` + : adminCheck), + ); + continue; + } + if (method !== "MigrationInitiated") continue; if (data.length !== 4) throw new Error(`MigrationInitiated in block ${block} has ${data.length} fields, expected 4`); - const [nonceValue, , recipientValue, amountValue] = data as [ + const [nonceValue, whoValue, recipientValue, amountValue] = data as [ { toBigInt(): bigint }, - unknown, + { toHex(): string }, { toHex(): string }, { toBigInt(): bigint }, ]; @@ -452,6 +1140,22 @@ async function ingestPendulumEvents(api: ApiPromise, draft: RuntimeState, finali throw new Error(`migration nonce gap: finalized event ${nonce}, expected ${draft.nextExpectedNonce}`); } draft.nextExpectedNonce++; + const expectedTreasury = config.expectedTreasuryDestination; + if ( + expectedTreasury && + whoValue.toHex().toLowerCase() === PENDULUM_TREASURY_ACCOUNT && + recipient.toLowerCase() !== expectedTreasury.toLowerCase() + ) { + // The vault is paused before the attestors can release it; while + // the pause is not confirmed, the monitor does not move past it. + const paused = await securityViolation( + "TREASURY DESTINATION VIOLATION", + `treasury migration nonce ${nonce} in finalized Pendulum block ${block} pays ${amountValue.toBigInt()} ` + + `pallet units to ${recipient}, not EXPECTED_TREASURY_DESTINATION ${expectedTreasury}; the monitor ` + + "holds at that block until the vault is paused", + ); + if (!paused) throw new PauseHold(`holding at Pendulum block ${block} until the vault is paused`); + } if (recipient === ZERO_ADDRESS || recipient.toLowerCase() === config.vaultAddress.toLowerCase()) { // The vault deterministically rejects this tuple (ZeroAddress / // RecipientIsVault) and the attestors skip it, so it can never be @@ -459,7 +1163,8 @@ async function ingestPendulumEvents(api: ApiPromise, draft: RuntimeState, finali // non-zero forever and page liveness every grace period (and make // RB-7's "zero outstanding nonces" unsatisfiable). Record it once // and leave it out of reconciliation. - await alert( + await pageOnce( + `unreleasable:${nonce}`, "CRITICAL: unreleasable migration burned on Pendulum", `nonce ${nonce} in block ${block} burned ${amountValue.toBigInt()} to ${recipient}; the vault can never release it`, ); @@ -474,6 +1179,7 @@ async function ingestPendulumEvents(api: ApiPromise, draft: RuntimeState, finali }); } draft.lastPendulumBlock = block; + draft.lastPendulumHash = blockHash; // Checkpoint a long catch-up as it goes: the Pendulum-side cursor is // self-consistent on its own (the Base cursor only ever trails it), so a // transient RPC error hours into a replay must not discard the progress @@ -486,12 +1192,49 @@ async function ingestPendulumEvents(api: ApiPromise, draft: RuntimeState, finali } } +/** A vault total at `blockNumber`: zero before the vault existed. */ +async function vaultTotalAt(functionName: "totalReleased" | "totalSwept", blockNumber: bigint): Promise { + if (blockNumber < 0n) return 0n; + try { + return await publicClient.readContract({ address: config.vaultAddress, abi: vaultAbi, functionName, blockNumber }); + } catch (readError) { + if (!(await publicClient.getBytecode({ address: config.vaultAddress, blockNumber }))) return 0n; + throw readError; + } +} + +/** The vault's events in one block range, and `gap` when they do not account + * for the change of its running totals over the range: the Released amounts + * must add up to the growth of totalReleased, the RemainderSwept amounts to + * that of totalSwept. A backend that serves a truncated log result fails it. */ +async function readVaultRange(start: bigint, end: bigint) { + const [events, releasedBefore, sweptBefore, releasedAfter, sweptAfter] = await Promise.all([ + publicClient.getContractEvents({ address: config.vaultAddress, abi: vaultAbi, fromBlock: start, toBlock: end }), + vaultTotalAt("totalReleased", start - 1n), + vaultTotalAt("totalSwept", start - 1n), + vaultTotalAt("totalReleased", end), + vaultTotalAt("totalSwept", end), + ]); + const approvals = events.filter((event) => event.eventName === "Approved"); + const releases = events.filter((event) => event.eventName === "Released"); + const released = releases.reduce((sum, event) => sum + (event.args as { tokenAmount: bigint }).tokenAmount, 0n); + const swept = events + .filter((event) => event.eventName === "RemainderSwept") + .reduce((sum, event) => sum + (event.args as { amount: bigint }).amount, 0n); + const gap = released !== releasedAfter - releasedBefore || swept !== sweptAfter - sweptBefore + ? `in Base blocks ${start}-${end} the Released events add up to ${released} and the RemainderSwept events to ` + + `${swept}, but totalReleased grew by ${releasedAfter - releasedBefore} and totalSwept by ${sweptAfter - sweptBefore}` + : undefined; + return { approvals, releases, gap }; +} + async function reconcileBaseEvents( draft: RuntimeState, toBlock: bigint, conversionFactor: bigint, sourceFinalizedTsMs: number, -): Promise<{ violation?: string; awaitingSource?: string }> { + threshold: bigint, +): Promise<{ violation?: string; awaitingSource?: string; incomplete?: string }> { const releasedInScan = new Set(); let firstViolation: string | undefined; for (const [start, end] of blockRanges(draft.baseFromBlock, toBlock, config.baseMaxBlockRange)) { @@ -501,36 +1244,46 @@ async function reconcileBaseEvents( // would advance the cursor past events this monitor never reconciled. A // lagging node fails this probe loudly instead, and the cycle replays. await publicClient.getBlock({ blockNumber: end }); - const approvals = await publicClient.getContractEvents({ - address: config.vaultAddress, - abi: vaultAbi, - eventName: "Approved", - fromBlock: start, - toBlock: end, - }); - const releases = await publicClient.getContractEvents({ - address: config.vaultAddress, - abi: vaultAbi, - eventName: "Released", - fromBlock: start, - toBlock: end, - }); + // The probe and the log query can still reach different backends, so the + // logs must also account for the vault's totals; a range whose logs do + // not is read once more, then held for the next poll. + let range = await readVaultRange(start, end); + if (range.gap) range = await readVaultRange(start, end); + if (range.gap) { + const heldMs = logScanHeldSinceMs === undefined ? 0 : Date.now() - logScanHeldSinceMs; + logScanHeldSinceMs ??= Date.now(); + if (Date.now() - lastLogScanAlertMs > config.unmatchedEventGraceSeconds * 1000) { + lastLogScanAlertMs = Date.now(); + // Held past the grace, no later Base event has been matched against its burn for that long. + const critical = heldMs > config.unmatchedEventGraceSeconds * 1000; + void alert( + `${critical ? "CRITICAL: " : ""}BASE LOG SCAN INCOMPLETE`, + `${range.gap}; the Base RPC served logs that miss events, so the range is read again every poll ` + + `and the Base cursor stays at block ${start}` + + (critical ? ` (for ${Math.round(heldMs / 1000)}s): no later Base event is matched against its burn` : "") + + " — check the Base RPC", + ); + } + return { violation: firstViolation, incomplete: range.gap }; + } + logScanHeldSinceMs = undefined; + const { approvals, releases } = range; const observedAt = Date.now(); let awaitingSource: string | undefined; // Preflight the complete range before deleting released migrations. If a // source RPC is merely behind, the range must be replayed intact later. for (const event of approvals) { - const actual = event.args as { nonce: bigint; recipient: `0x${string}`; palletAmount: bigint }; + const actual = event.args as { nonce: bigint; recipient: `0x${string}`; palletAmount: bigint; attestor: string }; const expected = draft.migrations.get(actual.nonce); - const mismatch = approvalMismatch(expected, actual); + let mismatch = approvalMismatch(expected, actual); const key = `Approved:${actual.nonce}`; if (!mismatch) { draft.unmatchedBaseFirstSeenAt.delete(key); continue; } if (!expected) { - const disposition = await unmatchedDisposition( + const disposition = unmatchedDisposition( "Approved", key, actual.nonce, @@ -539,17 +1292,35 @@ async function reconcileBaseEvents( observedAt, sourceFinalizedTsMs, ); - if (disposition === "await") { - awaitingSource ??= `Approved nonce ${actual.nonce} at Base block ${event.blockNumber}`; - continue; - } - firstViolation ??= `Approved at Base block ${event.blockNumber}: ${mismatch} (` + - (disposition === "proven" - ? "source view has passed the event's timestamp — fabricated" - : "source view still behind after the full grace period") + ")"; + if (disposition !== "proven") awaitingSource ??= `Approved nonce ${actual.nonce} at Base block ${event.blockNumber}`; + if (disposition === "await") continue; + mismatch += disposition === "proven" + ? " (source view has passed the event's timestamp — fabricated)" + : " (source view still behind after the full grace period)"; + } + // Approvals that disagree with the burn pause the vault once they come + // from two attestors, or from one at a threshold of two or less, where + // one more approval releases. One attestor's approval alone pages. + const { nonce, recipient, palletAmount, attestor } = actual; + const seen = draft.mismatchedApprovals.some((approval) => + approval.nonce === nonce && approval.attestor.toLowerCase() === attestor.toLowerCase() && + !approvalMismatch(approval, actual) + ); + if (!seen) draft.mismatchedApprovals.push({ nonce, recipient, palletAmount, attestor }); + const attestors = disagreeingAttestors(expected, nonce, draft.mismatchedApprovals); + if (attestors.length >= (threshold > 2n ? 2 : 1)) { + firstViolation ??= `Approved at Base block ${event.blockNumber}: ${mismatch}; attestor` + + `${attestors.length > 1 ? "s" : ""} ${attestors.join(", ")} approved tuples for nonce ${nonce} that ` + + "disagree with the finalized Pendulum view"; continue; } - firstViolation ??= `Approved at Base block ${event.blockNumber}: ${mismatch}`; + void pageOnce( + `approval:${event.transactionHash}:${event.logIndex}`, + "CRITICAL: MISMATCHED APPROVAL", + `Approved at Base block ${event.blockNumber} by attestor ${attestor}: ${mismatch}; one attestor's approval ` + + "cannot release, so the vault is not paused — check that attestor's key and Pendulum node; a second " + + "attestor approving this nonce against its burn pauses the vault", + ); } for (const event of releases) { const actual = event.args as { @@ -566,7 +1337,7 @@ async function reconcileBaseEvents( continue; } if (!expected) { - const disposition = await unmatchedDisposition( + const disposition = unmatchedDisposition( "Released", key, actual.nonce, @@ -575,10 +1346,8 @@ async function reconcileBaseEvents( observedAt, sourceFinalizedTsMs, ); - if (disposition === "await") { - awaitingSource ??= `Released nonce ${actual.nonce} at Base block ${event.blockNumber}`; - continue; - } + if (disposition !== "proven") awaitingSource ??= `Released nonce ${actual.nonce} at Base block ${event.blockNumber}`; + if (disposition === "await") continue; firstViolation ??= `Released at Base block ${event.blockNumber}: ${mismatch} (` + (disposition === "proven" ? "source view has passed the event's timestamp — fabricated" @@ -587,7 +1356,10 @@ async function reconcileBaseEvents( } firstViolation ??= `Released at Base block ${event.blockNumber}: ${mismatch}`; } - if (awaitingSource && !firstViolation) return { awaitingSource }; + // A range holding an event the source may still reveal (awaiting it, or + // past the grace) is replayed later even after a violation: passing it + // would lose a legitimate release for good. + if (awaitingSource) return { violation: firstViolation, awaitingSource }; for (const event of releases) { const actual = event.args as { @@ -608,12 +1380,293 @@ async function reconcileBaseEvents( releasedInScan.add(actual.nonce); draft.migrations.delete(actual.nonce); draft.livenessAlertedAt.delete(actual.nonce); + draft.mismatchedApprovals = draft.mismatchedApprovals.filter((approval) => approval.nonce !== actual.nonce); } draft.baseFromBlock = end + 1n; } return { violation: firstViolation }; } +/** Page every vault admin event, and every operation the governance Timelock + * scheduled, up to `toBlock`, the poll's `safe`/`finalized` Base block. + * Before the governance handover the vault admin acts without a delay, so + * its events are the only trace of a guardian, attestor or cap change. The + * scan keeps its own cursor in `state`, so a restart resumes where it stopped. */ +async function watchBaseAdmin(toBlock: bigint): Promise { + const timelock = config.timelockAddress; + /** Scheduled calls by operation id and block: a cancelled operation can be + * scheduled again with the same id, and then pages again. */ + const operations = new Map(); + const pages: Promise[] = []; + const fromBlock = state.baseAdminFromBlock; + for (const [start, end] of blockRanges(fromBlock, toBlock, config.baseMaxBlockRange)) { + await publicClient.getBlock({ blockNumber: end }); // same lagging-node probe as the vault scan + const adminEvents = await publicClient.getContractEvents({ + address: config.vaultAddress, + abi: vaultAdminEventsAbi, + fromBlock: start, + toBlock: end, + }); + for (const event of adminEvents) { + const args = Object.entries(event.args).map(([name, value]) => `${name}=${value}`).join(", "); + pages.push(pageOnce( + `vault:${event.transactionHash}:${event.logIndex}`, + "VAULT ADMIN EVENT", + `${event.eventName}(${args}) at Base block ${event.blockNumber}, tx ${event.transactionHash}`, + )); + } + if (!timelock) continue; + const scheduled = await publicClient.getContractEvents({ + address: timelock, + abi: timelockAbi, + eventName: "CallScheduled", + fromBlock: start, + toBlock: end, + }); + for (const event of scheduled) { + const { id, index, target, value, data } = event.args as { + id: `0x${string}`; + index: bigint; + target: `0x${string}`; + value: bigint; + data: `0x${string}`; + }; + const key = `timelock:${id}:${event.blockNumber}`; + const operation = operations.get(key) ?? { id, calls: [] }; + operation.calls.push({ index, target, value, data }); + operations.set(key, operation); + } + } + if (timelock && operations.size > 0) { + const token = await publicClient.readContract({ + address: config.vaultAddress, + abi: vaultAbi, + functionName: "token", + blockNumber: toBlock, + }); + const labels = { + [config.vaultAddress.toLowerCase()]: "vault", + [token.toLowerCase()]: "token", + [timelock.toLowerCase()]: "timelock", + }; + for (const [key, { id, calls }] of operations) { + if (onceOnlyPages.has(key)) continue; + const readyAt = await publicClient.readContract({ + address: timelock, + abi: timelockAbi, + functionName: "getTimestamp", + args: [id], + blockNumber: toBlock, + }); + pages.push(pageOnce( + key, + "GOVERNANCE OPERATION SCHEDULED", + describeTimelockOperation(id, calls, readyAt, Date.now(), labels, KNOWN_FUNCTIONS), + )); + } + } + await Promise.all(pages); + if (toBlock >= fromBlock) state.baseAdminFromBlock = toBlock + 1n; +} + +type ApiAt = Awaited>; + +/** A democracy `Bounded`, as much of it as the watcher reads. */ +interface BoundedCall { + isInline: boolean; + asInline: Uint8Array; + isLookup: boolean; + asLookup: { hash_: { toHex(): string }; len: { toNumber(): number } }; + asLegacy: { hash_: { toHex(): string } }; + toString(): string; +} + +/** A governance item's call: decoded, `pending` while it cannot be read yet + * (re-checked every poll), or neither when it is unreadable for good. */ +interface CallRead { + node?: CallNode; + text: string; + pending?: boolean; +} + +/** + * Largest governance call the watcher fetches and decodes. Anyone can note a + * preimage and propose it, and the decoding cost grows with its size; 64 KiB + * decodes in about 0.15 s even when packed with the smallest calls. + * Legitimate calls stay far below it; a bigger one is never fetched and pages + * as unreadable. + */ +const MAX_GOVERNANCE_CALL_BYTES = 64 * 1024; + +function readCall(decode: () => unknown): CallRead { + try { + const node = toCallNode(decode() as Parameters[0]); + return { node, text: describeCall(node) }; + } catch (decodeError) { + return { text: `the call cannot be decoded: ${String(decodeError).slice(0, 300)}` }; + } +} + +const oversizedCall = (len: number): CallRead => ({ + text: `the call is ${len} bytes, over the ${MAX_GOVERNANCE_CALL_BYTES} bytes the monitor decodes`, +}); + +/** The length of a noted preimage, which a legacy call reference leaves out. + * One noted before the pallet's status migration stays in the deprecated + * `statusFor` until something touches it. */ +async function notedLength(apiAt: ApiAt, hash: { toHex(): string }): Promise { + const status = (await apiAt.query.preimage.requestStatusFor(hash)).toJSON() as { + unrequested?: { len: number }; + requested?: { maybeLen: number | null }; + } | null; + if (status) return status.unrequested?.len ?? status.requested?.maybeLen ?? undefined; + const old = (await apiAt.query.preimage.statusFor(hash)).toJSON() as { + unrequested?: { len: number }; + requested?: { len: number | null }; + } | null; + return old?.unrequested?.len ?? old?.requested?.len ?? undefined; +} + +/** Reads a proposal's call. Its preimage may be noted at any time before + * enactment, so until then the call is pending, not unreadable. */ +async function boundedCall(apiAt: ApiAt, bounded: BoundedCall): Promise { + if (bounded.isInline) return readCall(() => apiAt.registry.createType("Call", bounded.asInline)); + const hash = bounded.isLookup ? bounded.asLookup.hash_ : bounded.asLegacy.hash_; + const len = bounded.isLookup ? bounded.asLookup.len.toNumber() : await notedLength(apiAt, hash); + if (len !== undefined && len > MAX_GOVERNANCE_CALL_BYTES) return oversizedCall(len); + const preimage = len === undefined + ? undefined + : (await apiAt.query.preimage.preimageFor([hash, len])) as unknown as { isSome: boolean; unwrap(): Uint8Array }; + if (!preimage?.isSome) return { text: `preimage ${hash.toHex()} is not noted yet`, pending: true }; + return readCall(() => apiAt.registry.createType("Call", preimage.unwrap())); +} + +/** Reads a council or technical-committee motion's call, sized before it is fetched. */ +async function motionCall(apiAt: ApiAt, collective: "council" | "technicalCommittee", hash: { toHex(): string }): Promise { + const proposalOf = apiAt.query[collective].proposalOf; + const len = Number((await proposalOf.size(hash)).toString()); + if (len > MAX_GOVERNANCE_CALL_BYTES) return oversizedCall(len); + const call = (await proposalOf(hash)) as unknown as { isSome: boolean; unwrap(): unknown }; + if (!call.isSome) return { text: "the motion's call is not stored", pending: true }; + return readCall(() => call.unwrap()); +} + +/** + * Page every pending Pendulum governance item once: public proposals, the + * council's external proposal, ongoing referenda and open council and + * technical-committee motions, with the call decoded. Calls that mint, move + * or unlock balances or change the runtime or the migration pallet page as + * CRITICAL: minted PEN burned through the migration drains the fixed-supply + * vault, and only Pendulum governance can stop the call before enactment. + * So does a call the watcher cannot read; until it can, it is re-checked + * every poll. Alert-only; it runs after the poll's conservation checks, so it + * cannot stop or delay them, and decodes within fixed bounds, so its cost to + * the next poll is bounded. + */ +async function watchPendulumGovernance(api: ApiPromise): Promise { + const finalizedHash = await api.rpc.chain.getFinalizedHead(); + const finalizedBlock = (await api.rpc.chain.getHeader(finalizedHash)).number.toNumber(); + const apiAt = await api.at(finalizedHash); + const democracy = apiAt.query.democracy; + const launchPeriod = Number(apiAt.consts.democracy.launchPeriod.toString()); + const nextLaunch = (Math.floor(finalizedBlock / launchPeriod) + 1) * launchPeriod; + const items: Array<{ key: string; label: string; hint: string; read(): Promise }> = []; + const proposals = (await democracy.publicProps()) as unknown as Array< + [{ toNumber(): number }, BoundedCall, { toString(): string }] + >; + for (const [index, bounded, proposer] of proposals) { + items.push({ + key: `proposal:${index.toNumber()}`, + label: `public proposal #${index.toNumber()} from ${proposer.toString()}`, + hint: `the best-backed public proposal becomes a referendum at block ${nextLaunch}; ` + + "a hostile one needs a council or technical-committee cancel before then, or nay votes after", + read: () => boundedCall(apiAt, bounded), + }); + } + const [count, lowest] = (await Promise.all([democracy.referendumCount(), democracy.lowestUnbaked()])).map((value) => + Number(value.toString()) + ); + /** The newest ongoing referendum of each call. */ + const referendumOf = new Map(); + for (let index = lowest; index < count; index++) { + const info = (await democracy.referendumInfoOf(index)) as unknown as { + isSome: boolean; + unwrap(): { isOngoing: boolean; asOngoing: { proposal: BoundedCall; end: { toNumber(): number } } }; + }; + if (!info.isSome || !info.unwrap().isOngoing) continue; + const ongoing = info.unwrap().asOngoing; + referendumOf.set(ongoing.proposal.toString(), index); + items.push({ + key: `referendum:${index}`, + label: `referendum #${index}`, + hint: `voting ends at block ${ongoing.end.toNumber()}; a hostile one needs nay votes or an emergency cancel`, + read: () => boundedCall(apiAt, ongoing.proposal), + }); + } + const external = (await democracy.nextExternal()) as unknown as { isSome: boolean; unwrap(): [BoundedCall, unknown] }; + if (external.isSome) { + const [bounded] = external.unwrap(); + // Only its call identifies an external proposal. A tabled one becomes a + // referendum with the same call, so that referendum keys the page of the + // same call proposed again. + items.push({ + key: `external:${referendumOf.get(bounded.toString()) ?? "-"}:${bounded.toString()}`, + label: "the council's external proposal", + hint: `it can become a referendum at block ${nextLaunch}`, + read: () => boundedCall(apiAt, bounded), + }); + } else { + // Once it has left the queue, the same call proposed again pages again. + for (const key of onceOnlyPages.keys()) if (key.startsWith("external:")) onceOnlyPages.delete(key); + } + for (const collective of ["council", "technicalCommittee"] as const) { + const motions = (await apiAt.query[collective].proposals()) as unknown as Array<{ toHex(): string }>; + for (const hash of motions) { + // A closed motion can be proposed again with the same call and hash, + // but never with the same index, so the index keys the page. + const votes = (await apiAt.query[collective].voting(hash)).toJSON() as { index: number } | null; + items.push({ + key: `${collective}:${votes?.index}:${hash.toHex()}`, + label: `${collective === "council" ? "council" : "technical committee"} motion ${hash.toHex()}`, + hint: "it executes as soon as enough members vote aye; a hostile one needs nay votes", + read: () => motionCall(apiAt, collective, hash), + }); + } + } + // Pages go out together, so a burst of new items costs one webhook round trip. + const pages: Promise[] = []; + for (const item of items) { + if (onceOnlyPages.has(item.key)) continue; + let call: CallRead; + try { + call = await item.read(); + } catch (readError) { + // One item must never hide the items after it; a failed read is retried. + call = { text: `reading the call failed: ${String(readError).slice(0, 300)}`, pending: true }; + } + if (!call.node) { + // A call nobody can review is critical. One that cannot be read yet + // is re-checked every poll and pages again once it can be read. + pages.push(pageOnce( + call.pending ? `${item.key}:unread` : item.key, + "CRITICAL: Pendulum governance item cannot be read", + `${item.label}: ${call.text}; ` + + `${call.pending ? "the monitor pages its call once it can read it" : "review it manually"}; ${item.hint}`, + )); + continue; + } + const risky = riskyCalls(call.node); + pages.push(pageOnce( + item.key, + risky.length + ? "CRITICAL: Pendulum governance item mints, moves or unlocks balances, or changes the runtime" + : "PENDULUM GOVERNANCE ITEM", + `${item.label}: ${call.text}${risky.length ? ` (flagged: ${risky.join(", ")})` : ""}; ${item.hint}`, + )); + } + await Promise.all(pages); +} + function payloadHash(migration: MigrationRecord): `0x${string}` { return keccak256(encodeAbiParameters( [{ type: "uint64" }, { type: "address" }, { type: "uint256" }], @@ -654,43 +1707,57 @@ async function activeApprovalCounts(migrations: MigrationRecord[], blockNumber: return counts; } -async function check(api: ApiPromise): Promise { - const confirmedBase = await publicClient.getBlock({ blockTag: config.baseFinalityTag }); - if (confirmedBase.number === null) throw new Error(`${config.baseFinalityTag} Base block has no number`); - const baseBlock = confirmedBase.number; +interface SourceView { + finalizedBlock: number; + timestampMs: number; + totalMigrated: bigint; +} +/** Reads finalized Pendulum and ingests its new migrations into `draft`. + * Throws on any read, decode or continuity failure. */ +async function readSource(api: ApiPromise, draft: RuntimeState): Promise { // Read Pendulum after choosing the Base snapshot. TotalMigrated can only // grow, so this ordering cannot false-positive if a burn finalizes mid-poll. const finalizedHash = await api.rpc.chain.getFinalizedHead(); const finalizedHeader = await api.rpc.chain.getHeader(finalizedHash); - const finalizedBlock = finalizedHeader.number.toNumber(); - const apiAt = await api.at(finalizedHash); + const finalized = { number: finalizedHeader.number.toNumber(), hash: finalizedHash.toHex() }; + const { hashes, confirmed, crossChecked, anchored } = await sourceHashes(api, draft, finalized); + // The view: the newest block the checkers confirm, or the block ingested + // last while they confirm none after it. Every value read from Pendulum + // below is read there. + const viewBlock = Math.max(confirmed, draft.lastPendulumBlock); + const viewHash = hashes.get(viewBlock) ?? draft.lastPendulumHash; + if (!viewHash) throw new Error(`no verified hash for Pendulum block ${viewBlock}`); + const apiAt = await api.at(viewHash); const sourceFinalizedTsMs = Number((await apiAt.query.timestamp.now()).toString().replaceAll(",", "")); // A stalling source view is this daemon going blind: it cannot verify any // new Base event, and once the unmatched-event grace expires that forces a // cannot-verify pause. Page as soon as the view goes stale, so operators - // get the entire grace window to restore the node before that happens. + // get the entire grace window to restore the node, or the checker that + // holds the view back, before that happens. if ( Date.now() - sourceFinalizedTsMs > config.sourceStaleAlertMs && Date.now() - lastSourceStallAlertMs > config.unmatchedEventGraceSeconds * 1000 ) { lastSourceStallAlertMs = Date.now(); + // The checkers that confirm less than the source finalized hold the view back. + const holding = finalized.number <= viewBlock ? "" : [ + crossChecked !== undefined && crossChecked < finalized.number && "the second node (PENDULUM_WS_CROSSCHECK)", + anchored !== undefined && anchored < finalized.number && "the relay RPCs (POLKADOT_WS)", + ].filter(Boolean).join(" and "); await alert( "PENDULUM SOURCE VIEW STALLED", - `finalized head ${finalizedBlock} is ${Math.round((Date.now() - sourceFinalizedTsMs) / 1000)}s old; ` + - `the monitor cannot verify new Base events against a stalled source — restore the node before ` + - `the ${config.unmatchedEventGraceSeconds}s unmatched-event grace forces a pause`, + `verified Pendulum block ${viewBlock} is ${Math.round((Date.now() - sourceFinalizedTsMs) / 1000)}s old` + + (holding ? ` (the source finalized ${finalized.number}, not confirmed yet by ${holding})` : "") + + `; the monitor cannot verify new Base events against a stalled view — ` + + `${holding ? `check ${holding}` : "restore the node"} before the ${config.unmatchedEventGraceSeconds}s ` + + "unmatched-event grace forces a pause", ); } - const draft = cloneState(state); - await ingestPendulumEvents(api, draft, finalizedBlock); - const [totalMigratedValue, chainNextNonceValue, totalIssuanceValue] = await Promise.all([ - apiAt.query.tokenMigration.totalMigrated(), - apiAt.query.tokenMigration.nextNonce(), - apiAt.query.balances.totalIssuance(), - ]); - const totalMigrated = BigInt(totalMigratedValue.toString().replaceAll(",", "")); - const chainNextNonce = BigInt(chainNextNonceValue.toString().replaceAll(",", "")); + const [totals, totalIssuanceValue] = await Promise.all([migrationTotals(apiAt), apiAt.query.balances.totalIssuance()]); + const { nextNonce: chainNextNonce, totalMigrated } = totals; + if (crossChecked !== undefined && crossChecked >= viewBlock) await crossCheckTotals(viewBlock, viewHash, totals); + await ingestPendulumEvents(api, draft, confirmed, hashes, crossChecked !== undefined); if (chainNextNonce !== draft.nextExpectedNonce) { throw new Error(`finalized nextNonce ${chainNextNonce} != monitor event cursor ${draft.nextExpectedNonce}`); } @@ -718,96 +1785,225 @@ async function check(api: ApiPromise): Promise { `source and can be burned against the fixed-supply vault — investigate before it migrates (RB-3)`, ); } + return { finalizedBlock: viewBlock, timestampMs: sourceFinalizedTsMs, totalMigrated }; +} + +/** When the vault balance check last started, and whether a poll is running. */ +let lastBalanceCheckMs = 0; +let polling = false; +/** The vault's totals at the confirmed Base block `blockNumber`, after the + * vault balance check (M2b) on them, which pauses the vault on a deficit. */ +async function checkVaultBalance(blockNumber: bigint) { + lastBalanceCheckMs = Date.now(); const [totalReleased, totalSwept, conversionFactor, tokenAddress] = await Promise.all([ - publicClient.readContract({ address: config.vaultAddress, abi: vaultAbi, functionName: "totalReleased", blockNumber: baseBlock }), - publicClient.readContract({ address: config.vaultAddress, abi: vaultAbi, functionName: "totalSwept", blockNumber: baseBlock }), - publicClient.readContract({ address: config.vaultAddress, abi: vaultAbi, functionName: "conversionFactor", blockNumber: baseBlock }), - publicClient.readContract({ address: config.vaultAddress, abi: vaultAbi, functionName: "token", blockNumber: baseBlock }), + publicClient.readContract({ address: config.vaultAddress, abi: vaultAbi, functionName: "totalReleased", blockNumber }), + publicClient.readContract({ address: config.vaultAddress, abi: vaultAbi, functionName: "totalSwept", blockNumber }), + publicClient.readContract({ address: config.vaultAddress, abi: vaultAbi, functionName: "conversionFactor", blockNumber }), + publicClient.readContract({ address: config.vaultAddress, abi: vaultAbi, functionName: "token", blockNumber }), ]); - - const reconciliation = await reconcileBaseEvents(draft, baseBlock, conversionFactor, sourceFinalizedTsMs); - if (reconciliation.violation) { - if (await securityViolation("MIGRATION TUPLE VIOLATION", reconciliation.violation)) { - state = draft; - persistState(state); - } - return; - } - const [totalSupply, vaultBalance] = await Promise.all([ - publicClient.readContract({ address: tokenAddress, abi: erc20Abi, functionName: "totalSupply", blockNumber: baseBlock }), + publicClient.readContract({ address: tokenAddress, abi: erc20Abi, functionName: "totalSupply", blockNumber }), publicClient.readContract({ address: tokenAddress, abi: erc20Abi, functionName: "balanceOf", args: [config.vaultAddress], - blockNumber: baseBlock, + blockNumber, }), ]); if (vaultConservationDeficit(vaultBalance, totalReleased, totalSwept, totalSupply)) { - if (await securityViolation( + await securityViolation( "VAULT BALANCE DEFICIT", `balance ${vaultBalance} + released ${totalReleased} + swept ${totalSwept} < supply ${totalSupply}`, - )) { + ); + } + return { totalReleased, totalSwept, conversionFactor, vaultBalance }; +} + +/** Runs the vault balance check on its own while a poll is still busy a poll + * interval after its last one started (a slow Pendulum source, a long + * catch-up). main() calls this every poll interval, so two checks are about + * three poll intervals apart at most. */ +export async function watchVaultBalance(): Promise { + if (!polling || Date.now() - lastBalanceCheckMs < config.pollIntervalMs) return; + lastBalanceCheckMs = Date.now(); + try { + const confirmed = await publicClient.getBlock({ blockTag: config.baseFinalityTag }); + if (confirmed.number === null) throw new Error(`${config.baseFinalityTag} Base block has no number`); + await checkVaultBalance(confirmed.number); + } catch (balanceError) { + await alert("monitor check failed", `the vault balance check failed: ${balanceError}`); + } +} + +/** One pass of the checks; resolves to whether every one of them ran. */ +async function check(api: ApiPromise): Promise { + const confirmedBase = await publicClient.getBlock({ blockTag: config.baseFinalityTag }); + if (confirmedBase.number === null) throw new Error(`${config.baseFinalityTag} Base block has no number`); + const baseBlock = confirmedBase.number; + // A Base view that stops advancing hides every vault event after it while + // all checks keep passing at the old block; page it like a stalled source. + const baseAgeMs = Date.now() - Number(confirmedBase.timestamp) * 1000; + if ( + baseAgeMs > config.baseStaleAlertMs && + Date.now() - lastBaseStallAlertMs > config.unmatchedEventGraceSeconds * 1000 + ) { + lastBaseStallAlertMs = Date.now(); + await alert( + "BASE VIEW STALLED", + `${config.baseFinalityTag} block ${baseBlock} is ${Math.round(baseAgeMs / 1000)}s old; vault events after it ` + + `are not being checked — restore the Base node, or check whether Base's ${config.baseFinalityTag} head stalled`, + ); + } + await checkAutoPauseReadiness(); + + // The vault balance check (M2b) reads Base only, so it runs before anything + // that waits on Pendulum or scans Base logs: a failing source or log scan + // cannot skip it, and while the rest of a poll takes longer than a poll + // interval, `watchVaultBalance` runs it on its own. Its pause does not end + // the poll. + const { totalReleased, totalSwept, conversionFactor, vaultBalance } = await checkVaultBalance(baseBlock); + + // The admin scan only pages and keeps its own cursor, so nothing else waits + // for it, and its failure skips no other check. + let adminScanned = true; + try { + await watchBaseAdmin(baseBlock); + adminScanFailingSinceMs = undefined; + } catch (adminError) { + adminScanned = false; + const failingMs = adminScanFailingSinceMs === undefined ? 0 : Date.now() - adminScanFailingSinceMs; + adminScanFailingSinceMs ??= Date.now(); + if (Date.now() - lastAdminScanFailureAlertMs > config.unmatchedEventGraceSeconds * 1000) { + lastAdminScanFailureAlertMs = Date.now(); + // Meanwhile the veto window of any operation scheduled since runs unseen. + const critical = failingMs > config.unmatchedEventGraceSeconds * 1000; + await alert( + `${critical ? "CRITICAL: " : ""}BASE ADMIN SCAN FAILED`, + `${adminError}; vault admin events and the governance queue are not paged until the scan reads again` + + (critical ? ` (failing for ${Math.round(failingMs / 1000)}s)` : "") + + ", and it then resumes where it stopped; every other check still runs", + ); + } + } + + // A Pendulum failure must not switch off the Base-side checks. Base events + // are then reconciled against the last view read in full, as against a + // lagging source but with no fabrication proof: a mismatch on a known nonce + // is judged at once, an unknown nonce after the grace. The vault balance + // check needs no source; M2a and liveness wait for it. + let draft = cloneState(state); + let source: SourceView | undefined; + try { + source = await readSource(api, draft); + } catch (sourceError) { + draft = cloneState(state); // drops a half-ingested block; the 200-block checkpoints already moved `state` + const contradicted = sourceError instanceof SourceIntegrityError; + const consequence = `Base events are checked against the last Pendulum view read in full: one whose nonce ` + + `that view lacks pauses the vault ${config.unmatchedEventGraceSeconds}s after its first sighting, and M2a ` + + "and liveness are skipped"; + if (sourceError instanceof PauseHold) log(sourceError.message); + else if (contradicted && Date.now() - lastSourceIntegrityAlertMs > config.unmatchedEventGraceSeconds * 1000) { + lastSourceIntegrityAlertMs = Date.now(); + await alert( + "CRITICAL: PENDULUM SOURCE CONTRADICTED", + `${sourceError}; nothing from the source is ingested until it verifies again, and until then ` + + `${consequence} — stop trusting the source node and compare it with other nodes and the relay chain`, + ); + } else if (!contradicted && Date.now() - lastSourceFailureAlertMs > config.unmatchedEventGraceSeconds * 1000) { + lastSourceFailureAlertMs = Date.now(); + await alert("PENDULUM SOURCE UNREADABLE", `${sourceError}; until the source reads again, ${consequence}`); + } + } + + const threshold = await publicClient.readContract({ + address: config.vaultAddress, + abi: vaultAbi, + functionName: "threshold", + blockNumber: baseBlock, + }); + const reconciliation = await reconcileBaseEvents(draft, baseBlock, conversionFactor, source?.timestampMs ?? 0, threshold); + if (reconciliation.violation) { + if (await securityViolation("MIGRATION TUPLE VIOLATION", reconciliation.violation)) { state = draft; persistState(state); } - return; + return false; } - if (reconciliation.awaitingSource) { - state = draft; + if (!source) { + state = draft; // reconciled against the last view read in full persistState(state); - log( - `waiting up to ${config.unmatchedEventGraceSeconds}s for Pendulum RPC to catch up with ` + - reconciliation.awaitingSource, - ); - return; + return false; } - if (releasedExceedsMigrated(totalReleased, totalMigrated, conversionFactor)) { + const { finalizedBlock, totalMigrated } = source; + // M2a needs only the totals, so a held log range does not stop it. A Base + // event whose burn the source view does not show yet would make it fire + // falsely, so it waits for that. + if ( + !reconciliation.awaitingSource && + releasedExceedsMigrated(totalReleased, totalMigrated, conversionFactor, config.acknowledgedReleaseExcess) + ) { if (await securityViolation( "CONSERVATION VIOLATION", - `released ${totalReleased} > migrated ${totalMigrated * conversionFactor} (token units)`, + `released ${totalReleased} > migrated ${totalMigrated * conversionFactor} + acknowledged excess ` + + `${config.acknowledgedReleaseExcess} (token units)`, )) { state = draft; persistState(state); } - return; + return false; + } + if (reconciliation.awaitingSource || reconciliation.incomplete) { + state = draft; + persistState(state); + log( + reconciliation.incomplete + ? `holding the Base cursor at ${draft.baseFromBlock}: ${reconciliation.incomplete}` + : `waiting up to ${config.unmatchedEventGraceSeconds}s for Pendulum RPC to catch up with ` + + reconciliation.awaitingSource, + ); + return false; } const pending = [...draft.migrations.values()]; - const threshold = await publicClient.readContract({ - address: config.vaultAddress, - abi: vaultAbi, - functionName: "threshold", - blockNumber: baseBlock, - }); const approvals = await activeApprovalCounts(pending, baseBlock); const now = Date.now(); + // One page per kind and poll however long the backlog, with the detail + // per nonce in the log: a page per nonce floods the receiver and, awaited + // in turn, delays the next poll's checks. + const quorum: MigrationRecord[] = []; + const belowQuorum: MigrationRecord[] = []; for (let index = 0; index < pending.length; index++) { const migration = pending[index]; if (!isStale(migration.firstSeenMs, now, config.graceSeconds)) continue; const lastAlerted = draft.livenessAlertedAt.get(migration.nonce); if (lastAlerted !== undefined && now - lastAlerted < config.graceSeconds * 1000) continue; - draft.livenessAlertedAt.set(migration.nonce, now); - if (approvals[index] >= threshold) { - // Quorum is present but nothing released it: cap-deferred and - // starved of daily allowance, paused, or made releasable by a - // threshold cut without ever emitting ReleasePending (so the - // releaser cannot see it). Silent indefinitely before round 9. - await alert( - "LIVENESS: quorum reached but not released", - `nonce ${migration.nonce} has ${approvals[index]}/${threshold} active approvals and is still unreleased ` + - `after ${config.graceSeconds}s — check the daily allowance / pause state, or call release() directly ` + - `if it qualified through a threshold cut (RB-6)`, - ); - continue; - } - await alert( - "LIVENESS: migration below approval quorum", - `nonce ${migration.nonce} has ${approvals[index]}/${threshold} active approvals after ${config.graceSeconds}s`, - ); + log(`liveness: nonce ${migration.nonce} has ${approvals[index]}/${threshold} active approvals and is unreleased`); + (approvals[index] >= threshold ? quorum : belowQuorum).push(migration); } + const pageLiveness = async (migrations: MigrationRecord[], subject: string, detail: string) => { + if (!migrations.length) return; + const nonces = migrations.slice(0, 10).map((migration) => migration.nonce).join(", ") + + (migrations.length > 10 ? ` and ${migrations.length - 10} more` : ""); + if (await alert(subject, `${migrations.length} migration(s) (nonce ${nonces}) ${detail}`)) { + for (const migration of migrations) draft.livenessAlertedAt.set(migration.nonce, now); + } + }; + // Quorum is present but nothing released it: cap-deferred and starved of + // daily allowance, paused, or made releasable by a threshold cut without + // ever emitting ReleasePending (so the releaser cannot see it). + await pageLiveness( + quorum, + "LIVENESS: quorum reached but not released", + `have ${threshold} or more active approvals and are still unreleased after ${config.graceSeconds}s — check the ` + + `daily allowance / pause state, or call release() directly if one qualified through a threshold cut (RB-6)`, + ); + await pageLiveness( + belowQuorum, + "LIVENESS: migration below approval quorum", + `have fewer than ${threshold} active approvals after ${config.graceSeconds}s`, + ); state = draft; persistState(state); @@ -815,16 +2011,32 @@ async function check(api: ApiPromise): Promise { `ok: migrated=${totalMigrated} released=${totalReleased} swept=${totalSwept} ` + `pending=${state.migrations.size} vaultBalance=${vaultBalance} base=${baseBlock} pendulum=${finalizedBlock}`, ); + return adminScanned; } -async function main(): Promise { +/** Tells an external dead-man's switch (Healthchecks.io style) that a poll + * ran every check; it pages when these requests stop. Never fatal. */ +async function heartbeat(): Promise { + if (!config.heartbeatUrl) return; + try { + const response = await fetch(config.heartbeatUrl, { signal: AbortSignal.timeout(10_000) }); + if (!response.ok) console.error(`heartbeat answered ${response.status}`); + } catch (heartbeatError) { + console.error(`heartbeat failed: ${redact(`${heartbeatError}`)}`); + } +} + +/** Startup once the Pendulum API is connected: durable state, the multicall + * probe, the state identity check and the auto-pause readiness. `connect` + * replaces how the nodes of the integrity checks are reached (tests). */ +export async function start(api: ApiPromise, connect?: (url: string) => Promise): Promise { + if (connect) connectNode = connect; state = hydrateState(); const multicallCode = await publicClient.getBytecode({ address: MULTICALL3_ADDRESS }); if (!multicallCode || multicallCode === "0x") { multicallUnavailable = true; await alert("multicall unavailable, using bounded individual reads", `no contract code at ${MULTICALL3_ADDRESS}`); } - const api = await ApiPromise.create({ provider: new WsProvider(config.pendulumWs) }); const pendulumGenesisHash = api.genesisHash.toHex(); if (state.pendulumGenesisHash) { assertMonitorStateIdentity( @@ -838,21 +2050,61 @@ async function main(): Promise { } else { state.pendulumGenesisHash = pendulumGenesisHash; } + if (!config.guardianPrivateKey) { + await pageOnce( + "autopause-disabled", + "AUTO-PAUSE DISABLED", + "no GUARDIAN_PRIVATE_KEY configured: violations page, but nothing pauses the vault automatically", + ); + } + const readiness = await checkAutoPauseReadiness(); log( `monitor started; Pendulum after ${state.lastPendulumBlock}, Base from ${state.baseFromBlock}, ` + - `polling every ${config.pollIntervalMs}ms`, + `polling every ${config.pollIntervalMs}ms; governance queue ` + + (config.timelockAddress ? `watched at ${config.timelockAddress}` : "not watched (TIMELOCK_ADDRESS unset)") + + `; second Pendulum node ${config.pendulumWsCrossCheck ? "cross-checked" : "not set"}` + + `; auto-pause ${!config.guardianPrivateKey ? "disabled" : readiness.length ? "NOT READY" : "ready"}` + + `; acknowledged release excess ${config.acknowledgedReleaseExcess} token units`, ); - for (;;) { - try { - await check(api); - } catch (checkError) { - await alert("monitor check failed", `${checkError}`); +} + +/** One poll; a failure pages and the next poll starts afresh. */ +export async function poll(api: ApiPromise): Promise { + polling = true; + let complete = false; + try { + complete = await check(api); + } catch (checkError) { + await alert("monitor check failed", `${checkError}`); + } + try { + await watchPendulumGovernance(api); + } catch (governanceError) { + complete = false; + if (Date.now() - lastGovernanceWatchFailureMs > config.graceSeconds * 1000) { + lastGovernanceWatchFailureMs = Date.now(); + await alert("pendulum governance watch failed", `${governanceError}`); } + } + await retryUndeliveredPages(); + if (complete) await heartbeat(); + polling = false; +} + +async function main(): Promise { + const api = await ApiPromise.create({ provider: new WsProvider(config.pendulumWs) }); + await start(api); + setInterval(() => void watchVaultBalance(), config.pollIntervalMs); + for (;;) { + await poll(api); await new Promise((resolve) => setTimeout(resolve, config.pollIntervalMs)); } } -main().catch(async (error) => { - await alert("monitor startup failed", `${error}`); - process.exit(1); -}); +// main.test.ts sets this before importing the module, to drive single polls. +if (!(globalThis as { penMonitorUnderTest?: boolean }).penMonitorUnderTest) { + main().catch(async (error) => { + await alert("monitor startup failed", `${error}`); + process.exit(1); + }); +} diff --git a/monitor/src/state.test.ts b/monitor/src/state.test.ts index ba6f96c5b..017bb11fb 100644 --- a/monitor/src/state.test.ts +++ b/monitor/src/state.test.ts @@ -69,3 +69,37 @@ test("missing unmatched-event history is rejected rather than resetting its grac rmSync(directory, { recursive: true, force: true }); } }); + +test("optional fields are checked when present", () => { + const directory = mkdtempSync(join(tmpdir(), "pen-monitor-state-")); + const path = join(directory, "state.json"); + const approval = { + nonce: "2", + recipient: "0x3333333333333333333333333333333333333333", + palletAmount: "100", + attestor: "0x4444444444444444444444444444444444444444", + } as const; + try { + const complete = { + ...state, + baseAdminFromBlock: "150", + mismatchedApprovals: [approval], + lastPendulumHash: `0x${"55".repeat(32)}` as const, + relayAnchor: { number: 100, hash: `0x${"66".repeat(32)}` as const }, + }; + saveMonitorState(path, complete); + assert.deepEqual(loadMonitorState(path), complete); + for (const bad of [ + { baseAdminFromBlock: "-1" }, + { mismatchedApprovals: [{ ...approval, attestor: "0x44" }] }, + { lastPendulumHash: "0x55" }, + { relayAnchor: { number: "100", hash: `0x${"66".repeat(32)}` } }, + { relayAnchor: { number: 100, hash: "0x66" } }, + ]) { + writeFileSync(path, JSON.stringify({ ...state, ...bad })); + assert.throws(() => loadMonitorState(path), /invalid or legacy schema/); + } + } finally { + rmSync(directory, { recursive: true, force: true }); + } +}); diff --git a/monitor/src/state.ts b/monitor/src/state.ts index caf936a90..320ae7cff 100644 --- a/monitor/src/state.ts +++ b/monitor/src/state.ts @@ -28,11 +28,32 @@ export interface PersistedMonitorState { migrations: PersistedMigration[]; livenessAlertedAt: Array<[string, number]>; unmatchedBaseFirstSeenAt: Array<[string, number]>; - /** Pendulum totalIssuance + TotalMigrated when first observed (round 9); - * absent in state written before it existed and re-anchored on load. */ + /** Pendulum totalIssuance + TotalMigrated when first observed; absent in + * state written before it existed and re-anchored on load. */ issuanceAnchor?: string; + /** Next Base block of the admin scan; absent in older state, which never + * persisted a Base cursor past it, and then `baseFromBlock`. */ + baseAdminFromBlock?: string; + /** Approvals that disagreed with the finalized burn, until their nonce is + * released; absent in older state. */ + mismatchedApprovals?: PersistedApproval[]; + /** The hash of `lastPendulumBlock`; absent in older state. */ + lastPendulumHash?: `0x${string}`; + /** The newest relay-chain record of the Pendulum head the source was + * verified against; absent without the relay-chain anchor. */ + relayAnchor?: { number: number; hash: `0x${string}` }; } +export interface PersistedApproval { + nonce: string; + recipient: `0x${string}`; + palletAmount: string; + attestor: `0x${string}`; +} + +const isAddress = (value: unknown) => typeof value === "string" && /^0x[0-9a-fA-F]{40}$/.test(value); +const isHash = (value: unknown) => typeof value === "string" && /^0x[0-9a-fA-F]{64}$/.test(value); + function isMissing(error: unknown): boolean { return (error as NodeJS.ErrnoException)?.code === "ENOENT"; } @@ -67,6 +88,12 @@ export function loadMonitorState(path: string): PersistedMonitorState | undefine const badAlerts = !Array.isArray(state.livenessAlertedAt) || state.livenessAlertedAt.some( (entry) => !Array.isArray(entry) || entry.length !== 2 || !isUint(entry[0]) || !Number.isSafeInteger(entry[1]), ); + const badApprovals = state.mismatchedApprovals !== undefined && ( + !Array.isArray(state.mismatchedApprovals) || state.mismatchedApprovals.some((approval) => + !isUint(approval?.nonce) || !isAddress(approval?.recipient) || !isUint(approval?.palletAmount) || + !isAddress(approval?.attestor) + ) + ); const badUnmatchedEvents = !Array.isArray(state.unmatchedBaseFirstSeenAt) || state.unmatchedBaseFirstSeenAt.some( (entry) => !Array.isArray(entry) || @@ -86,8 +113,13 @@ export function loadMonitorState(path: string): PersistedMonitorState | undefine !isUint(state.nextExpectedNonce) || !isUint(state.baseFromBlock) || (state.issuanceAnchor !== undefined && !isUint(state.issuanceAnchor)) || + (state.baseAdminFromBlock !== undefined && !isUint(state.baseAdminFromBlock)) || + (state.lastPendulumHash !== undefined && !isHash(state.lastPendulumHash)) || + (state.relayAnchor !== undefined && + (!Number.isSafeInteger(state.relayAnchor?.number) || !isHash(state.relayAnchor?.hash))) || badMigration || badAlerts || + badApprovals || badUnmatchedEvents ) { throw new Error(`monitor state ${path} has an invalid or legacy schema; refusing to reset security history`);