From 16ff4cf2c24287cacb6ebb947502821f593f381b Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Tue, 27 Oct 2026 12:00:44 +0100 Subject: [PATCH 01/16] fix(attestor): Refill a nonce gap left by a dropped transaction An own approval stays at its nonce until it is mined. When the account's mined nonce stays below it and the node holds nothing in between (a mined approval whose block was reorged away, or a skipped foreign transaction that later left the pool), nothing will ever fill the empty nonce, and every later approval queues behind it until the process restarts. The follow-up now detects that case and moves the approval down to the empty nonce, or, when the migration no longer needs it, leaves the nonce to the next approval. The attempt left behind merely reverts if it is ever mined. --- attestor/README.md | 6 ++- attestor/src/approval.test.ts | 70 +++++++++++++++++++++++++++++++++++ attestor/src/approval.ts | 16 +++++++- 3 files changed, 90 insertions(+), 2 deletions(-) diff --git a/attestor/README.md b/attestor/README.md index e82df80f3..44890cc5d 100644 --- a/attestor/README.md +++ b/attestor/README.md @@ -129,7 +129,11 @@ WantedBy=multi-user.target counting `FEE_BUMP_AFTER_SECONDS` from its last send. A first attempt whose nonce is taken (mined already, or held by a transaction this process did not send) moves on to the next nonce at once, at most five - in a row, without spending a bump. A node refusing a send for its price, an + in a row, without spending a bump. An approval that cannot be mined because + the node holds nothing at a nonce below it (a transaction of ours dropped by + a reorg, or a skipped one that left the pool) is sent again at that first + empty nonce, or the next approval is, so nothing waits on a nonce no + transaction will fill. A node refusing a send for its price, an underpriced replacement or a full pool is retried at the next bump instead of being fatal. Transactions in flight are kept in memory only: after a restart, an approval whose transaction is still pending is sent again at the diff --git a/attestor/src/approval.test.ts b/attestor/src/approval.test.ts index bc137daee..e3ec0f0ae 100644 --- a/attestor/src/approval.test.ts +++ b/attestor/src/approval.test.ts @@ -1119,6 +1119,76 @@ test("the new transaction after another took the nonce starts from the current e assert.ok(chain.approved.has(payloadHash(EVENT))); }); +test("an own approval reorged out of Base leaves no empty nonce the next approval waits behind", async () => { + const chain = new FakeVault(); + const { attestor, alerts, logs } = attestorOn(chain); + assert.equal(await approve(attestor, EVENT), true); + // The block holding it is reorged away, and the transaction is not re-injected. + chain.minedNonce = 5; + chain.approved.delete(payloadHash(EVENT)); + chain.receipts.clear(); + assert.equal(await approve(attestor, OTHER), true); + assert.deepEqual( + chain.sent.map((tx) => [approveArgs(tx)[0], tx.nonce]), + [ + [EVENT.nonce, 5], + [OTHER.nonce, 6], + [OTHER.nonce, 5], + ], + ); + assert.ok(chain.approved.has(payloadHash(OTHER))); + assert.equal(chain.receipts.get(chain.sent[1].hash)?.status, "reverted", "the attempt left behind merely reverts"); + assert.ok(logs.some((line) => line.startsWith("nonce 5 below 6 holds no transaction; sending the approval again there: nonce=8 "))); + assert.equal(alerts.length, 0); +}); + +test("a nonce skipped for a transaction this process did not send is filled once that transaction leaves the pool", async () => { + const chain = new FakeVault(); + chain.pool.set(5, { hash: `0x${"ee".repeat(32)}`, nonce: 5, value: 0n, maxFeePerGas: 50_000_000n, maxPriorityFeePerGas: 2_000_000n }); + chain.pendingNonce = 5; // the backend answering the count has not seen it + chain.onSend = () => { + if (chain.sent.length === 2) { + // Evicted before it was mined. + chain.pool.delete(5); + chain.pendingNonce = undefined; + } + }; + const { attestor } = attestorOn(chain); + assert.equal(await approve(attestor, EVENT), true); + assert.deepEqual( + chain.sent.map((tx) => tx.nonce), + [5, 6, 5], + ); + assert.ok(chain.receipts.has(chain.sent[2].hash)); + assert.ok(chain.approved.has(payloadHash(EVENT))); +}); + +test("an approval no longer needed behind an empty nonce is not sent again, and the next approval fills that nonce", async () => { + const chain = new FakeVault(); + chain.pool.set(5, { hash: `0x${"ee".repeat(32)}`, nonce: 5, value: 0n, maxFeePerGas: 50_000_000n, maxPriorityFeePerGas: 2_000_000n }); + chain.pendingNonce = 5; + chain.onSend = () => { + if (chain.sent.length === 2) { + chain.pool.delete(5); + chain.pendingNonce = undefined; + chain.consumed.add(EVENT.nonce); // released by the others meanwhile + } + }; + const { attestor } = attestorOn(chain); + assert.equal(await approve(attestor, EVENT), true); + assert.equal(attestor.missedApprovals, 1); + assert.equal(await approve(attestor, OTHER), true); + assert.deepEqual( + chain.sent.map((tx) => [approveArgs(tx)[0], tx.nonce]), + [ + [EVENT.nonce, 5], + [EVENT.nonce, 6], + [OTHER.nonce, 5], + ], + ); + assert.ok(chain.approved.has(payloadHash(OTHER))); +}); + test("unreleasable events are not tracked for durability; every other event is", async () => { const chain = new FakeVault(); const { attestor } = attestorOn(chain); diff --git a/attestor/src/approval.ts b/attestor/src/approval.ts index 8cef92230..035929f6b 100644 --- a/attestor/src/approval.ts +++ b/attestor/src/approval.ts @@ -197,7 +197,9 @@ export interface Attestor { /** The lowest nonce no transaction of this process has been accepted at (or * sent at without an answer). The node's pending count can trail it (a * load-balanced endpoint, a pool that dropped the transaction), and a nonce - * one payload was sent at is never signed for another. */ + * one payload was sent at is never signed for another, unless an empty + * nonce below kept that attempt from ever being mined and the payload was + * sent again there (see `sendUntilMined`). */ nextNonce: number; } @@ -402,6 +404,18 @@ async function sendUntilMined( Object.assign(submission, { nonce: undefined, fees: undefined, hashes: [], bumps: 0, paged: false }); continue; } + if (nonce !== undefined && minedNonce !== undefined && minedNonce < nonce && (await attestor.chain.accountNonce("pending")) <= minedNonce) { + // The node holds nothing below our nonce, so ours can never be mined: + // a transaction of ours there was dropped (its block reorged away), or + // one this process skipped left the pool. Fill that nonce instead of + // waiting on it, for this approval or else the next one; the attempt + // left behind merely reverts should it ever be mined. + attestor.nextNonce = minedNonce; + if (!needed) return undefined; + attestor.log(`nonce ${minedNonce} below ${nonce} holds no transaction; sending the approval again there: ${label}`); + Object.assign(submission, { nonce: undefined, fees: undefined, hashes: [], bumps: 0, paged: false }); + continue; + } const last = submission.fees as Fees; const fees = submission.paged || submission.bumps >= policy.max From 787bba34468d04a51d32a48e730e962d17d5799b Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Tue, 27 Oct 2026 12:02:09 +0100 Subject: [PATCH 02/16] fix(attestor): Require both signer fields in a checkpoint A checkpoint records the attestor address and its vault attestor generation together. One without either still loads as written before the fields existed and is bound to the current key on startup, but one with only one of them was edited, and binding it would skip the rewind the identity check asks for. Refuse such a checkpoint with a message pointing at the runbook. --- attestor/README.md | 5 +++-- attestor/src/state.test.ts | 24 ++++++++++++++++++++++++ attestor/src/state.ts | 15 ++++++++++++++- 3 files changed, 41 insertions(+), 3 deletions(-) diff --git a/attestor/README.md b/attestor/README.md index 44890cc5d..c14c10c01 100644 --- a/attestor/README.md +++ b/attestor/README.md @@ -150,8 +150,9 @@ WantedBy=multi-user.target with no checkpoint and `START_BLOCK` before the oldest unreleased migration; a re-added key lowers `lastProcessedBlock` to such a block and sets `attestorGeneration` to the current value, keeping every other field. - Checkpoints written before these fields existed are bound to the first key - that starts on them. + Checkpoints written before these fields existed (neither field present) are + bound to the first key that starts on them; a checkpoint with only one of + the two was edited and is refused. - The daemon verifies at startup that its address is in the vault's attestor set, that `BASE_RPC_URL` serves `BASE_CHAIN_ID` (`eth_chainId`, which the checkpoint's Base chain must match too) and that `PENDULUM_WS` serves diff --git a/attestor/src/state.test.ts b/attestor/src/state.test.ts index 6bd0b438d..85c774d0f 100644 --- a/attestor/src/state.test.ts +++ b/attestor/src/state.test.ts @@ -84,6 +84,30 @@ test("a checkpoint from before the signer was recorded still loads", () => { } }); +test("a checkpoint with only one of the signer fields is refused, not taken for one from before they existed", () => { + const directory = mkdtempSync(join(tmpdir(), "pen-attestor-state-")); + const path = join(directory, "checkpoint.json"); + try { + const { attestorAddress, attestorGeneration, ...legacy } = checkpoint; + writeFileSync(path, JSON.stringify({ ...legacy, attestorAddress })); + assert.throws( + () => assertCheckpointIdentity(loadCheckpoint(path)!, { ...checkpoint, attestorGeneration: 2 }), + /has attestorAddress but no attestorGeneration; .*\(runbook RB-6\)/, + ); + writeFileSync(path, JSON.stringify({ ...legacy, attestorGeneration })); + assert.throws( + () => + assertCheckpointIdentity(loadCheckpoint(path)!, { + ...checkpoint, + attestorAddress: "0x5555555555555555555555555555555555555555", + }), + /has attestorGeneration but no attestorAddress/, + ); + } finally { + rmSync(directory, { recursive: true, force: true }); + } +}); + test("the Base endpoint must serve the configured chain and the Pendulum node the pinned genesis", () => { assert.doesNotThrow(() => assertBaseChain(8453, 8453)); assert.throws(() => assertBaseChain(84532, 8453), /BASE_RPC_URL serves chain 84532, but BASE_CHAIN_ID is 8453/); diff --git a/attestor/src/state.ts b/attestor/src/state.ts index dfa3e1a0a..47d02a176 100644 --- a/attestor/src/state.ts +++ b/attestor/src/state.ts @@ -16,7 +16,8 @@ export interface Checkpoint { pendulumGenesisHash: `0x${string}`; /** The signer, and its vault attestor generation, whose approvals this * progress stands for. Optional only so that checkpoints written before - * they were recorded still load; the daemon binds those on startup. */ + * they were recorded still load (neither field); the daemon binds those on + * startup. */ attestorAddress?: `0x${string}`; attestorGeneration?: number; lastProcessedBlock: number; @@ -136,6 +137,18 @@ export function assertCheckpointIdentity( // Progress made by another key, or by this key before it was removed and // re-added, says nothing about the approvals that count now: resuming from // it would silently skip every unreleased migration before it. + // Both fields are always written together, so a checkpoint with only one of + // them was edited and is not one from before they existed. + if ((checkpoint.attestorAddress === undefined) !== (checkpoint.attestorGeneration === undefined)) { + const [present, missing] = + checkpoint.attestorAddress === undefined + ? ["attestorGeneration", "attestorAddress"] + : ["attestorAddress", "attestorGeneration"]; + throw new Error( + `checkpoint has ${present} but no ${missing}; the two are always written together, so the file was edited: ` + + "restore the removed field and follow the error that gives (runbook RB-6), never remove either one", + ); + } if ( checkpoint.attestorAddress !== undefined && checkpoint.attestorAddress.toLowerCase() !== expected.attestorAddress.toLowerCase() From eb2e06f716c34541eeebab4e1f6e679bf8d00d72 Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Tue, 27 Oct 2026 12:03:48 +0100 Subject: [PATCH 03/16] fix(attestor): Repeat the page for an own approval that stays stuck The "own approval not mined" page went out once per transaction. While an attestor waits on such a transaction it processes nothing else, so no other alert follows, and a page whose delivery failed left it silent. Page again 10 minutes after the first page, then at doubling intervals up to every 4 hours, for as long as the transaction stays unmined. Repeat pages say "still not mined (page N)". --- attestor/README.md | 10 ++++++---- attestor/src/approval.test.ts | 29 +++++++++++++++++++++++++++++ attestor/src/approval.ts | 34 ++++++++++++++++++++++------------ 3 files changed, 57 insertions(+), 16 deletions(-) diff --git a/attestor/README.md b/attestor/README.md index c14c10c01..0b994e4fe 100644 --- a/attestor/README.md +++ b/attestor/README.md @@ -117,10 +117,12 @@ WantedBy=multi-user.target approval is recorded, a transaction still pending is no longer needed, but it is followed and bumped the same way until its nonce is used: every later approval waits behind that nonce, and mined it merely reverts. When no bump - is left, or the cap stops one, it pages `own approval not mined` once (with - the nonce, the account's next mined nonce, the fees and whether the migration - is already resolved) and keeps waiting, since later approvals would queue - behind that nonce anyway. Whenever it stops bumping, it re-sends the last + is left, or the cap stops one, it pages `own approval not mined` (with the + nonce, the account's next mined nonce, the fees and whether the migration is + already resolved) and keeps waiting, since later approvals would queue + behind that nonce anyway. While it stays stuck, the page repeats 10, 20 and + 40 minutes later and so on, at most every 4 hours, marked `still not mined + (page N)`. Whenever it stops bumping, it re-sends the last attempt unchanged, so a node that dropped the transaction while still counting its nonce gets it back instead of leaving a gap. An approval that an RPC error interrupted keeps its transaction, also one whose send got no diff --git a/attestor/src/approval.test.ts b/attestor/src/approval.test.ts index e3ec0f0ae..b06a57b75 100644 --- a/attestor/src/approval.test.ts +++ b/attestor/src/approval.test.ts @@ -663,6 +663,35 @@ test("after the last bump it pages once and keeps waiting until the approval is assert.equal(attestor.missedApprovals, 0); }); +test("an approval that stays stuck after its page pages again, at growing intervals", async () => { + const chain = new FakeVault(); + chain.baseFee = 100_000_000n; + const pages: { at: number; detail: string }[] = []; + const { attestor } = attestorOn( + chain, + { + alert: async (subject, detail) => { + assert.equal(subject, "own approval not mined"); + pages.push({ at: attestor.now(), detail: `${detail}` }); + }, + }, + (elapsed) => { + // Fees fall again two hours in. + if (elapsed >= 120 * 60_000) chain.baseFee = 5_000_000n; + chain.mine(); + }, + ); + assert.equal(await approve(attestor, EVENT), true); + // First paged 4 min after the send, then 10, 20 and 40 min apart. + assert.deepEqual( + pages.slice(1).map((page, i) => (page.at - pages[i].at) / 60_000), + [10, 20, 40], + ); + assert.match(pages[0].detail, /^3 fee bump\(s\) used up: transaction nonce 5, /); + assert.match(pages[1].detail, /^still not mined \(page 2\), 3 fee bump\(s\) used up: transaction nonce 5, /); + assert.ok(chain.approved.has(payloadHash(EVENT))); +}); + test("the fee cap stops a bump and pages once; no attempt offers more than the cap", async () => { const chain = new FakeVault(); chain.baseFee = 100_000_000n; diff --git a/attestor/src/approval.ts b/attestor/src/approval.ts index 035929f6b..0d335fbb2 100644 --- a/attestor/src/approval.ts +++ b/attestor/src/approval.ts @@ -42,6 +42,12 @@ export const MIN_REPLACEMENT_PERCENT = 10; /** How often a sent approval's receipt is polled: about one Base block. */ const RECEIPT_POLL_MS = 2000; +/** An own approval still stuck after its page pages again this much later, + * then at twice the previous interval up to `REPAGE_MAX_MS`: a single page + * whose delivery failed must not leave a stuck attestor silent. */ +const REPAGE_FIRST_MS = 10 * 60_000; +const REPAGE_MAX_MS = 4 * 3600_000; + /** How many nonces in a row a first attempt tries at once when each is taken: * enough for a backend whose pending count trails by a few blocks, and a * bound for one that refuses every nonce. */ @@ -82,8 +88,9 @@ export interface Submission { * resumes the bump interval from there instead of starting it anew. */ sentAtMs?: number; bumps: number; - /** The page that it is not being mined went out. */ - paged: boolean; + /** How often the page that it is not being mined went out, and when last. */ + pages: number; + pagedAtMs?: number; } /** A block processed against the latest Base state whose releasable events are @@ -369,9 +376,9 @@ async function resendLatest(attestor: Attestor, submission: Submission, nonce: n * An attempt still not mined `feeBump.afterMs` after it was sent, also across * a retry after an error, is re-sent: the identical call at the same nonce with * both fees raised (`bumpedFees`), at most `feeBump.max` times. When no bump is - * left or the cap stops one, it pages once - * and keeps waiting, re-sending the latest attempt unchanged: later approvals - * would queue behind this nonce anyway. An approval no longer needed (released, + * left or the cap stops one, it pages (again at growing intervals while it + * stays stuck) and keeps waiting, re-sending the latest attempt unchanged: + * later approvals would queue behind this nonce anyway. An approval no longer needed (released, * or ours recorded) is followed the same way while its nonce is pending, as * every later approval waits behind that nonce; mined, it merely reverts. * Returns undefined when the approval is no longer needed and no attempt holds @@ -401,7 +408,7 @@ async function sendUntilMined( if (late) return late; if (!needed) return undefined; attestor.log(`nonce ${nonce} was used by another transaction; sending the approval again: ${label}`); - Object.assign(submission, { nonce: undefined, fees: undefined, hashes: [], bumps: 0, paged: false }); + Object.assign(submission, { nonce: undefined, fees: undefined, hashes: [], bumps: 0, pages: 0 }); continue; } if (nonce !== undefined && minedNonce !== undefined && minedNonce < nonce && (await attestor.chain.accountNonce("pending")) <= minedNonce) { @@ -413,22 +420,25 @@ async function sendUntilMined( attestor.nextNonce = minedNonce; if (!needed) return undefined; attestor.log(`nonce ${minedNonce} below ${nonce} holds no transaction; sending the approval again there: ${label}`); - Object.assign(submission, { nonce: undefined, fees: undefined, hashes: [], bumps: 0, paged: false }); + Object.assign(submission, { nonce: undefined, fees: undefined, hashes: [], bumps: 0, pages: 0 }); continue; } const last = submission.fees as Fees; const fees = - submission.paged || submission.bumps >= policy.max + submission.pages > 0 || submission.bumps >= policy.max ? undefined : bumpedFees(last, await attestor.chain.fees(), policy); if (fees === undefined) { - if (!submission.paged) { - submission.paged = true; + const repageMs = Math.min(REPAGE_MAX_MS, REPAGE_FIRST_MS * 2 ** (submission.pages - 1)); + if (submission.pages === 0 || attestor.now() - (submission.pagedAtMs as number) >= repageMs) { + submission.pages += 1; + submission.pagedAtMs = attestor.now(); const reason = submission.bumps >= policy.max ? `${submission.bumps} fee bump(s) used up` : "the fee cap stops the next bump"; await attestor.alert( "own approval not mined", - `${reason}: transaction nonce ${nonce ?? "not accepted by the node"}, account's next mined nonce ${minedNonce ?? "unknown"}, ` + + `${submission.pages > 1 ? `still not mined (page ${submission.pages}), ` : ""}` + + `${reason}: transaction nonce ${nonce ?? "not accepted by the node"}, account's next mined nonce ${minedNonce ?? "unknown"}, ` + `maxFeePerGas ${formatGwei(last.maxFeePerGas)} gwei (cap ${formatGwei(policy.maxFeePerGas)} gwei)` + `${needed ? "" : ", migration already resolved"}; ${label}`, "critical", @@ -494,7 +504,7 @@ export async function approve(attestor: Attestor, event: MigrationEvent): Promis functionName: "approve", args: [event.nonce, event.recipient, event.palletAmount], }); - submission = { event, data, gas, hashes: [], bumps: 0, paged: false }; + submission = { event, data, gas, hashes: [], bumps: 0, pages: 0 }; attestor.submissions.set(key, submission); } const receipt = await sendUntilMined(attestor, event, submission, label); From f318f3e55ab190aa3df33cd3ba3ba6d6bad78f7d Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Tue, 27 Oct 2026 12:05:13 +0100 Subject: [PATCH 04/16] fix(attestor,releaser): Retry a failed alert delivery A page whose webhook delivery failed (relay down, rate limited, timed out) was logged and dropped. Try a failed delivery again after 2 s and after 10 s more. A 4xx answer other than 408 and 429 is not retried: the relay refused the request itself, which repeating cannot change. --- attestor/README.md | 2 +- attestor/src/alert.test.ts | 32 +++++++++++++++++++++++--- attestor/src/alert.ts | 46 ++++++++++++++++++++++++++------------ releaser/README.md | 2 +- releaser/src/alert.test.ts | 32 +++++++++++++++++++++++--- releaser/src/alert.ts | 46 ++++++++++++++++++++++++++------------ 6 files changed, 124 insertions(+), 36 deletions(-) diff --git a/attestor/README.md b/attestor/README.md index 0b994e4fe..9a4168eec 100644 --- a/attestor/README.md +++ b/attestor/README.md @@ -42,7 +42,7 @@ point. | `CHECKPOINT_FILE` | Path persisting the last processed block (default `./checkpoint.json`) | | `START_BLOCK` | First Pendulum block to scan on the very first run | | `MIN_GAS_BALANCE_WEI` | Low-gas alert threshold (default 0.01 ETH) | -| `ALERT_WEBHOOK_URL` | Optional webhook receiving JSON alerts in Slack incoming-webhook form: `text` holds `[CRITICAL]` or `[WARNING]`, the attestor and the subject, then the detail; `severity`, `subject` and `detail` are also sent as fields. A non-2xx answer, a timeout (10 s) or a connection error is logged as a failed delivery | +| `ALERT_WEBHOOK_URL` | Optional webhook receiving JSON alerts in Slack incoming-webhook form: `text` holds `[CRITICAL]` or `[WARNING]`, the attestor and the subject, then the detail; `severity`, `subject` and `detail` are also sent as fields. A non-2xx answer, a timeout (10 s) or a connection error is logged as a failed delivery and tried again after 2 s and after 10 s more; a 4xx other than 408 and 429 is not retried, as repeating the request cannot change it | | `HEARTBEAT_URL` | Optional http(s) URL requested (GET) after a finalized head processed without error, at most once a minute; point it at a dead-man's switch that pages when the requests stop. Startup refuses any other value. A failed request is logged (error name and code only, never the URL), never fatal | | `BASE_CHAIN_ID` | Default 8453 (Base mainnet) | | `BASE_FINALITY_TAG` | Base confirmation boundary the checkpoint waits for: `safe` (default) or `finalized` | diff --git a/attestor/src/alert.test.ts b/attestor/src/alert.test.ts index 6da992645..8df13970f 100644 --- a/attestor/src/alert.test.ts +++ b/attestor/src/alert.test.ts @@ -42,24 +42,50 @@ test("a webhook error status, a timeout or a dead relay is a failed delivery, an for (const status of [500, 429, 404]) { const { url, server } = await webhook(status); try { - assert.equal(await postAlert(url, { subject: "s" }), false, `HTTP ${status}`); + assert.equal(await postAlert(url, { subject: "s" }, 10_000, []), false, `HTTP ${status}`); } finally { server.close(); } } const hanging = await webhook(null); try { - assert.equal(await postAlert(hanging.url, { subject: "s" }, 200), false, "timeout"); + assert.equal(await postAlert(hanging.url, { subject: "s" }, 200, []), false, "timeout"); } finally { hanging.server.closeAllConnections(); hanging.server.close(); } const closed = await webhook(200); closed.server.close(); - assert.equal(await postAlert(closed.url, { subject: "s" }), false, "connection refused"); + assert.equal(await postAlert(closed.url, { subject: "s" }, 10_000, []), false, "connection refused"); assert.equal(logged.mock.callCount(), 5); }); +test("a failed delivery is tried again after each wait, a request the relay refused is not", async (t) => { + t.mock.method(console, "error", () => {}); + let answers: number[] = []; + let posts = 0; + const server = createServer((request, response) => { + request.resume(); + request.on("end", () => response.writeHead(answers[posts++]).end()); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const url = `http://127.0.0.1:${(server.address() as AddressInfo).port}`; + try { + for (const [statuses, delivered] of [ + [[503, 429, 200], true], + [[500, 502, 504], false], + [[404], false], + ] as const) { + answers = [...statuses]; + posts = 0; + assert.equal(await postAlert(url, { subject: "s" }, 10_000, [10, 10]), delivered, `${statuses}`); + assert.equal(posts, statuses.length, `${statuses}`); + } + } finally { + server.close(); + } +}); + test("a heartbeat requests its URL, and a failing endpoint is logged without the URL, never thrown", async (t) => { const logged = t.mock.method(console, "error", () => {}); const seen: string[] = []; diff --git a/attestor/src/alert.ts b/attestor/src/alert.ts index c5af62001..376769f49 100644 --- a/attestor/src/alert.ts +++ b/attestor/src/alert.ts @@ -66,24 +66,42 @@ export function throttledHeartbeat( }; } +/** A failed alert delivery is tried again after each of these waits: a relay + * that is briefly down or rate-limiting must not swallow a page. */ +const ALERT_RETRY_DELAYS_MS = [2_000, 10_000]; + /** * POST one alert to the webhook and report whether it was delivered. `fetch` * resolves on any HTTP answer, so a non-2xx status is checked explicitly: a * relay answering 4xx/5xx, a timeout and a connection failure are all logged - * and count as not delivered. + * and count as not delivered. Each is tried again after the next of + * `retryDelaysMs`, except a 4xx other than 408 and 429: the relay refused the + * request itself (a wrong URL, a rejected payload), which repeating it cannot + * change. */ -export async function postAlert(url: string, body: unknown, timeoutMs = 10_000): Promise { - try { - const response = await fetch(url, { - method: "POST", - headers: { "content-type": "application/json" }, - body: JSON.stringify(body), - signal: AbortSignal.timeout(timeoutMs), - }); - if (response.ok) return true; - console.error(`alert webhook failed: HTTP ${response.status}`); - } catch (webhookError) { - console.error("alert webhook failed", webhookError); +export async function postAlert( + url: string, + body: unknown, + timeoutMs = 10_000, + retryDelaysMs = ALERT_RETRY_DELAYS_MS, +): Promise { + for (let attempt = 0; ; attempt++) { + let status: number | undefined; + try { + const response = await fetch(url, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(body), + signal: AbortSignal.timeout(timeoutMs), + }); + if (response.ok) return true; + status = response.status; + console.error(`alert webhook failed: HTTP ${response.status}`); + } catch (webhookError) { + console.error("alert webhook failed", webhookError); + } + const refused = status !== undefined && status < 500 && status !== 408 && status !== 429; + if (refused || attempt === retryDelaysMs.length) return false; + await new Promise((resolve) => setTimeout(resolve, retryDelaysMs[attempt])); } - return false; } diff --git a/releaser/README.md b/releaser/README.md index 0ead43908..c0c79dd5a 100644 --- a/releaser/README.md +++ b/releaser/README.md @@ -37,7 +37,7 @@ consumed nonce and drops it. | `POLL_INTERVAL_MS` | Default 60s | | `MAX_BLOCK_RANGE` | Max blocks per `eth_getLogs` (default 10,000) | | `MIN_GAS_BALANCE_WEI` | Low-gas alert threshold | -| `ALERT_WEBHOOK_URL` | Optional JSON alert webhook in Slack incoming-webhook form: `text` holds `[CRITICAL]` or `[WARNING]`, the releaser and the subject, then the detail; `severity`, `subject` and `detail` are also sent as fields. A non-2xx answer, a timeout (10 s) or a connection error is logged as a failed delivery | +| `ALERT_WEBHOOK_URL` | Optional JSON alert webhook in Slack incoming-webhook form: `text` holds `[CRITICAL]` or `[WARNING]`, the releaser and the subject, then the detail; `severity`, `subject` and `detail` are also sent as fields. A non-2xx answer, a timeout (10 s) or a connection error is logged as a failed delivery and tried again after 2 s and after 10 s more; a 4xx other than 408 and 429 is not retried, as repeating the request cannot change it | | `HEARTBEAT_URL` | Optional http(s) URL requested (GET) after a cycle that completed without error, at most once a minute; point it at a dead-man's switch that pages when the requests stop. Startup refuses any other value. A failed request is logged (error name and code only, never the URL), never fatal | | `BASE_CHAIN_ID` | Default 8453; startup refuses a `BASE_RPC_URL` that serves another chain (`eth_chainId`) | | `BASE_FINALITY_TAG` | `safe` (default) or `finalized`; the scan cursor stays inside this boundary and a pending entry is dropped only once its nonce is consumed there | diff --git a/releaser/src/alert.test.ts b/releaser/src/alert.test.ts index 9a65363d2..bd4480cfa 100644 --- a/releaser/src/alert.test.ts +++ b/releaser/src/alert.test.ts @@ -42,24 +42,50 @@ test("a webhook error status, a timeout or a dead relay is a failed delivery, an for (const status of [500, 429, 404]) { const { url, server } = await webhook(status); try { - assert.equal(await postAlert(url, { subject: "s" }), false, `HTTP ${status}`); + assert.equal(await postAlert(url, { subject: "s" }, 10_000, []), false, `HTTP ${status}`); } finally { server.close(); } } const hanging = await webhook(null); try { - assert.equal(await postAlert(hanging.url, { subject: "s" }, 200), false, "timeout"); + assert.equal(await postAlert(hanging.url, { subject: "s" }, 200, []), false, "timeout"); } finally { hanging.server.closeAllConnections(); hanging.server.close(); } const closed = await webhook(200); closed.server.close(); - assert.equal(await postAlert(closed.url, { subject: "s" }), false, "connection refused"); + assert.equal(await postAlert(closed.url, { subject: "s" }, 10_000, []), false, "connection refused"); assert.equal(logged.mock.callCount(), 5); }); +test("a failed delivery is tried again after each wait, a request the relay refused is not", async (t) => { + t.mock.method(console, "error", () => {}); + let answers: number[] = []; + let posts = 0; + const server = createServer((request, response) => { + request.resume(); + request.on("end", () => response.writeHead(answers[posts++]).end()); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const url = `http://127.0.0.1:${(server.address() as AddressInfo).port}`; + try { + for (const [statuses, delivered] of [ + [[503, 429, 200], true], + [[500, 502, 504], false], + [[404], false], + ] as const) { + answers = [...statuses]; + posts = 0; + assert.equal(await postAlert(url, { subject: "s" }, 10_000, [10, 10]), delivered, `${statuses}`); + assert.equal(posts, statuses.length, `${statuses}`); + } + } finally { + server.close(); + } +}); + test("a heartbeat requests its URL, and a failing endpoint is logged without the URL, never thrown", async (t) => { const logged = t.mock.method(console, "error", () => {}); const seen: string[] = []; diff --git a/releaser/src/alert.ts b/releaser/src/alert.ts index c5af62001..376769f49 100644 --- a/releaser/src/alert.ts +++ b/releaser/src/alert.ts @@ -66,24 +66,42 @@ export function throttledHeartbeat( }; } +/** A failed alert delivery is tried again after each of these waits: a relay + * that is briefly down or rate-limiting must not swallow a page. */ +const ALERT_RETRY_DELAYS_MS = [2_000, 10_000]; + /** * POST one alert to the webhook and report whether it was delivered. `fetch` * resolves on any HTTP answer, so a non-2xx status is checked explicitly: a * relay answering 4xx/5xx, a timeout and a connection failure are all logged - * and count as not delivered. + * and count as not delivered. Each is tried again after the next of + * `retryDelaysMs`, except a 4xx other than 408 and 429: the relay refused the + * request itself (a wrong URL, a rejected payload), which repeating it cannot + * change. */ -export async function postAlert(url: string, body: unknown, timeoutMs = 10_000): Promise { - try { - const response = await fetch(url, { - method: "POST", - headers: { "content-type": "application/json" }, - body: JSON.stringify(body), - signal: AbortSignal.timeout(timeoutMs), - }); - if (response.ok) return true; - console.error(`alert webhook failed: HTTP ${response.status}`); - } catch (webhookError) { - console.error("alert webhook failed", webhookError); +export async function postAlert( + url: string, + body: unknown, + timeoutMs = 10_000, + retryDelaysMs = ALERT_RETRY_DELAYS_MS, +): Promise { + for (let attempt = 0; ; attempt++) { + let status: number | undefined; + try { + const response = await fetch(url, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(body), + signal: AbortSignal.timeout(timeoutMs), + }); + if (response.ok) return true; + status = response.status; + console.error(`alert webhook failed: HTTP ${response.status}`); + } catch (webhookError) { + console.error("alert webhook failed", webhookError); + } + const refused = status !== undefined && status < 500 && status !== 408 && status !== 429; + if (refused || attempt === retryDelaysMs.length) return false; + await new Promise((resolve) => setTimeout(resolve, retryDelaysMs[attempt])); } - return false; } From e8b4f7106606c747ffb5d669e895539e1b136448 Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Tue, 27 Oct 2026 12:06:24 +0100 Subject: [PATCH 05/16] fix(attestor): Refuse a non-Pendulum genesis on Base mainnet PENDULUM_GENESIS_HASH lets rehearsals run against a local test chain. Nothing tied the override to a test Base chain, so an environment carried over from a rehearsal could pin a test chain's genesis while pointing at the mainnet vault. Refuse any genesis other than Pendulum's when BASE_CHAIN_ID is 8453. --- attestor/README.md | 2 +- attestor/src/config.test.ts | 15 +++++++++++++-- attestor/src/config.ts | 11 +++++++++-- 3 files changed, 23 insertions(+), 5 deletions(-) diff --git a/attestor/README.md b/attestor/README.md index 9a4168eec..939df3ec7 100644 --- a/attestor/README.md +++ b/attestor/README.md @@ -35,7 +35,7 @@ point. | Variable | Meaning | |---|---| | `PENDULUM_WS` | WebSocket of your own Pendulum node, e.g. `ws://127.0.0.1:9944` | -| `PENDULUM_GENESIS_HASH` | Genesis hash that node must serve (default Pendulum's, `0x5d3c2986…462c86`); set it only for a local test chain | +| `PENDULUM_GENESIS_HASH` | Genesis hash that node must serve (default Pendulum's, `0x5d3c2986…462c86`); set it only for a local test chain. With `BASE_CHAIN_ID` 8453 (Base mainnet) any other value is refused at startup | | `BASE_RPC_URL` | Base JSON-RPC endpoint | | `VAULT_ADDRESS` | MigrationVault address on Base | | `ATTESTOR_PRIVATE_KEY` | This attestor's signing key (0x-prefixed) | diff --git a/attestor/src/config.test.ts b/attestor/src/config.test.ts index f1947499f..d1c6f0e21 100644 --- a/attestor/src/config.test.ts +++ b/attestor/src/config.test.ts @@ -36,14 +36,25 @@ test("the expected Pendulum genesis defaults to Pendulum's and must be a block h "0x5d3c298622d5634ed019bf61ea4b71655030015bde9beb0d6a24743714462c86", ); const local = `0x${"ab".repeat(32)}`; - assert.equal(loadConfig({ PENDULUM_GENESIS_HASH: local }, "config.pendulumGenesisHash").stdout, local); + assert.equal(loadConfig({ BASE_CHAIN_ID: "84532", PENDULUM_GENESIS_HASH: local }, "config.pendulumGenesisHash").stdout, local); for (const value of ["0x1234", "pendulum", `${"ab".repeat(32)}`]) { - const { status, stderr } = loadConfig({ PENDULUM_GENESIS_HASH: value }); + const { status, stderr } = loadConfig({ BASE_CHAIN_ID: "84532", PENDULUM_GENESIS_HASH: value }); assert.notEqual(status, 0, value); assert.match(stderr, /invalid PENDULUM_GENESIS_HASH/, value); } }); +test("on Base mainnet the expected genesis can only be Pendulum's", () => { + const local = `0x${"ab".repeat(32)}`; + for (const chain of [{}, { BASE_CHAIN_ID: "8453" }] as Record[]) { + const { status, stderr } = loadConfig({ ...chain, PENDULUM_GENESIS_HASH: local }); + assert.notEqual(status, 0); + assert.match(stderr, /PENDULUM_GENESIS_HASH 0xabab.* is not Pendulum's, which BASE_CHAIN_ID 8453 \(Base mainnet\) requires/); + } + const pendulum = "0x5D3C298622D5634ED019BF61EA4B71655030015BDE9BEB0D6A24743714462C86"; + assert.equal(loadConfig({ PENDULUM_GENESIS_HASH: pendulum }, "config.pendulumGenesisHash").stdout, pendulum); +}); + test("HEARTBEAT_URL must be an http(s) URL, and a bad one is refused without echoing it", () => { assert.equal(loadConfig({ HEARTBEAT_URL: "https://hc-ping.com/secret-token" }, "config.heartbeatUrl").stdout, "https://hc-ping.com/secret-token"); const { status, stderr } = loadConfig({ HEARTBEAT_URL: "hc-ping.com/secret-token" }); diff --git a/attestor/src/config.ts b/attestor/src/config.ts index 820d9ec7d..1b0cb4fe2 100644 --- a/attestor/src/config.ts +++ b/attestor/src/config.ts @@ -38,12 +38,19 @@ function finalityTag(): "safe" | "finalized" { const baseFinalityTag = finalityTag(); +const baseChainId = envNumber("BASE_CHAIN_ID", 8453); + /** Pendulum's genesis block hash. */ const PENDULUM_GENESIS_HASH = "0x5d3c298622d5634ed019bf61ea4b71655030015bde9beb0d6a24743714462c86"; function genesisHash(): `0x${string}` { const value = process.env.PENDULUM_GENESIS_HASH || PENDULUM_GENESIS_HASH; if (!/^0x[0-9a-fA-F]{64}$/.test(value)) throw new Error(`invalid PENDULUM_GENESIS_HASH ${value}`); + // Another genesis is for rehearsals against a test chain; the mainnet vault + // only ever pays for burns on Pendulum itself. + if (baseChainId === 8453 && value.toLowerCase() !== PENDULUM_GENESIS_HASH) { + throw new Error(`PENDULUM_GENESIS_HASH ${value} is not Pendulum's, which BASE_CHAIN_ID 8453 (Base mainnet) requires`); + } return value as `0x${string}`; } @@ -54,7 +61,7 @@ export const config = { * daemon outage longer than the pruning horizon wedges the catch-up. */ pendulumWs: required("PENDULUM_WS"), /** The genesis that node must serve; startup refuses any other chain. A - * local test chain sets its own. */ + * local test chain sets its own, never on Base mainnet. */ pendulumGenesisHash: genesisHash(), /** Base JSON-RPC endpoint. */ baseRpcUrl: required("BASE_RPC_URL"), @@ -77,7 +84,7 @@ export const config = { * requests stop. */ heartbeatUrl: parseHeartbeatUrl(process.env.HEARTBEAT_URL), /** Base chain id: 8453 mainnet. */ - baseChainId: envNumber("BASE_CHAIN_ID", 8453), + baseChainId, /** Confirmation boundary required before the checkpoint may pass a block. */ baseFinalityTag, /** How long a block's approvals may stay outside the finality boundary From cde5bdbe1f5a470a04d62848ae7131ae9e1f4859 Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Tue, 27 Oct 2026 12:08:03 +0100 Subject: [PATCH 06/16] fix(attestor): Stop when the signer's attestor standing changes while running Membership and attestor generation were read once, at startup. A key removed and re-added while the daemon ran kept making progress although its earlier approvals no longer counted, and only the next restart refused the checkpoint. Read both again on every finalized head (two calls) and exit when either differs from startup; the next start then asks for the checkpoint to be rewound. The startup check uses the same reads. --- attestor/README.md | 6 +++++- attestor/src/approval.test.ts | 35 ++++++++++++++++++++++++++++++++++- attestor/src/approval.ts | 34 ++++++++++++++++++++++++++++++++++ attestor/src/main.ts | 30 +++++++++++++----------------- 4 files changed, 86 insertions(+), 19 deletions(-) diff --git a/attestor/README.md b/attestor/README.md index 939df3ec7..6c83500ea 100644 --- a/attestor/README.md +++ b/attestor/README.md @@ -159,7 +159,11 @@ WantedBy=multi-user.target set, that `BASE_RPC_URL` serves `BASE_CHAIN_ID` (`eth_chainId`, which the checkpoint's Base chain must match too) and that `PENDULUM_WS` serves `PENDULUM_GENESIS_HASH`, and refuses to run otherwise. These checks run on - every start, the first one without a checkpoint included. + every start, the first one without a checkpoint included. While running, it + re-reads its attestor membership and generation on every finalized head and + exits with `fatal error, exiting` when either changed (a key removed, or + removed and re-added): the next start then refuses the checkpoint until it + is rewound as described above. - After a Pendulum **runtime upgrade**, verify event decoding against the new metadata on a staging node before letting the fleet advance past the upgrade block. diff --git a/attestor/src/approval.test.ts b/attestor/src/approval.test.ts index b06a57b75..bdb5cbdf8 100644 --- a/attestor/src/approval.test.ts +++ b/attestor/src/approval.test.ts @@ -28,6 +28,7 @@ import { privateKeyToAccount } from "viem/accounts"; import { advanceCheckpoint, approve, + assertSignerUnchanged, bumpedFees, cappedFees, GAS_LIMIT_MULTIPLIER, @@ -39,7 +40,7 @@ import { type UnconfirmedBlock, type VaultChain, } from "./approval.js"; -import type { MigrationEvent } from "./checks.js"; +import { isTransientRpcError, type MigrationEvent } from "./checks.js"; import { vaultAbi } from "./vaultAbi.js"; const VAULT = "0x1111111111111111111111111111111111111111"; @@ -177,6 +178,14 @@ class FakeVault implements VaultChain { async boundaryBlock() { return this.boundary; } + member = true; + generation = 1; + async isAttestor() { + return this.member; + } + async attestorGeneration() { + return this.generation; + } /** Mine every pooled transaction that is next in line and pays the base fee. * An approval records ours unless the migration is already resolved. */ mine() { @@ -296,6 +305,14 @@ test("the viem chain reads the boundary at the finality tag and everything else const [simulated] = pick("simulateContract"); assert.deepEqual([simulated.account, simulated.functionName, simulated.args], [account, "approve", [EVENT.nonce, EVENT.recipient, EVENT.palletAmount]]); assert.deepEqual(pick("getTransactionCount"), [{ address: account.address, blockTag: "pending" }]); + answers.readContract = (args) => (args.functionName === "attestorGeneration" ? 2n : true); + assert.equal(await chain.isAttestor(), true); + assert.equal(await chain.attestorGeneration(), 2); + const [, , member, generation] = pick("readContract"); + assert.deepEqual( + [member.functionName, member.args, generation.functionName, generation.args, generation.address], + ["isAttestor", [account.address], "attestorGeneration", [account.address], VAULT], + ); answers.getBlock = () => ({ number: null }); await assert.rejects(chain.boundaryBlock(), /finalized Base block has no number/); @@ -1218,6 +1235,22 @@ test("an approval no longer needed behind an empty nonce is not sent again, and assert.ok(chain.approved.has(payloadHash(OTHER))); }); +test("a signer removed, or removed and re-added, while running stops the daemon instead of going on", async () => { + const chain = new FakeVault(); + const { attestor } = attestorOn(chain); + await assertSignerUnchanged(attestor, 1); + // Removed and re-added between two heads: its generation-1 approvals no longer count. + chain.generation = 2; + await assert.rejects(assertSignerUnchanged(attestor, 1), (error: Error) => { + assert.match(error.message, /went from attestor generation 1 to 2 while running, .*\(runbook RB-6\)/); + assert.equal(isTransientRpcError(error), false, "fatal, not retried on the next head"); + return true; + }); + chain.generation = 1; + chain.member = false; + await assert.rejects(assertSignerUnchanged(attestor, 1), /is no longer an attestor of 0x1111/); +}); + test("unreleasable events are not tracked for durability; every other event is", async () => { const chain = new FakeVault(); const { attestor } = attestorOn(chain); diff --git a/attestor/src/approval.ts b/attestor/src/approval.ts index 0d335fbb2..b8e12d2db 100644 --- a/attestor/src/approval.ts +++ b/attestor/src/approval.ts @@ -123,6 +123,10 @@ export interface VaultChain { receipt(hash: Hash): Promise; /** The newest block inside the Base finality boundary. */ boundaryBlock(): Promise; + /** Whether the signer is in the vault's attestor set, and its attestor + * generation there (bumped by every `addAttestor`). */ + isAttestor(): Promise; + attestorGeneration(): Promise; } /** `VaultChain` on a viem client, reading and sending as `account`. */ @@ -177,6 +181,17 @@ export function viemVaultChain( if (confirmed.number === null) throw new Error(`${finalityTag} Base block has no number`); return confirmed.number; }, + isAttestor: () => + client.readContract({ address: vaultAddress, abi: vaultAbi, functionName: "isAttestor", args: [account.address] }), + attestorGeneration: async () => + Number( + await client.readContract({ + address: vaultAddress, + abi: vaultAbi, + functionName: "attestorGeneration", + args: [account.address], + }), + ), }; } @@ -560,6 +575,25 @@ export async function approve(attestor: Attestor, event: MigrationEvent): Promis return true; } +/** Refuse to go on once the signer's standing in the vault changed since + * startup, when it was at `generation`: its key removed, or removed and + * re-added, while the daemon ran. Its approvals from before no longer count, + * so progress made under them has to be rewound first, which the checkpoint + * check at the next start asks for. */ +export async function assertSignerUnchanged(attestor: Attestor, generation: number): Promise { + const signer = attestor.account.address; + if (!(await attestor.chain.isAttestor())) { + throw new Error(`${signer} is no longer an attestor of ${attestor.vaultAddress}; stopping`); + } + const current = await attestor.chain.attestorGeneration(); + if (current !== generation) { + throw new Error( + `${signer} went from attestor generation ${generation} to ${current} while running, so its earlier approvals ` + + "no longer count; stopping: rewind the checkpoint as the next start asks (runbook RB-6)", + ); + } +} + /** Approve every event of one finalized Pendulum block and queue the block for * the durability check. */ export async function processBlock( diff --git a/attestor/src/main.ts b/attestor/src/main.ts index 9baf6d1b5..9a0318d6d 100644 --- a/attestor/src/main.ts +++ b/attestor/src/main.ts @@ -25,7 +25,14 @@ import { ApiPromise, WsProvider } from "@polkadot/api"; import { createPublicClient, defineChain, http } from "viem"; import { privateKeyToAccount } from "viem/accounts"; import { alertText, postAlert, throttledHeartbeat, type Severity } from "./alert.js"; -import { advanceCheckpoint, processBlock, viemVaultChain, type Attestor, type UnconfirmedBlock } from "./approval.js"; +import { + advanceCheckpoint, + assertSignerUnchanged, + processBlock, + viemVaultChain, + type Attestor, + type UnconfirmedBlock, +} from "./approval.js"; import { decodeMigrationEvent, isTransientRpcError, type MigrationEvent } from "./checks.js"; import { config } from "./config.js"; import { @@ -36,7 +43,6 @@ import { saveCheckpoint, type Checkpoint, } from "./state.js"; -import { vaultAbi } from "./vaultAbi.js"; /** Repeated transient failures (a sustained RPC outage) page at most this * often; every occurrence is still logged. */ @@ -121,23 +127,10 @@ async function checkGasBalance(): Promise { async function main(): Promise { const baseChainId = await publicClient.getChainId(); assertBaseChain(baseChainId, config.baseChainId); - const isAttestor = await publicClient.readContract({ - address: config.vaultAddress, - abi: vaultAbi, - functionName: "isAttestor", - args: [account.address], - }); - if (!isAttestor) { + if (!(await attestor.chain.isAttestor())) { throw new Error(`${account.address} is not an attestor of ${config.vaultAddress}`); } - const attestorGeneration = Number( - await publicClient.readContract({ - address: config.vaultAddress, - abi: vaultAbi, - functionName: "attestorGeneration", - args: [account.address], - }), - ); + const attestorGeneration = await attestor.chain.attestorGeneration(); await checkGasBalance(); const api = await ApiPromise.create({ provider: new WsProvider(config.pendulumWs) }); @@ -180,6 +173,9 @@ async function main(): Promise { const finalized = head.number.toNumber(); // Serialize: a slow Base transaction must not let block processing overlap. processing = processing.then(async () => { + // Two reads per head: a key removed or re-added while running must + // not go on making progress that its new generation does not back. + await assertSignerUnchanged(attestor, attestorGeneration); for (let block = processedThrough + 1; block <= finalized; block++) { await processBlock(attestor, block, await migrationEventsInBlock(api, block), unconfirmed); processedThrough = block; From cfea5ae7be11b9cde1da8b01d93cb40c9983feee Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Tue, 27 Oct 2026 12:11:22 +0100 Subject: [PATCH 07/16] fix(releaser): Shrink log ranges an endpoint refuses as too large The ReleasePending scan split its blocks into ranges of MAX_BLOCK_RANGE (default 10,000) and failed the whole cycle when the endpoint refused that size. Base Sepolia's public endpoint caps eth_getLogs at 200 blocks, so every cycle failed and paged until the setting was lowered by hand. A range the endpoint refuses for its size is now read again at half the size, down to a single block. After ten full-size reads in a row succeed the size doubles again, never above MAX_BLOCK_RANGE, which stays the maximum. Other failures propagate as before. MAX_BLOCK_RANGE must now be a whole number of at least 1, and each cycle's log line shows the current size. --- releaser/README.md | 2 +- releaser/src/checks.test.ts | 102 ++++++++++++++++++++++++++++++++---- releaser/src/checks.ts | 66 ++++++++++++++++++----- releaser/src/config.test.ts | 10 ++++ releaser/src/config.ts | 11 ++-- releaser/src/main.ts | 15 ++++-- 6 files changed, 175 insertions(+), 31 deletions(-) diff --git a/releaser/README.md b/releaser/README.md index c0c79dd5a..37ee447a8 100644 --- a/releaser/README.md +++ b/releaser/README.md @@ -35,7 +35,7 @@ consumed nonce and drops it. | `START_BLOCK` | Vault deployment block — where log scanning begins on a first run | | `STATE_FILE` | Scan checkpoint + pending set (default `./releaser-state.json`) | | `POLL_INTERVAL_MS` | Default 60s | -| `MAX_BLOCK_RANGE` | Max blocks per `eth_getLogs` (default 10,000) | +| `MAX_BLOCK_RANGE` | Max blocks per `eth_getLogs` (default 10,000; a whole number of at least 1). A range the endpoint refuses as too large is read again at half the size, down to one block, and the size doubles again after 10 full-size reads in a row, never above this value; the current size is in each cycle's `ok:` log line. Set it to the endpoint's cap to avoid the refused reads (Base Sepolia's public endpoint allows 200) | | `MIN_GAS_BALANCE_WEI` | Low-gas alert threshold | | `ALERT_WEBHOOK_URL` | Optional JSON alert webhook in Slack incoming-webhook form: `text` holds `[CRITICAL]` or `[WARNING]`, the releaser and the subject, then the detail; `severity`, `subject` and `detail` are also sent as fields. A non-2xx answer, a timeout (10 s) or a connection error is logged as a failed delivery and tried again after 2 s and after 10 s more; a 4xx other than 408 and 429 is not retried, as repeating the request cannot change it | | `HEARTBEAT_URL` | Optional http(s) URL requested (GET) after a cycle that completed without error, at most once a minute; point it at a dead-man's switch that pages when the requests stop. Startup refuses any other value. A failed request is logged (error name and code only, never the URL), never fatal | diff --git a/releaser/src/checks.test.ts b/releaser/src/checks.test.ts index 89b065df6..a9a69276e 100644 --- a/releaser/src/checks.test.ts +++ b/releaser/src/checks.test.ts @@ -1,7 +1,16 @@ import { strict as assert } from "node:assert"; import { test } from "node:test"; -import { ContractFunctionRevertedError, encodeErrorResult } from "viem"; -import { blockRanges, chunks, classifyReleaseFailure, contractErrorName, toPalletAmount } from "./checks.js"; +import { + ContractFunctionRevertedError, + encodeErrorResult, + HttpRequestError, + InvalidInputRpcError, + InvalidParamsRpcError, + LimitExceededRpcError, + RpcRequestError, + TimeoutError, +} from "viem"; +import { chunks, classifyReleaseFailure, contractErrorName, isRangeTooLarge, readInRanges, toPalletAmount } from "./checks.js"; import { vaultAbi } from "./vaultAbi.js"; test("a cap-deferred release retries quietly — the case this service exists for", () => { @@ -46,16 +55,89 @@ test("a non-exact conversion throws rather than truncating into a wrong payload assert.throws(() => toPalletAmount(10n, 0n), /invalid conversionFactor/); }); -test("block ranges are chunked for RPC log limits, inclusive and gapless", () => { - assert.deepEqual(blockRanges(1n, 10n, 4n), [ - [1n, 4n], - [5n, 8n], - [9n, 10n], +/** An endpoint's refusal of a log range, as viem raises a JSON-RPC error. */ +const rpcError = (code: number, message: string) => new RpcRequestError({ body: {}, error: { code, message }, url: "http://rpc" }); + +test("an eth_getLogs refusal for the size of its range is recognised, other failures are not", () => { + // Base's public endpoints answer it with HTTP 400. + const base = { code: -32602, message: "block range greater than 200 max" }; + assert.equal(isRangeTooLarge(new HttpRequestError({ url: "http://rpc", status: 400, details: JSON.stringify(base) })), true); + assert.equal(isRangeTooLarge(new InvalidParamsRpcError(rpcError(base.code, base.message))), true); + for (const message of [ + "exceed maximum block range: 50000", + "query exceeds max block range 100000", + "eth_getLogs is limited to a 10,000 range", + "query returned more than 10000 results", + "Log response size exceeded.", + ]) { + assert.equal(isRangeTooLarge(new InvalidInputRpcError(rpcError(-32000, message))), true, message); + } + assert.equal(isRangeTooLarge(new HttpRequestError({ url: "http://rpc", status: 429, details: "Too Many Requests" })), false); + assert.equal(isRangeTooLarge(new LimitExceededRpcError(rpcError(-32005, "request limit reached"))), false); + assert.equal(isRangeTooLarge(new TimeoutError({ body: {}, url: "http://rpc" })), false); +}); + +test("log ranges are read in order and gapless, shrinking while the endpoint refuses their size and growing back to the maximum", async () => { + let cap = 200n; + const reads: [bigint, bigint][] = []; + const read = async (start: bigint, end: bigint) => { + if (end - start + 1n > cap) throw new InvalidParamsRpcError(rpcError(-32602, `block range greater than ${cap} max`)); + reads.push([start, end]); + }; + const span = { size: 10_000n, max: 10_000n, streak: 0 }; + await readInRanges(1n, 500n, span, read); + // 10,000 halved until accepted. + assert.deepEqual(reads, [ + [1n, 156n], + [157n, 312n], + [313n, 468n], + [469n, 500n], ]); - // A single block still produces one range. - assert.deepEqual(blockRanges(7n, 7n, 100n), [[7n, 7n]]); + // Ten full-size reads in a row try twice the size again, refused here. + reads.length = 0; + await readInRanges(501n, 2060n, span, read); + assert.deepEqual( + reads.map(([start, end]) => end - start + 1n), + Array(10).fill(156n), + ); + assert.equal(reads.at(-1)?.[1], 2060n); + assert.equal(span.size, 156n); + // On an endpoint without the cap it grows back, never above the maximum. + cap = 1_000_000n; + reads.length = 0; + await readInRanges(2061n, 1_000_000n, span, read); + const sizes = reads.map(([start, end]) => end - start + 1n); + assert.ok(sizes.every((size, i) => size <= 10_000n && (i === 0 || reads[i][0] === reads[i - 1][1] + 1n))); + assert.equal(span.size, 10_000n); + assert.equal(sizes.at(-2), 10_000n); // Nothing new to scan. - assert.deepEqual(blockRanges(11n, 10n, 100n), []); + reads.length = 0; + await readInRanges(11n, 10n, span, read); + assert.deepEqual(reads, []); +}); + +test("a refused single block and any other failure propagate", async () => { + const refused = async () => { + throw new InvalidParamsRpcError(rpcError(-32602, "block range greater than 0 max")); + }; + const tried: bigint[] = []; + const span = { size: 4n, max: 4n, streak: 0 }; + await assert.rejects( + readInRanges(1n, 10n, span, async (start, end) => { + tried.push(end - start + 1n); + await refused(); + }), + InvalidParamsRpcError, + ); + assert.deepEqual(tried, [4n, 2n, 1n]); + const unchanged = { size: 100n, max: 100n, streak: 0 }; + await assert.rejects( + readInRanges(1n, 10n, unchanged, async () => { + throw new HttpRequestError({ url: "http://rpc", status: 503 }); + }), + HttpRequestError, + ); + assert.equal(unchanged.size, 100n, "not a reason to shrink"); }); test("an amount larger than the daily cap itself is blocked, not a refill wait", () => { diff --git a/releaser/src/checks.ts b/releaser/src/checks.ts index d7d336f42..a3d82decd 100644 --- a/releaser/src/checks.ts +++ b/releaser/src/checks.ts @@ -94,19 +94,61 @@ export function toPalletAmount(tokenAmount: bigint, conversionFactor: bigint): b return tokenAmount / conversionFactor; } +/** How endpoints word a refused `eth_getLogs` span: Base's public endpoints + * (`block range greater than 200 max`), geth and reth (`exceed(s) maximum + * block range`), and hosted providers (`limited to a 10,000 range`, `ranges + * over 10000 blocks`, `query returned more than 10000 results`, `Log response + * size exceeded`). */ +const RANGE_TOO_LARGE = /\branges?\b|more than [\d,]+ results|response size/i; + +/** True when an endpoint refused an `eth_getLogs` call for the size of its + * block range (or of its answer), which a smaller range avoids. */ +export function isRangeTooLarge(error: unknown): boolean { + let depth = 0; + for (let cause: unknown = error; cause && depth < 10; cause = (cause as { cause?: unknown }).cause, depth++) { + for (const key of ["details", "shortMessage", "message"] as const) { + const text = (cause as Record)[key]; + if (typeof text === "string" && RANGE_TOO_LARGE.test(text)) return true; + } + } + return false; +} + +/** Full-size reads in a row after which a reduced span is tried at twice its + * size again. */ +const GROW_AFTER_READS = 10; + /** - * Split [fromBlock, toBlock] into chunks no larger than `maxRange` blocks. - * - * Public Base RPCs cap `eth_getLogs` ranges; a releaser restarting after a long - * outage would otherwise request a span the provider rejects and make no - * progress at all. Returns [] when there is nothing new to scan. + * Read [fromBlock, toBlock] in order, in consecutive ranges of at most + * `span.size` blocks. Public Base RPCs cap `eth_getLogs` ranges, some far below + * any sensible default (Base Sepolia's at 200 blocks), so a range the endpoint + * refuses as too large is read again at half the size, down to one block. After + * `GROW_AFTER_READS` full-size reads in a row succeed, the size doubles again, + * never above `span.max`. `span` persists across calls; any other failure, or + * a refused single block, propagates. */ -export function blockRanges(fromBlock: bigint, toBlock: bigint, maxRange: bigint): Array<[bigint, bigint]> { - if (maxRange <= 0n) throw new Error(`invalid maxRange ${maxRange}`); - const ranges: Array<[bigint, bigint]> = []; - for (let start = fromBlock; start <= toBlock; start += maxRange) { - const end = start + maxRange - 1n; - ranges.push([start, end > toBlock ? toBlock : end]); +export async function readInRanges( + fromBlock: bigint, + toBlock: bigint, + span: { size: bigint; max: bigint; streak: number }, + read: (start: bigint, end: bigint) => Promise, +): Promise { + if (span.size <= 0n) throw new Error(`invalid block range size ${span.size}`); + for (let start = fromBlock; start <= toBlock; ) { + const end = start + span.size - 1n < toBlock ? start + span.size - 1n : toBlock; + try { + await read(start, end); + } catch (error) { + if (span.size === 1n || !isRangeTooLarge(error)) throw error; + span.size /= 2n; + span.streak = 0; + continue; + } + // Only a read of the full size shows that the size is accepted. + if (end - start + 1n === span.size && ++span.streak >= GROW_AFTER_READS) { + span.size = span.size * 2n < span.max ? span.size * 2n : span.max; + span.streak = 0; + } + start = end + 1n; } - return ranges; } diff --git a/releaser/src/config.test.ts b/releaser/src/config.test.ts index 73fcbdbe7..bc56558e1 100644 --- a/releaser/src/config.test.ts +++ b/releaser/src/config.test.ts @@ -27,3 +27,13 @@ test("HEARTBEAT_URL must be an http(s) URL, and a bad one is refused without ech assert.match(stderr, /invalid HEARTBEAT_URL/); assert.doesNotMatch(stderr, /secret-token/); }); + +test("MAX_BLOCK_RANGE must be a whole number of at least 1", () => { + assert.equal(loadConfig({}, "String(config.maxBlockRange)").stdout, "10000"); + assert.equal(loadConfig({ MAX_BLOCK_RANGE: "200" }, "String(config.maxBlockRange)").stdout, "200"); + for (const value of ["0", "-1", "2.5", "lots"]) { + const { status, stderr } = loadConfig({ MAX_BLOCK_RANGE: value }); + assert.notEqual(status, 0, value); + assert.match(stderr, /invalid MAX_BLOCK_RANGE/, value); + } +}); diff --git a/releaser/src/config.ts b/releaser/src/config.ts index 3ed978da9..6ab013487 100644 --- a/releaser/src/config.ts +++ b/releaser/src/config.ts @@ -10,11 +10,13 @@ function required(name: string): string { /** A numeric environment value, validated at startup: a typo must fail loudly * here, not surface later as NaN-driven behavior. */ -function envNumber(name: string, fallback: number): number { +function envNumber(name: string, fallback: number, min = 0, integer = false): number { const raw = process.env[name]; if (raw === undefined || raw === "") return fallback; const value = Number(raw); - if (!Number.isFinite(value) || value < 0) throw new Error(`invalid ${name}: ${raw}`); + if (!Number.isFinite(value) || value < min || (integer && !Number.isInteger(value))) { + throw new Error(`invalid ${name}: ${raw}`); + } return value; } @@ -39,8 +41,9 @@ export const config = { * set to the vault's deployment block. */ startBlock: BigInt(process.env.START_BLOCK ?? "0"), pollIntervalMs: envNumber("POLL_INTERVAL_MS", 60_000), - /** Max blocks per `eth_getLogs` call; public RPCs cap this. */ - maxBlockRange: BigInt(process.env.MAX_BLOCK_RANGE ?? "10000"), + /** Max blocks per `eth_getLogs` call. Public RPCs cap this, often lower: + * a range the endpoint refuses is read again in smaller ones. */ + maxBlockRange: BigInt(envNumber("MAX_BLOCK_RANGE", 10_000, 1, true)), /** Alert when the gas wallet drops below this balance (wei). */ minGasBalanceWei: BigInt(process.env.MIN_GAS_BALANCE_WEI ?? "5000000000000000"), alertWebhookUrl: process.env.ALERT_WEBHOOK_URL, diff --git a/releaser/src/main.ts b/releaser/src/main.ts index 383ef9224..57901bf67 100644 --- a/releaser/src/main.ts +++ b/releaser/src/main.ts @@ -23,7 +23,7 @@ import { createPublicClient, createWalletClient, defineChain, http } from "viem"; import { privateKeyToAccount } from "viem/accounts"; import { alertText, postAlert, throttledHeartbeat, type Severity } from "./alert.js"; -import { blockRanges, chunks, toPalletAmount } from "./checks.js"; +import { chunks, readInRanges, toPalletAmount } from "./checks.js"; import { config } from "./config.js"; import { drainPending, type PendingRelease, type Releaser } from "./release.js"; import { @@ -112,9 +112,13 @@ function saveState(): void { writePersistedState(config.stateFile, state); } +/** The `eth_getLogs` span, MAX_BLOCK_RANGE at most: it shrinks while the + * endpoint refuses ranges as too large (see `readInRanges`). */ +const logSpan = { size: config.maxBlockRange, max: config.maxBlockRange, streak: 0 }; + /** Scan for newly deferred releases and add them to the pending set. */ async function ingestNewPending(toBlock: bigint, conversionFactor: bigint): Promise { - for (const [start, end] of blockRanges(fromBlock, toBlock, config.maxBlockRange)) { + await readInRanges(fromBlock, toBlock, logSpan, async (start, end) => { // Probe the range's end block first. Against a load-balanced endpoint, // `eth_getLogs` can be served by a node that has not reached `end` yet // and some providers then silently truncate rather than error — which @@ -140,7 +144,7 @@ async function ingestNewPending(toBlock: bigint, conversionFactor: bigint): Prom log(`deferred release observed: nonce=${nonce} recipient=${recipient}`); } fromBlock = end + 1n; - } + }); } /** Read `nonceConsumed` for many nonces, batched where possible. @@ -262,7 +266,10 @@ async function main(): Promise { }); await drainPending(releaser, conversionFactor, dailyCap); saveState(); - log(`ok: ${pending.size} pending, scanned ${config.baseFinalityTag} through block ${fromBlock - 1n}`); + log( + `ok: ${pending.size} pending, scanned ${config.baseFinalityTag} through block ${fromBlock - 1n} ` + + `(eth_getLogs span ${logSpan.size})`, + ); void heartbeat(); } catch (cycleError) { await alert("releaser cycle failed", `${cycleError}`); From fc5fab1b1412e707943de15eab9edbdddc2b8f55 Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Tue, 27 Oct 2026 12:12:57 +0100 Subject: [PATCH 08/16] test(releaser): Pin when a pending release may be dropped Cover the rule that only a consumption read at the finality boundary drops an entry: a release consumed only at the latest state, or a NonceAlreadyConsumed from a latest-state simulation, keeps the entry; a boundary read drops it. Also cover the interval throttle of blocked pages and the vault binding of the state file. --- releaser/src/release.test.ts | 83 ++++++++++++++++++++++++++++++++---- releaser/src/state.test.ts | 5 ++- 2 files changed, 79 insertions(+), 9 deletions(-) diff --git a/releaser/src/release.test.ts b/releaser/src/release.test.ts index c33f73aac..f9261c3c4 100644 --- a/releaser/src/release.test.ts +++ b/releaser/src/release.test.ts @@ -15,16 +15,19 @@ const RECIPIENT = "0x2222222222222222222222222222222222222222"; const CONVERSION_FACTOR = 1_000_000n; const DAILY_CAP = 10_000n * 10n ** 18n; -/** A releaser whose public client answers with `answers` (the finality - * boundary at block 100 and every nonce unconsumed unless overridden). */ -function releaserWith(answers: Record unknown>, entries: PendingRelease[]) { +type Answer = (args: Record) => unknown; + +/** A releaser whose public client answers each call with `answers` (the + * finality boundary at block 100 and every nonce unconsumed unless + * overridden). */ +function releaserWith(answers: Record, entries: PendingRelease[]) { const alerts: string[] = []; const logs: string[] = []; const writes: unknown[] = []; - const methods: Record unknown> = { getBlock: () => ({ number: 100n }), readContract: () => false, ...answers }; + const methods: Record = { getBlock: () => ({ number: 100n }), readContract: () => false, ...answers }; const releaser: Releaser = { publicClient: Object.fromEntries( - Object.entries(methods).map(([name, answer]) => [name, async () => answer()]), + Object.entries(methods).map(([name, answer]) => [name, async (args: Record = {}) => answer(args)]), ) as unknown as PublicClient, walletClient: { writeContract: async (request: unknown) => { @@ -46,15 +49,23 @@ function releaserWith(answers: Record unknown>, entries: PendingRe return { releaser, alerts, logs, writes }; } -/** What simulation raises for the vault's `ExceedsDailyCap`. */ -function dailyCapRevert(): Error { - const data = encodeErrorResult({ abi: vaultAbi, errorName: "ExceedsDailyCap", args: [DAILY_CAP + 1n, 0n] }); +/** What simulation raises for a vault revert. */ +function vaultRevert( + error: + | { errorName: "ExceedsDailyCap" | "ExceedsPerReleaseCap"; args: [bigint, bigint] } + | { errorName: "NonceAlreadyConsumed"; args: [bigint] }, +): Error { + const data = encodeErrorResult({ abi: vaultAbi, ...error }); return new ContractFunctionExecutionError( new ContractFunctionRevertedError({ abi: vaultAbi, data, functionName: "release" }), { abi: vaultAbi, functionName: "release", args: [7n, RECIPIENT, 0n] }, ); } +const dailyCapRevert = () => vaultRevert({ errorName: "ExceedsDailyCap", args: [DAILY_CAP + 1n, 0n] }); + +const ENTRY = { nonce: 7n, recipient: RECIPIENT, palletAmount: 5_000n * 10n ** 12n } as const; + test("a successful release keeps its entry until the nonce reads consumed at the finality boundary", async () => { const entry = { nonce: 7n, recipient: RECIPIENT, palletAmount: 5_000n * 10n ** 12n } as const; const { releaser, alerts, logs, writes } = releaserWith( @@ -90,3 +101,59 @@ test("a daily-cap revert pages as blocked when the token amount is above the dai ]); assert.equal(releaser.pending.size, 2); }); + +test("a consumption seen only at the latest state keeps the entry until the finality boundary shows it", async () => { + const atBoundary = (args: Record) => args.blockNumber === 100n; + for (const [simulated, consumedLatest] of [ + // Our release, or a peer's, is mined but not yet inside the boundary. + [new Error("execution reverted"), true], + // A simulation served by a node ahead of the ones the reads reach. + [vaultRevert({ errorName: "NonceAlreadyConsumed", args: [7n] }), false], + ] as const) { + const { releaser, alerts, writes } = releaserWith( + { + simulateContract: () => { + throw simulated; + }, + readContract: (args) => !atBoundary(args) && consumedLatest, + }, + [ENTRY], + ); + await drainPending(releaser, CONVERSION_FACTOR, DAILY_CAP); + assert.deepEqual([...releaser.pending.keys()], [7n], `${simulated.name}`); + assert.deepEqual([alerts, writes], [[], []]); + } + const { releaser, logs } = releaserWith( + { + simulateContract: () => { + throw vaultRevert({ errorName: "NonceAlreadyConsumed", args: [7n] }); + }, + readContract: atBoundary, + }, + [ENTRY], + ); + await drainPending(releaser, CONVERSION_FACTOR, DAILY_CAP); + assert.equal(releaser.pending.size, 0, "consumed at the boundary: dropped"); + assert.ok(logs.some((line) => line.startsWith("skip (already consumed): nonce=7 "))); +}); + +test("a blocked release pages once per interval, not once per cycle", async () => { + const { releaser, alerts } = releaserWith( + { + simulateContract: () => { + throw vaultRevert({ errorName: "ExceedsPerReleaseCap", args: [ENTRY.palletAmount * CONVERSION_FACTOR, 1n] }); + }, + }, + [ENTRY], + ); + await drainPending(releaser, CONVERSION_FACTOR, DAILY_CAP); + await drainPending(releaser, CONVERSION_FACTOR, DAILY_CAP); + assert.deepEqual(alerts, [ + `critical release blocked and needs operator action: nonce=7 recipient=${RECIPIENT} amount=${ENTRY.palletAmount} — ExceedsPerReleaseCap; it cannot self-heal`, + ]); + // The interval has passed. + releaser.blockedAlertedAt.set(7n, Date.now() - releaser.blockedAlertIntervalMs); + await drainPending(releaser, CONVERSION_FACTOR, DAILY_CAP); + assert.equal(alerts.length, 2); + assert.equal(releaser.pending.size, 1); +}); diff --git a/releaser/src/state.test.ts b/releaser/src/state.test.ts index 020671935..4a225b41a 100644 --- a/releaser/src/state.test.ts +++ b/releaser/src/state.test.ts @@ -23,7 +23,10 @@ test("releaser state is atomically persisted and identity-bound", () => { assert.deepEqual(loaded, state); assert.doesNotThrow(() => assertStateIdentity(loaded!, state)); assert.throws(() => assertStateIdentity(loaded!, { ...state, baseChainId: 1 }), /does not match/); - } finally { + assert.throws( + () => assertStateIdentity(loaded!, { ...state, vaultAddress: "0x3333333333333333333333333333333333333333" }), + /state vault 0x1111111111111111111111111111111111111111 does not match configured vault 0x3333/, + ); } finally { rmSync(directory, { recursive: true, force: true }); } }); From 05a5f7f7fbe7bb3d02044daa8dbe32c57c870429 Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Tue, 27 Oct 2026 12:15:27 +0100 Subject: [PATCH 09/16] fix(releaser): Throttle repeated unexpected release failures per release An unexpected release failure paged on every cycle for every affected entry, so one persistent condition produced a critical page per entry per minute on the webhook the attestors share. Throttle it per release like a blocked one: the same page about a release goes out again only after BLOCKED_ALERT_INTERVAL_MS, and a different page about it at once. --- releaser/README.md | 4 ++-- releaser/src/main.ts | 7 +++--- releaser/src/release.test.ts | 35 +++++++++++++++++++++++---- releaser/src/release.ts | 46 ++++++++++++++++++++---------------- 4 files changed, 62 insertions(+), 30 deletions(-) diff --git a/releaser/README.md b/releaser/README.md index 37ee447a8..6cf08c302 100644 --- a/releaser/README.md +++ b/releaser/README.md @@ -41,7 +41,7 @@ consumed nonce and drops it. | `HEARTBEAT_URL` | Optional http(s) URL requested (GET) after a cycle that completed without error, at most once a minute; point it at a dead-man's switch that pages when the requests stop. Startup refuses any other value. A failed request is logged (error name and code only, never the URL), never fatal | | `BASE_CHAIN_ID` | Default 8453; startup refuses a `BASE_RPC_URL` that serves another chain (`eth_chainId`) | | `BASE_FINALITY_TAG` | `safe` (default) or `finalized`; the scan cursor stays inside this boundary and a pending entry is dropped only once its nonce is consumed there | -| `BLOCKED_ALERT_INTERVAL_MS` | Re-page interval for governance-blocked releases (default 6h — the condition needs a ≥48h timelocked action, so per-poll paging would bury the signal) | +| `BLOCKED_ALERT_INTERVAL_MS` | Re-page interval for the same page about one release: governance-blocked, or failing unexpectedly (default 6h — a blocked release needs a ≥48h timelocked action, so per-poll paging would bury the signal). A different page about it goes out at once | | `READ_BATCH_SIZE` | Bound on Multicall calldata and individual fallback concurrency (default 100) | ## Run @@ -63,7 +63,7 @@ Failures are classified rather than treated alike: | `ExceedsDailyCap`, `EnforcedPause`, `NotEnoughApprovals` | **Retry quietly** — expected temporary conditions | | `NonceAlreadyConsumed` (at the finality boundary) | **Done** — drop it | | `ExceedsPerReleaseCap`, `InsufficientVaultBalance` | **Alert, throttled** to `BLOCKED_ALERT_INTERVAL_MS` — cannot self-heal without governance or operator action | -| anything else | **Alert** | +| anything else | **Alert** `unexpected release failure`, **throttled** to `BLOCKED_ALERT_INTERVAL_MS` per release | A successful `release()` does **not** drop the entry immediately: it leaves the durable pending set only once the nonce reads as consumed at the diff --git a/releaser/src/main.ts b/releaser/src/main.ts index 57901bf67..a1cd555b4 100644 --- a/releaser/src/main.ts +++ b/releaser/src/main.ts @@ -57,7 +57,7 @@ const walletClient = createWalletClient({ account, chain: baseChain, transport: const pending = new Map(); let fromBlock = config.startBlock; -const blockedAlertedAt = new Map(); +const alertedAt: Releaser["alertedAt"] = new Map(); function log(message: string): void { console.log(`${new Date().toISOString()} ${message}`); @@ -202,7 +202,7 @@ async function pruneConsumed(blockNumber: bigint): Promise { entries.forEach((entry, i) => { if (consumed[i]) { pending.delete(entry.nonce); - blockedAlertedAt.delete(entry.nonce); + alertedAt.delete(entry.nonce); } }); } @@ -215,9 +215,10 @@ const releaser: Releaser = { baseFinalityTag: config.baseFinalityTag, blockedAlertIntervalMs: config.blockedAlertIntervalMs, pending, - blockedAlertedAt, + alertedAt, alert, log, + now: Date.now, }; async function checkGasBalance(): Promise { diff --git a/releaser/src/release.test.ts b/releaser/src/release.test.ts index f9261c3c4..dbe75b5d5 100644 --- a/releaser/src/release.test.ts +++ b/releaser/src/release.test.ts @@ -25,6 +25,7 @@ function releaserWith(answers: Record, entries: PendingRelease[] const logs: string[] = []; const writes: unknown[] = []; const methods: Record = { getBlock: () => ({ number: 100n }), readContract: () => false, ...answers }; + let clock = 1_000_000; const releaser: Releaser = { publicClient: Object.fromEntries( Object.entries(methods).map(([name, answer]) => [name, async (args: Record = {}) => answer(args)]), @@ -40,13 +41,14 @@ function releaserWith(answers: Record, entries: PendingRelease[] baseFinalityTag: "safe", blockedAlertIntervalMs: 6 * 3600_000, pending: new Map(entries.map((entry) => [entry.nonce, entry])), - blockedAlertedAt: new Map(), + alertedAt: new Map(), alert: async (subject, detail, severity = "warning") => { alerts.push(`${severity} ${subject}: ${detail}`); }, log: (message) => logs.push(message), + now: () => clock, }; - return { releaser, alerts, logs, writes }; + return { releaser, alerts, logs, writes, advance: (ms: number) => (clock += ms) }; } /** What simulation raises for a vault revert. */ @@ -138,7 +140,7 @@ test("a consumption seen only at the latest state keeps the entry until the fina }); test("a blocked release pages once per interval, not once per cycle", async () => { - const { releaser, alerts } = releaserWith( + const { releaser, alerts, advance } = releaserWith( { simulateContract: () => { throw vaultRevert({ errorName: "ExceedsPerReleaseCap", args: [ENTRY.palletAmount * CONVERSION_FACTOR, 1n] }); @@ -151,9 +153,32 @@ test("a blocked release pages once per interval, not once per cycle", async () = assert.deepEqual(alerts, [ `critical release blocked and needs operator action: nonce=7 recipient=${RECIPIENT} amount=${ENTRY.palletAmount} — ExceedsPerReleaseCap; it cannot self-heal`, ]); - // The interval has passed. - releaser.blockedAlertedAt.set(7n, Date.now() - releaser.blockedAlertIntervalMs); + advance(releaser.blockedAlertIntervalMs); await drainPending(releaser, CONVERSION_FACTOR, DAILY_CAP); assert.equal(alerts.length, 2); assert.equal(releaser.pending.size, 1); }); + +test("an unexpected release failure pages once per interval, and again at once when the failure changes", async () => { + let failure: Error = new Error("execution reverted: TokenNotSet"); + const { releaser, alerts, advance } = releaserWith( + { + simulateContract: () => { + throw failure; + }, + }, + [ENTRY], + ); + for (let cycle = 0; cycle < 5; cycle++) await drainPending(releaser, CONVERSION_FACTOR, DAILY_CAP); + assert.deepEqual(alerts, [ + `critical unexpected release failure: nonce=7 recipient=${RECIPIENT} amount=${ENTRY.palletAmount} — execution reverted: TokenNotSet`, + ]); + advance(releaser.blockedAlertIntervalMs); + await drainPending(releaser, CONVERSION_FACTOR, DAILY_CAP); + assert.equal(alerts.length, 2); + failure = vaultRevert({ errorName: "ExceedsPerReleaseCap", args: [ENTRY.palletAmount * CONVERSION_FACTOR, 1n] }); + await drainPending(releaser, CONVERSION_FACTOR, DAILY_CAP); + assert.equal(alerts.length, 3); + assert.match(alerts[2], /^critical release blocked and needs operator action: nonce=7 /); + assert.equal(releaser.pending.size, 1); +}); diff --git a/releaser/src/release.ts b/releaser/src/release.ts index a873f18e5..b60a2401e 100644 --- a/releaser/src/release.ts +++ b/releaser/src/release.ts @@ -25,13 +25,24 @@ export interface Releaser { blockedAlertIntervalMs: number; /** nonce -> pending release. Keyed by nonce: the vault consumes a nonce once. */ pending: Map; - /** nonce -> when its "blocked" state was last alerted. A blocked release needs - * a timelocked governance action (>= 48h) to clear, so re-paging it every - * poll would emit thousands of identical alerts and train on-call to ignore - * them; re-alert at a bounded interval instead. */ - blockedAlertedAt: Map; + /** nonce -> the last page about it, and when it went out. A blocked release + * needs a timelocked governance action (>= 48h) to clear, and an unexpected + * failure repeats every poll as well, so re-paging either every poll would + * emit thousands of identical alerts and train on-call to ignore them; the + * same page re-alerts at a bounded interval instead. */ + alertedAt: Map; alert(subject: string, detail: string, severity?: Severity): Promise; log(message: string): void; + now(): number; +} + +/** Page `subject` about the release of `nonce`, unless the same page about it + * went out less than `blockedAlertIntervalMs` ago. */ +async function pageThrottled(releaser: Releaser, nonce: bigint, subject: string, detail: string): Promise { + const last = releaser.alertedAt.get(nonce); + if (last?.subject === subject && releaser.now() - last.at < releaser.blockedAlertIntervalMs) return; + releaser.alertedAt.set(nonce, { subject, at: releaser.now() }); + await releaser.alert(subject, detail, "critical"); } /** A release transaction that was mined and reverted. The receipt carries no @@ -41,8 +52,7 @@ class MinedRevert extends Error {} /** Try to push each pending release through; classify what comes back. */ export async function drainPending(releaser: Releaser, conversionFactor: bigint, dailyCap: bigint): Promise { - const { publicClient, walletClient, account, vaultAddress, baseFinalityTag, pending, blockedAlertedAt, alert, log } = - releaser; + const { publicClient, walletClient, account, vaultAddress, baseFinalityTag, pending, alertedAt, log } = releaser; for (const p of [...pending.values()]) { const label = `nonce=${p.nonce} recipient=${p.recipient} amount=${p.palletAmount}`; try { @@ -118,28 +128,24 @@ export async function drainPending(releaser: Releaser, conversionFactor: bigint, switch (classifyReleaseFailure(errorName)) { case "done": pending.delete(p.nonce); - blockedAlertedAt.delete(p.nonce); + alertedAt.delete(p.nonce); log(`skip (already consumed): ${label}`); break; case "retry": // Expected while the daily bucket refills — stay quiet. break; - case "blocked": { + case "blocked": // Clearing this needs a timelocked governance action, so the // condition persists for days; page at a bounded interval. - const lastAlerted = blockedAlertedAt.get(p.nonce) ?? 0; - if (Date.now() - lastAlerted >= releaser.blockedAlertIntervalMs) { - blockedAlertedAt.set(p.nonce, Date.now()); - await alert( - "release blocked and needs operator action", - `${label} — ${errorName}; it cannot self-heal`, - "critical", - ); - } + await pageThrottled( + releaser, + p.nonce, + "release blocked and needs operator action", + `${label} — ${errorName}; it cannot self-heal`, + ); break; - } case "unexpected": - await alert("unexpected release failure", `${label} — ${reason}`, "critical"); + await pageThrottled(releaser, p.nonce, "unexpected release failure", `${label} — ${reason}`); break; } } From 1d5cf8db2263a2a255444b8b904b46081434962c Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Tue, 27 Oct 2026 12:20:19 +0100 Subject: [PATCH 10/16] fix(releaser): Follow a stuck release transaction with bounded fee bumps A release transaction not mined within viem's 180 s receipt wait paged "unexpected release failure", and the next cycle signed another release at the next nonce, queued behind the stuck one, until fees fell again. Sign each release at the pending nonce with the capped fee estimate and wait FEE_BUMP_AFTER_SECONDS for it. One not mined by then stays in flight: no other release is sent behind it, and from cycle to cycle it is re-sent as the identical call at the same nonce with both fees raised, at most FEE_BUMP_MAX times and never above MAX_FEE_PER_GAS_GWEI, then it pages "release transaction not mined", throttled like the other release pages. Once its nonce is used by another transaction, or nothing below it can be mined, the release is simulated and sent anew. The settings and defaults match the attestor's. --- releaser/README.md | 18 +++ releaser/src/config.test.ts | 22 ++++ releaser/src/config.ts | 21 +++ releaser/src/main.ts | 1 + releaser/src/release.test.ts | 181 +++++++++++++++++++++++--- releaser/src/release.ts | 240 +++++++++++++++++++++++++++++------ 6 files changed, 424 insertions(+), 59 deletions(-) diff --git a/releaser/README.md b/releaser/README.md index 6cf08c302..cfd13a8e1 100644 --- a/releaser/README.md +++ b/releaser/README.md @@ -43,6 +43,10 @@ consumed nonce and drops it. | `BASE_FINALITY_TAG` | `safe` (default) or `finalized`; the scan cursor stays inside this boundary and a pending entry is dropped only once its nonce is consumed there | | `BLOCKED_ALERT_INTERVAL_MS` | Re-page interval for the same page about one release: governance-blocked, or failing unexpectedly (default 6h — a blocked release needs a ≥48h timelocked action, so per-poll paging would bury the signal). A different page about it goes out at once | | `READ_BATCH_SIZE` | Bound on Multicall calldata and individual fallback concurrency (default 100) | +| `FEE_BUMP_AFTER_SECONDS` | Re-send a release transaction that is still not mined after this long (default 60) | +| `FEE_BUMP_PERCENT` | Raise both fees by at least this many percent per re-send (default 20; a whole number of at least 10, the minimum a node accepts for a replacement) | +| `FEE_BUMP_MAX` | Re-sends per transaction before the `release transaction not mined` page (default 3) | +| `MAX_FEE_PER_GAS_GWEI` | No release transaction ever offers a higher `maxFeePerGas` (default 1) | ## Run @@ -70,6 +74,20 @@ durable pending set only once the nonce reads as consumed at the `safe`/`finalized` boundary, so a reorged-away release is still ours to retry. Until then re-attempts are gas-free simulations classified as pending finality. +Each release is signed at the account's pending nonce with the node's fee +estimate, capped at `MAX_FEE_PER_GAS_GWEI`. One not mined within +`FEE_BUMP_AFTER_SECONDS` is followed from cycle to cycle instead of being +signed again, and no other release is sent meanwhile, since it would only +queue behind that nonce. Every `FEE_BUMP_AFTER_SECONDS` it is re-sent as the +identical call at the same nonce with both fees raised by `FEE_BUMP_PERCENT`, +and at least to the current estimate, at most `FEE_BUMP_MAX` times. When no +bump is left, or the cap stops one, it pages `release transaction not mined` +(with the nonce, the account's next mined nonce and the fees; throttled to +`BLOCKED_ALERT_INTERVAL_MS`) and keeps waiting. Once another transaction uses +its nonce, or the node holds nothing at a nonce below it, the release is +simulated and sent again. The transaction in flight is kept in memory only: +after a restart, a pending one stuck below the cap has to be replaced by hand. + State (safe/finalized scan checkpoint + pending set) is atomically persisted and bound to the configured chain and vault, so a restart resumes without trusting unsafe logs or losing pending work. Malformed state is fatal, and so is diff --git a/releaser/src/config.test.ts b/releaser/src/config.test.ts index bc56558e1..d1ed1a717 100644 --- a/releaser/src/config.test.ts +++ b/releaser/src/config.test.ts @@ -28,6 +28,28 @@ test("HEARTBEAT_URL must be an http(s) URL, and a bad one is refused without ech assert.doesNotMatch(stderr, /secret-token/); }); +test("fee bumping defaults to 60 s, 20 %, 3 bumps and a 1 gwei cap, and refuses invalid settings", () => { + const print = "[config.feeBump.afterMs, config.feeBump.percent, config.feeBump.max, config.feeBump.maxFeePerGas].join(' ')"; + assert.equal(loadConfig({}, print).stdout, "60000 20 3 1000000000"); + assert.equal( + loadConfig({ FEE_BUMP_AFTER_SECONDS: "30", FEE_BUMP_PERCENT: "10", FEE_BUMP_MAX: "0", MAX_FEE_PER_GAS_GWEI: "0.25" }, print) + .stdout, + "30000 10 0 250000000", + ); + for (const [name, value] of [ + ["FEE_BUMP_PERCENT", "9"], + ["FEE_BUMP_PERCENT", "12.5"], + ["FEE_BUMP_MAX", "-1"], + ["FEE_BUMP_AFTER_SECONDS", "0"], + ["MAX_FEE_PER_GAS_GWEI", "0"], + ["MAX_FEE_PER_GAS_GWEI", "1e9"], + ]) { + const { status, stderr } = loadConfig({ [name]: value }); + assert.notEqual(status, 0, `${name}=${value}`); + assert.match(stderr, new RegExp(`invalid ${name}`), `${name}=${value}`); + } +}); + test("MAX_BLOCK_RANGE must be a whole number of at least 1", () => { assert.equal(loadConfig({}, "String(config.maxBlockRange)").stdout, "10000"); assert.equal(loadConfig({ MAX_BLOCK_RANGE: "200" }, "String(config.maxBlockRange)").stdout, "200"); diff --git a/releaser/src/config.ts b/releaser/src/config.ts index 6ab013487..c6c8dde81 100644 --- a/releaser/src/config.ts +++ b/releaser/src/config.ts @@ -1,4 +1,6 @@ +import { parseGwei } from "viem"; import { parseHeartbeatUrl } from "./alert.js"; +import { MIN_REPLACEMENT_PERCENT } from "./release.js"; function required(name: string): string { const value = process.env[name]; @@ -20,6 +22,13 @@ function envNumber(name: string, fallback: number, min = 0, integer = false): nu return value; } +/** A positive gwei amount given as a decimal, e.g. `0.5`. */ +function envGwei(name: string, fallback: string): bigint { + const raw = process.env[name] || fallback; + if (!/^\d+(\.\d{1,9})?$/.test(raw) || parseGwei(raw) === 0n) throw new Error(`invalid ${name}: ${raw}`); + return parseGwei(raw); +} + function finalityTag(): "safe" | "finalized" { const value = process.env.BASE_FINALITY_TAG ?? "safe"; if (value !== "safe" && value !== "finalized") throw new Error(`invalid BASE_FINALITY_TAG ${value}`); @@ -62,4 +71,16 @@ export const config = { blockedAlertIntervalMs: envNumber("BLOCKED_ALERT_INTERVAL_MS", 6 * 60 * 60 * 1000), /** Bound both Multicall calldata and individual fallback concurrency. */ readBatchSize: envNumber("READ_BATCH_SIZE", 100), + /** A release transaction still not mined after `afterMs` is re-sent at the + * same nonce with both fees raised by `percent` (at least the 10 % a + * replacement needs, and at least to the current estimate), at most `max` + * times and never above `maxFeePerGas`; then it pages. No other release is + * sent meanwhile, as it would only queue behind that nonce. Same settings + * and defaults as the attestor's. */ + feeBump: { + afterMs: envNumber("FEE_BUMP_AFTER_SECONDS", 60, 1) * 1000, + percent: envNumber("FEE_BUMP_PERCENT", 20, MIN_REPLACEMENT_PERCENT, true), + max: envNumber("FEE_BUMP_MAX", 3, 0, true), + maxFeePerGas: envGwei("MAX_FEE_PER_GAS_GWEI", "1"), + }, }; diff --git a/releaser/src/main.ts b/releaser/src/main.ts index a1cd555b4..c4e2cf4d4 100644 --- a/releaser/src/main.ts +++ b/releaser/src/main.ts @@ -216,6 +216,7 @@ const releaser: Releaser = { blockedAlertIntervalMs: config.blockedAlertIntervalMs, pending, alertedAt, + feeBump: config.feeBump, alert, log, now: Date.now, diff --git a/releaser/src/release.test.ts b/releaser/src/release.test.ts index dbe75b5d5..75ceb03e1 100644 --- a/releaser/src/release.test.ts +++ b/releaser/src/release.test.ts @@ -4,10 +4,13 @@ import { ContractFunctionExecutionError, ContractFunctionRevertedError, encodeErrorResult, + TransactionReceiptNotFoundError, + WaitForTransactionReceiptTimeoutError, + type Hash, type PublicClient, } from "viem"; import { privateKeyToAccount } from "viem/accounts"; -import { drainPending, type PendingRelease, type Releaser } from "./release.js"; +import { drainPending, type InFlightRelease, type PendingRelease, type Releaser } from "./release.js"; import { vaultAbi } from "./vaultAbi.js"; const RECIPIENT = "0x2222222222222222222222222222222222222222"; @@ -16,26 +19,41 @@ const CONVERSION_FACTOR = 1_000_000n; const DAILY_CAP = 10_000n * 10n ** 18n; type Answer = (args: Record) => unknown; +type Write = InFlightRelease["call"] & { nonce: number; maxFeePerGas: bigint; maxPriorityFeePerGas: bigint }; -/** A releaser whose public client answers each call with `answers` (the - * finality boundary at block 100 and every nonce unconsumed unless - * overridden). */ +/** The hash the wallet answers for its nth send. */ +const txHash = (index: number): Hash => `0x${(index + 1).toString(16).padStart(64, "0")}`; + +/** A releaser whose public client answers each call with `answers`. Unless + * overridden: the finality boundary is at block 100, every nonce unconsumed, + * a release simulates fine, the account's nonce is 4, and a sent release is + * mined at once. */ function releaserWith(answers: Record, entries: PendingRelease[]) { const alerts: string[] = []; const logs: string[] = []; - const writes: unknown[] = []; - const methods: Record = { getBlock: () => ({ number: 100n }), readContract: () => false, ...answers }; + const writes: Write[] = []; + const methods: Record = { + getBlock: () => ({ number: 100n }), + readContract: () => false, + simulateContract: () => ({}), + estimateContractGas: () => 50_000n, + getTransactionCount: () => 4, + estimateFeesPerGas: () => ({ maxFeePerGas: 7_000_000n, maxPriorityFeePerGas: 1_000_000n }), + waitForTransactionReceipt: ({ hash }) => ({ status: "success", transactionHash: hash }), + ...answers, + }; let clock = 1_000_000; const releaser: Releaser = { publicClient: Object.fromEntries( Object.entries(methods).map(([name, answer]) => [name, async (args: Record = {}) => answer(args)]), ) as unknown as PublicClient, walletClient: { - writeContract: async (request: unknown) => { - writes.push(request); - return `0x${"ab".repeat(32)}`; + writeContract: async (call: Write) => { + writes.push(call); + return txHash(writes.length - 1); }, } as unknown as Releaser["walletClient"], + feeBump: { afterMs: 60_000, percent: 20, max: 3, maxFeePerGas: 1_000_000_000n }, account: privateKeyToAccount(`0x${"11".repeat(32)}`), vaultAddress: "0x1111111111111111111111111111111111111111", baseFinalityTag: "safe", @@ -69,20 +87,143 @@ const dailyCapRevert = () => vaultRevert({ errorName: "ExceedsDailyCap", args: [ const ENTRY = { nonce: 7n, recipient: RECIPIENT, palletAmount: 5_000n * 10n ** 12n } as const; test("a successful release keeps its entry until the nonce reads consumed at the finality boundary", async () => { - const entry = { nonce: 7n, recipient: RECIPIENT, palletAmount: 5_000n * 10n ** 12n } as const; - const { releaser, alerts, logs, writes } = releaserWith( - { - simulateContract: () => ({ request: { functionName: "release" } }), - estimateContractGas: () => 50_000n, - waitForTransactionReceipt: () => ({ status: "success" }), - }, - [entry], - ); + const { releaser, alerts, logs, writes } = releaserWith({}, [ENTRY]); await drainPending(releaser, CONVERSION_FACTOR, DAILY_CAP); - assert.deepEqual(writes, [{ functionName: "release", gas: 100_000n }]); - assert.deepEqual([...releaser.pending.values()], [entry]); + // The vault's release with the entry's tuple, at the pending nonce, the + // estimated fees and a doubled gas limit. + assert.deepEqual( + writes.map(({ address, functionName, args, gas, nonce, maxFeePerGas, maxPriorityFeePerGas }) => ({ + address, + functionName, + args, + gas, + nonce, + maxFeePerGas, + maxPriorityFeePerGas, + })), + [ + { + address: releaser.vaultAddress, + functionName: "release", + args: [ENTRY.nonce, ENTRY.recipient, ENTRY.palletAmount], + gas: 100_000n, + nonce: 4, + maxFeePerGas: 7_000_000n, + maxPriorityFeePerGas: 1_000_000n, + }, + ], + ); + assert.deepEqual([...releaser.pending.values()], [ENTRY]); assert.ok(logs.some((line) => line.startsWith("released (awaiting safe): nonce=7 "))); assert.deepEqual(alerts, []); + assert.equal(releaser.inFlight, undefined); +}); + +/** A node that never mines what it is sent. */ +const notMined: Record = { + waitForTransactionReceipt: ({ hash }) => { + throw new WaitForTransactionReceiptTimeoutError({ hash: hash as Hash }); + }, + getTransactionReceipt: ({ hash }) => { + throw new TransactionReceiptNotFoundError({ hash: hash as Hash }); + }, +}; + +test("a release not mined in time is followed at its nonce: nothing is sent behind it, it is bumped, then paged", async () => { + let minedHash: Hash | undefined; + const { releaser, alerts, writes, advance } = releaserWith( + { + ...notMined, + simulateContract: ({ args }) => { + // Once it is mined, the latest state has the nonce consumed. + if (minedHash && (args as bigint[])[0] === ENTRY.nonce) { + throw vaultRevert({ errorName: "NonceAlreadyConsumed", args: [ENTRY.nonce] }); + } + return {}; + }, + getTransactionReceipt: ({ hash }) => { + if (hash !== minedHash) throw new TransactionReceiptNotFoundError({ hash: hash as Hash }); + return { status: "success", transactionHash: hash }; + }, + }, + [ENTRY, { ...ENTRY, nonce: 8n }], + ); + const drain = () => drainPending(releaser, CONVERSION_FACTOR, DAILY_CAP); + await drain(); + advance(30_000); + await drain(); + assert.equal(writes.length, 1, "not sent again before the bump interval, and nothing sent behind it"); + for (let cycle = 0; cycle < 5; cycle++) { + advance(60_000); + await drain(); + } + // The identical call at the same nonce, both fees raised, three times. + assert.deepEqual( + writes.map((w) => [w.args[0], w.nonce, w.gas, w.maxFeePerGas, w.maxPriorityFeePerGas]), + [ + [7n, 4, 100_000n, 7_000_000n, 1_000_000n], + [7n, 4, 100_000n, 8_400_000n, 1_200_000n], + [7n, 4, 100_000n, 10_080_000n, 1_440_000n], + [7n, 4, 100_000n, 12_096_000n, 1_728_000n], + ], + ); + // Then one page, not repeated within the interval. + assert.deepEqual(alerts, [ + `critical release transaction not mined: 3 fee bump(s) used up: transaction nonce 4, account's next mined nonce 4, ` + + `maxFeePerGas 0.012096 gwei (cap 1 gwei); nonce=7 recipient=${RECIPIENT} amount=${ENTRY.palletAmount}`, + ]); + // The second bump is mined: its entry waits for the boundary, and the next release goes out. + minedHash = txHash(2); + await drain(); + assert.deepEqual( + writes.map((w) => w.args[0]), + [7n, 7n, 7n, 7n, 8n], + ); + assert.equal(releaser.inFlight?.entry.nonce, 8n); + assert.equal(releaser.pending.size, 2); + assert.equal(alerts.length, 1); +}); + +test("a release whose nonce another transaction used, or that an empty nonce below keeps from being mined, is sent anew", async () => { + let latest = 4; + let pendingCount = 5; + const { releaser, writes, logs } = releaserWith( + { ...notMined, getTransactionCount: ({ blockTag }) => (blockTag === "latest" ? latest : pendingCount) }, + [ENTRY], + ); + const drain = () => drainPending(releaser, CONVERSION_FACTOR, DAILY_CAP); + await drain(); + // Queued behind a transaction at nonce 4, which then leaves the pool unmined. + pendingCount = 4; + await drain(); + // Another transaction takes nonce 4. + latest = 5; + pendingCount = 5; + await drain(); + assert.deepEqual( + writes.map((w) => w.nonce), + [5, 4, 5], + ); + assert.equal(logs.filter((line) => line.startsWith("release transaction at nonce ")).length, 2); +}); + +test("the fee cap stops a release's bump and pages; no attempt offers more than the cap", async () => { + const { releaser, alerts, writes, advance } = releaserWith(notMined, [ENTRY]); + releaser.feeBump = { ...releaser.feeBump, maxFeePerGas: 8_000_000n }; + for (let cycle = 0; cycle < 3; cycle++) { + await drainPending(releaser, CONVERSION_FACTOR, DAILY_CAP); + advance(60_000); + } + // 7 → 8.4 is capped at 8; 8 → 8 is no replacement a pool takes. + assert.deepEqual( + writes.map((w) => [w.maxFeePerGas, w.maxPriorityFeePerGas]), + [ + [7_000_000n, 1_000_000n], + [8_000_000n, 1_200_000n], + ], + ); + assert.equal(alerts.length, 1); + assert.match(alerts[0], /^critical release transaction not mined: the fee cap stops the next bump: transaction nonce 4, .*\(cap 0\.008 gwei\); nonce=7 /); }); test("a daily-cap revert pages as blocked when the token amount is above the daily cap itself, and waits quietly otherwise", async () => { diff --git a/releaser/src/release.ts b/releaser/src/release.ts index b60a2401e..5beb76301 100644 --- a/releaser/src/release.ts +++ b/releaser/src/release.ts @@ -1,21 +1,77 @@ /** * The releaser's release attempts: simulate and send `release()` for each - * pending entry, and classify what a failed attempt means. `main.ts` injects - * the clients and the pending set, so the decisions here are unit tested (see - * release.test.ts). + * pending entry, follow a sent transaction until it is mined (re-sent with + * higher fees at its nonce when it is not), and classify what a failed attempt + * means. `main.ts` injects the clients and the pending set, so the decisions + * here are unit tested (see release.test.ts). */ -import type { Account, Chain, PublicClient, Transport, WalletClient } from "viem"; +import { + BaseError, + formatGwei, + TransactionReceiptNotFoundError, + type Account, + type Chain, + type Hash, + type PublicClient, + type TransactionReceipt, + type Transport, + type WalletClient, +} from "viem"; import type { Severity } from "./alert.js"; import { classifyReleaseFailure, contractErrorName } from "./checks.js"; import { vaultAbi } from "./vaultAbi.js"; +/** op-geth replaces a pending transaction only when the new one raises both + * its fee cap and its tip by at least this many percent. */ +export const MIN_REPLACEMENT_PERCENT = 10; + export interface PendingRelease { nonce: bigint; recipient: `0x${string}`; palletAmount: bigint; } +export interface Fees { + maxFeePerGas: bigint; + maxPriorityFeePerGas: bigint; +} + +/** When, how often and how far a release transaction that is not mined is + * re-sent with higher fees. */ +export interface FeeBumpPolicy { + /** Re-send an attempt still not mined after this long. */ + afterMs: number; + /** Raise both fees by at least this many percent per re-send. */ + percent: number; + /** Re-sends per transaction. */ + max: number; + /** No attempt ever offers a higher maxFeePerGas. */ + maxFeePerGas: bigint; +} + +/** A release transaction sent and not seen mined: the identical call, kept at + * its nonce and re-sent there with higher fees until an attempt is mined. */ +export interface InFlightRelease { + entry: PendingRelease; + call: { + account: Account; + address: `0x${string}`; + abi: typeof vaultAbi; + functionName: "release"; + args: readonly [bigint, `0x${string}`, bigint]; + gas: bigint; + }; + nonce: number; + /** The latest attempt's fees. */ + fees: Fees; + /** Every attempt at `nonce`; any of them may be the one that is mined. */ + hashes: Hash[]; + /** When the latest attempt was sent. */ + sentAtMs: number; + bumps: number; +} + export interface Releaser { publicClient: PublicClient; walletClient: WalletClient; @@ -31,6 +87,10 @@ export interface Releaser { * emit thousands of identical alerts and train on-call to ignore them; the * same page re-alerts at a bounded interval instead. */ alertedAt: Map; + feeBump: FeeBumpPolicy; + /** The release transaction not seen mined yet, if any. Kept in memory + * only. */ + inFlight?: InFlightRelease; alert(subject: string, detail: string, severity?: Severity): Promise; log(message: string): void; now(): number; @@ -45,57 +105,159 @@ async function pageThrottled(releaser: Releaser, nonce: bigint, subject: string, await releaser.alert(subject, detail, "critical"); } -/** A release transaction that was mined and reverted. The receipt carries no - * revert data, so the reason is unknowable here; the next cycle's simulation - * classifies it properly (and pruneConsumed drops it if a peer won). */ -class MinedRevert extends Error {} +const min = (a: bigint, b: bigint) => (a < b ? a : b); +const max = (a: bigint, b: bigint) => (a > b ? a : b); + +/** `fee` raised by `percent`, rounded up and by at least 1 wei. */ +function raised(fee: bigint, percent: number): bigint { + return max(fee + 1n, (fee * BigInt(100 + percent) + 99n) / 100n); +} + +/** The first attempt's fees: the node's estimate, never above the cap. */ +export function cappedFees(estimate: Fees, cap: bigint): Fees { + return { maxFeePerGas: min(estimate.maxFeePerGas, cap), maxPriorityFeePerGas: min(estimate.maxPriorityFeePerGas, cap) }; +} + +/** The fees to re-send an attempt that was not mined with: both raised by the + * policy's percent, and at least to the node's current estimate, never above + * the cap. Undefined when the cap leaves no room for a replacement the pool + * accepts (both fees raised by at least `MIN_REPLACEMENT_PERCENT`). */ +export function bumpedFees(last: Fees, estimate: Fees, policy: FeeBumpPolicy): Fees | undefined { + const next = (fee: bigint, current: bigint) => min(policy.maxFeePerGas, max(raised(fee, policy.percent), current)); + const fees = { + maxFeePerGas: next(last.maxFeePerGas, estimate.maxFeePerGas), + maxPriorityFeePerGas: next(last.maxPriorityFeePerGas, estimate.maxPriorityFeePerGas), + }; + const replaces = + fees.maxFeePerGas >= raised(last.maxFeePerGas, MIN_REPLACEMENT_PERCENT) && + fees.maxPriorityFeePerGas >= raised(last.maxPriorityFeePerGas, MIN_REPLACEMENT_PERCENT); + return replaces ? fees : undefined; +} + +const releaseLabel = (p: PendingRelease) => `nonce=${p.nonce} recipient=${p.recipient} amount=${p.palletAmount}`; + +/** A mined release leaves the pending set only once its nonce reads consumed at + * the finality boundary (see `pruneConsumed`); until then a re-attempt is a + * cheap simulation that classifies as PendingFinality and stays quiet, and a + * release reorged away is still ours to retry. A mined revert carries no + * reason (usually a lost race: a peer released, or the daily allowance went + * between simulation and inclusion); the next cycle's simulation classifies + * it, and the prune drops it if a peer won. */ +function logMined(releaser: Releaser, receipt: TransactionReceipt, label: string): void { + releaser.log( + receipt.status === "success" + ? `released (awaiting ${releaser.baseFinalityTag}): ${label} tx=${receipt.transactionHash}` + : `release reverted on-chain, re-evaluating next cycle: ${label}`, + ); +} + +/** + * Follow the release transaction in flight and say whether it is settled: + * an attempt is mined, its nonce was used by another transaction, or the node + * holds nothing at a nonce below it, so nothing will ever let it be mined. The + * entry is then simulated again like any other. While it is pending, it is + * re-sent `feeBump.afterMs` after the last send: the identical call at the + * same nonce with both fees raised (`bumpedFees`), at most `feeBump.max` + * times. When no bump is left or the cap stops one, it pages (throttled like + * every page about a release) and keeps waiting. + */ +async function followInFlight(releaser: Releaser, tx: InFlightRelease): Promise { + const { publicClient, walletClient, account, feeBump, log } = releaser; + const label = releaseLabel(tx.entry); + for (const hash of tx.hashes) { + const receipt = await publicClient.getTransactionReceipt({ hash }).catch((error: unknown) => { + if (error instanceof TransactionReceiptNotFoundError) return undefined; + throw error; + }); + if (receipt) { + releaser.inFlight = undefined; + logMined(releaser, receipt, label); + return true; + } + } + const count = (blockTag: "latest" | "pending") => publicClient.getTransactionCount({ address: account.address, blockTag }); + const mined = await count("latest"); + // Its nonce was used (by an attempt whose receipt is not visible yet, or by + // another transaction), or the node holds nothing below it: decide afresh. + if (mined > tx.nonce || (mined < tx.nonce && (await count("pending")) <= mined)) { + releaser.inFlight = undefined; + log( + `release transaction at nonce ${tx.nonce} can no longer be mined there ` + + `(account's next mined nonce ${mined}); re-evaluating: ${label}`, + ); + return true; + } + if (releaser.now() - tx.sentAtMs < feeBump.afterMs) return false; + const fees = tx.bumps < feeBump.max ? bumpedFees(tx.fees, await publicClient.estimateFeesPerGas(), feeBump) : undefined; + if (fees === undefined) { + const reason = tx.bumps >= feeBump.max ? `${tx.bumps} fee bump(s) used up` : "the fee cap stops the next bump"; + await pageThrottled( + releaser, + tx.entry.nonce, + "release transaction not mined", + `${reason}: transaction nonce ${tx.nonce}, account's next mined nonce ${mined}, ` + + `maxFeePerGas ${formatGwei(tx.fees.maxFeePerGas)} gwei (cap ${formatGwei(feeBump.maxFeePerGas)} gwei); ${label}`, + ); + return false; + } + tx.bumps += 1; + tx.fees = fees; + tx.sentAtMs = releaser.now(); + try { + const hash = await walletClient.writeContract({ ...tx.call, nonce: tx.nonce, ...fees }); + tx.hashes.push(hash); + log(`fee bump ${tx.bumps} at nonce ${tx.nonce}: ${label} tx=${hash}`); + } catch (error) { + // The earlier attempts at this nonce stay valid; the next bump goes higher. + const reason = error instanceof BaseError ? error.shortMessage : error; + log(`fee bump ${tx.bumps} at nonce ${tx.nonce} not accepted: ${label} — ${reason}`); + } + return false; +} -/** Try to push each pending release through; classify what comes back. */ +/** Try to push each pending release through; classify what comes back. One + * release transaction is in flight at a time: a later one would only queue + * behind its nonce. */ export async function drainPending(releaser: Releaser, conversionFactor: bigint, dailyCap: bigint): Promise { - const { publicClient, walletClient, account, vaultAddress, baseFinalityTag, pending, alertedAt, log } = releaser; + const { publicClient, walletClient, account, vaultAddress, baseFinalityTag, pending, alertedAt, log, feeBump } = releaser; + if (releaser.inFlight !== undefined && !(await followInFlight(releaser, releaser.inFlight))) return; for (const p of [...pending.values()]) { - const label = `nonce=${p.nonce} recipient=${p.recipient} amount=${p.palletAmount}`; + const label = releaseLabel(p); try { - // Simulate first so a still-capped release costs no gas. - const { request } = await publicClient.simulateContract({ + const release = { account, address: vaultAddress, abi: vaultAbi, functionName: "release", args: [p.nonce, p.recipient, p.palletAmount], - }); + } as const; + // Simulate first so a still-capped release costs no gas. + await publicClient.simulateContract(release); // Pad the gas limit: a release writes an ERC20Votes checkpoint for the // vault's vote sink, and whether that is an overwrite or a new entry // depends on the block timestamp the transaction lands in — a state // the estimate cannot know. An exact estimate can run out of gas in // that write (seen in phase 3); doubling it is cheap insurance. - const gasLimit = (await publicClient.estimateContractGas({ - account, - address: vaultAddress, - abi: vaultAbi, - functionName: "release", - args: [p.nonce, p.recipient, p.palletAmount], - })) * 2n; - const txHash = await walletClient.writeContract({ ...request, gas: gasLimit }); - const receipt = await publicClient.waitForTransactionReceipt({ hash: txHash }); - if (receipt.status !== "success") { - throw new MinedRevert(`release reverted on-chain: ${txHash}`); + const call = { ...release, gas: (await publicClient.estimateContractGas(release)) * 2n }; + const nonce = await publicClient.getTransactionCount({ address: account.address, blockTag: "pending" }); + const fees = cappedFees(await publicClient.estimateFeesPerGas(), feeBump.maxFeePerGas); + const sentAtMs = releaser.now(); + const hash = await walletClient.writeContract({ ...call, nonce, ...fees }); + let receipt: TransactionReceipt; + try { + receipt = await publicClient.waitForTransactionReceipt({ hash, timeout: feeBump.afterMs }); + } catch (waitError) { + // Sent, so it may be pending: followed from the next cycle on + // instead of signing another release behind its nonce. + releaser.inFlight = { entry: p, call, nonce, fees, hashes: [hash], sentAtMs, bumps: 0 }; + log( + `release not seen mined yet, following it at nonce ${nonce}: ${label} tx=${hash} — ` + + `${waitError instanceof BaseError ? waitError.shortMessage : waitError}`, + ); + return; } - // Deliberately NOT deleted here: the entry leaves the durable pending - // set only when `pruneConsumed` sees the nonce consumed at the - // finality boundary. Until then a re-attempt is a cheap simulation - // that classifies as PendingFinality and stays quiet — and if the - // release were reorged away, the entry is still ours to retry. - log(`released (awaiting ${baseFinalityTag}): ${label} tx=${txHash}`); + logMined(releaser, receipt, label); } catch (error) { - if (error instanceof MinedRevert) { - // Usually a benign race (a peer released, or the daily allowance - // was consumed between simulation and inclusion). Stay quiet: the - // next cycle prunes it if consumed and re-simulates otherwise, - // which yields a decodable, classifiable reason. - log(`release reverted on-chain, re-evaluating next cycle: ${label}`); - continue; - } const reason = error instanceof Error ? error.message : String(error); // A peer may have consumed the nonce after our simulation. Settle // that race from state before relying on decoded custom errors. From 1674bc5a2865a0262e8b4df3a22e3d2746a325c8 Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Tue, 27 Oct 2026 12:21:22 +0100 Subject: [PATCH 11/16] fix(releaser): Refuse a zero poll interval or read batch size POLL_INTERVAL_MS=0 turned the cycle loop into a busy loop against the RPC, and READ_BATCH_SIZE=0 failed every cycle that had anything pending. Require a poll interval of at least one second and a whole read batch size of at least 1 at startup. --- releaser/README.md | 4 ++-- releaser/src/config.test.ts | 14 ++++++++++++++ releaser/src/config.ts | 4 ++-- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/releaser/README.md b/releaser/README.md index cfd13a8e1..3e0dd92c0 100644 --- a/releaser/README.md +++ b/releaser/README.md @@ -34,7 +34,7 @@ consumed nonce and drops it. | `RELEASER_PRIVATE_KEY` | Gas-only signing key (no privileges) | | `START_BLOCK` | Vault deployment block — where log scanning begins on a first run | | `STATE_FILE` | Scan checkpoint + pending set (default `./releaser-state.json`) | -| `POLL_INTERVAL_MS` | Default 60s | +| `POLL_INTERVAL_MS` | Default 60s; at least 1000 | | `MAX_BLOCK_RANGE` | Max blocks per `eth_getLogs` (default 10,000; a whole number of at least 1). A range the endpoint refuses as too large is read again at half the size, down to one block, and the size doubles again after 10 full-size reads in a row, never above this value; the current size is in each cycle's `ok:` log line. Set it to the endpoint's cap to avoid the refused reads (Base Sepolia's public endpoint allows 200) | | `MIN_GAS_BALANCE_WEI` | Low-gas alert threshold | | `ALERT_WEBHOOK_URL` | Optional JSON alert webhook in Slack incoming-webhook form: `text` holds `[CRITICAL]` or `[WARNING]`, the releaser and the subject, then the detail; `severity`, `subject` and `detail` are also sent as fields. A non-2xx answer, a timeout (10 s) or a connection error is logged as a failed delivery and tried again after 2 s and after 10 s more; a 4xx other than 408 and 429 is not retried, as repeating the request cannot change it | @@ -42,7 +42,7 @@ consumed nonce and drops it. | `BASE_CHAIN_ID` | Default 8453; startup refuses a `BASE_RPC_URL` that serves another chain (`eth_chainId`) | | `BASE_FINALITY_TAG` | `safe` (default) or `finalized`; the scan cursor stays inside this boundary and a pending entry is dropped only once its nonce is consumed there | | `BLOCKED_ALERT_INTERVAL_MS` | Re-page interval for the same page about one release: governance-blocked, or failing unexpectedly (default 6h — a blocked release needs a ≥48h timelocked action, so per-poll paging would bury the signal). A different page about it goes out at once | -| `READ_BATCH_SIZE` | Bound on Multicall calldata and individual fallback concurrency (default 100) | +| `READ_BATCH_SIZE` | Bound on Multicall calldata and individual fallback concurrency (default 100; a whole number of at least 1) | | `FEE_BUMP_AFTER_SECONDS` | Re-send a release transaction that is still not mined after this long (default 60) | | `FEE_BUMP_PERCENT` | Raise both fees by at least this many percent per re-send (default 20; a whole number of at least 10, the minimum a node accepts for a replacement) | | `FEE_BUMP_MAX` | Re-sends per transaction before the `release transaction not mined` page (default 3) | diff --git a/releaser/src/config.test.ts b/releaser/src/config.test.ts index d1ed1a717..29737a671 100644 --- a/releaser/src/config.test.ts +++ b/releaser/src/config.test.ts @@ -50,6 +50,20 @@ test("fee bumping defaults to 60 s, 20 %, 3 bumps and a 1 gwei cap, and refuses } }); +test("a zero poll interval or read batch size is refused instead of looping or failing every cycle", () => { + assert.equal(loadConfig({}, "[config.pollIntervalMs, config.readBatchSize].join(' ')").stdout, "60000 100"); + for (const [name, value] of [ + ["POLL_INTERVAL_MS", "0"], + ["POLL_INTERVAL_MS", "999"], + ["READ_BATCH_SIZE", "0"], + ["READ_BATCH_SIZE", "2.5"], + ]) { + const { status, stderr } = loadConfig({ [name]: value }); + assert.notEqual(status, 0, `${name}=${value}`); + assert.match(stderr, new RegExp(`invalid ${name}`), `${name}=${value}`); + } +}); + test("MAX_BLOCK_RANGE must be a whole number of at least 1", () => { assert.equal(loadConfig({}, "String(config.maxBlockRange)").stdout, "10000"); assert.equal(loadConfig({ MAX_BLOCK_RANGE: "200" }, "String(config.maxBlockRange)").stdout, "200"); diff --git a/releaser/src/config.ts b/releaser/src/config.ts index c6c8dde81..bcd66d9e1 100644 --- a/releaser/src/config.ts +++ b/releaser/src/config.ts @@ -49,7 +49,7 @@ export const config = { /** Base block to begin scanning `ReleasePending` from on a first run — * set to the vault's deployment block. */ startBlock: BigInt(process.env.START_BLOCK ?? "0"), - pollIntervalMs: envNumber("POLL_INTERVAL_MS", 60_000), + pollIntervalMs: envNumber("POLL_INTERVAL_MS", 60_000, 1000), /** Max blocks per `eth_getLogs` call. Public RPCs cap this, often lower: * a range the endpoint refuses is read again in smaller ones. */ maxBlockRange: BigInt(envNumber("MAX_BLOCK_RANGE", 10_000, 1, true)), @@ -70,7 +70,7 @@ export const config = { * timelock, so per-poll paging would only bury the signal. */ blockedAlertIntervalMs: envNumber("BLOCKED_ALERT_INTERVAL_MS", 6 * 60 * 60 * 1000), /** Bound both Multicall calldata and individual fallback concurrency. */ - readBatchSize: envNumber("READ_BATCH_SIZE", 100), + readBatchSize: envNumber("READ_BATCH_SIZE", 100, 1, true), /** A release transaction still not mined after `afterMs` is re-sent at the * same nonce with both fees raised by `percent` (at least the 10 % a * replacement needs, and at least to the current estimate), at most `max` From 8b6e679459dc29febeab77db2029f42bdc608f55 Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Wed, 11 Nov 2026 16:24:45 +0100 Subject: [PATCH 12/16] fix(attestor): Bump and page an approval whose nonce below only looks empty The follow-up took a pending count at or below the mined count as proof that the nonce in between was empty, and started the approval over there: its bumps, pages and earlier hashes were reset, and a refused send moved it straight back up. When the backend answering the count did not hold a transaction stuck at that nonce, the approval went down and back up every bump interval and was never bumped or paged. A latest count that lagged also dropped the earlier attempts from the follow-up. Re-send the latest attempt at the empty nonce instead, keeping the bumps, pages and hashes. The approval moves there only once the node takes it; a refusal leaves it at its nonce, bumped and paged as before. Receipts are polled newest first, so once mined, the attempt that moved down is reported rather than one it left behind. --- attestor/README.md | 7 +++-- attestor/src/approval.test.ts | 51 ++++++++++++++++++++++++++++++++++- attestor/src/approval.ts | 31 +++++++++++++++------ 3 files changed, 78 insertions(+), 11 deletions(-) diff --git a/attestor/README.md b/attestor/README.md index 6c83500ea..33f129f85 100644 --- a/attestor/README.md +++ b/attestor/README.md @@ -135,8 +135,11 @@ WantedBy=multi-user.target the node holds nothing at a nonce below it (a transaction of ours dropped by a reorg, or a skipped one that left the pool) is sent again at that first empty nonce, or the next approval is, so nothing waits on a nonce no - transaction will fill. A node refusing a send for its price, an - underpriced replacement or a full pool is retried at the next bump instead + transaction will fill. It moves there, keeping its bumps and pages, only + once the node takes it there: a count from a lagging backend can show a + nonce empty that a transaction does hold, and a refused approval stays at + its nonce, bumped and paged as before. A node refusing a send for its + price, an underpriced replacement or a full pool is retried at the next bump instead of being fatal. Transactions in flight are kept in memory only: after a restart, an approval whose transaction is still pending is sent again at the next nonce, so a pending transaction stuck below the cap has to be replaced diff --git a/attestor/src/approval.test.ts b/attestor/src/approval.test.ts index bdb5cbdf8..f2b7c6377 100644 --- a/attestor/src/approval.test.ts +++ b/attestor/src/approval.test.ts @@ -1184,7 +1184,8 @@ test("an own approval reorged out of Base leaves no empty nonce the next approva ); assert.ok(chain.approved.has(payloadHash(OTHER))); assert.equal(chain.receipts.get(chain.sent[1].hash)?.status, "reverted", "the attempt left behind merely reverts"); - assert.ok(logs.some((line) => line.startsWith("nonce 5 below 6 holds no transaction; sending the approval again there: nonce=8 "))); + assert.ok(logs.some((line) => line.startsWith("nonce 5 below 6 holds no transaction; sent the approval again there: nonce=8 "))); + assert.ok(logs.some((line) => line.startsWith("approved: nonce=8 ") && line.endsWith(`tx=${chain.sent[2].hash}`))); assert.equal(alerts.length, 0); }); @@ -1235,6 +1236,54 @@ test("an approval no longer needed behind an empty nonce is not sent again, and assert.ok(chain.approved.has(payloadHash(OTHER))); }); +test("a nonce below that the pending count shows empty but a stuck transaction holds leaves the approval to be bumped and paged", async () => { + const chain = new FakeVault(); + // A transaction from before a restart, stuck below the base fee with a higher tip than the estimate. + const stuck: Sent = { hash: `0x${"ee".repeat(32)}`, nonce: 5, value: 0n, maxFeePerGas: 50_000_000n, maxPriorityFeePerGas: 2_000_000n }; + chain.pool.set(5, stuck); + chain.baseFee = 100_000_000n; + chain.pendingNonce = 5; // the backend answering the count does not hold it + const { attestor, alerts, logs } = attestorOn(chain, {}, (elapsed) => { + if (elapsed >= 6 * 3600_000) chain.baseFee = 5_000_000n; // fees fall after 6 h + chain.mine(); + }); + assert.equal(await approve(attestor, EVENT), true); + assert.ok(chain.sent.every((tx) => tx.nonce === 5 || tx.nonce === 6)); + assert.ok(!logs.some((line) => line.includes("holds no transaction")), "never moved down to nonce 5"); + assert.equal(logs.filter((line) => /^fee bump \d at nonce 6: /.test(line)).length, 3); + assert.ok(alerts.length > 1 && alerts.every((a) => a.subject === "own approval not mined")); + assert.match(alerts[0].detail, /^3 fee bump\(s\) used up: transaction nonce 6, account's next mined nonce 5, /); + assert.ok(chain.receipts.has(stuck.hash), "the stuck transaction kept its nonce"); + assert.ok(chain.approved.has(payloadHash(EVENT))); +}); + +test("a latest count that lags behind keeps the approval at its nonce and its earlier attempts tracked", async () => { + const chain = new FakeVault(); + chain.baseFee = 9_000_000n; // above the first attempt and the first bump + // No room in the pool for a second bump. + chain.refuse = (_, tx) => (tx.maxFeePerGas > 8_400_000n ? "txpool is full" : undefined); + let lagging = false; + const accountNonce = chain.accountNonce.bind(chain); + // A backend a block behind: nonce 4 not yet mined there, and nothing of ours held. + chain.accountNonce = async (blockTag) => (lagging ? 4 : accountNonce(blockTag)); + const { attestor, logs } = attestorOn(chain, {}, (elapsed) => { + lagging = elapsed > 1.5 * AFTER_MS && elapsed <= 2 * AFTER_MS; + if (elapsed > 2 * AFTER_MS) chain.baseFee = 5_000_000n; + chain.mine(); + }); + assert.equal(await approve(attestor, EVENT), true); + assert.deepEqual( + chain.sent.map((tx) => [tx.nonce, tx.maxFeePerGas]), + [ + [5, 7_000_000n], + [5, 8_400_000n], + [4, 8_400_000n], + [5, 10_080_000n], + ], + ); + assert.ok(logs.some((line) => line.startsWith("approved: nonce=7 ") && line.endsWith(`tx=${chain.sent[1].hash}`))); +}); + test("a signer removed, or removed and re-added, while running stops the daemon instead of going on", async () => { const chain = new FakeVault(); const { attestor } = attestorOn(chain); diff --git a/attestor/src/approval.ts b/attestor/src/approval.ts index b8e12d2db..eb607060f 100644 --- a/attestor/src/approval.ts +++ b/attestor/src/approval.ts @@ -82,7 +82,8 @@ export interface Submission { nonce?: number; /** The latest attempt's fees. */ fees?: Fees; - /** Every attempt at `nonce`; any of them may be the one that is mined. */ + /** Every attempt at `nonce`, and those left above it when the approval + * moved down to an empty nonce; any of them may be the one that is mined. */ hashes: Hash[]; /** When the latest attempt was sent: a follow-up an error interrupted * resumes the bump interval from there instead of starting it anew. */ @@ -300,11 +301,13 @@ export function bumpedFees(last: Fees, estimate: Fees, policy: FeeBumpPolicy): F return replaces ? fees : undefined; } -/** Poll the receipts of `hashes` until one is mined or `timeoutMs` has passed. */ +/** Poll the receipts of `hashes` until one is mined or `timeoutMs` has passed. + * Newest first: an attempt moved down to an empty nonce is mined before the + * ones it left above, which then merely revert. */ async function minedReceipt(attestor: Attestor, hashes: Hash[], timeoutMs: number): Promise { const deadline = attestor.now() + timeoutMs; for (;;) { - for (const hash of hashes) { + for (const hash of [...hashes].reverse()) { const receipt = await attestor.chain.receipt(hash); if (receipt) return receipt; } @@ -432,11 +435,23 @@ async function sendUntilMined( // one this process skipped left the pool. Fill that nonce instead of // waiting on it, for this approval or else the next one; the attempt // left behind merely reverts should it ever be mined. - attestor.nextNonce = minedNonce; - if (!needed) return undefined; - attestor.log(`nonce ${minedNonce} below ${nonce} holds no transaction; sending the approval again there: ${label}`); - Object.assign(submission, { nonce: undefined, fees: undefined, hashes: [], bumps: 0, pages: 0 }); - continue; + if (!needed) { + attestor.nextNonce = minedNonce; + return undefined; + } + // Counts from a backend that lags can hide a transaction that does + // hold that nonce, so the latest attempt moves down only once the node + // takes it there, with its bumps and pages; refused, it stays where it + // is and is bumped and paged as before. + const { hash, error } = await sendAttempt(attestor, submission, minedNonce, submission.fees as Fees); + if (error === undefined) { + attestor.log(`nonce ${minedNonce} below ${nonce} holds no transaction; sent the approval again there: ${label} tx=${hash}`); + submission.nonce = minedNonce; + submission.hashes.push(hash); + attestor.nextNonce = minedNonce + 1; + continue; + } + attestor.log(`approval not accepted at nonce ${minedNonce}, so it stays at nonce ${nonce}: ${label}`, error); } const last = submission.fees as Fees; const fees = From 2947fc92a3750bc5e506e20f8c5beb4a2855c65a Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Wed, 11 Nov 2026 16:26:25 +0100 Subject: [PATCH 13/16] fix(attestor): Refuse a checkpoint without the signer fields A checkpoint with neither attestorAddress nor attestorGeneration loaded as one from before the fields existed and was bound to whichever key started on it, so that key resumed from progress nothing tied to its approvals. Refuse it like a checkpoint with only one of the fields. The error says to restore removed fields, or else to backfill as a new key does (runbook RB-6). --- attestor/README.md | 5 ++--- attestor/src/main.ts | 4 ---- attestor/src/state.test.ts | 21 +++++++++++---------- attestor/src/state.ts | 26 +++++++++++++++----------- 4 files changed, 28 insertions(+), 28 deletions(-) diff --git a/attestor/README.md b/attestor/README.md index 33f129f85..c58203e18 100644 --- a/attestor/README.md +++ b/attestor/README.md @@ -155,9 +155,8 @@ WantedBy=multi-user.target with no checkpoint and `START_BLOCK` before the oldest unreleased migration; a re-added key lowers `lastProcessedBlock` to such a block and sets `attestorGeneration` to the current value, keeping every other field. - Checkpoints written before these fields existed (neither field present) are - bound to the first key that starts on them; a checkpoint with only one of - the two was edited and is refused. + A checkpoint without these fields, or with only one of them, was edited or + not written by this daemon and is refused; the error says how to recover. - The daemon verifies at startup that its address is in the vault's attestor set, that `BASE_RPC_URL` serves `BASE_CHAIN_ID` (`eth_chainId`, which the checkpoint's Base chain must match too) and that `PENDULUM_WS` serves diff --git a/attestor/src/main.ts b/attestor/src/main.ts index 9a0318d6d..6f9985ed1 100644 --- a/attestor/src/main.ts +++ b/attestor/src/main.ts @@ -152,10 +152,6 @@ async function main(): Promise { attestorAddress: account.address, attestorGeneration, }); - // A checkpoint written before the signer was recorded is bound to it from - // the next save on. - checkpoint.attestorAddress ??= account.address; - checkpoint.attestorGeneration ??= attestorGeneration; log(`attestor ${account.address} (generation ${attestorGeneration}) starting after block ${checkpoint.lastProcessedBlock}`); /** Blocks processed at the latest Base state, oldest first, that the diff --git a/attestor/src/state.test.ts b/attestor/src/state.test.ts index 85c774d0f..c65ecb39f 100644 --- a/attestor/src/state.test.ts +++ b/attestor/src/state.test.ts @@ -66,15 +66,16 @@ test("a checkpoint made by another key or an earlier generation of this key is r ); }); -test("a checkpoint from before the signer was recorded still loads", () => { +test("a checkpoint without the signer fields is refused, not bound to the key that starts on it", () => { const directory = mkdtempSync(join(tmpdir(), "pen-attestor-state-")); const path = join(directory, "checkpoint.json"); try { - const { attestorAddress, attestorGeneration, ...legacy } = checkpoint; - writeFileSync(path, JSON.stringify(legacy)); - const loaded = loadCheckpoint(path); - assert.deepEqual(loaded, legacy); - assert.doesNotThrow(() => assertCheckpointIdentity(loaded!, { ...legacy, attestorAddress, attestorGeneration })); + const { attestorAddress, attestorGeneration, ...unbound } = checkpoint; + writeFileSync(path, JSON.stringify(unbound)); + assert.throws( + () => assertCheckpointIdentity(loadCheckpoint(path)!, { ...unbound, attestorAddress, attestorGeneration }), + /has neither attestorAddress nor attestorGeneration, .*\(runbook RB-6\): .*start with no checkpoint/, + ); writeFileSync(path, JSON.stringify({ ...checkpoint, attestorGeneration: 0 })); assert.throws(() => loadCheckpoint(path), /invalid or legacy schema/); writeFileSync(path, JSON.stringify({ ...checkpoint, attestorAddress: [checkpoint.attestorAddress] })); @@ -84,17 +85,17 @@ test("a checkpoint from before the signer was recorded still loads", () => { } }); -test("a checkpoint with only one of the signer fields is refused, not taken for one from before they existed", () => { +test("a checkpoint with only one of the signer fields is refused as edited", () => { const directory = mkdtempSync(join(tmpdir(), "pen-attestor-state-")); const path = join(directory, "checkpoint.json"); try { - const { attestorAddress, attestorGeneration, ...legacy } = checkpoint; - writeFileSync(path, JSON.stringify({ ...legacy, attestorAddress })); + const { attestorAddress, attestorGeneration, ...unbound } = checkpoint; + writeFileSync(path, JSON.stringify({ ...unbound, attestorAddress })); assert.throws( () => assertCheckpointIdentity(loadCheckpoint(path)!, { ...checkpoint, attestorGeneration: 2 }), /has attestorAddress but no attestorGeneration; .*\(runbook RB-6\)/, ); - writeFileSync(path, JSON.stringify({ ...legacy, attestorGeneration })); + writeFileSync(path, JSON.stringify({ ...unbound, attestorGeneration })); assert.throws( () => assertCheckpointIdentity(loadCheckpoint(path)!, { diff --git a/attestor/src/state.ts b/attestor/src/state.ts index 47d02a176..2ac89687d 100644 --- a/attestor/src/state.ts +++ b/attestor/src/state.ts @@ -15,9 +15,8 @@ export interface Checkpoint { vaultAddress: `0x${string}`; pendulumGenesisHash: `0x${string}`; /** The signer, and its vault attestor generation, whose approvals this - * progress stands for. Optional only so that checkpoints written before - * they were recorded still load (neither field); the daemon binds those on - * startup. */ + * progress stands for. Optional only so that a file without them loads + * and `assertCheckpointIdentity` refuses it with the way out. */ attestorAddress?: `0x${string}`; attestorGeneration?: number; lastProcessedBlock: number; @@ -137,9 +136,17 @@ export function assertCheckpointIdentity( // Progress made by another key, or by this key before it was removed and // re-added, says nothing about the approvals that count now: resuming from // it would silently skip every unreleased migration before it. - // Both fields are always written together, so a checkpoint with only one of - // them was edited and is not one from before they existed. - if ((checkpoint.attestorAddress === undefined) !== (checkpoint.attestorGeneration === undefined)) { + // Both fields are always written together, so a checkpoint without them, + // or with only one of them, was edited or not written by this daemon. + if (checkpoint.attestorAddress === undefined && checkpoint.attestorGeneration === undefined) { + throw new Error( + "checkpoint has neither attestorAddress nor attestorGeneration, so nothing shows whose approvals its " + + "progress stands for: restore both if they were removed and follow the error that gives, or else " + + "backfill as a new key does (runbook RB-6): move the checkpoint file aside and start with no checkpoint " + + "and START_BLOCK before the oldest unreleased migration", + ); + } + if (checkpoint.attestorAddress === undefined || checkpoint.attestorGeneration === undefined) { const [present, missing] = checkpoint.attestorAddress === undefined ? ["attestorGeneration", "attestorAddress"] @@ -149,17 +156,14 @@ export function assertCheckpointIdentity( "restore the removed field and follow the error that gives (runbook RB-6), never remove either one", ); } - if ( - checkpoint.attestorAddress !== undefined && - checkpoint.attestorAddress.toLowerCase() !== expected.attestorAddress.toLowerCase() - ) { + if (checkpoint.attestorAddress.toLowerCase() !== expected.attestorAddress.toLowerCase()) { throw new Error( `checkpoint belongs to attestor ${checkpoint.attestorAddress}, but this daemon signs as ${expected.attestorAddress}; ` + "a new key must backfill (runbook RB-6): move the checkpoint file aside and start with no checkpoint " + "and START_BLOCK before the oldest unreleased migration", ); } - if (checkpoint.attestorGeneration !== undefined && checkpoint.attestorGeneration !== expected.attestorGeneration) { + if (checkpoint.attestorGeneration !== expected.attestorGeneration) { throw new Error( `checkpoint was written under attestor generation ${checkpoint.attestorGeneration}, but ` + `${expected.attestorAddress} is now at generation ${expected.attestorGeneration}, so its earlier approvals ` + From 39a78e86bab2f57239c55612b808e9810ff89ea9 Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Wed, 11 Nov 2026 16:29:11 +0100 Subject: [PATCH 14/16] fix(releaser): Send a stuck release again unchanged once bumps stop Once its bumps were used up, or the fee cap stopped the next one, a release transaction in flight was never sent again. One the pool then dropped, leaving the account's latest and pending counts both at its nonce, was neither settled nor sent again, so every deferred release waited behind it until a restart. When it is not bumped, send the latest attempt again unchanged each cycle, as the attestor does, so a pool that dropped it takes it back. The answers "already known" and "nonce too low" are expected and not logged; any other failure is. --- releaser/README.md | 10 ++-- releaser/src/release.test.ts | 97 +++++++++++++++++++++++++++++++++--- releaser/src/release.ts | 20 +++++++- 3 files changed, 114 insertions(+), 13 deletions(-) diff --git a/releaser/README.md b/releaser/README.md index 3e0dd92c0..9187669d1 100644 --- a/releaser/README.md +++ b/releaser/README.md @@ -83,10 +83,12 @@ identical call at the same nonce with both fees raised by `FEE_BUMP_PERCENT`, and at least to the current estimate, at most `FEE_BUMP_MAX` times. When no bump is left, or the cap stops one, it pages `release transaction not mined` (with the nonce, the account's next mined nonce and the fees; throttled to -`BLOCKED_ALERT_INTERVAL_MS`) and keeps waiting. Once another transaction uses -its nonce, or the node holds nothing at a nonce below it, the release is -simulated and sent again. The transaction in flight is kept in memory only: -after a restart, a pending one stuck below the cap has to be replaced by hand. +`BLOCKED_ALERT_INTERVAL_MS`) and keeps waiting, sending the last attempt +again unchanged every cycle, so a pool that dropped it gets it back. Once +another transaction uses its nonce, or the node holds nothing at a nonce +below it, the release is simulated and sent again. The transaction in flight +is kept in memory only: after a restart, a pending one stuck below the cap +has to be replaced by hand. State (safe/finalized scan checkpoint + pending set) is atomically persisted and bound to the configured chain and vault, so a restart resumes without diff --git a/releaser/src/release.test.ts b/releaser/src/release.test.ts index 75ceb03e1..e5ac6f021 100644 --- a/releaser/src/release.test.ts +++ b/releaser/src/release.test.ts @@ -4,12 +4,16 @@ import { ContractFunctionExecutionError, ContractFunctionRevertedError, encodeErrorResult, + getContractError, + InvalidInputRpcError, + RpcRequestError, TransactionReceiptNotFoundError, WaitForTransactionReceiptTimeoutError, type Hash, type PublicClient, } from "viem"; import { privateKeyToAccount } from "viem/accounts"; +import { getTransactionError } from "viem/utils"; import { drainPending, type InFlightRelease, type PendingRelease, type Releaser } from "./release.js"; import { vaultAbi } from "./vaultAbi.js"; @@ -27,7 +31,8 @@ const txHash = (index: number): Hash => `0x${(index + 1).toString(16).padStart(6 /** A releaser whose public client answers each call with `answers`. Unless * overridden: the finality boundary is at block 100, every nonce unconsumed, * a release simulates fine, the account's nonce is 4, and a sent release is - * mined at once. */ + * mined at once. A `writeContract` answer is the node's to each send, given + * the call and its hash: it may throw a refusal. */ function releaserWith(answers: Record, entries: PendingRelease[]) { const alerts: string[] = []; const logs: string[] = []; @@ -50,7 +55,9 @@ function releaserWith(answers: Record, entries: PendingRelease[] walletClient: { writeContract: async (call: Write) => { writes.push(call); - return txHash(writes.length - 1); + const hash = txHash(writes.length - 1); + await methods.writeContract?.({ ...call, hash }); + return hash; }, } as unknown as Releaser["walletClient"], feeBump: { afterMs: 60_000, percent: 20, max: 3, maxFeePerGas: 1_000_000_000n }, @@ -84,6 +91,15 @@ function vaultRevert( const dailyCapRevert = () => vaultRevert({ errorName: "ExceedsDailyCap", args: [DAILY_CAP + 1n, 0n] }); +/** What writeContract raises when the node refuses the transaction. */ +const sendRefusal = (message: string) => + getContractError( + getTransactionError(new InvalidInputRpcError(new RpcRequestError({ body: {}, error: { code: -32000, message }, url: "http://rpc" })), { + account: null, + }), + { abi: vaultAbi, functionName: "release", args: [] }, + ); + const ENTRY = { nonce: 7n, recipient: RECIPIENT, palletAmount: 5_000n * 10n ** 12n } as const; test("a successful release keeps its entry until the nonce reads consumed at the finality boundary", async () => { @@ -129,9 +145,9 @@ const notMined: Record = { }, }; -test("a release not mined in time is followed at its nonce: nothing is sent behind it, it is bumped, then paged", async () => { +test("a release not mined in time is followed at its nonce: nothing is sent behind it, it is bumped, then paged and sent again unchanged", async () => { let minedHash: Hash | undefined; - const { releaser, alerts, writes, advance } = releaserWith( + const { releaser, alerts, logs, writes, advance } = releaserWith( { ...notMined, simulateContract: ({ args }) => { @@ -145,6 +161,11 @@ test("a release not mined in time is followed at its nonce: nothing is sent behi if (hash !== minedHash) throw new TransactionReceiptNotFoundError({ hash: hash as Hash }); return { status: "success", transactionHash: hash }; }, + // The node still holds the last bump, then has the second one mined. + writeContract: () => { + if (writes.length === 5) throw sendRefusal("already known"); + if (writes.length === 6) throw sendRefusal("nonce too low"); + }, }, [ENTRY, { ...ENTRY, nonce: 8n }], ); @@ -157,7 +178,8 @@ test("a release not mined in time is followed at its nonce: nothing is sent behi advance(60_000); await drain(); } - // The identical call at the same nonce, both fees raised, three times. + // The identical call at the same nonce, both fees raised, three times; then + // the last one again, unchanged, every cycle. assert.deepEqual( writes.map((w) => [w.args[0], w.nonce, w.gas, w.maxFeePerGas, w.maxPriorityFeePerGas]), [ @@ -165,19 +187,23 @@ test("a release not mined in time is followed at its nonce: nothing is sent behi [7n, 4, 100_000n, 8_400_000n, 1_200_000n], [7n, 4, 100_000n, 10_080_000n, 1_440_000n], [7n, 4, 100_000n, 12_096_000n, 1_728_000n], + [7n, 4, 100_000n, 12_096_000n, 1_728_000n], + [7n, 4, 100_000n, 12_096_000n, 1_728_000n], ], ); - // Then one page, not repeated within the interval. + // One page, not repeated within the interval; the node's answers to the + // sends again are no failure. assert.deepEqual(alerts, [ `critical release transaction not mined: 3 fee bump(s) used up: transaction nonce 4, account's next mined nonce 4, ` + `maxFeePerGas 0.012096 gwei (cap 1 gwei); nonce=7 recipient=${RECIPIENT} amount=${ENTRY.palletAmount}`, ]); + assert.ok(!logs.some((line) => line.startsWith("re-sending "))); // The second bump is mined: its entry waits for the boundary, and the next release goes out. minedHash = txHash(2); await drain(); assert.deepEqual( writes.map((w) => w.args[0]), - [7n, 7n, 7n, 7n, 8n], + [7n, 7n, 7n, 7n, 7n, 7n, 8n], ); assert.equal(releaser.inFlight?.entry.nonce, 8n); assert.equal(releaser.pending.size, 2); @@ -214,18 +240,73 @@ test("the fee cap stops a release's bump and pages; no attempt offers more than await drainPending(releaser, CONVERSION_FACTOR, DAILY_CAP); advance(60_000); } - // 7 → 8.4 is capped at 8; 8 → 8 is no replacement a pool takes. + // 7 → 8.4 is capped at 8; 8 → 8 is no replacement a pool takes, so the + // capped attempt is sent again unchanged. assert.deepEqual( writes.map((w) => [w.maxFeePerGas, w.maxPriorityFeePerGas]), [ [7_000_000n, 1_000_000n], [8_000_000n, 1_200_000n], + [8_000_000n, 1_200_000n], ], ); assert.equal(alerts.length, 1); assert.match(alerts[0], /^critical release transaction not mined: the fee cap stops the next bump: transaction nonce 4, .*\(cap 0\.008 gwei\); nonce=7 /); }); +test("a release the pool dropped after its last bump is sent again unchanged, so the releases behind it go out", async () => { + const mined = new Set(); + const receipt = (hash: Hash, missing: Error) => { + if (!mined.has(hash)) throw missing; + return { status: "success", transactionHash: hash }; + }; + let feesFell = false; + const { releaser, alerts, logs, writes, advance } = releaserWith( + { + simulateContract: ({ args }) => { + if (mined.size > 0 && (args as bigint[])[0] === ENTRY.nonce) { + throw vaultRevert({ errorName: "NonceAlreadyConsumed", args: [ENTRY.nonce] }); + } + return {}; + }, + // Dropped from the pool: the node counts nothing pending at its nonce. + getTransactionCount: () => 4 + mined.size, + getTransactionReceipt: ({ hash }) => receipt(hash as Hash, new TransactionReceiptNotFoundError({ hash: hash as Hash })), + waitForTransactionReceipt: ({ hash }) => + receipt(hash as Hash, new WaitForTransactionReceiptTimeoutError({ hash: hash as Hash })), + // Nothing is mined until fees fall; then whatever is sent is. + writeContract: ({ hash }) => { + if (feesFell) mined.add(hash as Hash); + }, + }, + [ENTRY, { ...ENTRY, nonce: 8n }], + ); + for (let minute = 0; minute < 7; minute++) { + feesFell = minute >= 5; + await drainPending(releaser, CONVERSION_FACTOR, DAILY_CAP); + advance(60_000); + } + assert.deepEqual( + writes.map((w) => [w.args[0], w.nonce, w.maxFeePerGas]), + [ + [7n, 4, 7_000_000n], + [7n, 4, 8_400_000n], + [7n, 4, 10_080_000n], + [7n, 4, 12_096_000n], + [7n, 4, 12_096_000n], + [7n, 4, 12_096_000n], + [8n, 5, 7_000_000n], + ], + ); + assert.ok(logs.some((line) => line.startsWith("released (awaiting safe): nonce=7 ") && line.endsWith(`tx=${txHash(5)}`))); + assert.ok(logs.some((line) => line.startsWith("released (awaiting safe): nonce=8 "))); + assert.deepEqual( + alerts.map((a) => a.split(":")[0]), + ["critical release transaction not mined"], + ); + assert.equal(releaser.inFlight, undefined); +}); + test("a daily-cap revert pages as blocked when the token amount is above the daily cap itself, and waits quietly otherwise", async () => { // Both pallet amounts are far below the token-denominated cap. const above = { nonce: 7n, recipient: RECIPIENT, palletAmount: 20_000n * 10n ** 12n } as const; diff --git a/releaser/src/release.ts b/releaser/src/release.ts index 5beb76301..5b4135d5d 100644 --- a/releaser/src/release.ts +++ b/releaser/src/release.ts @@ -9,6 +9,7 @@ import { BaseError, formatGwei, + NonceTooLowError, TransactionReceiptNotFoundError, type Account, type Chain, @@ -159,7 +160,9 @@ function logMined(releaser: Releaser, receipt: TransactionReceipt, label: string * re-sent `feeBump.afterMs` after the last send: the identical call at the * same nonce with both fees raised (`bumpedFees`), at most `feeBump.max` * times. When no bump is left or the cap stops one, it pages (throttled like - * every page about a release) and keeps waiting. + * every page about a release) and keeps waiting, sending the latest attempt + * again unchanged each cycle: that is no bump, but it brings back a + * transaction the pool dropped, which nothing else would ever send again. */ async function followInFlight(releaser: Releaser, tx: InFlightRelease): Promise { const { publicClient, walletClient, account, feeBump, log } = releaser; @@ -198,6 +201,21 @@ async function followInFlight(releaser: Releaser, tx: InFlightRelease): Promise< `${reason}: transaction nonce ${tx.nonce}, account's next mined nonce ${mined}, ` + `maxFeePerGas ${formatGwei(tx.fees.maxFeePerGas)} gwei (cap ${formatGwei(feeBump.maxFeePerGas)} gwei); ${label}`, ); + try { + const hash = await walletClient.writeContract({ ...tx.call, nonce: tx.nonce, ...tx.fees }); + // New only when the last bump was refused: that is the attempt sent again. + if (!tx.hashes.includes(hash)) tx.hashes.push(hash); + } catch (error) { + // A node that still holds it answers that it knows it, and one that + // saw its nonce used that the nonce is too low; viem raises both as + // NonceTooLowError, and neither is a failure. + if (!(error instanceof BaseError && error.walk((cause) => cause instanceof NonceTooLowError))) { + log( + `re-sending the latest attempt at nonce ${tx.nonce} failed: ${label} — ` + + `${error instanceof BaseError ? error.shortMessage : error}`, + ); + } + } return false; } tx.bumps += 1; From d3f3791cdebe4f458a5005a701efb963e5d5a577 Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Wed, 11 Nov 2026 16:32:55 +0100 Subject: [PATCH 15/16] fix(releaser): Throttle each release page per reason, not only per subject The page throttle kept one entry per release, compared by subject only. An unexpected release failure whose reason changed (another revert, or an RPC error instead of a revert) stayed muted for BLOCKED_ALERT_INTERVAL_MS, and a transient RPC error that paged first muted the real failure that followed it for as long. Key each page by its subject and a normalized reason, each with its own interval: for an unexpected failure the revert (its reason, error name or selector), or else the names of the errors it was raised through, never the message, which names requests and endpoints. A blocked page is keyed by its error name, and the not-mined page by why bumping stopped. --- releaser/README.md | 4 +-- releaser/src/checks.ts | 15 +++++++++++ releaser/src/release.test.ts | 48 ++++++++++++++++++++++++++++-------- releaser/src/release.ts | 47 ++++++++++++++++++++++++----------- 4 files changed, 88 insertions(+), 26 deletions(-) diff --git a/releaser/README.md b/releaser/README.md index 9187669d1..1e7a69ccd 100644 --- a/releaser/README.md +++ b/releaser/README.md @@ -41,7 +41,7 @@ consumed nonce and drops it. | `HEARTBEAT_URL` | Optional http(s) URL requested (GET) after a cycle that completed without error, at most once a minute; point it at a dead-man's switch that pages when the requests stop. Startup refuses any other value. A failed request is logged (error name and code only, never the URL), never fatal | | `BASE_CHAIN_ID` | Default 8453; startup refuses a `BASE_RPC_URL` that serves another chain (`eth_chainId`) | | `BASE_FINALITY_TAG` | `safe` (default) or `finalized`; the scan cursor stays inside this boundary and a pending entry is dropped only once its nonce is consumed there | -| `BLOCKED_ALERT_INTERVAL_MS` | Re-page interval for the same page about one release: governance-blocked, or failing unexpectedly (default 6h — a blocked release needs a ≥48h timelocked action, so per-poll paging would bury the signal). A different page about it goes out at once | +| `BLOCKED_ALERT_INTERVAL_MS` | Re-page interval for the same page about one release, for the same reason: governance-blocked, or failing unexpectedly (default 6h — a blocked release needs a ≥48h timelocked action, so per-poll paging would bury the signal). A different page about it, or the same page for another reason, goes out at once; each reason keeps its own interval | | `READ_BATCH_SIZE` | Bound on Multicall calldata and individual fallback concurrency (default 100; a whole number of at least 1) | | `FEE_BUMP_AFTER_SECONDS` | Re-send a release transaction that is still not mined after this long (default 60) | | `FEE_BUMP_PERCENT` | Raise both fees by at least this many percent per re-send (default 20; a whole number of at least 10, the minimum a node accepts for a replacement) | @@ -67,7 +67,7 @@ Failures are classified rather than treated alike: | `ExceedsDailyCap`, `EnforcedPause`, `NotEnoughApprovals` | **Retry quietly** — expected temporary conditions | | `NonceAlreadyConsumed` (at the finality boundary) | **Done** — drop it | | `ExceedsPerReleaseCap`, `InsufficientVaultBalance` | **Alert, throttled** to `BLOCKED_ALERT_INTERVAL_MS` — cannot self-heal without governance or operator action | -| anything else | **Alert** `unexpected release failure`, **throttled** to `BLOCKED_ALERT_INTERVAL_MS` per release | +| anything else | **Alert** `unexpected release failure`, **throttled** to `BLOCKED_ALERT_INTERVAL_MS` per release and reason: the revert, or the kind of error, never its text (which names requests and endpoints), so a transient RPC error neither mutes a revert nor makes it page again | A successful `release()` does **not** drop the entry immediately: it leaves the durable pending set only once the nonce reads as consumed at the diff --git a/releaser/src/checks.ts b/releaser/src/checks.ts index a3d82decd..1b4a827a5 100644 --- a/releaser/src/checks.ts +++ b/releaser/src/checks.ts @@ -67,6 +67,21 @@ export function classifyReleaseFailure(errorName: string | undefined): ReleaseOu return "unexpected"; } +/** What a failed release is about, without the volatile parts of its text + * (request ids, URLs, block numbers): the vault's revert (the reason it gave, + * its error name, or the selector of an error the ABI lacks), or else the + * names of the errors it was raised through. */ +export function failureReason(error: unknown): string { + const names: string[] = []; + for (let cause: unknown = error, depth = 0; cause instanceof Error && depth < 10; cause = cause.cause, depth++) { + if (cause instanceof ContractFunctionRevertedError) { + return `reverted: ${cause.reason ?? cause.data?.errorName ?? cause.signature ?? "no reason"}`; + } + names.push(cause.name); + } + return names.join(" < ") || String(error); +} + /** Divide work into bounded batches. Both Multicall calldata and individual * fallback concurrency use the same explicit limit. */ export function chunks(items: T[], size: number): T[][] { diff --git a/releaser/src/release.test.ts b/releaser/src/release.test.ts index e5ac6f021..f6d70cfbb 100644 --- a/releaser/src/release.test.ts +++ b/releaser/src/release.test.ts @@ -5,6 +5,7 @@ import { ContractFunctionRevertedError, encodeErrorResult, getContractError, + HttpRequestError, InvalidInputRpcError, RpcRequestError, TransactionReceiptNotFoundError, @@ -91,6 +92,18 @@ function vaultRevert( const dailyCapRevert = () => vaultRevert({ errorName: "ExceedsDailyCap", args: [DAILY_CAP + 1n, 0n] }); +/** What simulation raises for a revert with a reason string. */ +const revertedWith = (reason: string) => + new ContractFunctionRevertedError({ + abi: vaultAbi, + data: encodeErrorResult({ + abi: [{ type: "error", name: "Error", inputs: [{ name: "message", type: "string" }] }], + errorName: "Error", + args: [reason], + }), + functionName: "release", + }); + /** What writeContract raises when the node refuses the transaction. */ const sendRefusal = (message: string) => getContractError( @@ -381,8 +394,8 @@ test("a blocked release pages once per interval, not once per cycle", async () = assert.equal(releaser.pending.size, 1); }); -test("an unexpected release failure pages once per interval, and again at once when the failure changes", async () => { - let failure: Error = new Error("execution reverted: TokenNotSet"); +test("an unexpected release failure pages once per interval for each reason, and again at once when the failure changes", async () => { + let failure: Error = revertedWith("TokenNotSet"); const { releaser, alerts, advance } = releaserWith( { simulateContract: () => { @@ -391,16 +404,31 @@ test("an unexpected release failure pages once per interval, and again at once w }, [ENTRY], ); - for (let cycle = 0; cycle < 5; cycle++) await drainPending(releaser, CONVERSION_FACTOR, DAILY_CAP); - assert.deepEqual(alerts, [ - `critical unexpected release failure: nonce=7 recipient=${RECIPIENT} amount=${ENTRY.palletAmount} — execution reverted: TokenNotSet`, - ]); + const drain = () => drainPending(releaser, CONVERSION_FACTOR, DAILY_CAP); + for (let cycle = 0; cycle < 5; cycle++) await drain(); + assert.equal(alerts.length, 1); + assert.match(alerts[0], /^critical unexpected release failure: nonce=7 .*reason:\nTokenNotSet\n/); advance(releaser.blockedAlertIntervalMs); - await drainPending(releaser, CONVERSION_FACTOR, DAILY_CAP); + await drain(); assert.equal(alerts.length, 2); - failure = vaultRevert({ errorName: "ExceedsPerReleaseCap", args: [ENTRY.palletAmount * CONVERSION_FACTOR, 1n] }); - await drainPending(releaser, CONVERSION_FACTOR, DAILY_CAP); + // Another revert pages at once. + failure = revertedWith("Pausable: paused"); + await drain(); assert.equal(alerts.length, 3); - assert.match(alerts[2], /^critical release blocked and needs operator action: nonce=7 /); + // So does an RPC error, once, whatever endpoint and request its text names. + // Each reason keeps its own interval: the RPC error neither mutes the revert + // nor makes it page again. + for (const id of [1, 2]) { + failure = new HttpRequestError({ url: `http://rpc-${id}`, status: 503, details: `request ${id}` }); + await drain(); + } + failure = revertedWith("Pausable: paused"); + await drain(); + assert.equal(alerts.length, 4); + assert.match(alerts[3], /^critical unexpected release failure: nonce=7 .* — HTTP request failed\./); + failure = vaultRevert({ errorName: "ExceedsPerReleaseCap", args: [ENTRY.palletAmount * CONVERSION_FACTOR, 1n] }); + await drain(); + assert.equal(alerts.length, 5); + assert.match(alerts[4], /^critical release blocked and needs operator action: nonce=7 /); assert.equal(releaser.pending.size, 1); }); diff --git a/releaser/src/release.ts b/releaser/src/release.ts index 5b4135d5d..acec0f49e 100644 --- a/releaser/src/release.ts +++ b/releaser/src/release.ts @@ -20,7 +20,7 @@ import { type WalletClient, } from "viem"; import type { Severity } from "./alert.js"; -import { classifyReleaseFailure, contractErrorName } from "./checks.js"; +import { classifyReleaseFailure, contractErrorName, failureReason } from "./checks.js"; import { vaultAbi } from "./vaultAbi.js"; /** op-geth replaces a pending transaction only when the new one raises both @@ -82,12 +82,13 @@ export interface Releaser { blockedAlertIntervalMs: number; /** nonce -> pending release. Keyed by nonce: the vault consumes a nonce once. */ pending: Map; - /** nonce -> the last page about it, and when it went out. A blocked release - * needs a timelocked governance action (>= 48h) to clear, and an unexpected - * failure repeats every poll as well, so re-paging either every poll would - * emit thousands of identical alerts and train on-call to ignore them; the - * same page re-alerts at a bounded interval instead. */ - alertedAt: Map; + /** nonce -> each page about it, by subject and reason, and when it last went + * out. A blocked release needs a timelocked governance action (>= 48h) to + * clear, and an unexpected failure repeats every poll as well, so re-paging + * either every poll would emit thousands of identical alerts and train + * on-call to ignore them; the same page re-alerts at a bounded interval + * instead. */ + alertedAt: Map>; feeBump: FeeBumpPolicy; /** The release transaction not seen mined yet, if any. Kept in memory * only. */ @@ -97,12 +98,22 @@ export interface Releaser { now(): number; } -/** Page `subject` about the release of `nonce`, unless the same page about it - * went out less than `blockedAlertIntervalMs` ago. */ -async function pageThrottled(releaser: Releaser, nonce: bigint, subject: string, detail: string): Promise { - const last = releaser.alertedAt.get(nonce); - if (last?.subject === subject && releaser.now() - last.at < releaser.blockedAlertIntervalMs) return; - releaser.alertedAt.set(nonce, { subject, at: releaser.now() }); +/** Page `subject` about the release of `nonce`, unless the same page about it, + * for the same `reason`, went out less than `blockedAlertIntervalMs` ago. Each + * reason keeps its own interval, so one that comes and goes, such as a + * transient RPC error, neither mutes another nor makes it page again. */ +async function pageThrottled( + releaser: Releaser, + nonce: bigint, + subject: string, + reason: string, + detail: string, +): Promise { + const pages = releaser.alertedAt.get(nonce) ?? new Map(); + const key = `${subject}: ${reason}`; + const last = pages.get(key); + if (last !== undefined && releaser.now() - last < releaser.blockedAlertIntervalMs) return; + releaser.alertedAt.set(nonce, pages.set(key, releaser.now())); await releaser.alert(subject, detail, "critical"); } @@ -198,6 +209,7 @@ async function followInFlight(releaser: Releaser, tx: InFlightRelease): Promise< releaser, tx.entry.nonce, "release transaction not mined", + reason, `${reason}: transaction nonce ${tx.nonce}, account's next mined nonce ${mined}, ` + `maxFeePerGas ${formatGwei(tx.fees.maxFeePerGas)} gwei (cap ${formatGwei(feeBump.maxFeePerGas)} gwei); ${label}`, ); @@ -321,11 +333,18 @@ export async function drainPending(releaser: Releaser, conversionFactor: bigint, releaser, p.nonce, "release blocked and needs operator action", + `${errorName}`, `${label} — ${errorName}; it cannot self-heal`, ); break; case "unexpected": - await pageThrottled(releaser, p.nonce, "unexpected release failure", `${label} — ${reason}`); + await pageThrottled( + releaser, + p.nonce, + "unexpected release failure", + failureReason(error), + `${label} — ${reason}`, + ); break; } } From f65299ab25f3e27b17443456983c17ff13a00d05 Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Wed, 11 Nov 2026 16:34:13 +0100 Subject: [PATCH 16/16] fix(releaser): Retry an undelivered release page after 10 minutes A page about a release was recorded as sent before its webhook delivery, so one whose delivery failed (relay down, rate limited, or timed out after the retries) muted the same page for BLOCKED_ALERT_INTERVAL_MS. Record whether each page was delivered. One that was not goes out again after 10 minutes, or after BLOCKED_ALERT_INTERVAL_MS when that is shorter, rather than on the next cycle: each cycle waits on a page's delivery attempts, up to about 42 s, which would hold back the releases that can go while the webhook is down. Without a webhook, the logged alert counts as delivered. --- releaser/README.md | 2 +- releaser/src/main.ts | 8 ++++--- releaser/src/release.test.ts | 28 +++++++++++++++++++++++ releaser/src/release.ts | 43 +++++++++++++++++++++++------------- 4 files changed, 62 insertions(+), 19 deletions(-) diff --git a/releaser/README.md b/releaser/README.md index 1e7a69ccd..f5d060db4 100644 --- a/releaser/README.md +++ b/releaser/README.md @@ -41,7 +41,7 @@ consumed nonce and drops it. | `HEARTBEAT_URL` | Optional http(s) URL requested (GET) after a cycle that completed without error, at most once a minute; point it at a dead-man's switch that pages when the requests stop. Startup refuses any other value. A failed request is logged (error name and code only, never the URL), never fatal | | `BASE_CHAIN_ID` | Default 8453; startup refuses a `BASE_RPC_URL` that serves another chain (`eth_chainId`) | | `BASE_FINALITY_TAG` | `safe` (default) or `finalized`; the scan cursor stays inside this boundary and a pending entry is dropped only once its nonce is consumed there | -| `BLOCKED_ALERT_INTERVAL_MS` | Re-page interval for the same page about one release, for the same reason: governance-blocked, or failing unexpectedly (default 6h — a blocked release needs a ≥48h timelocked action, so per-poll paging would bury the signal). A different page about it, or the same page for another reason, goes out at once; each reason keeps its own interval | +| `BLOCKED_ALERT_INTERVAL_MS` | Re-page interval for the same page about one release, for the same reason: governance-blocked, or failing unexpectedly (default 6h — a blocked release needs a ≥48h timelocked action, so per-poll paging would bury the signal). A different page about it, or the same page for another reason, goes out at once; each reason keeps its own interval. A page whose webhook delivery failed goes out again after 10 minutes, or after this interval when it is shorter | | `READ_BATCH_SIZE` | Bound on Multicall calldata and individual fallback concurrency (default 100; a whole number of at least 1) | | `FEE_BUMP_AFTER_SECONDS` | Re-send a release transaction that is still not mined after this long (default 60) | | `FEE_BUMP_PERCENT` | Raise both fees by at least this many percent per re-send (default 20; a whole number of at least 10, the minimum a node accepts for a replacement) | diff --git a/releaser/src/main.ts b/releaser/src/main.ts index c4e2cf4d4..d857caa4a 100644 --- a/releaser/src/main.ts +++ b/releaser/src/main.ts @@ -69,11 +69,13 @@ function redact(text: string): string { return text.replace(/https?:\/\/[^\s"')]+/gi, ""); } -async function alert(subject: string, detail: string, severity: Severity = "warning"): Promise { +/** Log the alert and post it to the webhook; resolves to whether it was + * delivered. Without a webhook the log line is the delivery. */ +async function alert(subject: string, detail: string, severity: Severity = "warning"): Promise { console.error(`${new Date().toISOString()} ALERT: ${subject} — ${detail}`); - if (!config.alertWebhookUrl) return; + if (!config.alertWebhookUrl) return true; const redacted = redact(detail); - await postAlert(config.alertWebhookUrl, { + return postAlert(config.alertWebhookUrl, { text: alertText(severity, `pen-releaser ${account.address}`, subject, redacted), severity, service: "pen-releaser", diff --git a/releaser/src/release.test.ts b/releaser/src/release.test.ts index f6d70cfbb..2d30cac60 100644 --- a/releaser/src/release.test.ts +++ b/releaser/src/release.test.ts @@ -70,6 +70,7 @@ function releaserWith(answers: Record, entries: PendingRelease[] alertedAt: new Map(), alert: async (subject, detail, severity = "warning") => { alerts.push(`${severity} ${subject}: ${detail}`); + return true; }, log: (message) => logs.push(message), now: () => clock, @@ -394,6 +395,33 @@ test("a blocked release pages once per interval, not once per cycle", async () = assert.equal(releaser.pending.size, 1); }); +test("a page whose delivery failed goes out again ten minutes later, neither every cycle nor an interval later", async () => { + const { releaser, advance } = releaserWith( + { + simulateContract: () => { + throw vaultRevert({ errorName: "ExceedsPerReleaseCap", args: [ENTRY.palletAmount * CONVERSION_FACTOR, 1n] }); + }, + }, + [ENTRY], + ); + // The webhook is down for the first page only. + const start = releaser.now(); + const pages: { minute: number; delivered: boolean }[] = []; + releaser.alert = async () => { + const delivered = pages.length > 0; + pages.push({ minute: (releaser.now() - start) / 60_000, delivered }); + return delivered; + }; + for (let minute = 0; minute < 15; minute++) { + await drainPending(releaser, CONVERSION_FACTOR, DAILY_CAP); + advance(60_000); + } + assert.deepEqual(pages, [ + { minute: 0, delivered: false }, + { minute: 10, delivered: true }, + ]); +}); + test("an unexpected release failure pages once per interval for each reason, and again at once when the failure changes", async () => { let failure: Error = revertedWith("TokenNotSet"); const { releaser, alerts, advance } = releaserWith( diff --git a/releaser/src/release.ts b/releaser/src/release.ts index acec0f49e..e34b664f7 100644 --- a/releaser/src/release.ts +++ b/releaser/src/release.ts @@ -82,26 +82,34 @@ export interface Releaser { blockedAlertIntervalMs: number; /** nonce -> pending release. Keyed by nonce: the vault consumes a nonce once. */ pending: Map; - /** nonce -> each page about it, by subject and reason, and when it last went - * out. A blocked release needs a timelocked governance action (>= 48h) to - * clear, and an unexpected failure repeats every poll as well, so re-paging - * either every poll would emit thousands of identical alerts and train - * on-call to ignore them; the same page re-alerts at a bounded interval - * instead. */ - alertedAt: Map>; + /** nonce -> each page about it, by subject and reason: when it last went + * out, and whether it was delivered. A blocked release needs a timelocked + * governance action (>= 48h) to clear, and an unexpected failure repeats + * every poll as well, so re-paging either every poll would emit thousands + * of identical alerts and train on-call to ignore them; the same page + * re-alerts at a bounded interval instead. */ + alertedAt: Map>; feeBump: FeeBumpPolicy; /** The release transaction not seen mined yet, if any. Kept in memory * only. */ inFlight?: InFlightRelease; - alert(subject: string, detail: string, severity?: Severity): Promise; + /** Resolves to whether the alert was delivered. */ + alert(subject: string, detail: string, severity?: Severity): Promise; log(message: string): void; now(): number; } +/** A page whose delivery failed goes out again after this long, or after + * `blockedAlertIntervalMs` when that is shorter. Not on the next cycle: the + * cycle waits on every delivery attempt, up to about 42 s with its retries, + * so a webhook that is down would hold back the releases that can go. */ +const UNDELIVERED_REPAGE_MS = 10 * 60_000; + /** Page `subject` about the release of `nonce`, unless the same page about it, - * for the same `reason`, went out less than `blockedAlertIntervalMs` ago. Each - * reason keeps its own interval, so one that comes and goes, such as a - * transient RPC error, neither mutes another nor makes it page again. */ + * for the same `reason`, went out less than `blockedAlertIntervalMs` ago, or + * less than `UNDELIVERED_REPAGE_MS` ago if its delivery failed. Each reason + * keeps its own interval, so one that comes and goes, such as a transient RPC + * error, neither mutes another nor makes it page again. */ async function pageThrottled( releaser: Releaser, nonce: bigint, @@ -109,12 +117,17 @@ async function pageThrottled( reason: string, detail: string, ): Promise { - const pages = releaser.alertedAt.get(nonce) ?? new Map(); + const pages = releaser.alertedAt.get(nonce) ?? new Map(); const key = `${subject}: ${reason}`; const last = pages.get(key); - if (last !== undefined && releaser.now() - last < releaser.blockedAlertIntervalMs) return; - releaser.alertedAt.set(nonce, pages.set(key, releaser.now())); - await releaser.alert(subject, detail, "critical"); + if (last !== undefined) { + const interval = last.delivered + ? releaser.blockedAlertIntervalMs + : Math.min(UNDELIVERED_REPAGE_MS, releaser.blockedAlertIntervalMs); + if (releaser.now() - last.at < interval) return; + } + const delivered = await releaser.alert(subject, detail, "critical"); + releaser.alertedAt.set(nonce, pages.set(key, { at: releaser.now(), delivered })); } const min = (a: bigint, b: bigint) => (a < b ? a : b);