From 7654d04b7aba5f32496f60b635eb5e029e62344d Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Thu, 12 Nov 2026 09:04:15 +0100 Subject: [PATCH 01/10] fix(attestor,releaser): Redact endpoint URLs from all console output Alerts replaced URLs with only in what they post to the webhook. Their console line, other logged errors and library log lines still printed endpoint URLs whole: viem's error texts quote the RPC URL, and polkadot.js logs the WebSocket endpoint on every disconnect. Route all console output through the same redaction, installed once at startup, and extend it to ws(s) URLs. The redaction moves into the alert module both services share. --- attestor/README.md | 3 +++ attestor/src/alert.test.ts | 13 ++++++++++++- attestor/src/alert.ts | 18 ++++++++++++++++++ attestor/src/main.ts | 10 +++------- releaser/README.md | 3 +++ releaser/src/alert.test.ts | 13 ++++++++++++- releaser/src/alert.ts | 18 ++++++++++++++++++ releaser/src/main.ts | 10 +++------- 8 files changed, 72 insertions(+), 16 deletions(-) diff --git a/attestor/README.md b/attestor/README.md index c58203e18..84a93b9d7 100644 --- a/attestor/README.md +++ b/attestor/README.md @@ -54,6 +54,9 @@ point. | `FEE_BUMP_MAX` | Re-sends per approval before the `own approval not mined` page (default 3) | | `MAX_FEE_PER_GAS_GWEI` | No approval ever offers a higher `maxFeePerGas` (default 1, about 200 times Base's usual base fee; one approval uses at most about 300k gas) | +Log lines and alerts show `` in place of every http(s) and ws(s) URL: +endpoint URLs often carry an access key. + ## Run ```sh diff --git a/attestor/src/alert.test.ts b/attestor/src/alert.test.ts index 8df13970f..07387c3cc 100644 --- a/attestor/src/alert.test.ts +++ b/attestor/src/alert.test.ts @@ -3,7 +3,7 @@ import { createServer, type Server } from "node:http"; import type { AddressInfo } from "node:net"; import { test } from "node:test"; import { inspect } from "node:util"; -import { alertText, parseHeartbeatUrl, pingHeartbeat, postAlert, throttledHeartbeat } from "./alert.js"; +import { alertText, parseHeartbeatUrl, pingHeartbeat, postAlert, redactConsole, throttledHeartbeat } from "./alert.js"; /** A local webhook answering every POST with `status`, or never when null. */ async function webhook(status: number | null): Promise<{ url: string; server: Server }> { @@ -166,3 +166,14 @@ test("heartbeats go out at most once a minute, however often they are due", asyn assert.deepEqual(seen, ["/ping", "/ping"]); assert.equal(throttledHeartbeat(undefined)(), undefined, "nothing without a URL"); }); + +test("console output shows in place of each endpoint, also inside a logged error", (t) => { + const mocks = (["debug", "info", "log", "warn", "error"] as const).map((method) => t.mock.method(console, method, () => {})); + redactConsole(); + console.log("scanning %s", "https://base.example/v2/secret-key"); + console.error("cycle failed", new Error("HTTP request failed.\n\nURL: https://base.example/v2/secret-key")); + console.warn("API-WS: disconnected from wss://pendulum.example/ws?apikey=secret-key: 1006"); + const lines = mocks.flatMap((mock) => mock.mock.calls.map((call) => `${call.arguments[0]}`)); + assert.equal(lines.length, 3); + for (const line of lines) assert.ok(line.includes("") && !line.includes("secret-key"), line); +}); diff --git a/attestor/src/alert.ts b/attestor/src/alert.ts index 376769f49..b90ed7255 100644 --- a/attestor/src/alert.ts +++ b/attestor/src/alert.ts @@ -1,5 +1,23 @@ +import { format } from "node:util"; + export type Severity = "critical" | "warning"; +/** Strip URLs before anything leaves the process: endpoints commonly embed API + * keys, and error texts quote them verbatim (viem's the RPC URL, polkadot.js + * the WebSocket one). */ +export function redact(text: string): string { + return text.replace(/(?:https?|wss?):\/\/[^\s"')]+/gi, ""); +} + +/** Pass all console output through `redact`, called once at startup: errors + * are logged whole, and libraries log on their own. */ +export function redactConsole(): void { + for (const method of ["debug", "info", "log", "warn", "error"] as const) { + const write = console[method].bind(console); + console[method] = (...args: unknown[]) => write(redact(format(...args))); + } +} + /** Longer details are cut, well inside what Slack shows of one message. */ const MAX_TEXT = 3000; diff --git a/attestor/src/main.ts b/attestor/src/main.ts index 6f9985ed1..d463ff148 100644 --- a/attestor/src/main.ts +++ b/attestor/src/main.ts @@ -24,7 +24,7 @@ import { ApiPromise, WsProvider } from "@polkadot/api"; import { createPublicClient, defineChain, http } from "viem"; import { privateKeyToAccount } from "viem/accounts"; -import { alertText, postAlert, throttledHeartbeat, type Severity } from "./alert.js"; +import { alertText, postAlert, redact, redactConsole, throttledHeartbeat, type Severity } from "./alert.js"; import { advanceCheckpoint, assertSignerUnchanged, @@ -44,6 +44,8 @@ import { type Checkpoint, } from "./state.js"; +redactConsole(); + /** Repeated transient failures (a sustained RPC outage) page at most this * often; every occurrence is still logged. */ const TRANSIENT_ALERT_INTERVAL_MS = 60_000; @@ -62,12 +64,6 @@ function log(message: string, extra?: unknown): void { console.log(`${new Date().toISOString()} ${message}`, extra ?? ""); } -/** Strip URLs before anything leaves the process: RPC endpoints commonly embed - * API keys, and viem error texts quote the endpoint verbatim. */ -function redact(text: string): string { - return text.replace(/https?:\/\/[^\s"')]+/gi, ""); -} - async function alert(subject: string, detail: unknown, severity: Severity = "warning"): Promise { console.error(`${new Date().toISOString()} ALERT: ${subject}`, detail); if (!config.alertWebhookUrl) return; diff --git a/releaser/README.md b/releaser/README.md index f5d060db4..bc501686c 100644 --- a/releaser/README.md +++ b/releaser/README.md @@ -48,6 +48,9 @@ consumed nonce and drops it. | `FEE_BUMP_MAX` | Re-sends per transaction before the `release transaction not mined` page (default 3) | | `MAX_FEE_PER_GAS_GWEI` | No release transaction ever offers a higher `maxFeePerGas` (default 1) | +Log lines and alerts show `` in place of every http(s) and ws(s) URL: +endpoint URLs often carry an access key. + ## Run ```sh diff --git a/releaser/src/alert.test.ts b/releaser/src/alert.test.ts index bd4480cfa..b6bd197b6 100644 --- a/releaser/src/alert.test.ts +++ b/releaser/src/alert.test.ts @@ -3,7 +3,7 @@ import { createServer, type Server } from "node:http"; import type { AddressInfo } from "node:net"; import { test } from "node:test"; import { inspect } from "node:util"; -import { alertText, parseHeartbeatUrl, pingHeartbeat, postAlert, throttledHeartbeat } from "./alert.js"; +import { alertText, parseHeartbeatUrl, pingHeartbeat, postAlert, redactConsole, throttledHeartbeat } from "./alert.js"; /** A local webhook answering every POST with `status`, or never when null. */ async function webhook(status: number | null): Promise<{ url: string; server: Server }> { @@ -166,3 +166,14 @@ test("heartbeats go out at most once a minute, however often they are due", asyn assert.deepEqual(seen, ["/ping", "/ping"]); assert.equal(throttledHeartbeat(undefined)(), undefined, "nothing without a URL"); }); + +test("console output shows in place of each endpoint, also inside a logged error", (t) => { + const mocks = (["debug", "info", "log", "warn", "error"] as const).map((method) => t.mock.method(console, method, () => {})); + redactConsole(); + console.log("scanning %s", "https://base.example/v2/secret-key"); + console.error("cycle failed", new Error("HTTP request failed.\n\nURL: https://base.example/v2/secret-key")); + console.warn("API-WS: disconnected from wss://pendulum.example/ws?apikey=secret-key: 1006"); + const lines = mocks.flatMap((mock) => mock.mock.calls.map((call) => `${call.arguments[0]}`)); + assert.equal(lines.length, 3); + for (const line of lines) assert.ok(line.includes("") && !line.includes("secret-key"), line); +}); diff --git a/releaser/src/alert.ts b/releaser/src/alert.ts index 376769f49..b90ed7255 100644 --- a/releaser/src/alert.ts +++ b/releaser/src/alert.ts @@ -1,5 +1,23 @@ +import { format } from "node:util"; + export type Severity = "critical" | "warning"; +/** Strip URLs before anything leaves the process: endpoints commonly embed API + * keys, and error texts quote them verbatim (viem's the RPC URL, polkadot.js + * the WebSocket one). */ +export function redact(text: string): string { + return text.replace(/(?:https?|wss?):\/\/[^\s"')]+/gi, ""); +} + +/** Pass all console output through `redact`, called once at startup: errors + * are logged whole, and libraries log on their own. */ +export function redactConsole(): void { + for (const method of ["debug", "info", "log", "warn", "error"] as const) { + const write = console[method].bind(console); + console[method] = (...args: unknown[]) => write(redact(format(...args))); + } +} + /** Longer details are cut, well inside what Slack shows of one message. */ const MAX_TEXT = 3000; diff --git a/releaser/src/main.ts b/releaser/src/main.ts index d857caa4a..f9a0ead57 100644 --- a/releaser/src/main.ts +++ b/releaser/src/main.ts @@ -22,7 +22,7 @@ import { createPublicClient, createWalletClient, defineChain, http } from "viem"; import { privateKeyToAccount } from "viem/accounts"; -import { alertText, postAlert, throttledHeartbeat, type Severity } from "./alert.js"; +import { alertText, postAlert, redact, redactConsole, throttledHeartbeat, type Severity } from "./alert.js"; import { chunks, readInRanges, toPalletAmount } from "./checks.js"; import { config } from "./config.js"; import { drainPending, type PendingRelease, type Releaser } from "./release.js"; @@ -34,6 +34,8 @@ import { } from "./state.js"; import { vaultAbi } from "./vaultAbi.js"; +redactConsole(); + /** Canonical Multicall3, deployed at the same address on Base and every major * chain. viem refuses to batch unless the chain definition declares it, even * when the contract is present on-chain, so it has to be named here. */ @@ -63,12 +65,6 @@ function log(message: string): void { console.log(`${new Date().toISOString()} ${message}`); } -/** Strip URLs before anything leaves the process: RPC endpoints commonly embed - * API keys, and viem error texts quote the endpoint verbatim. */ -function redact(text: string): string { - return text.replace(/https?:\/\/[^\s"')]+/gi, ""); -} - /** Log the alert and post it to the webhook; resolves to whether it was * delivered. Without a webhook the log line is the delivery. */ async function alert(subject: string, detail: string, severity: Severity = "warning"): Promise { From 8124a10cb1100abe65d24b5e1f4edf1fae739dae Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Thu, 12 Nov 2026 09:06:10 +0100 Subject: [PATCH 02/10] fix(releaser): Page a failing cycle at most every 10 minutes Every failed cycle paged "releaser cycle failed", so an endpoint that stays unreachable paged on every poll, and each page held the next cycle back for its delivery attempts. Page the first failure, then at most every 10 minutes, the same way the attestor limits its transient-failure page; the failures in between are logged. A dead-man's switch on HEARTBEAT_URL still notices a lasting failure, as the heartbeat goes out only after a completed cycle. --- releaser/README.md | 4 ++- releaser/src/main.test.ts | 70 +++++++++++++++++++++++++++++++++++---- releaser/src/main.ts | 12 ++++++- 3 files changed, 78 insertions(+), 8 deletions(-) diff --git a/releaser/README.md b/releaser/README.md index bc501686c..f7e21b48f 100644 --- a/releaser/README.md +++ b/releaser/README.md @@ -61,7 +61,9 @@ npm install && npm run build && npm start Each cycle it scans new `ReleasePending` logs, drops nonces the vault has already consumed, then simulates and submits `release()` for the rest. -Simulating first means a still-capped release costs no gas. +Simulating first means a still-capped release costs no gas. A cycle that +fails, on an RPC error for instance, is logged and runs again after +`POLL_INTERVAL_MS`; it pages `releaser cycle failed` at most every 10 minutes. Failures are classified rather than treated alike: diff --git a/releaser/src/main.test.ts b/releaser/src/main.test.ts index 691da9cfb..691a66b06 100644 --- a/releaser/src/main.test.ts +++ b/releaser/src/main.test.ts @@ -1,31 +1,39 @@ import assert from "node:assert/strict"; import { spawn } from "node:child_process"; import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; -import { createServer } from "node:http"; +import { createServer, type Server } from "node:http"; import type { AddressInfo } from "node:net"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { test } from "node:test"; import { fileURLToPath } from "node:url"; -test("startup refuses a Base endpoint serving another chain before it reads the state", { timeout: 30_000 }, async () => { - // Answers eth_chainId with Base Sepolia, and nothing else. +/** A local JSON-RPC endpoint giving these results by method, and an error for + * any other method. */ +async function rpcServer(results: Record): Promise { const server = createServer((request, response) => { let body = ""; request.on("data", (chunk) => (body += chunk)); request.on("end", () => { const { id, method } = JSON.parse(body); + const result = results[method]; response.setHeader("content-type", "application/json"); response.end( JSON.stringify( - method === "eth_chainId" - ? { jsonrpc: "2.0", id, result: "0x14a34" } - : { jsonrpc: "2.0", id, error: { code: -32601, message: `unexpected ${method}` } }, + result === undefined + ? { jsonrpc: "2.0", id, error: { code: -32601, message: `unexpected ${method}` } } + : { jsonrpc: "2.0", id, result }, ), ); }); }); await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + return server; +} + +test("startup refuses a Base endpoint serving another chain before it reads the state", { timeout: 30_000 }, async () => { + // Answers eth_chainId with Base Sepolia, and nothing else. + const server = await rpcServer({ eth_chainId: "0x14a34" }); const directory = mkdtempSync(join(tmpdir(), "pen-releaser-main-")); const stateFile = join(directory, "state.json"); // Read first, this would fail on the state instead. @@ -52,3 +60,53 @@ test("startup refuses a Base endpoint serving another chain before it reads the rmSync(directory, { recursive: true, force: true }); } }); + +test("a failing cycle pages once, then is only logged, and no output shows the endpoint", { timeout: 30_000 }, async () => { + // The startup reads succeed; then every cycle fails on its first read. + const server = await rpcServer({ + eth_chainId: "0x2105", + eth_call: `0x${"0".repeat(63)}1`, // conversionFactor + eth_getCode: "0x60", // Multicall3 + eth_getBalance: "0xde0b6b3a7640000", // 1 ETH + }); + const directory = mkdtempSync(join(tmpdir(), "pen-releaser-main-")); + const child = spawn(process.execPath, [fileURLToPath(new URL("./main.js", import.meta.url))], { + env: { + PATH: process.env.PATH, + BASE_RPC_URL: `http://127.0.0.1:${(server.address() as AddressInfo).port}/v2/secret-key`, + VAULT_ADDRESS: "0x1111111111111111111111111111111111111111", + RELEASER_PRIVATE_KEY: `0x${"11".repeat(32)}`, + STATE_FILE: join(directory, "state.json"), + POLL_INTERVAL_MS: "1000", + }, + }); + try { + let stdout = ""; + let stderr = ""; + // Until the third failed cycle, paged or logged, or an early exit. + await new Promise((resolve) => { + const seen = () => { + if (`${stdout}${stderr}`.split("releaser cycle failed").length > 3) resolve(undefined); + }; + child.stdout.on("data", (chunk) => { + stdout += chunk; + seen(); + }); + child.stderr.on("data", (chunk) => { + stderr += chunk; + seen(); + }); + child.on("exit", resolve); + }); + const output = `${stdout}${stderr}`; + assert.equal(stderr.split("ALERT: releaser cycle failed").length, 2, output); + assert.equal(stdout.split("releaser cycle failed:").length, 3, output); + assert.match(stderr, /URL: /); + assert.ok(!output.includes("secret-key"), output); + } finally { + child.kill(); + server.closeAllConnections(); + server.close(); + rmSync(directory, { recursive: true, force: true }); + } +}); diff --git a/releaser/src/main.ts b/releaser/src/main.ts index f9a0ead57..d3169c4b9 100644 --- a/releaser/src/main.ts +++ b/releaser/src/main.ts @@ -36,6 +36,10 @@ import { vaultAbi } from "./vaultAbi.js"; redactConsole(); +/** Repeated cycle failures (a sustained RPC outage) page at most this often; + * every failure is still logged. */ +const CYCLE_ALERT_INTERVAL_MS = 10 * 60_000; + /** Canonical Multicall3, deployed at the same address on Base and every major * chain. viem refuses to batch unless the chain definition declares it, even * when the contract is present on-chain, so it has to be named here. */ @@ -251,6 +255,7 @@ async function main(): Promise { log(`releaser ${account.address} started; scanning from block ${fromBlock}, ${pending.size} pending`); await checkGasBalance(); const heartbeat = throttledHeartbeat(config.heartbeatUrl); + let lastCycleAlertMs = 0; for (;;) { try { @@ -272,7 +277,12 @@ async function main(): Promise { ); void heartbeat(); } catch (cycleError) { - await alert("releaser cycle failed", `${cycleError}`); + if (Date.now() - lastCycleAlertMs >= CYCLE_ALERT_INTERVAL_MS) { + lastCycleAlertMs = Date.now(); + await alert("releaser cycle failed", `${cycleError}`); + } else { + log(`releaser cycle failed: ${cycleError}`); + } } await new Promise((resolve) => setTimeout(resolve, config.pollIntervalMs)); } From d61ac4e03c67e3df3aa6f2f06ad06b21c98efe4d Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Thu, 12 Nov 2026 09:06:15 +0100 Subject: [PATCH 03/10] docs(attestor): Base mines a block every 200 ms --- attestor/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/attestor/README.md b/attestor/README.md index 84a93b9d7..2b56b708c 100644 --- a/attestor/README.md +++ b/attestor/README.md @@ -49,7 +49,7 @@ point. | `BASE_FINALITY_TIMEOUT_MS` | How long a block's approvals may stay outside that boundary before an alert + idempotent re-submission (default 15 min for `safe`, 45 min for `finalized`) | | `HEAD_STALL_ALERT_MS` | Page when no finalized Pendulum head has arrived for this long (default 5 min): the daemon is push-driven, so a node that stops finalizing would otherwise idle undetected | | `MISSED_APPROVAL_ALERT_COUNT` | Page after this many consecutive migrations resolved on-chain without this attestor's own approval landing, and again every as many more (default 3, a whole number of at least 1) | -| `FEE_BUMP_AFTER_SECONDS` | Re-send an own approval that is still not mined after this long (default 60: Base mines a block every 2 s, so this means stuck, not slow) | +| `FEE_BUMP_AFTER_SECONDS` | Re-send an own approval that is still not mined after this long (default 60: Base mines a block every 200 ms, so this means stuck, not slow) | | `FEE_BUMP_PERCENT` | Raise both fees by at least this many percent per re-send (default 20; a whole number of at least 10, the minimum a node accepts for a replacement) | | `FEE_BUMP_MAX` | Re-sends per approval before the `own approval not mined` page (default 3) | | `MAX_FEE_PER_GAS_GWEI` | No approval ever offers a higher `maxFeePerGas` (default 1, about 200 times Base's usual base fee; one approval uses at most about 300k gas) | From 27fce5fc6131e12c96a572c877a899299f232ab2 Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Thu, 12 Nov 2026 09:32:57 +0100 Subject: [PATCH 04/10] fix(attestor,releaser): Refuse an endpoint URL setting without its scheme BASE_RPC_URL, PENDULUM_WS and ALERT_WEBHOOK_URL were used as given. A value without its scheme cannot work, but it failed later and echoed the value: viem's and polkadot.js's errors quote it, and the URL redaction cannot recognise a URL without a scheme. A webhook URL without one let the daemon run with every alert failing delivery. Check them at startup like HEARTBEAT_URL, without echoing the value: http(s) for the RPC and webhook URLs, ws(s) for the Pendulum node. --- attestor/README.md | 3 ++- attestor/src/alert.ts | 17 ++++++++++++----- attestor/src/config.test.ts | 10 ++++++++++ attestor/src/config.ts | 8 ++++---- releaser/README.md | 3 ++- releaser/src/alert.ts | 17 ++++++++++++----- releaser/src/config.test.ts | 10 ++++++++++ releaser/src/config.ts | 6 +++--- 8 files changed, 55 insertions(+), 19 deletions(-) diff --git a/attestor/README.md b/attestor/README.md index 2b56b708c..be159907b 100644 --- a/attestor/README.md +++ b/attestor/README.md @@ -55,7 +55,8 @@ point. | `MAX_FEE_PER_GAS_GWEI` | No approval ever offers a higher `maxFeePerGas` (default 1, about 200 times Base's usual base fee; one approval uses at most about 300k gas) | Log lines and alerts show `` in place of every http(s) and ws(s) URL: -endpoint URLs often carry an access key. +endpoint URLs often carry an access key. Startup refuses an endpoint setting +without the right scheme, and does not echo it. ## Run diff --git a/attestor/src/alert.ts b/attestor/src/alert.ts index b90ed7255..20d7443a5 100644 --- a/attestor/src/alert.ts +++ b/attestor/src/alert.ts @@ -37,20 +37,27 @@ export function alertText(severity: Severity, source: string, subject: string, d * refuse more than a few requests a minute. */ const HEARTBEAT_INTERVAL_MS = 60_000; -/** HEARTBEAT_URL, checked once at startup: unset, or an http(s) URL. The value - * is never echoed, as it usually holds a token. */ -export function parseHeartbeatUrl(value: string | undefined): string | undefined { - if (!value) return undefined; +/** The URL setting `name`, checked once at startup: an http(s) URL, or a ws(s) + * one when `scheme` says so. The value is never echoed, as it usually holds a + * token. */ +export function checkUrl(name: string, value: string, scheme: "http" | "ws" = "http"): string { let protocol: string | undefined; try { protocol = new URL(value).protocol; } catch { // Not a URL at all: refused below. } - if (protocol !== "http:" && protocol !== "https:") throw new Error("invalid HEARTBEAT_URL: expected an http(s) URL"); + if (protocol !== `${scheme}:` && protocol !== `${scheme}s:`) { + throw new Error(`invalid ${name}: expected ${scheme === "ws" ? "a ws(s)" : "an http(s)"} URL`); + } return value; } +/** HEARTBEAT_URL, checked once at startup: unset, or an http(s) URL. */ +export function parseHeartbeatUrl(value: string | undefined): string | undefined { + return value ? checkUrl("HEARTBEAT_URL", value) : undefined; +} + /** * Request the URL of an external dead-man's switch, which pages once these * requests stop. A failed request is logged, never thrown: the missing diff --git a/attestor/src/config.test.ts b/attestor/src/config.test.ts index d1c6f0e21..56591b26b 100644 --- a/attestor/src/config.test.ts +++ b/attestor/src/config.test.ts @@ -55,6 +55,16 @@ test("on Base mainnet the expected genesis can only be Pendulum's", () => { assert.equal(loadConfig({ PENDULUM_GENESIS_HASH: pendulum }, "config.pendulumGenesisHash").stdout, pendulum); }); +test("an endpoint URL needs the right scheme, and a bad one is refused without echoing it", () => { + assert.equal(loadConfig({ ALERT_WEBHOOK_URL: "https://hooks.example/secret-key" }, "config.alertWebhookUrl").stdout, "https://hooks.example/secret-key"); + for (const [name, value] of Object.entries({ PENDULUM_WS: "https://node.example/secret-key", BASE_RPC_URL: "base.example/secret-key", ALERT_WEBHOOK_URL: "hooks.example/secret-key" })) { + const { status, stderr } = loadConfig({ [name]: value }); + assert.notEqual(status, 0, name); + assert.match(stderr, new RegExp(`invalid ${name}`), name); + assert.doesNotMatch(stderr, /secret-key/, name); + } +}); + test("HEARTBEAT_URL must be an http(s) URL, and a bad one is refused without echoing it", () => { assert.equal(loadConfig({ HEARTBEAT_URL: "https://hc-ping.com/secret-token" }, "config.heartbeatUrl").stdout, "https://hc-ping.com/secret-token"); const { status, stderr } = loadConfig({ HEARTBEAT_URL: "hc-ping.com/secret-token" }); diff --git a/attestor/src/config.ts b/attestor/src/config.ts index 1b0cb4fe2..e59235656 100644 --- a/attestor/src/config.ts +++ b/attestor/src/config.ts @@ -1,5 +1,5 @@ import { parseGwei } from "viem"; -import { parseHeartbeatUrl } from "./alert.js"; +import { checkUrl, parseHeartbeatUrl } from "./alert.js"; import { MIN_REPLACEMENT_PERCENT } from "./approval.js"; function required(name: string): string { @@ -59,12 +59,12 @@ export const config = { * Never point this at a public RPC: the attestor would inherit its honesty. * The node must retain historical state (`--state-pruning archive`), or any * daemon outage longer than the pruning horizon wedges the catch-up. */ - pendulumWs: required("PENDULUM_WS"), + pendulumWs: checkUrl("PENDULUM_WS", required("PENDULUM_WS"), "ws"), /** The genesis that node must serve; startup refuses any other chain. A * local test chain sets its own, never on Base mainnet. */ pendulumGenesisHash: genesisHash(), /** Base JSON-RPC endpoint. */ - baseRpcUrl: required("BASE_RPC_URL"), + baseRpcUrl: checkUrl("BASE_RPC_URL", required("BASE_RPC_URL")), /** MigrationVault contract address on Base. */ vaultAddress: required("VAULT_ADDRESS") as `0x${string}`, /** This attestor's transaction-signing key (0x-prefixed, 32 bytes). @@ -78,7 +78,7 @@ export const config = { /** Alert when the gas wallet drops below this balance (wei). */ minGasBalanceWei: BigInt(process.env.MIN_GAS_BALANCE_WEI ?? "10000000000000000"), // 0.01 ETH /** Optional webhook that receives JSON alerts (low gas, fatal errors). */ - alertWebhookUrl: process.env.ALERT_WEBHOOK_URL, + alertWebhookUrl: process.env.ALERT_WEBHOOK_URL ? checkUrl("ALERT_WEBHOOK_URL", process.env.ALERT_WEBHOOK_URL) : undefined, /** Optional http(s) URL requested after a finalized head processed without * error, at most once a minute: a dead-man's switch pages when the * requests stop. */ diff --git a/releaser/README.md b/releaser/README.md index f7e21b48f..e18a2f952 100644 --- a/releaser/README.md +++ b/releaser/README.md @@ -49,7 +49,8 @@ consumed nonce and drops it. | `MAX_FEE_PER_GAS_GWEI` | No release transaction ever offers a higher `maxFeePerGas` (default 1) | Log lines and alerts show `` in place of every http(s) and ws(s) URL: -endpoint URLs often carry an access key. +endpoint URLs often carry an access key. Startup refuses an endpoint setting +without the right scheme, and does not echo it. ## Run diff --git a/releaser/src/alert.ts b/releaser/src/alert.ts index b90ed7255..20d7443a5 100644 --- a/releaser/src/alert.ts +++ b/releaser/src/alert.ts @@ -37,20 +37,27 @@ export function alertText(severity: Severity, source: string, subject: string, d * refuse more than a few requests a minute. */ const HEARTBEAT_INTERVAL_MS = 60_000; -/** HEARTBEAT_URL, checked once at startup: unset, or an http(s) URL. The value - * is never echoed, as it usually holds a token. */ -export function parseHeartbeatUrl(value: string | undefined): string | undefined { - if (!value) return undefined; +/** The URL setting `name`, checked once at startup: an http(s) URL, or a ws(s) + * one when `scheme` says so. The value is never echoed, as it usually holds a + * token. */ +export function checkUrl(name: string, value: string, scheme: "http" | "ws" = "http"): string { let protocol: string | undefined; try { protocol = new URL(value).protocol; } catch { // Not a URL at all: refused below. } - if (protocol !== "http:" && protocol !== "https:") throw new Error("invalid HEARTBEAT_URL: expected an http(s) URL"); + if (protocol !== `${scheme}:` && protocol !== `${scheme}s:`) { + throw new Error(`invalid ${name}: expected ${scheme === "ws" ? "a ws(s)" : "an http(s)"} URL`); + } return value; } +/** HEARTBEAT_URL, checked once at startup: unset, or an http(s) URL. */ +export function parseHeartbeatUrl(value: string | undefined): string | undefined { + return value ? checkUrl("HEARTBEAT_URL", value) : undefined; +} + /** * Request the URL of an external dead-man's switch, which pages once these * requests stop. A failed request is logged, never thrown: the missing diff --git a/releaser/src/config.test.ts b/releaser/src/config.test.ts index 29737a671..2d2c7910c 100644 --- a/releaser/src/config.test.ts +++ b/releaser/src/config.test.ts @@ -20,6 +20,16 @@ function loadConfig(extra: Record, print = "''"): { status: numb return { status: result.status, stdout: result.stdout.trim(), stderr: result.stderr }; } +test("an endpoint URL needs the right scheme, and a bad one is refused without echoing it", () => { + assert.equal(loadConfig({ ALERT_WEBHOOK_URL: "https://hooks.example/secret-key" }, "config.alertWebhookUrl").stdout, "https://hooks.example/secret-key"); + for (const [name, value] of Object.entries({ BASE_RPC_URL: "wss://base.example/secret-key", ALERT_WEBHOOK_URL: "hooks.example/secret-key" })) { + const { status, stderr } = loadConfig({ [name]: value }); + assert.notEqual(status, 0, name); + assert.match(stderr, new RegExp(`invalid ${name}`), name); + assert.doesNotMatch(stderr, /secret-key/, name); + } +}); + test("HEARTBEAT_URL must be an http(s) URL, and a bad one is refused without echoing it", () => { assert.equal(loadConfig({ HEARTBEAT_URL: "https://hc-ping.com/secret-token" }, "config.heartbeatUrl").stdout, "https://hc-ping.com/secret-token"); const { status, stderr } = loadConfig({ HEARTBEAT_URL: "hc-ping.com/secret-token" }); diff --git a/releaser/src/config.ts b/releaser/src/config.ts index bcd66d9e1..14ffaf3e0 100644 --- a/releaser/src/config.ts +++ b/releaser/src/config.ts @@ -1,5 +1,5 @@ import { parseGwei } from "viem"; -import { parseHeartbeatUrl } from "./alert.js"; +import { checkUrl, parseHeartbeatUrl } from "./alert.js"; import { MIN_REPLACEMENT_PERCENT } from "./release.js"; function required(name: string): string { @@ -37,7 +37,7 @@ function finalityTag(): "safe" | "finalized" { export const config = { /** Base JSON-RPC endpoint. */ - baseRpcUrl: required("BASE_RPC_URL"), + baseRpcUrl: checkUrl("BASE_RPC_URL", required("BASE_RPC_URL")), /** MigrationVault contract address on Base. */ vaultAddress: required("VAULT_ADDRESS") as `0x${string}`, /** Gas-only signing key. `release()` is permissionless and can only pay the @@ -55,7 +55,7 @@ export const config = { maxBlockRange: BigInt(envNumber("MAX_BLOCK_RANGE", 10_000, 1, true)), /** Alert when the gas wallet drops below this balance (wei). */ minGasBalanceWei: BigInt(process.env.MIN_GAS_BALANCE_WEI ?? "5000000000000000"), - alertWebhookUrl: process.env.ALERT_WEBHOOK_URL, + alertWebhookUrl: process.env.ALERT_WEBHOOK_URL ? checkUrl("ALERT_WEBHOOK_URL", process.env.ALERT_WEBHOOK_URL) : undefined, /** Optional http(s) URL requested after a cycle that completed without * error, at most once a minute: a dead-man's switch pages when the * requests stop. */ From ff49bed27df12424d249f586c9c5b11a81ae8afe Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Thu, 12 Nov 2026 09:33:57 +0100 Subject: [PATCH 05/10] fix(attestor,releaser): Print an uncaught error through the redacted console Node prints an uncaught exception or unhandled rejection itself, past the console, so the URL redaction did not reach it. Every async path has a handler today; this covers a later one. The process still exits with status 1, as it does without the handler. --- attestor/src/main.ts | 6 ++++++ releaser/src/main.ts | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/attestor/src/main.ts b/attestor/src/main.ts index d463ff148..dcf8c604b 100644 --- a/attestor/src/main.ts +++ b/attestor/src/main.ts @@ -45,6 +45,12 @@ import { } from "./state.js"; redactConsole(); +// Node prints an uncaught error or unhandled rejection itself, past the +// console: print it redacted instead, then exit as Node would. +process.on("uncaughtException", (error) => { + console.error(`${new Date().toISOString()} fatal error`, error); + process.exit(1); +}); /** Repeated transient failures (a sustained RPC outage) page at most this * often; every occurrence is still logged. */ diff --git a/releaser/src/main.ts b/releaser/src/main.ts index d3169c4b9..96e4eb6a4 100644 --- a/releaser/src/main.ts +++ b/releaser/src/main.ts @@ -35,6 +35,12 @@ import { import { vaultAbi } from "./vaultAbi.js"; redactConsole(); +// Node prints an uncaught error or unhandled rejection itself, past the +// console: print it redacted instead, then exit as Node would. +process.on("uncaughtException", (error) => { + console.error(`${new Date().toISOString()} fatal error`, error); + process.exit(1); +}); /** Repeated cycle failures (a sustained RPC outage) page at most this often; * every failure is still logged. */ From d39935d77068bcc426117e21cf08c1710d32d047 Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Thu, 12 Nov 2026 09:34:45 +0100 Subject: [PATCH 06/10] fix(releaser): Log a failed multicall batch instead of paging A failed Multicall batch paged on every cycle, once per batch, although the bounded per-nonce reads that follow it still complete the cycle. Log it instead: if those reads fail too, the cycle fails and pages. --- releaser/README.md | 3 ++- releaser/src/main.ts | 7 +++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/releaser/README.md b/releaser/README.md index e18a2f952..9431d6c75 100644 --- a/releaser/README.md +++ b/releaser/README.md @@ -102,7 +102,8 @@ trusting unsafe logs or losing pending work. Malformed state is fatal, and so is an endpoint that does not serve `BASE_CHAIN_ID`, checked before the state is read on every start. Multicall is disabled permanently only when an on-chain code check proves it absent; a -temporary provider error falls back in bounded batches for that cycle. +temporary provider error falls back in bounded batches for that cycle and is +logged, not paged. **Known limitation:** the pending set is driven by `ReleasePending`, which the vault emits when a threshold is crossed inside `approve()`. A payload made diff --git a/releaser/src/main.ts b/releaser/src/main.ts index 96e4eb6a4..069e77c4f 100644 --- a/releaser/src/main.ts +++ b/releaser/src/main.ts @@ -189,10 +189,9 @@ async function readConsumed(nonces: bigint[], blockNumber: bigint): Promise Date: Thu, 12 Nov 2026 09:35:29 +0100 Subject: [PATCH 07/10] test(releaser): End the cycle-failure test when its lines never come The test waited for three failed cycles or the releaser's exit, with nothing to stop the releaser, so a regression hung the test run, and the CI job with it. Stop the releaser after 20 s, as the startup test does, so the assertions run and fail with its output. --- releaser/src/main.test.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/releaser/src/main.test.ts b/releaser/src/main.test.ts index 691a66b06..4090df1a6 100644 --- a/releaser/src/main.test.ts +++ b/releaser/src/main.test.ts @@ -80,6 +80,8 @@ test("a failing cycle pages once, then is only logged, and no output shows the e POLL_INTERVAL_MS: "1000", }, }); + // A regression must fail the test, not leave the releaser running. + const kill = setTimeout(() => child.kill(), 20_000); try { let stdout = ""; let stderr = ""; @@ -104,6 +106,7 @@ test("a failing cycle pages once, then is only logged, and no output shows the e assert.match(stderr, /URL: /); assert.ok(!output.includes("secret-key"), output); } finally { + clearTimeout(kill); child.kill(); server.closeAllConnections(); server.close(); From 7d87590444baa569cef0905cd9d3db1aa6b4a6ff Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Thu, 12 Nov 2026 09:35:56 +0100 Subject: [PATCH 08/10] test(attestor,releaser): Cover every scheme and console method in the redaction test The test only logged https and wss URLs through log, error and warn, so a redaction that missed http, ws, info or debug still passed it. --- attestor/src/alert.test.ts | 4 +++- releaser/src/alert.test.ts | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/attestor/src/alert.test.ts b/attestor/src/alert.test.ts index 07387c3cc..00645d274 100644 --- a/attestor/src/alert.test.ts +++ b/attestor/src/alert.test.ts @@ -173,7 +173,9 @@ test("console output shows in place of each endpoint, also inside a logged console.log("scanning %s", "https://base.example/v2/secret-key"); console.error("cycle failed", new Error("HTTP request failed.\n\nURL: https://base.example/v2/secret-key")); console.warn("API-WS: disconnected from wss://pendulum.example/ws?apikey=secret-key: 1006"); + console.info("API-WS: disconnected from ws://127.0.0.1:9944/secret-key: 1006"); + console.debug("posting to http://hooks.example/secret-key"); const lines = mocks.flatMap((mock) => mock.mock.calls.map((call) => `${call.arguments[0]}`)); - assert.equal(lines.length, 3); + assert.equal(lines.length, 5); for (const line of lines) assert.ok(line.includes("") && !line.includes("secret-key"), line); }); diff --git a/releaser/src/alert.test.ts b/releaser/src/alert.test.ts index b6bd197b6..8754a3672 100644 --- a/releaser/src/alert.test.ts +++ b/releaser/src/alert.test.ts @@ -173,7 +173,9 @@ test("console output shows in place of each endpoint, also inside a logged console.log("scanning %s", "https://base.example/v2/secret-key"); console.error("cycle failed", new Error("HTTP request failed.\n\nURL: https://base.example/v2/secret-key")); console.warn("API-WS: disconnected from wss://pendulum.example/ws?apikey=secret-key: 1006"); + console.info("API-WS: disconnected from ws://127.0.0.1:9944/secret-key: 1006"); + console.debug("posting to http://hooks.example/secret-key"); const lines = mocks.flatMap((mock) => mock.mock.calls.map((call) => `${call.arguments[0]}`)); - assert.equal(lines.length, 3); + assert.equal(lines.length, 5); for (const line of lines) assert.ok(line.includes("") && !line.includes("secret-key"), line); }); From dd7a1fd720c1175dec0a6e1a685a4a582d9eeae6 Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Thu, 12 Nov 2026 09:36:10 +0100 Subject: [PATCH 09/10] docs(attestor): Base block time in the fee-bump comment --- attestor/src/config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/attestor/src/config.ts b/attestor/src/config.ts index e59235656..f112d012b 100644 --- a/attestor/src/config.ts +++ b/attestor/src/config.ts @@ -108,7 +108,7 @@ export const config = { * nonce with both fees raised by `percent` (at least the 10 % a replacement * needs, and at least to the current estimate), at most `max` times and * never above `maxFeePerGas`; then it pages once. Base mines a block every - * 2 s and an approval priced at the estimate lands in the next one or two, + * 200 ms and an approval priced at the estimate lands in the next one or two, * so 60 s means stuck, not slow. Since each bump also catches up with the * estimate, one normally suffices; when three do not, the price is not the * problem and the page (about 4 min after the first send) goes to a person. From 9dd64ed489b57d8af1f325e7be26c18cee86a8b7 Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Thu, 12 Nov 2026 09:49:58 +0100 Subject: [PATCH 10/10] fix(attestor,releaser): Wait for the Base endpoint at startup instead of exiting A Base read that failed at startup ended the process with a critical "startup failed" page, so a daemon restarted during an RPC outage paged again on every restart: about every 11 s with the systemd example. Retry the startup reads in the process instead, paging the way the running daemon does during the same outage: the releaser shares the 10-minute interval of its cycle-failure page, the attestor the one-minute interval of its transient-failure page, and every failure is still logged. Nothing is processed before these reads, so no event can be skipped. A wrong chain, a key that is not an attestor and unreadable state stay fatal. --- attestor/README.md | 5 +- attestor/src/main.test.ts | 53 +++++++++++++++ attestor/src/main.ts | 51 ++++++++++---- releaser/README.md | 5 +- releaser/src/main.test.ts | 137 ++++++++++++++++++++++---------------- releaser/src/main.ts | 59 +++++++++++----- 6 files changed, 221 insertions(+), 89 deletions(-) create mode 100644 attestor/src/main.test.ts diff --git a/attestor/README.md b/attestor/README.md index be159907b..b23d7dcf6 100644 --- a/attestor/README.md +++ b/attestor/README.md @@ -165,7 +165,10 @@ WantedBy=multi-user.target set, that `BASE_RPC_URL` serves `BASE_CHAIN_ID` (`eth_chainId`, which the checkpoint's Base chain must match too) and that `PENDULUM_WS` serves `PENDULUM_GENESIS_HASH`, and refuses to run otherwise. These checks run on - every start, the first one without a checkpoint included. While running, it + every start, the first one without a checkpoint included. A Base read they + need that fails, as during an RPC outage, is tried again every 5 s and pages + `Base read failed at startup, retrying` at most once a minute, so a restart + during an outage waits for the endpoint instead of exiting. While running, it re-reads its attestor membership and generation on every finalized head and exits with `fatal error, exiting` when either changed (a key removed, or removed and re-added): the next start then refuses the checkpoint until it diff --git a/attestor/src/main.test.ts b/attestor/src/main.test.ts new file mode 100644 index 000000000..4fa34f1ba --- /dev/null +++ b/attestor/src/main.test.ts @@ -0,0 +1,53 @@ +import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import { once } from "node:events"; +import { createServer } from "node:http"; +import type { AddressInfo } from "node:net"; +import { test } from "node:test"; +import { fileURLToPath } from "node:url"; + +test("a failed Base read at startup is retried, paging once, instead of exiting", { timeout: 30_000 }, async () => { + // A Base endpoint that refuses connections, as during an outage. + const closed = createServer().listen(0, "127.0.0.1"); + await once(closed, "listening"); + const port = (closed.address() as AddressInfo).port; + closed.close(); + const child = spawn(process.execPath, [fileURLToPath(new URL("./main.js", import.meta.url))], { + env: { + PATH: process.env.PATH, + PENDULUM_WS: "ws://127.0.0.1:9944", + BASE_RPC_URL: `http://127.0.0.1:${port}/v2/secret-key`, + VAULT_ADDRESS: "0x1111111111111111111111111111111111111111", + ATTESTOR_PRIVATE_KEY: `0x${"11".repeat(32)}`, + }, + }); + let stdout = ""; + let stderr = ""; + // A regression must fail the test, not leave the attestor running. + const kill = setTimeout(() => child.kill(), 20_000); + try { + // Until the second failed read, paged or logged, or an early exit. + await new Promise((resolve) => { + const seen = () => { + if (`${stdout}${stderr}`.split("Base read failed at startup").length > 2) resolve(undefined); + }; + child.stdout.on("data", (chunk) => { + stdout += chunk; + seen(); + }); + child.stderr.on("data", (chunk) => { + stderr += chunk; + seen(); + }); + child.on("exit", resolve); + }); + const all = `${stdout}${stderr}`; + assert.equal(child.exitCode, null, all); + assert.equal(stderr.split("ALERT: Base read failed at startup").length, 2, all); + assert.equal(stdout.split("Base read failed at startup").length, 2, all); + assert.ok(!all.includes("secret-key"), all); + } finally { + clearTimeout(kill); + child.kill(); + } +}); diff --git a/attestor/src/main.ts b/attestor/src/main.ts index dcf8c604b..f500c6995 100644 --- a/attestor/src/main.ts +++ b/attestor/src/main.ts @@ -52,9 +52,12 @@ process.on("uncaughtException", (error) => { process.exit(1); }); -/** Repeated transient failures (a sustained RPC outage) page at most this - * often; every occurrence is still logged. */ +/** Repeated failures that are retried (a sustained RPC outage, at startup or + * on a finalized head) page at most this often; every occurrence is still + * logged. */ const TRANSIENT_ALERT_INTERVAL_MS = 60_000; +/** A Base read at startup that fails is tried again after this long. */ +const STARTUP_RETRY_MS = 5_000; const baseChain = defineChain({ id: config.baseChainId, @@ -84,6 +87,34 @@ async function alert(subject: string, detail: unknown, severity: Severity = "war }); } +let lastRetryAlertMs = 0; + +/** Page a failure that will be retried, or only log it if such a page went out + * less than TRANSIENT_ALERT_INTERVAL_MS ago. */ +async function alertRetried(subject: string, error: unknown): Promise { + if (Date.now() - lastRetryAlertMs >= TRANSIENT_ALERT_INTERVAL_MS) { + lastRetryAlertMs = Date.now(); + await alert(subject, error); + } else { + log(subject, error); + } +} + +/** A Base read at startup, tried again every STARTUP_RETRY_MS until it answers: + * a restart during an RPC outage waits for the endpoint, paging as a running + * attestor does, instead of exiting and paging again on every restart. + * Nothing is processed before these reads, so no event can be skipped. */ +async function startupRead(read: () => Promise): Promise { + for (;;) { + try { + return await read(); + } catch (error) { + await alertRetried("Base read failed at startup, retrying", error); + await new Promise((resolve) => setTimeout(resolve, STARTUP_RETRY_MS)); + } + } +} + /** Extract MigrationInitiated events from one finalized Pendulum block. */ async function migrationEventsInBlock(api: ApiPromise, blockNumber: number): Promise { const blockHash = await api.rpc.chain.getBlockHash(blockNumber); @@ -127,13 +158,13 @@ async function checkGasBalance(): Promise { } async function main(): Promise { - const baseChainId = await publicClient.getChainId(); + const baseChainId = await startupRead(() => publicClient.getChainId()); assertBaseChain(baseChainId, config.baseChainId); - if (!(await attestor.chain.isAttestor())) { + if (!(await startupRead(() => attestor.chain.isAttestor()))) { throw new Error(`${account.address} is not an attestor of ${config.vaultAddress}`); } - const attestorGeneration = await attestor.chain.attestorGeneration(); - await checkGasBalance(); + const attestorGeneration = await startupRead(() => attestor.chain.attestorGeneration()); + await startupRead(checkGasBalance); const api = await ApiPromise.create({ provider: new WsProvider(config.pendulumWs) }); const pendulumGenesisHash = api.genesisHash.toHex(); @@ -160,7 +191,6 @@ async function main(): Promise { * checkpoint has not yet passed. Bounded by the Base finality lag. */ const unconfirmed: UnconfirmedBlock[] = []; let processedThrough = checkpoint.lastProcessedBlock; - let lastTransientAlertMs = 0; const heartbeat = throttledHeartbeat(config.heartbeatUrl); let lastHeadAt = Date.now(); @@ -185,12 +215,7 @@ async function main(): Promise { // Not a statement about any event — resume from the in-memory // position on the next finalized head. Sustained outages page // at a bounded rate instead of once per head. - if (Date.now() - lastTransientAlertMs >= TRANSIENT_ALERT_INTERVAL_MS) { - lastTransientAlertMs = Date.now(); - await alert("transient RPC failure, retrying on the next finalized head", error); - } else { - log("transient RPC failure, retrying on the next finalized head", error); - } + await alertRetried("transient RPC failure, retrying on the next finalized head", error); return; } // PRD A5: never skip an event silently. Alert and exit; the process diff --git a/releaser/README.md b/releaser/README.md index 9431d6c75..0b234b378 100644 --- a/releaser/README.md +++ b/releaser/README.md @@ -64,7 +64,10 @@ Each cycle it scans new `ReleasePending` logs, drops nonces the vault has already consumed, then simulates and submits `release()` for the rest. Simulating first means a still-capped release costs no gas. A cycle that fails, on an RPC error for instance, is logged and runs again after -`POLL_INTERVAL_MS`; it pages `releaser cycle failed` at most every 10 minutes. +`POLL_INTERVAL_MS`, and so does a failed Base read at startup: a restart during +an outage waits for the endpoint instead of exiting. These failures page +(`releaser cycle failed`, `releaser startup read failed, retrying`) at most +once every 10 minutes. Failures are classified rather than treated alike: diff --git a/releaser/src/main.test.ts b/releaser/src/main.test.ts index 4090df1a6..eee8ca184 100644 --- a/releaser/src/main.test.ts +++ b/releaser/src/main.test.ts @@ -31,6 +31,48 @@ async function rpcServer(results: Record): Promise { return server; } +/** What the startup reads ask for, answered so that startup completes. */ +const STARTUP_RESULTS = { + eth_chainId: "0x2105", + eth_call: `0x${"0".repeat(63)}1`, // conversionFactor + eth_getCode: "0x60", // Multicall3 + eth_getBalance: "0xde0b6b3a7640000", // 1 ETH +}; + +const count = (text: string, marker: string) => text.split(marker).length - 1; + +/** The releaser, started against `server`, with its output so far. `until` + * resolves once `seen` holds for that output or the releaser has exited. The + * releaser is stopped after 20 s, so a regression fails a test, not hangs it. */ +function startReleaser(server: Server, stateFile: string) { + const child = spawn(process.execPath, [fileURLToPath(new URL("./main.js", import.meta.url))], { + env: { + PATH: process.env.PATH, + BASE_RPC_URL: `http://127.0.0.1:${(server.address() as AddressInfo).port}/v2/secret-key`, + VAULT_ADDRESS: "0x1111111111111111111111111111111111111111", + RELEASER_PRIVATE_KEY: `0x${"11".repeat(32)}`, + STATE_FILE: stateFile, + POLL_INTERVAL_MS: "1000", + }, + }); + const output = { stdout: "", stderr: "" }; + child.stdout.on("data", (chunk) => (output.stdout += chunk)); + child.stderr.on("data", (chunk) => (output.stderr += chunk)); + const kill = setTimeout(() => child.kill(), 20_000); + child.on("exit", () => clearTimeout(kill)); + const until = (seen: (all: string) => boolean) => + new Promise((resolve) => { + const check = () => { + if (seen(`${output.stdout}${output.stderr}`) || child.exitCode !== null || child.signalCode !== null) resolve(); + }; + check(); + child.stdout.on("data", check); + child.stderr.on("data", check); + child.on("exit", check); + }); + return { child, output, until }; +} + test("startup refuses a Base endpoint serving another chain before it reads the state", { timeout: 30_000 }, async () => { // Answers eth_chainId with Base Sepolia, and nothing else. const server = await rpcServer({ eth_chainId: "0x14a34" }); @@ -38,24 +80,38 @@ test("startup refuses a Base endpoint serving another chain before it reads the const stateFile = join(directory, "state.json"); // Read first, this would fail on the state instead. writeFileSync(stateFile, "{"); + const releaser = startReleaser(server, stateFile); try { - const child = spawn(process.execPath, [fileURLToPath(new URL("./main.js", import.meta.url))], { - env: { - PATH: process.env.PATH, - BASE_RPC_URL: `http://127.0.0.1:${(server.address() as AddressInfo).port}`, - VAULT_ADDRESS: "0x1111111111111111111111111111111111111111", - RELEASER_PRIVATE_KEY: `0x${"11".repeat(32)}`, - STATE_FILE: stateFile, - }, - }); - let stderr = ""; - child.stderr.on("data", (chunk) => (stderr += chunk)); - const kill = setTimeout(() => child.kill(), 20_000); - const status = await new Promise((resolve) => child.on("exit", resolve)); - clearTimeout(kill); - assert.equal(status, 1); - assert.match(stderr, /releaser startup failed — Error: BASE_RPC_URL serves chain 84532, but BASE_CHAIN_ID is 8453/); + await releaser.until(() => false); + assert.equal(releaser.child.exitCode, 1); + assert.match(releaser.output.stderr, /releaser startup failed — Error: BASE_RPC_URL serves chain 84532, but BASE_CHAIN_ID is 8453/); } finally { + releaser.child.kill(); + server.close(); + rmSync(directory, { recursive: true, force: true }); + } +}); + +test("a failed startup read is retried, paging once, until the endpoint answers", { timeout: 30_000 }, async () => { + // Nothing answers at first, as during an outage. + const results: Record = {}; + const server = await rpcServer(results); + const directory = mkdtempSync(join(tmpdir(), "pen-releaser-main-")); + const releaser = startReleaser(server, join(directory, "state.json")); + try { + await releaser.until((all) => count(all, "releaser startup read failed") >= 2); + Object.assign(results, STARTUP_RESULTS); + await releaser.until((all) => all.includes(" started; ")); + const { stdout, stderr } = releaser.output; + const all = `${stdout}${stderr}`; + assert.equal(releaser.child.exitCode, null, all); + assert.ok(all.includes(" started; "), all); + assert.equal(count(stderr, "ALERT: releaser startup read failed"), 1, all); + assert.ok(count(stdout, "releaser startup read failed, retrying:") >= 1, all); + assert.ok(!all.includes("secret-key"), all); + } finally { + releaser.child.kill(); + server.closeAllConnections(); server.close(); rmSync(directory, { recursive: true, force: true }); } @@ -63,51 +119,20 @@ test("startup refuses a Base endpoint serving another chain before it reads the test("a failing cycle pages once, then is only logged, and no output shows the endpoint", { timeout: 30_000 }, async () => { // The startup reads succeed; then every cycle fails on its first read. - const server = await rpcServer({ - eth_chainId: "0x2105", - eth_call: `0x${"0".repeat(63)}1`, // conversionFactor - eth_getCode: "0x60", // Multicall3 - eth_getBalance: "0xde0b6b3a7640000", // 1 ETH - }); + const server = await rpcServer(STARTUP_RESULTS); const directory = mkdtempSync(join(tmpdir(), "pen-releaser-main-")); - const child = spawn(process.execPath, [fileURLToPath(new URL("./main.js", import.meta.url))], { - env: { - PATH: process.env.PATH, - BASE_RPC_URL: `http://127.0.0.1:${(server.address() as AddressInfo).port}/v2/secret-key`, - VAULT_ADDRESS: "0x1111111111111111111111111111111111111111", - RELEASER_PRIVATE_KEY: `0x${"11".repeat(32)}`, - STATE_FILE: join(directory, "state.json"), - POLL_INTERVAL_MS: "1000", - }, - }); - // A regression must fail the test, not leave the releaser running. - const kill = setTimeout(() => child.kill(), 20_000); + const releaser = startReleaser(server, join(directory, "state.json")); try { - let stdout = ""; - let stderr = ""; // Until the third failed cycle, paged or logged, or an early exit. - await new Promise((resolve) => { - const seen = () => { - if (`${stdout}${stderr}`.split("releaser cycle failed").length > 3) resolve(undefined); - }; - child.stdout.on("data", (chunk) => { - stdout += chunk; - seen(); - }); - child.stderr.on("data", (chunk) => { - stderr += chunk; - seen(); - }); - child.on("exit", resolve); - }); - const output = `${stdout}${stderr}`; - assert.equal(stderr.split("ALERT: releaser cycle failed").length, 2, output); - assert.equal(stdout.split("releaser cycle failed:").length, 3, output); + await releaser.until((all) => count(all, "releaser cycle failed") >= 3); + const { stdout, stderr } = releaser.output; + const all = `${stdout}${stderr}`; + assert.equal(count(stderr, "ALERT: releaser cycle failed"), 1, all); + assert.equal(count(stdout, "releaser cycle failed:"), 2, all); assert.match(stderr, /URL: /); - assert.ok(!output.includes("secret-key"), output); + assert.ok(!all.includes("secret-key"), all); } finally { - clearTimeout(kill); - child.kill(); + releaser.child.kill(); server.closeAllConnections(); server.close(); rmSync(directory, { recursive: true, force: true }); diff --git a/releaser/src/main.ts b/releaser/src/main.ts index 069e77c4f..018a0832d 100644 --- a/releaser/src/main.ts +++ b/releaser/src/main.ts @@ -42,9 +42,9 @@ process.on("uncaughtException", (error) => { process.exit(1); }); -/** Repeated cycle failures (a sustained RPC outage) page at most this often; - * every failure is still logged. */ -const CYCLE_ALERT_INTERVAL_MS = 10 * 60_000; +/** A failure that is retried, at startup or in a cycle (a sustained RPC + * outage, say), pages at most this often; every failure is still logged. */ +const RETRY_ALERT_INTERVAL_MS = 10 * 60_000; /** Canonical Multicall3, deployed at the same address on Base and every major * chain. viem refuses to batch unless the chain definition declares it, even @@ -91,6 +91,33 @@ async function alert(subject: string, detail: string, severity: Severity = "warn }); } +let lastRetryAlertMs = 0; + +/** Page a failure that will be retried, or only log it if such a page went out + * less than RETRY_ALERT_INTERVAL_MS ago. */ +async function alertRetried(subject: string, error: unknown): Promise { + if (Date.now() - lastRetryAlertMs >= RETRY_ALERT_INTERVAL_MS) { + lastRetryAlertMs = Date.now(); + await alert(subject, `${error}`); + } else { + log(`${subject}: ${error}`); + } +} + +/** A Base read at startup, tried again every POLL_INTERVAL_MS until it answers: + * a restart during an RPC outage waits for the endpoint, as a running releaser + * does, instead of exiting and paging again on every restart. */ +async function startupRead(read: () => Promise): Promise { + for (;;) { + try { + return await read(); + } catch (error) { + await alertRetried("releaser startup read failed, retrying", error); + await new Promise((resolve) => setTimeout(resolve, config.pollIntervalMs)); + } + } +} + function loadState(): void { const state = readPersistedState(config.stateFile); if (!state) return; @@ -239,17 +266,19 @@ async function checkGasBalance(): Promise { async function main(): Promise { // The state is bound to BASE_CHAIN_ID: an endpoint serving another chain // would scan its logs into this state. - const baseChainId = await publicClient.getChainId(); + const baseChainId = await startupRead(() => publicClient.getChainId()); if (baseChainId !== config.baseChainId) { throw new Error(`BASE_RPC_URL serves chain ${baseChainId}, but BASE_CHAIN_ID is ${config.baseChainId}`); } loadState(); - const conversionFactor = await publicClient.readContract({ - address: config.vaultAddress, - abi: vaultAbi, - functionName: "conversionFactor", - }); - const multicallCode = await publicClient.getBytecode({ address: MULTICALL3 }); + const conversionFactor = await startupRead(() => + publicClient.readContract({ + address: config.vaultAddress, + abi: vaultAbi, + functionName: "conversionFactor", + }), + ); + const multicallCode = await startupRead(() => publicClient.getBytecode({ address: MULTICALL3 })); if (!multicallCode || multicallCode === "0x") { multicallUnavailable = true; await alert( @@ -258,9 +287,8 @@ async function main(): Promise { ); } log(`releaser ${account.address} started; scanning from block ${fromBlock}, ${pending.size} pending`); - await checkGasBalance(); + await startupRead(checkGasBalance); const heartbeat = throttledHeartbeat(config.heartbeatUrl); - let lastCycleAlertMs = 0; for (;;) { try { @@ -282,12 +310,7 @@ async function main(): Promise { ); void heartbeat(); } catch (cycleError) { - if (Date.now() - lastCycleAlertMs >= CYCLE_ALERT_INTERVAL_MS) { - lastCycleAlertMs = Date.now(); - await alert("releaser cycle failed", `${cycleError}`); - } else { - log(`releaser cycle failed: ${cycleError}`); - } + await alertRetried("releaser cycle failed", cycleError); } await new Promise((resolve) => setTimeout(resolve, config.pollIntervalMs)); }