diff --git a/attestor/README.md b/attestor/README.md index c58203e18..b23d7dcf6 100644 --- a/attestor/README.md +++ b/attestor/README.md @@ -49,11 +49,15 @@ point. | `BASE_FINALITY_TIMEOUT_MS` | How long a block's approvals may stay outside that boundary before an alert + idempotent re-submission (default 15 min for `safe`, 45 min for `finalized`) | | `HEAD_STALL_ALERT_MS` | Page when no finalized Pendulum head has arrived for this long (default 5 min): the daemon is push-driven, so a node that stops finalizing would otherwise idle undetected | | `MISSED_APPROVAL_ALERT_COUNT` | Page after this many consecutive migrations resolved on-chain without this attestor's own approval landing, and again every as many more (default 3, a whole number of at least 1) | -| `FEE_BUMP_AFTER_SECONDS` | Re-send an own approval that is still not mined after this long (default 60: Base mines a block every 2 s, so this means stuck, not slow) | +| `FEE_BUMP_AFTER_SECONDS` | Re-send an own approval that is still not mined after this long (default 60: Base mines a block every 200 ms, so this means stuck, not slow) | | `FEE_BUMP_PERCENT` | Raise both fees by at least this many percent per re-send (default 20; a whole number of at least 10, the minimum a node accepts for a replacement) | | `FEE_BUMP_MAX` | Re-sends per approval before the `own approval not mined` page (default 3) | | `MAX_FEE_PER_GAS_GWEI` | No approval ever offers a higher `maxFeePerGas` (default 1, about 200 times Base's usual base fee; one approval uses at most about 300k gas) | +Log lines and alerts show `` in place of every http(s) and ws(s) URL: +endpoint URLs often carry an access key. Startup refuses an endpoint setting +without the right scheme, and does not echo it. + ## Run ```sh @@ -161,7 +165,10 @@ WantedBy=multi-user.target set, that `BASE_RPC_URL` serves `BASE_CHAIN_ID` (`eth_chainId`, which the checkpoint's Base chain must match too) and that `PENDULUM_WS` serves `PENDULUM_GENESIS_HASH`, and refuses to run otherwise. These checks run on - every start, the first one without a checkpoint included. While running, it + every start, the first one without a checkpoint included. A Base read they + need that fails, as during an RPC outage, is tried again every 5 s and pages + `Base read failed at startup, retrying` at most once a minute, so a restart + during an outage waits for the endpoint instead of exiting. While running, it re-reads its attestor membership and generation on every finalized head and exits with `fatal error, exiting` when either changed (a key removed, or removed and re-added): the next start then refuses the checkpoint until it diff --git a/attestor/src/alert.test.ts b/attestor/src/alert.test.ts index 8df13970f..00645d274 100644 --- a/attestor/src/alert.test.ts +++ b/attestor/src/alert.test.ts @@ -3,7 +3,7 @@ import { createServer, type Server } from "node:http"; import type { AddressInfo } from "node:net"; import { test } from "node:test"; import { inspect } from "node:util"; -import { alertText, parseHeartbeatUrl, pingHeartbeat, postAlert, throttledHeartbeat } from "./alert.js"; +import { alertText, parseHeartbeatUrl, pingHeartbeat, postAlert, redactConsole, throttledHeartbeat } from "./alert.js"; /** A local webhook answering every POST with `status`, or never when null. */ async function webhook(status: number | null): Promise<{ url: string; server: Server }> { @@ -166,3 +166,16 @@ test("heartbeats go out at most once a minute, however often they are due", asyn assert.deepEqual(seen, ["/ping", "/ping"]); assert.equal(throttledHeartbeat(undefined)(), undefined, "nothing without a URL"); }); + +test("console output shows in place of each endpoint, also inside a logged error", (t) => { + const mocks = (["debug", "info", "log", "warn", "error"] as const).map((method) => t.mock.method(console, method, () => {})); + redactConsole(); + console.log("scanning %s", "https://base.example/v2/secret-key"); + console.error("cycle failed", new Error("HTTP request failed.\n\nURL: https://base.example/v2/secret-key")); + console.warn("API-WS: disconnected from wss://pendulum.example/ws?apikey=secret-key: 1006"); + console.info("API-WS: disconnected from ws://127.0.0.1:9944/secret-key: 1006"); + console.debug("posting to http://hooks.example/secret-key"); + const lines = mocks.flatMap((mock) => mock.mock.calls.map((call) => `${call.arguments[0]}`)); + assert.equal(lines.length, 5); + for (const line of lines) assert.ok(line.includes("") && !line.includes("secret-key"), line); +}); diff --git a/attestor/src/alert.ts b/attestor/src/alert.ts index 376769f49..20d7443a5 100644 --- a/attestor/src/alert.ts +++ b/attestor/src/alert.ts @@ -1,5 +1,23 @@ +import { format } from "node:util"; + export type Severity = "critical" | "warning"; +/** Strip URLs before anything leaves the process: endpoints commonly embed API + * keys, and error texts quote them verbatim (viem's the RPC URL, polkadot.js + * the WebSocket one). */ +export function redact(text: string): string { + return text.replace(/(?:https?|wss?):\/\/[^\s"')]+/gi, ""); +} + +/** Pass all console output through `redact`, called once at startup: errors + * are logged whole, and libraries log on their own. */ +export function redactConsole(): void { + for (const method of ["debug", "info", "log", "warn", "error"] as const) { + const write = console[method].bind(console); + console[method] = (...args: unknown[]) => write(redact(format(...args))); + } +} + /** Longer details are cut, well inside what Slack shows of one message. */ const MAX_TEXT = 3000; @@ -19,20 +37,27 @@ export function alertText(severity: Severity, source: string, subject: string, d * refuse more than a few requests a minute. */ const HEARTBEAT_INTERVAL_MS = 60_000; -/** HEARTBEAT_URL, checked once at startup: unset, or an http(s) URL. The value - * is never echoed, as it usually holds a token. */ -export function parseHeartbeatUrl(value: string | undefined): string | undefined { - if (!value) return undefined; +/** The URL setting `name`, checked once at startup: an http(s) URL, or a ws(s) + * one when `scheme` says so. The value is never echoed, as it usually holds a + * token. */ +export function checkUrl(name: string, value: string, scheme: "http" | "ws" = "http"): string { let protocol: string | undefined; try { protocol = new URL(value).protocol; } catch { // Not a URL at all: refused below. } - if (protocol !== "http:" && protocol !== "https:") throw new Error("invalid HEARTBEAT_URL: expected an http(s) URL"); + if (protocol !== `${scheme}:` && protocol !== `${scheme}s:`) { + throw new Error(`invalid ${name}: expected ${scheme === "ws" ? "a ws(s)" : "an http(s)"} URL`); + } return value; } +/** HEARTBEAT_URL, checked once at startup: unset, or an http(s) URL. */ +export function parseHeartbeatUrl(value: string | undefined): string | undefined { + return value ? checkUrl("HEARTBEAT_URL", value) : undefined; +} + /** * Request the URL of an external dead-man's switch, which pages once these * requests stop. A failed request is logged, never thrown: the missing diff --git a/attestor/src/config.test.ts b/attestor/src/config.test.ts index d1c6f0e21..56591b26b 100644 --- a/attestor/src/config.test.ts +++ b/attestor/src/config.test.ts @@ -55,6 +55,16 @@ test("on Base mainnet the expected genesis can only be Pendulum's", () => { assert.equal(loadConfig({ PENDULUM_GENESIS_HASH: pendulum }, "config.pendulumGenesisHash").stdout, pendulum); }); +test("an endpoint URL needs the right scheme, and a bad one is refused without echoing it", () => { + assert.equal(loadConfig({ ALERT_WEBHOOK_URL: "https://hooks.example/secret-key" }, "config.alertWebhookUrl").stdout, "https://hooks.example/secret-key"); + for (const [name, value] of Object.entries({ PENDULUM_WS: "https://node.example/secret-key", BASE_RPC_URL: "base.example/secret-key", ALERT_WEBHOOK_URL: "hooks.example/secret-key" })) { + const { status, stderr } = loadConfig({ [name]: value }); + assert.notEqual(status, 0, name); + assert.match(stderr, new RegExp(`invalid ${name}`), name); + assert.doesNotMatch(stderr, /secret-key/, name); + } +}); + test("HEARTBEAT_URL must be an http(s) URL, and a bad one is refused without echoing it", () => { assert.equal(loadConfig({ HEARTBEAT_URL: "https://hc-ping.com/secret-token" }, "config.heartbeatUrl").stdout, "https://hc-ping.com/secret-token"); const { status, stderr } = loadConfig({ HEARTBEAT_URL: "hc-ping.com/secret-token" }); diff --git a/attestor/src/config.ts b/attestor/src/config.ts index 1b0cb4fe2..f112d012b 100644 --- a/attestor/src/config.ts +++ b/attestor/src/config.ts @@ -1,5 +1,5 @@ import { parseGwei } from "viem"; -import { parseHeartbeatUrl } from "./alert.js"; +import { checkUrl, parseHeartbeatUrl } from "./alert.js"; import { MIN_REPLACEMENT_PERCENT } from "./approval.js"; function required(name: string): string { @@ -59,12 +59,12 @@ export const config = { * Never point this at a public RPC: the attestor would inherit its honesty. * The node must retain historical state (`--state-pruning archive`), or any * daemon outage longer than the pruning horizon wedges the catch-up. */ - pendulumWs: required("PENDULUM_WS"), + pendulumWs: checkUrl("PENDULUM_WS", required("PENDULUM_WS"), "ws"), /** The genesis that node must serve; startup refuses any other chain. A * local test chain sets its own, never on Base mainnet. */ pendulumGenesisHash: genesisHash(), /** Base JSON-RPC endpoint. */ - baseRpcUrl: required("BASE_RPC_URL"), + baseRpcUrl: checkUrl("BASE_RPC_URL", required("BASE_RPC_URL")), /** MigrationVault contract address on Base. */ vaultAddress: required("VAULT_ADDRESS") as `0x${string}`, /** This attestor's transaction-signing key (0x-prefixed, 32 bytes). @@ -78,7 +78,7 @@ export const config = { /** Alert when the gas wallet drops below this balance (wei). */ minGasBalanceWei: BigInt(process.env.MIN_GAS_BALANCE_WEI ?? "10000000000000000"), // 0.01 ETH /** Optional webhook that receives JSON alerts (low gas, fatal errors). */ - alertWebhookUrl: process.env.ALERT_WEBHOOK_URL, + alertWebhookUrl: process.env.ALERT_WEBHOOK_URL ? checkUrl("ALERT_WEBHOOK_URL", process.env.ALERT_WEBHOOK_URL) : undefined, /** Optional http(s) URL requested after a finalized head processed without * error, at most once a minute: a dead-man's switch pages when the * requests stop. */ @@ -108,7 +108,7 @@ export const config = { * nonce with both fees raised by `percent` (at least the 10 % a replacement * needs, and at least to the current estimate), at most `max` times and * never above `maxFeePerGas`; then it pages once. Base mines a block every - * 2 s and an approval priced at the estimate lands in the next one or two, + * 200 ms and an approval priced at the estimate lands in the next one or two, * so 60 s means stuck, not slow. Since each bump also catches up with the * estimate, one normally suffices; when three do not, the price is not the * problem and the page (about 4 min after the first send) goes to a person. diff --git a/attestor/src/main.test.ts b/attestor/src/main.test.ts new file mode 100644 index 000000000..4fa34f1ba --- /dev/null +++ b/attestor/src/main.test.ts @@ -0,0 +1,53 @@ +import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import { once } from "node:events"; +import { createServer } from "node:http"; +import type { AddressInfo } from "node:net"; +import { test } from "node:test"; +import { fileURLToPath } from "node:url"; + +test("a failed Base read at startup is retried, paging once, instead of exiting", { timeout: 30_000 }, async () => { + // A Base endpoint that refuses connections, as during an outage. + const closed = createServer().listen(0, "127.0.0.1"); + await once(closed, "listening"); + const port = (closed.address() as AddressInfo).port; + closed.close(); + const child = spawn(process.execPath, [fileURLToPath(new URL("./main.js", import.meta.url))], { + env: { + PATH: process.env.PATH, + PENDULUM_WS: "ws://127.0.0.1:9944", + BASE_RPC_URL: `http://127.0.0.1:${port}/v2/secret-key`, + VAULT_ADDRESS: "0x1111111111111111111111111111111111111111", + ATTESTOR_PRIVATE_KEY: `0x${"11".repeat(32)}`, + }, + }); + let stdout = ""; + let stderr = ""; + // A regression must fail the test, not leave the attestor running. + const kill = setTimeout(() => child.kill(), 20_000); + try { + // Until the second failed read, paged or logged, or an early exit. + await new Promise((resolve) => { + const seen = () => { + if (`${stdout}${stderr}`.split("Base read failed at startup").length > 2) resolve(undefined); + }; + child.stdout.on("data", (chunk) => { + stdout += chunk; + seen(); + }); + child.stderr.on("data", (chunk) => { + stderr += chunk; + seen(); + }); + child.on("exit", resolve); + }); + const all = `${stdout}${stderr}`; + assert.equal(child.exitCode, null, all); + assert.equal(stderr.split("ALERT: Base read failed at startup").length, 2, all); + assert.equal(stdout.split("Base read failed at startup").length, 2, all); + assert.ok(!all.includes("secret-key"), all); + } finally { + clearTimeout(kill); + child.kill(); + } +}); diff --git a/attestor/src/main.ts b/attestor/src/main.ts index 6f9985ed1..f500c6995 100644 --- a/attestor/src/main.ts +++ b/attestor/src/main.ts @@ -24,7 +24,7 @@ import { ApiPromise, WsProvider } from "@polkadot/api"; import { createPublicClient, defineChain, http } from "viem"; import { privateKeyToAccount } from "viem/accounts"; -import { alertText, postAlert, throttledHeartbeat, type Severity } from "./alert.js"; +import { alertText, postAlert, redact, redactConsole, throttledHeartbeat, type Severity } from "./alert.js"; import { advanceCheckpoint, assertSignerUnchanged, @@ -44,9 +44,20 @@ import { type Checkpoint, } from "./state.js"; -/** Repeated transient failures (a sustained RPC outage) page at most this - * often; every occurrence is still logged. */ +redactConsole(); +// Node prints an uncaught error or unhandled rejection itself, past the +// console: print it redacted instead, then exit as Node would. +process.on("uncaughtException", (error) => { + console.error(`${new Date().toISOString()} fatal error`, error); + process.exit(1); +}); + +/** Repeated failures that are retried (a sustained RPC outage, at startup or + * on a finalized head) page at most this often; every occurrence is still + * logged. */ const TRANSIENT_ALERT_INTERVAL_MS = 60_000; +/** A Base read at startup that fails is tried again after this long. */ +const STARTUP_RETRY_MS = 5_000; const baseChain = defineChain({ id: config.baseChainId, @@ -62,12 +73,6 @@ function log(message: string, extra?: unknown): void { console.log(`${new Date().toISOString()} ${message}`, extra ?? ""); } -/** Strip URLs before anything leaves the process: RPC endpoints commonly embed - * API keys, and viem error texts quote the endpoint verbatim. */ -function redact(text: string): string { - return text.replace(/https?:\/\/[^\s"')]+/gi, ""); -} - async function alert(subject: string, detail: unknown, severity: Severity = "warning"): Promise { console.error(`${new Date().toISOString()} ALERT: ${subject}`, detail); if (!config.alertWebhookUrl) return; @@ -82,6 +87,34 @@ async function alert(subject: string, detail: unknown, severity: Severity = "war }); } +let lastRetryAlertMs = 0; + +/** Page a failure that will be retried, or only log it if such a page went out + * less than TRANSIENT_ALERT_INTERVAL_MS ago. */ +async function alertRetried(subject: string, error: unknown): Promise { + if (Date.now() - lastRetryAlertMs >= TRANSIENT_ALERT_INTERVAL_MS) { + lastRetryAlertMs = Date.now(); + await alert(subject, error); + } else { + log(subject, error); + } +} + +/** A Base read at startup, tried again every STARTUP_RETRY_MS until it answers: + * a restart during an RPC outage waits for the endpoint, paging as a running + * attestor does, instead of exiting and paging again on every restart. + * Nothing is processed before these reads, so no event can be skipped. */ +async function startupRead(read: () => Promise): Promise { + for (;;) { + try { + return await read(); + } catch (error) { + await alertRetried("Base read failed at startup, retrying", error); + await new Promise((resolve) => setTimeout(resolve, STARTUP_RETRY_MS)); + } + } +} + /** Extract MigrationInitiated events from one finalized Pendulum block. */ async function migrationEventsInBlock(api: ApiPromise, blockNumber: number): Promise { const blockHash = await api.rpc.chain.getBlockHash(blockNumber); @@ -125,13 +158,13 @@ async function checkGasBalance(): Promise { } async function main(): Promise { - const baseChainId = await publicClient.getChainId(); + const baseChainId = await startupRead(() => publicClient.getChainId()); assertBaseChain(baseChainId, config.baseChainId); - if (!(await attestor.chain.isAttestor())) { + if (!(await startupRead(() => attestor.chain.isAttestor()))) { throw new Error(`${account.address} is not an attestor of ${config.vaultAddress}`); } - const attestorGeneration = await attestor.chain.attestorGeneration(); - await checkGasBalance(); + const attestorGeneration = await startupRead(() => attestor.chain.attestorGeneration()); + await startupRead(checkGasBalance); const api = await ApiPromise.create({ provider: new WsProvider(config.pendulumWs) }); const pendulumGenesisHash = api.genesisHash.toHex(); @@ -158,7 +191,6 @@ async function main(): Promise { * checkpoint has not yet passed. Bounded by the Base finality lag. */ const unconfirmed: UnconfirmedBlock[] = []; let processedThrough = checkpoint.lastProcessedBlock; - let lastTransientAlertMs = 0; const heartbeat = throttledHeartbeat(config.heartbeatUrl); let lastHeadAt = Date.now(); @@ -183,12 +215,7 @@ async function main(): Promise { // Not a statement about any event — resume from the in-memory // position on the next finalized head. Sustained outages page // at a bounded rate instead of once per head. - if (Date.now() - lastTransientAlertMs >= TRANSIENT_ALERT_INTERVAL_MS) { - lastTransientAlertMs = Date.now(); - await alert("transient RPC failure, retrying on the next finalized head", error); - } else { - log("transient RPC failure, retrying on the next finalized head", error); - } + await alertRetried("transient RPC failure, retrying on the next finalized head", error); return; } // PRD A5: never skip an event silently. Alert and exit; the process diff --git a/releaser/README.md b/releaser/README.md index f5d060db4..0b234b378 100644 --- a/releaser/README.md +++ b/releaser/README.md @@ -48,6 +48,10 @@ consumed nonce and drops it. | `FEE_BUMP_MAX` | Re-sends per transaction before the `release transaction not mined` page (default 3) | | `MAX_FEE_PER_GAS_GWEI` | No release transaction ever offers a higher `maxFeePerGas` (default 1) | +Log lines and alerts show `` in place of every http(s) and ws(s) URL: +endpoint URLs often carry an access key. Startup refuses an endpoint setting +without the right scheme, and does not echo it. + ## Run ```sh @@ -58,7 +62,12 @@ npm install && npm run build && npm start Each cycle it scans new `ReleasePending` logs, drops nonces the vault has already consumed, then simulates and submits `release()` for the rest. -Simulating first means a still-capped release costs no gas. +Simulating first means a still-capped release costs no gas. A cycle that +fails, on an RPC error for instance, is logged and runs again after +`POLL_INTERVAL_MS`, and so does a failed Base read at startup: a restart during +an outage waits for the endpoint instead of exiting. These failures page +(`releaser cycle failed`, `releaser startup read failed, retrying`) at most +once every 10 minutes. Failures are classified rather than treated alike: @@ -96,7 +105,8 @@ trusting unsafe logs or losing pending work. Malformed state is fatal, and so is an endpoint that does not serve `BASE_CHAIN_ID`, checked before the state is read on every start. Multicall is disabled permanently only when an on-chain code check proves it absent; a -temporary provider error falls back in bounded batches for that cycle. +temporary provider error falls back in bounded batches for that cycle and is +logged, not paged. **Known limitation:** the pending set is driven by `ReleasePending`, which the vault emits when a threshold is crossed inside `approve()`. A payload made diff --git a/releaser/src/alert.test.ts b/releaser/src/alert.test.ts index bd4480cfa..8754a3672 100644 --- a/releaser/src/alert.test.ts +++ b/releaser/src/alert.test.ts @@ -3,7 +3,7 @@ import { createServer, type Server } from "node:http"; import type { AddressInfo } from "node:net"; import { test } from "node:test"; import { inspect } from "node:util"; -import { alertText, parseHeartbeatUrl, pingHeartbeat, postAlert, throttledHeartbeat } from "./alert.js"; +import { alertText, parseHeartbeatUrl, pingHeartbeat, postAlert, redactConsole, throttledHeartbeat } from "./alert.js"; /** A local webhook answering every POST with `status`, or never when null. */ async function webhook(status: number | null): Promise<{ url: string; server: Server }> { @@ -166,3 +166,16 @@ test("heartbeats go out at most once a minute, however often they are due", asyn assert.deepEqual(seen, ["/ping", "/ping"]); assert.equal(throttledHeartbeat(undefined)(), undefined, "nothing without a URL"); }); + +test("console output shows in place of each endpoint, also inside a logged error", (t) => { + const mocks = (["debug", "info", "log", "warn", "error"] as const).map((method) => t.mock.method(console, method, () => {})); + redactConsole(); + console.log("scanning %s", "https://base.example/v2/secret-key"); + console.error("cycle failed", new Error("HTTP request failed.\n\nURL: https://base.example/v2/secret-key")); + console.warn("API-WS: disconnected from wss://pendulum.example/ws?apikey=secret-key: 1006"); + console.info("API-WS: disconnected from ws://127.0.0.1:9944/secret-key: 1006"); + console.debug("posting to http://hooks.example/secret-key"); + const lines = mocks.flatMap((mock) => mock.mock.calls.map((call) => `${call.arguments[0]}`)); + assert.equal(lines.length, 5); + for (const line of lines) assert.ok(line.includes("") && !line.includes("secret-key"), line); +}); diff --git a/releaser/src/alert.ts b/releaser/src/alert.ts index 376769f49..20d7443a5 100644 --- a/releaser/src/alert.ts +++ b/releaser/src/alert.ts @@ -1,5 +1,23 @@ +import { format } from "node:util"; + export type Severity = "critical" | "warning"; +/** Strip URLs before anything leaves the process: endpoints commonly embed API + * keys, and error texts quote them verbatim (viem's the RPC URL, polkadot.js + * the WebSocket one). */ +export function redact(text: string): string { + return text.replace(/(?:https?|wss?):\/\/[^\s"')]+/gi, ""); +} + +/** Pass all console output through `redact`, called once at startup: errors + * are logged whole, and libraries log on their own. */ +export function redactConsole(): void { + for (const method of ["debug", "info", "log", "warn", "error"] as const) { + const write = console[method].bind(console); + console[method] = (...args: unknown[]) => write(redact(format(...args))); + } +} + /** Longer details are cut, well inside what Slack shows of one message. */ const MAX_TEXT = 3000; @@ -19,20 +37,27 @@ export function alertText(severity: Severity, source: string, subject: string, d * refuse more than a few requests a minute. */ const HEARTBEAT_INTERVAL_MS = 60_000; -/** HEARTBEAT_URL, checked once at startup: unset, or an http(s) URL. The value - * is never echoed, as it usually holds a token. */ -export function parseHeartbeatUrl(value: string | undefined): string | undefined { - if (!value) return undefined; +/** The URL setting `name`, checked once at startup: an http(s) URL, or a ws(s) + * one when `scheme` says so. The value is never echoed, as it usually holds a + * token. */ +export function checkUrl(name: string, value: string, scheme: "http" | "ws" = "http"): string { let protocol: string | undefined; try { protocol = new URL(value).protocol; } catch { // Not a URL at all: refused below. } - if (protocol !== "http:" && protocol !== "https:") throw new Error("invalid HEARTBEAT_URL: expected an http(s) URL"); + if (protocol !== `${scheme}:` && protocol !== `${scheme}s:`) { + throw new Error(`invalid ${name}: expected ${scheme === "ws" ? "a ws(s)" : "an http(s)"} URL`); + } return value; } +/** HEARTBEAT_URL, checked once at startup: unset, or an http(s) URL. */ +export function parseHeartbeatUrl(value: string | undefined): string | undefined { + return value ? checkUrl("HEARTBEAT_URL", value) : undefined; +} + /** * Request the URL of an external dead-man's switch, which pages once these * requests stop. A failed request is logged, never thrown: the missing diff --git a/releaser/src/config.test.ts b/releaser/src/config.test.ts index 29737a671..2d2c7910c 100644 --- a/releaser/src/config.test.ts +++ b/releaser/src/config.test.ts @@ -20,6 +20,16 @@ function loadConfig(extra: Record, print = "''"): { status: numb return { status: result.status, stdout: result.stdout.trim(), stderr: result.stderr }; } +test("an endpoint URL needs the right scheme, and a bad one is refused without echoing it", () => { + assert.equal(loadConfig({ ALERT_WEBHOOK_URL: "https://hooks.example/secret-key" }, "config.alertWebhookUrl").stdout, "https://hooks.example/secret-key"); + for (const [name, value] of Object.entries({ BASE_RPC_URL: "wss://base.example/secret-key", ALERT_WEBHOOK_URL: "hooks.example/secret-key" })) { + const { status, stderr } = loadConfig({ [name]: value }); + assert.notEqual(status, 0, name); + assert.match(stderr, new RegExp(`invalid ${name}`), name); + assert.doesNotMatch(stderr, /secret-key/, name); + } +}); + test("HEARTBEAT_URL must be an http(s) URL, and a bad one is refused without echoing it", () => { assert.equal(loadConfig({ HEARTBEAT_URL: "https://hc-ping.com/secret-token" }, "config.heartbeatUrl").stdout, "https://hc-ping.com/secret-token"); const { status, stderr } = loadConfig({ HEARTBEAT_URL: "hc-ping.com/secret-token" }); diff --git a/releaser/src/config.ts b/releaser/src/config.ts index bcd66d9e1..14ffaf3e0 100644 --- a/releaser/src/config.ts +++ b/releaser/src/config.ts @@ -1,5 +1,5 @@ import { parseGwei } from "viem"; -import { parseHeartbeatUrl } from "./alert.js"; +import { checkUrl, parseHeartbeatUrl } from "./alert.js"; import { MIN_REPLACEMENT_PERCENT } from "./release.js"; function required(name: string): string { @@ -37,7 +37,7 @@ function finalityTag(): "safe" | "finalized" { export const config = { /** Base JSON-RPC endpoint. */ - baseRpcUrl: required("BASE_RPC_URL"), + baseRpcUrl: checkUrl("BASE_RPC_URL", required("BASE_RPC_URL")), /** MigrationVault contract address on Base. */ vaultAddress: required("VAULT_ADDRESS") as `0x${string}`, /** Gas-only signing key. `release()` is permissionless and can only pay the @@ -55,7 +55,7 @@ export const config = { maxBlockRange: BigInt(envNumber("MAX_BLOCK_RANGE", 10_000, 1, true)), /** Alert when the gas wallet drops below this balance (wei). */ minGasBalanceWei: BigInt(process.env.MIN_GAS_BALANCE_WEI ?? "5000000000000000"), - alertWebhookUrl: process.env.ALERT_WEBHOOK_URL, + alertWebhookUrl: process.env.ALERT_WEBHOOK_URL ? checkUrl("ALERT_WEBHOOK_URL", process.env.ALERT_WEBHOOK_URL) : undefined, /** Optional http(s) URL requested after a cycle that completed without * error, at most once a minute: a dead-man's switch pages when the * requests stop. */ diff --git a/releaser/src/main.test.ts b/releaser/src/main.test.ts index 691da9cfb..eee8ca184 100644 --- a/releaser/src/main.test.ts +++ b/releaser/src/main.test.ts @@ -1,53 +1,139 @@ import assert from "node:assert/strict"; import { spawn } from "node:child_process"; import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; -import { createServer } from "node:http"; +import { createServer, type Server } from "node:http"; import type { AddressInfo } from "node:net"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { test } from "node:test"; import { fileURLToPath } from "node:url"; -test("startup refuses a Base endpoint serving another chain before it reads the state", { timeout: 30_000 }, async () => { - // Answers eth_chainId with Base Sepolia, and nothing else. +/** A local JSON-RPC endpoint giving these results by method, and an error for + * any other method. */ +async function rpcServer(results: Record): Promise { const server = createServer((request, response) => { let body = ""; request.on("data", (chunk) => (body += chunk)); request.on("end", () => { const { id, method } = JSON.parse(body); + const result = results[method]; response.setHeader("content-type", "application/json"); response.end( JSON.stringify( - method === "eth_chainId" - ? { jsonrpc: "2.0", id, result: "0x14a34" } - : { jsonrpc: "2.0", id, error: { code: -32601, message: `unexpected ${method}` } }, + result === undefined + ? { jsonrpc: "2.0", id, error: { code: -32601, message: `unexpected ${method}` } } + : { jsonrpc: "2.0", id, result }, ), ); }); }); await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + return server; +} + +/** What the startup reads ask for, answered so that startup completes. */ +const STARTUP_RESULTS = { + eth_chainId: "0x2105", + eth_call: `0x${"0".repeat(63)}1`, // conversionFactor + eth_getCode: "0x60", // Multicall3 + eth_getBalance: "0xde0b6b3a7640000", // 1 ETH +}; + +const count = (text: string, marker: string) => text.split(marker).length - 1; + +/** The releaser, started against `server`, with its output so far. `until` + * resolves once `seen` holds for that output or the releaser has exited. The + * releaser is stopped after 20 s, so a regression fails a test, not hangs it. */ +function startReleaser(server: Server, stateFile: string) { + const child = spawn(process.execPath, [fileURLToPath(new URL("./main.js", import.meta.url))], { + env: { + PATH: process.env.PATH, + BASE_RPC_URL: `http://127.0.0.1:${(server.address() as AddressInfo).port}/v2/secret-key`, + VAULT_ADDRESS: "0x1111111111111111111111111111111111111111", + RELEASER_PRIVATE_KEY: `0x${"11".repeat(32)}`, + STATE_FILE: stateFile, + POLL_INTERVAL_MS: "1000", + }, + }); + const output = { stdout: "", stderr: "" }; + child.stdout.on("data", (chunk) => (output.stdout += chunk)); + child.stderr.on("data", (chunk) => (output.stderr += chunk)); + const kill = setTimeout(() => child.kill(), 20_000); + child.on("exit", () => clearTimeout(kill)); + const until = (seen: (all: string) => boolean) => + new Promise((resolve) => { + const check = () => { + if (seen(`${output.stdout}${output.stderr}`) || child.exitCode !== null || child.signalCode !== null) resolve(); + }; + check(); + child.stdout.on("data", check); + child.stderr.on("data", check); + child.on("exit", check); + }); + return { child, output, until }; +} + +test("startup refuses a Base endpoint serving another chain before it reads the state", { timeout: 30_000 }, async () => { + // Answers eth_chainId with Base Sepolia, and nothing else. + const server = await rpcServer({ eth_chainId: "0x14a34" }); const directory = mkdtempSync(join(tmpdir(), "pen-releaser-main-")); const stateFile = join(directory, "state.json"); // Read first, this would fail on the state instead. writeFileSync(stateFile, "{"); + const releaser = startReleaser(server, stateFile); try { - const child = spawn(process.execPath, [fileURLToPath(new URL("./main.js", import.meta.url))], { - env: { - PATH: process.env.PATH, - BASE_RPC_URL: `http://127.0.0.1:${(server.address() as AddressInfo).port}`, - VAULT_ADDRESS: "0x1111111111111111111111111111111111111111", - RELEASER_PRIVATE_KEY: `0x${"11".repeat(32)}`, - STATE_FILE: stateFile, - }, - }); - let stderr = ""; - child.stderr.on("data", (chunk) => (stderr += chunk)); - const kill = setTimeout(() => child.kill(), 20_000); - const status = await new Promise((resolve) => child.on("exit", resolve)); - clearTimeout(kill); - assert.equal(status, 1); - assert.match(stderr, /releaser startup failed — Error: BASE_RPC_URL serves chain 84532, but BASE_CHAIN_ID is 8453/); + await releaser.until(() => false); + assert.equal(releaser.child.exitCode, 1); + assert.match(releaser.output.stderr, /releaser startup failed — Error: BASE_RPC_URL serves chain 84532, but BASE_CHAIN_ID is 8453/); + } finally { + releaser.child.kill(); + server.close(); + rmSync(directory, { recursive: true, force: true }); + } +}); + +test("a failed startup read is retried, paging once, until the endpoint answers", { timeout: 30_000 }, async () => { + // Nothing answers at first, as during an outage. + const results: Record = {}; + const server = await rpcServer(results); + const directory = mkdtempSync(join(tmpdir(), "pen-releaser-main-")); + const releaser = startReleaser(server, join(directory, "state.json")); + try { + await releaser.until((all) => count(all, "releaser startup read failed") >= 2); + Object.assign(results, STARTUP_RESULTS); + await releaser.until((all) => all.includes(" started; ")); + const { stdout, stderr } = releaser.output; + const all = `${stdout}${stderr}`; + assert.equal(releaser.child.exitCode, null, all); + assert.ok(all.includes(" started; "), all); + assert.equal(count(stderr, "ALERT: releaser startup read failed"), 1, all); + assert.ok(count(stdout, "releaser startup read failed, retrying:") >= 1, all); + assert.ok(!all.includes("secret-key"), all); + } finally { + releaser.child.kill(); + server.closeAllConnections(); + server.close(); + rmSync(directory, { recursive: true, force: true }); + } +}); + +test("a failing cycle pages once, then is only logged, and no output shows the endpoint", { timeout: 30_000 }, async () => { + // The startup reads succeed; then every cycle fails on its first read. + const server = await rpcServer(STARTUP_RESULTS); + const directory = mkdtempSync(join(tmpdir(), "pen-releaser-main-")); + const releaser = startReleaser(server, join(directory, "state.json")); + try { + // Until the third failed cycle, paged or logged, or an early exit. + await releaser.until((all) => count(all, "releaser cycle failed") >= 3); + const { stdout, stderr } = releaser.output; + const all = `${stdout}${stderr}`; + assert.equal(count(stderr, "ALERT: releaser cycle failed"), 1, all); + assert.equal(count(stdout, "releaser cycle failed:"), 2, all); + assert.match(stderr, /URL: /); + assert.ok(!all.includes("secret-key"), all); } finally { + releaser.child.kill(); + server.closeAllConnections(); server.close(); rmSync(directory, { recursive: true, force: true }); } diff --git a/releaser/src/main.ts b/releaser/src/main.ts index d857caa4a..018a0832d 100644 --- a/releaser/src/main.ts +++ b/releaser/src/main.ts @@ -22,7 +22,7 @@ import { createPublicClient, createWalletClient, defineChain, http } from "viem"; import { privateKeyToAccount } from "viem/accounts"; -import { alertText, postAlert, throttledHeartbeat, type Severity } from "./alert.js"; +import { alertText, postAlert, redact, redactConsole, throttledHeartbeat, type Severity } from "./alert.js"; import { chunks, readInRanges, toPalletAmount } from "./checks.js"; import { config } from "./config.js"; import { drainPending, type PendingRelease, type Releaser } from "./release.js"; @@ -34,6 +34,18 @@ import { } from "./state.js"; import { vaultAbi } from "./vaultAbi.js"; +redactConsole(); +// Node prints an uncaught error or unhandled rejection itself, past the +// console: print it redacted instead, then exit as Node would. +process.on("uncaughtException", (error) => { + console.error(`${new Date().toISOString()} fatal error`, error); + process.exit(1); +}); + +/** A failure that is retried, at startup or in a cycle (a sustained RPC + * outage, say), pages at most this often; every failure is still logged. */ +const RETRY_ALERT_INTERVAL_MS = 10 * 60_000; + /** Canonical Multicall3, deployed at the same address on Base and every major * chain. viem refuses to batch unless the chain definition declares it, even * when the contract is present on-chain, so it has to be named here. */ @@ -63,12 +75,6 @@ function log(message: string): void { console.log(`${new Date().toISOString()} ${message}`); } -/** Strip URLs before anything leaves the process: RPC endpoints commonly embed - * API keys, and viem error texts quote the endpoint verbatim. */ -function redact(text: string): string { - return text.replace(/https?:\/\/[^\s"')]+/gi, ""); -} - /** Log the alert and post it to the webhook; resolves to whether it was * delivered. Without a webhook the log line is the delivery. */ async function alert(subject: string, detail: string, severity: Severity = "warning"): Promise { @@ -85,6 +91,33 @@ async function alert(subject: string, detail: string, severity: Severity = "warn }); } +let lastRetryAlertMs = 0; + +/** Page a failure that will be retried, or only log it if such a page went out + * less than RETRY_ALERT_INTERVAL_MS ago. */ +async function alertRetried(subject: string, error: unknown): Promise { + if (Date.now() - lastRetryAlertMs >= RETRY_ALERT_INTERVAL_MS) { + lastRetryAlertMs = Date.now(); + await alert(subject, `${error}`); + } else { + log(`${subject}: ${error}`); + } +} + +/** A Base read at startup, tried again every POLL_INTERVAL_MS until it answers: + * a restart during an RPC outage waits for the endpoint, as a running releaser + * does, instead of exiting and paging again on every restart. */ +async function startupRead(read: () => Promise): Promise { + for (;;) { + try { + return await read(); + } catch (error) { + await alertRetried("releaser startup read failed, retrying", error); + await new Promise((resolve) => setTimeout(resolve, config.pollIntervalMs)); + } + } +} + function loadState(): void { const state = readPersistedState(config.stateFile); if (!state) return; @@ -183,10 +216,9 @@ async function readConsumed(nonces: bigint[], blockNumber: bigint): Promise { async function main(): Promise { // The state is bound to BASE_CHAIN_ID: an endpoint serving another chain // would scan its logs into this state. - const baseChainId = await publicClient.getChainId(); + const baseChainId = await startupRead(() => publicClient.getChainId()); if (baseChainId !== config.baseChainId) { throw new Error(`BASE_RPC_URL serves chain ${baseChainId}, but BASE_CHAIN_ID is ${config.baseChainId}`); } loadState(); - const conversionFactor = await publicClient.readContract({ - address: config.vaultAddress, - abi: vaultAbi, - functionName: "conversionFactor", - }); - const multicallCode = await publicClient.getBytecode({ address: MULTICALL3 }); + const conversionFactor = await startupRead(() => + publicClient.readContract({ + address: config.vaultAddress, + abi: vaultAbi, + functionName: "conversionFactor", + }), + ); + const multicallCode = await startupRead(() => publicClient.getBytecode({ address: MULTICALL3 })); if (!multicallCode || multicallCode === "0x") { multicallUnavailable = true; await alert( @@ -253,7 +287,7 @@ async function main(): Promise { ); } log(`releaser ${account.address} started; scanning from block ${fromBlock}, ${pending.size} pending`); - await checkGasBalance(); + await startupRead(checkGasBalance); const heartbeat = throttledHeartbeat(config.heartbeatUrl); for (;;) { @@ -276,7 +310,7 @@ async function main(): Promise { ); void heartbeat(); } catch (cycleError) { - await alert("releaser cycle failed", `${cycleError}`); + await alertRetried("releaser cycle failed", cycleError); } await new Promise((resolve) => setTimeout(resolve, config.pollIntervalMs)); }