diff --git a/daemons/README.md b/daemons/README.md index 816e4ef..fca3777 100644 --- a/daemons/README.md +++ b/daemons/README.md @@ -205,6 +205,11 @@ cancel of a pending one. Runbook: `docs/pq-rotation.md`. ## Audit +`rand-bridge-audit --config --headroom` counts the guardians that are signing +against the quorum, for releases and for mints, and exits non-zero without a spare (BR-402; +`docs/guardian-hosts.md`, "Quorum headroom"). A guardian's `/v1/health` names the sha256 of the +PQ key it co-signs with (`pq_key_sha256`) for that count. + `rand-bridge-audit --config ` reconciles custody: for each of the seven approved backings it reads the endpoint's `custody`, its accrued protocol fees and the token balance it really holds, and — once the Rand node serves per-backing `locked` rows — what Rand says is outstanding. It diff --git a/daemons/mainnet/provision-guardian.sh b/daemons/mainnet/provision-guardian.sh new file mode 100755 index 0000000..d1922f0 --- /dev/null +++ b/daemons/mainnet/provision-guardian.sh @@ -0,0 +1,122 @@ +#!/usr/bin/env bash +# Lays a guardian down on a NEW host (BR-402): the binary, its config, the systemd unit, an +# empty secrets file and the data directory. Run it ON that host, as root. It contacts nothing, +# reads no key, writes no key, and does not start the service: the key holder puts the keys in +# the secrets file by hand, on that host, and starts it (docs/guardian-hosts.md, "Bringing a +# guardian up on its own host"). +# +# provision-guardian.sh --index 7 --binary ./rand-guardian --sha256 <64 hex> \ +# --config ./guardian-7.toml [--root ] [--dry-run] +# +# --sha256 the checksum of the binary as built and recorded elsewhere; a mismatch stops here. +# --root install under instead of / (a rehearsal or a test): no user is created and +# systemd is not touched. +# It never overwrites: an existing binary, config, unit or secrets file that differs stops it. +set -euo pipefail + +index="" binary="" sha="" config="" root="" dry=0 +while [[ $# -gt 0 ]]; do + case "$1" in + --index) index="$2"; shift 2 ;; + --binary) binary="$2"; shift 2 ;; + --sha256) sha="$2"; shift 2 ;; + --config) config="$2"; shift 2 ;; + --root) root="${2%/}"; shift 2 ;; + --dry-run) dry=1; shift ;; + *) echo "error: unknown argument $1" >&2; exit 2 ;; + esac +done +die() { echo "error: $*" >&2; exit 1; } +[[ "$index" =~ ^[1-9][0-9]?$ ]] || die "--index is required" +[[ -f "$binary" ]] || die "--binary: $binary is not a file" +[[ "$sha" =~ ^[0-9a-fA-F]{64}$ ]] || die "--sha256 takes the binary's 64-hex checksum" +[[ -f "$config" ]] || die "--config: $config is not a file" + +if command -v sha256sum >/dev/null 2>&1; then got=$(sha256sum "$binary" | cut -d' ' -f1) +else got=$(shasum -a 256 "$binary" | cut -d' ' -f1); fi +[[ "$(printf %s "$got" | tr A-F a-f)" == "$(printf %s "$sha" | tr A-F a-f)" ]] \ + || die "the binary's sha256 is $got, not the one given: do not install it" + +# The config must be this guardian's: its key variables carry its number, and its API stays on +# loopback. Nothing in it is a secret. +key_env="GUARDIAN${index}_PRIV_KEY" +pq_env="GUARDIAN${index}_PQ_SEED" +grep -Eq "^key_env *= *\"$key_env\"" "$config" || die "the config's key_env is not \"$key_env\": wrong guardian's file?" +grep -Eq '^listen *= *"127\.0\.0\.1:[0-9]+"' "$config" || die "the config's [guardian] listen is not on 127.0.0.1" +grep -Eqi '(priv_key|pq_seed) *= *"[0-9a-f]{64}' "$config" && die "the config seems to carry key material; keys go in the secrets file only" + +bin_to="$root/usr/local/bin/rand-guardian" +cfg_to="$root/etc/rand-guardian.toml" +env_to="$root/etc/rand-guardian/env" +unit_to="$root/etc/systemd/system/rand-guardian.service" +data_to="$root/var/lib/rand-guardian/data" + +unit=$(cat < for a rehearsal)" + id rand-guardian >/dev/null 2>&1 || useradd --system --home /var/lib/rand-guardian --shell /usr/sbin/nologin rand-guardian +fi +install -d -m 755 "$root/usr/local/bin" "$root/etc/systemd/system" +install -d -m 700 "$root/etc/rand-guardian" "$root/var/lib/rand-guardian" "$data_to" +install -m 755 "$binary" "$bin_to" +install -m 640 "$config" "$cfg_to" +printf '%s\n' "$unit" >"$unit_to" +if [[ ! -e "$env_to" ]]; then (umask 077; printf '%s\n' "$envfile" >"$env_to"); fi +chmod 600 "$env_to" +if [[ -z "$root" ]]; then + chown -R rand-guardian:rand-guardian /var/lib/rand-guardian + chown root:rand-guardian "$cfg_to" + systemctl daemon-reload +fi + +cat </v1/health # names this guardian's address + 4. from the relayer host: rand-bridge-audit --config --headroom +NEXT diff --git a/daemons/mainnet/systemd/rand-bridge-headroom.service b/daemons/mainnet/systemd/rand-bridge-headroom.service new file mode 100644 index 0000000..a170b28 --- /dev/null +++ b/daemons/mainnet/systemd/rand-bridge-headroom.service @@ -0,0 +1,12 @@ +# Quorum headroom check (BR-402), for the relayer host: fails when fewer than one guardian could +# stop before signing stops. Pair it with rand-bridge-headroom.timer. OnFailure names the unit +# that tells someone; rand-bridge-alert@.service is an example to replace with your own +# (mail, a webhook) — this repository ships no notifier. +[Unit] +Description=Rand bridge guardian quorum headroom (BR-402) +OnFailure=rand-bridge-alert@%n.service + +[Service] +Type=oneshot +User=rand-relayer +ExecStart=/usr/local/bin/rand-bridge-audit --config /etc/rand-relayer.toml --headroom --min-spare 1 diff --git a/daemons/mainnet/systemd/rand-bridge-headroom.timer b/daemons/mainnet/systemd/rand-bridge-headroom.timer new file mode 100644 index 0000000..a4cffdd --- /dev/null +++ b/daemons/mainnet/systemd/rand-bridge-headroom.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Check the Rand bridge guardian quorum headroom every five minutes (BR-402) + +[Timer] +OnBootSec=2min +OnUnitActiveSec=5min + +[Install] +WantedBy=timers.target diff --git a/daemons/src/api.rs b/daemons/src/api.rs index 4022006..dd695af 100644 --- a/daemons/src/api.rs +++ b/daemons/src/api.rs @@ -53,17 +53,29 @@ pub struct Health { /// Transfers waiting for room in their window, or for the peer check, `chain/sequence`. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub deferred: Vec, + /// The sha256 (hex) of the Dilithium2 public key this guardian co-signs mints with; + /// absent when it holds none, and from a build before BR-402's headroom check. Public: + /// the key itself is in `rand_getBridgeState`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub pq_key_sha256: Option, } struct ApiState { store: Store, guardian: String, + pq_key_sha256: Option, } pub fn router(store: Store, guardian: [u8; 20]) -> Router { + router_with_pq(store, guardian, None) +} + +/// [`router`], naming in `/v1/health` the PQ public key the guardian co-signs with. +pub fn router_with_pq(store: Store, guardian: [u8; 20], pq_public_key: Option<&[u8]>) -> Router { let state = Arc::new(ApiState { store, guardian: hex::encode(guardian), + pq_key_sha256: pq_public_key.map(crate::headroom::pq_key_id), }); Router::new() .route("/v1/health", get(health)) @@ -89,6 +101,7 @@ async fn health(State(state): State>) -> Json { paused, held, deferred, + pq_key_sha256: state.pq_key_sha256.clone(), }) } @@ -132,6 +145,21 @@ impl GuardianClient { &self.origin } + /// `GET /v1/health`. + pub async fn health(&self) -> Result { + let url = format!("{}/v1/health", self.origin); + self.http + .get(&url) + .send() + .await + .with_context(|| format!("GET {url}"))? + .error_for_status() + .with_context(|| format!("GET {url}"))? + .json() + .await + .with_context(|| format!("GET {url}: not a health record")) + } + /// Whatever record this guardian serves for `(chain, sequence)`, unchecked: /// the caller decides what it proves (BR-502). pub async fn record(&self, chain: u16, sequence: u64) -> Result> { diff --git a/daemons/src/bin/audit.rs b/daemons/src/bin/audit.rs index b764583..0e408c5 100644 --- a/daemons/src/bin/audit.rs +++ b/daemons/src/bin/audit.rs @@ -46,6 +46,19 @@ struct Cli { /// multisig and upgrade authority on every endpoint. #[arg(long)] governance: bool, + /// Check how many guardians are signing against the quorum (BR-402) + /// instead of custody: asks each guardian API for `/v1/health` and the + /// Rand node for the registered set. Read-only; exits non-zero when + /// fewer than `--min-spare` guardians could stop before signing stops. + #[arg(long, conflicts_with = "governance")] + headroom: bool, + /// The spare `--headroom` asks for, for releases and for mints. + #[arg(long, default_value_t = 1)] + min_spare: u32, + /// The guardian API origins `--headroom` asks, comma-separated; default + /// `[relayer].guardians` of the config. + #[arg(long, value_delimiter = ',')] + guardian_origins: Vec, } /// The approved backings: (bridge chain, symbol, token, decimals). @@ -640,6 +653,47 @@ async fn solana_governance( /// can change the endpoints, not who can move custody. const GOVERNANCE_CAVEAT: &str = "caveat: custody is still authorised by the guardian quorum, whose keys are not yet distributed (BR-4)"; +// ---- --headroom -------------------------------------------------------------- + +/// The `--headroom` report. `Ok(false)` when the spare is under `min_spare`. +async fn headroom_report(config: &Config, origins: &[String], min_spare: i64, out: &mut impl std::io::Write) -> Result { + use bridge_daemons::headroom; + let origins: Vec = if origins.is_empty() { + config.relayer.as_ref().map(|r| r.guardians.clone()).unwrap_or_default() + } else { + origins.to_vec() + }; + if origins.is_empty() { + bail!("no guardian origins: pass --guardian-origins, or a config with [relayer].guardians"); + } + let state = bridge_daemons::sources::rand::bridge_state(&JsonRpc::new(&config.rand.rpc)) + .await + .context("reading the registered guardian set from the Rand node")? + .ok_or_else(|| anyhow!("the Rand node serves no bridge"))?; + let mut asks = tokio::task::JoinSet::new(); + for (i, origin) in origins.iter().enumerate() { + let client = bridge_daemons::api::GuardianClient::new(origin); + asks.spawn(async move { (i, client.health().await.map_err(|e| format!("{e:#}"))) }); + } + let mut answers: Vec>> = vec![None; origins.len()]; + while let Some(joined) = asks.join_next().await { + let (i, health) = joined?; + answers[i] = Some(health); + } + let probes: Vec = origins + .iter() + .zip(answers) + .map(|(origin, health)| headroom::Probe { + origin: JsonRpc::new(origin).shown(), + health: health.unwrap_or_else(|| Err("not asked".into())), + }) + .collect(); + let report = headroom::report(&state.guardians, &state.pq_guardians, &probes); + writeln!(out, "guardian set {} on the Rand node", state.guardian_set_index)?; + write!(out, "{}", headroom::render(&report, min_spare))?; + Ok(report.ok(min_spare)) +} + /// Writes the BR-4 caveat after a `--governance` run, however it ended (an /// early error included), and hands the verdict back unchanged. fn with_caveat(result: Result, out: &mut dyn std::io::Write) -> Result { @@ -736,6 +790,12 @@ async fn main() -> Result<()> { let result = governance(&config).await; return with_caveat(result, &mut std::io::stdout()); } + if cli.headroom { + if !headroom_report(&config, &cli.guardian_origins, i64::from(cli.min_spare), &mut std::io::stdout()).await? { + bail!("guardian quorum headroom is under {} spare (BR-402)", cli.min_spare); + } + return Ok(()); + } let mut rows = evm_rows(&config).await?; rows.extend(solana_rows(&config).await?); let rand = rand_side(&config, &cli.token).await; diff --git a/daemons/src/bin/guardian.rs b/daemons/src/bin/guardian.rs index 202dac1..ffebcf3 100644 --- a/daemons/src/bin/guardian.rs +++ b/daemons/src/bin/guardian.rs @@ -234,7 +234,8 @@ async fn main() -> Result<()> { .await .with_context(|| format!("binding {}", section.listen))?; tracing::info!("signature API on {}", section.listen); - let router = api::router(store.clone(), key.address()); + let pq_public = pq.as_ref().map(|p| p.key.public_key()); + let router = api::router_with_pq(store.clone(), key.address(), pq_public.as_deref()); tokio::spawn(async move { if let Err(e) = axum::serve(listener, router).await { tracing::error!("signature API stopped: {e}"); diff --git a/daemons/src/headroom.rs b/daemons/src/headroom.rs new file mode 100644 index 0000000..0c38538 --- /dev/null +++ b/daemons/src/headroom.rs @@ -0,0 +1,188 @@ +//! How many guardians are signing, against how many the quorum needs (BR-402): +//! `rand-bridge-audit --headroom`. Read-only. +//! +//! A registered set of `n` needs `n * 2 / 3 + 1` signatures. The spare is the +//! number of guardians that can stop before nothing is signed. A guardian +//! counts when its API answers `/v1/health`, the address it answers with is in +//! the registered set, and its signing is not paused. Two origins answering as +//! one guardian count once. For mints the chain also needs a quorum of +//! Dilithium2 co-signatures: a guardian counts for that only when its health +//! names a PQ key that is the registered one at its own index. A guardian +//! whose build does not report its PQ key is counted as unknown, never as +//! present. + +use std::collections::BTreeMap; + +use crate::api::Health; + +/// `n * 2 / 3 + 1`: the quorum of a set of `n`, on every verifier. +pub fn quorum(n: usize) -> usize { + n * 2 / 3 + 1 +} + +/// What one configured origin answered. +#[derive(Clone, Debug)] +pub struct Probe { + /// The origin as configured (host and port; guardian APIs carry no secret). + pub origin: String, + pub health: Result, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Line { + /// Counted: registered, answering, not paused. `pq`: whether its PQ key is the registered + /// one at its index (`None`: its build does not say). + Signing { index: usize, pq: Option, held: usize, deferred: usize }, + Paused { index: usize, reason: String }, + /// It answers as an address the chain does not have in its set. + NotRegistered { guardian: String }, + /// A second origin answering as a guardian already counted. + Duplicate { index: usize }, + Down { error: String }, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Report { + pub registered: usize, + pub quorum: usize, + pub lines: Vec<(String, Line)>, + /// Registered guardians no configured origin answered as, by index. + pub missing: Vec, + pub signing: usize, + /// The PQ set's size and quorum; `None` on a chain without the co-signature. + pub pq: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PqReport { + pub registered: usize, + pub quorum: usize, + /// Signing guardians whose PQ key is the registered one. + pub signing: usize, + /// Signing guardians whose build does not report a PQ key. + pub unknown: usize, +} + +impl Report { + /// Guardians that can stop before releases stop; negative: below quorum. + pub fn spare(&self) -> i64 { + self.signing as i64 - self.quorum as i64 + } + /// The same for mints, counting only PQ keys known to be right. + pub fn pq_spare(&self) -> Option { + self.pq.as_ref().map(|p| p.signing as i64 - p.quorum as i64) + } + /// Whether both spares are at least `min_spare`. Unknown PQ keys do not count. + pub fn ok(&self, min_spare: i64) -> bool { + self.spare() >= min_spare && self.pq_spare().is_none_or(|s| s >= min_spare) + } +} + +fn sha256_hex(data: &[u8]) -> String { + use sha2::{Digest, Sha256}; + hex::encode(Sha256::digest(data)) +} + +/// What a guardian's health names as its PQ key: the sha256 of the public key, hex. +pub fn pq_key_id(public_key: &[u8]) -> String { + sha256_hex(public_key) +} + +/// `guardians`: the registered ECDSA set, in index order. `pq_guardians`: the +/// registered PQ keys, index-aligned with it; empty on a chain without them. +pub fn report(guardians: &[[u8; 20]], pq_guardians: &[Vec], probes: &[Probe]) -> Report { + let mut seen: BTreeMap = BTreeMap::new(); + let mut lines = Vec::new(); + let (mut signing, mut pq_signing, mut pq_unknown) = (0, 0, 0); + for p in probes { + let line = match &p.health { + Err(e) => Line::Down { error: e.clone() }, + Ok(h) => { + let address = h.guardian.trim_start_matches("0x").to_ascii_lowercase(); + match guardians.iter().position(|g| hex::encode(g) == address) { + None => Line::NotRegistered { guardian: address }, + Some(index) if seen.contains_key(&index) => Line::Duplicate { index }, + Some(index) => { + seen.insert(index, ()); + match &h.paused { + Some(reason) => Line::Paused { index, reason: reason.clone() }, + None => { + signing += 1; + let pq = h.pq_key_sha256.as_ref().map(|id| { + pq_guardians.get(index).is_some_and(|k| pq_key_id(k) == id.to_ascii_lowercase()) + }); + match pq { + Some(true) => pq_signing += 1, + None => pq_unknown += 1, + Some(false) => {} + } + Line::Signing { index, pq, held: h.held.len(), deferred: h.deferred.len() } + } + } + } + } + } + }; + lines.push((p.origin.clone(), line)); + } + Report { + registered: guardians.len(), + quorum: quorum(guardians.len()), + lines, + missing: (0..guardians.len()).filter(|i| !seen.contains_key(i)).collect(), + signing, + pq: (!pq_guardians.is_empty()).then(|| PqReport { + registered: pq_guardians.len(), + quorum: quorum(pq_guardians.len()), + signing: pq_signing, + unknown: pq_unknown, + }), + } +} + +/// The report as text. +pub fn render(r: &Report, min_spare: i64) -> String { + let mut out = format!("registered guardians: {}; signatures required: {}\n", r.registered, r.quorum); + for (origin, line) in &r.lines { + let text = match line { + Line::Signing { index, pq, held, deferred } => format!( + "guardian {index}: signing; PQ key {}{}", + match pq { + Some(true) => "registered", + Some(false) => "NOT the registered one (its co-signatures do not count)", + None => "not reported by this build", + }, + if *held + *deferred > 0 { format!("; {held} held, {deferred} deferred") } else { String::new() } + ), + Line::Paused { index, reason } => format!("guardian {index}: PAUSED ({reason})"), + Line::NotRegistered { guardian } => format!("answers as 0x{guardian}, which is not in the registered set"), + Line::Duplicate { index } => format!("guardian {index} again (counted once)"), + Line::Down { error } => format!("DOWN: {error}"), + }; + out.push_str(&format!(" {origin:<28} {text}\n")); + } + if !r.missing.is_empty() { + let list: Vec = r.missing.iter().map(|i| i.to_string()).collect(); + out.push_str(&format!(" no origin answered as registered guardian(s) {}\n", list.join(", "))); + } + let word = |spare: i64| match spare { + s if s < 0 => format!("BELOW QUORUM by {}: nothing is signed", -s), + 0 => "NO SPARE: any one guardian stopping stops it".to_string(), + s => format!("{s} spare"), + }; + out.push_str(&format!("releases: {} of {} required are signing; {}\n", r.signing, r.quorum, word(r.spare()))); + if let Some(p) = &r.pq { + out.push_str(&format!( + "mints: {} of {} required PQ co-signers are signing{}; {}\n", + p.signing, + p.quorum, + if p.unknown > 0 { format!(" ({} more unknown, not counted)", p.unknown) } else { String::new() }, + word(p.signing as i64 - p.quorum as i64) + )); + } + out.push_str(&format!( + "{}: at least {min_spare} spare is asked for (BR-402)\n", + if r.ok(min_spare) { "OK" } else { "ALERT" } + )); + out +} diff --git a/daemons/src/lib.rs b/daemons/src/lib.rs index 0388b07..8ae3b85 100644 --- a/daemons/src/lib.rs +++ b/daemons/src/lib.rs @@ -14,6 +14,7 @@ pub mod crypto; pub mod gov_audit; pub mod governance; pub mod guardian; +pub mod headroom; pub mod isolate; pub mod limits; pub mod message; diff --git a/daemons/tests/headroom.rs b/daemons/tests/headroom.rs new file mode 100644 index 0000000..c0c5368 --- /dev/null +++ b/daemons/tests/headroom.rs @@ -0,0 +1,201 @@ +//! BR-402: how many guardians are signing, against the quorum — computed from +//! each guardian's `/v1/health` and the registered set, and reported by +//! `rand-bridge-audit --headroom`. + +mod common; + +use std::process::Command; + +use bridge_daemons::api::{self, Health}; +use bridge_daemons::headroom::{self, pq_key_id, quorum, Line, Probe}; +use bridge_daemons::pq::PqKey; +use bridge_daemons::store::Store; +use common::*; +use serde_json::json; + +fn pk(i: u8) -> Vec { + PqKey::from_seed_hex(&format!("{:064x}", 0x300 + u64::from(i))).unwrap().public_key() +} + +fn addr(i: u8) -> [u8; 20] { + [i; 20] +} + +fn health(i: u8, pq: Option, paused: bool) -> Result { + Ok(Health { + guardian: hex::encode(addr(i)), + paused: paused.then(|| "since 1: drill".to_string()), + held: vec![], + deferred: vec![], + pq_key_sha256: pq.map(|k| pq_key_id(&pk(k))), + }) +} + +fn probes(list: Vec>) -> Vec { + list.into_iter().enumerate().map(|(i, health)| Probe { origin: format!("g{i}"), health }).collect() +} + +fn set(n: u8) -> (Vec<[u8; 20]>, Vec>) { + ((1..=n).map(addr).collect(), (1..=n).map(pk).collect()) +} + +#[test] +fn the_quorum_is_two_thirds_plus_one() { + assert_eq!((quorum(8), quorum(6), quorum(10), quorum(1)), (6, 5, 7, 1)); +} + +#[test] +fn six_of_eight_running_is_exactly_the_quorum_with_no_spare() { + let (guardians, pq) = set(8); + // The live shape: six answer, 7 and 8 do not. + let mut list: Vec<_> = (1..=6).map(|i| health(i, Some(i), false)).collect(); + list.push(Err("connection refused".into())); + list.push(Err("connection refused".into())); + let r = headroom::report(&guardians, &pq, &probes(list)); + assert_eq!((r.registered, r.quorum, r.signing, r.spare()), (8, 6, 6, 0)); + assert_eq!(r.missing, vec![6, 7]); + assert_eq!(r.pq_spare(), Some(0)); + assert!(r.ok(0) && !r.ok(1), "no spare is an alert at the default"); + let text = headroom::render(&r, 1); + assert!(text.contains("registered guardians: 8; signatures required: 6"), "{text}"); + assert!(text.contains("releases: 6 of 6 required are signing; NO SPARE"), "{text}"); + assert!(text.contains("mints: 6 of 6 required PQ co-signers are signing; NO SPARE"), "{text}"); + assert!(text.contains("no origin answered as registered guardian(s) 6, 7") && text.contains("ALERT"), "{text}"); + // All eight: two spare. + let r = headroom::report(&guardians, &pq, &probes((1..=8).map(|i| health(i, Some(i), false)).collect())); + assert_eq!((r.spare(), r.pq_spare()), (2, Some(2))); + assert!(r.ok(1) && headroom::render(&r, 1).contains("OK")); + // Five: below quorum. + let r = headroom::report(&guardians, &pq, &probes((1..=5).map(|i| health(i, Some(i), false)).collect())); + assert_eq!(r.spare(), -1); + assert!(headroom::render(&r, 1).contains("BELOW QUORUM by 1")); +} + +#[test] +fn only_a_registered_unpaused_guardian_counts_and_only_once() { + let (guardians, pq) = set(4); // quorum 3 + let r = headroom::report( + &guardians, + &pq, + &probes(vec![ + health(1, Some(1), false), + health(1, Some(1), false), // a second origin for guardian 1 + health(2, Some(2), true), // paused + health(9, Some(9), false), // not in the set + health(3, Some(3), false), + ]), + ); + assert_eq!(r.signing, 2); + assert!(matches!(r.lines[1].1, Line::Duplicate { index: 0 })); + assert!(matches!(r.lines[2].1, Line::Paused { index: 1, .. })); + assert!(matches!(r.lines[3].1, Line::NotRegistered { .. })); + assert_eq!(r.missing, vec![3], "a paused guardian answered; it is not missing"); + assert_eq!(r.spare(), -1); +} + +#[test] +fn a_pq_key_counts_only_when_it_is_the_registered_one_at_that_index() { + let (guardians, pq) = set(4); + let r = headroom::report( + &guardians, + &pq, + &probes(vec![ + health(1, Some(1), false), + health(2, Some(3), false), // guardian 2 runs guardian 3's key: wrong index + health(3, None, false), // an older build, or no seed: not known + health(4, Some(4), false), + ]), + ); + assert_eq!(r.signing, 4); + let p = r.pq.clone().unwrap(); + assert_eq!((p.signing, p.unknown, p.quorum), (2, 1, 3)); + assert_eq!(r.pq_spare(), Some(-1), "unknown is not counted as present"); + assert!(!r.ok(0), "releases have a spare; mints do not"); + let text = headroom::render(&r, 0); + assert!(text.contains("NOT the registered one") && text.contains("not reported by this build") && text.contains("1 more unknown"), "{text}"); + // A chain without the co-signature has no PQ line. + let r = headroom::report(&guardians, &[], &probes((1..=4).map(|i| health(i, None, false)).collect())); + assert!(r.pq.is_none() && r.ok(1)); +} + +// ---- the guardian's health, and the binary -------------------------------------- + +async fn serve(router: axum::Router) -> String { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = format!("http://{}", listener.local_addr().unwrap()); + tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); + url +} + +#[tokio::test(flavor = "multi_thread")] +async fn the_audit_counts_live_guardians_against_the_registered_set() { + let keys = guardian_keys(4); + let dirs: Vec<_> = (0..4).map(|_| tempfile::tempdir().unwrap()).collect(); + let pqs: Vec> = (1..=4).map(pk).collect(); + // Guardians 0-2 run; 0 and 1 report their PQ key, 2 is a build that does not; 3 is down. + let mut origins = Vec::new(); + for i in 0..3 { + let store = Store::open(dirs[i].path()).unwrap(); + let router = if i < 2 { + api::router_with_pq(store, keys[i].address(), Some(&pqs[i])) + } else { + api::router(store, keys[i].address()) + }; + origins.push(serve(router).await); + } + origins.push(format!("http://127.0.0.1:{}", free_port())); + // The health record names the key by its hash; a client of an older build ignores the field. + let h = api::GuardianClient::new(&origins[0]).health().await.unwrap(); + assert_eq!(h.pq_key_sha256, Some(pq_key_id(&pqs[0]))); + assert!(api::GuardianClient::new(&origins[2]).health().await.unwrap().pq_key_sha256.is_none()); + + let addresses: Vec = keys.iter().map(|k| format!("0x{}", hex::encode(k.address()))).collect(); + let pq_hex: Vec = pqs.iter().map(hex::encode).collect(); + let node = rpc(move |m, _| match m { + "rand_getBridgeState" => Some(Ok(json!({ + "enabled": true, "guardian_set_index": 1, "guardians": addresses, "burn_sequence": 0, "pq_guardians": pq_hex, + }))), + _ => None, + }); + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("config.toml"); + let list = origins.iter().map(|o| format!("{o:?}")).collect::>().join(", "); + std::fs::write( + &path, + format!( + "data_dir = {:?}\n[rand]\nrpc = {:?}\nemitter = {:?}\n[relayer]\nguardians = [{list}]\n", + dir.path().join("data"), + node.url, + hex::encode(RAND_EMITTER) + ), + ) + .unwrap(); + let run = |extra: Vec| { + let path = path.clone(); + tokio::task::spawn_blocking(move || { + let o = Command::new(env!("CARGO_BIN_EXE_rand-bridge-audit")) + .env_remove("RAND_BRIDGE_CONFIG") + .arg("--config") + .arg(&path) + .arg("--headroom") + .args(extra) + .output() + .unwrap(); + (o.status.success(), String::from_utf8_lossy(&o.stdout).to_string(), String::from_utf8_lossy(&o.stderr).to_string()) + }) + }; + // 3 of 4 signing at a quorum of 3: no spare, which is an alert at the default. + let (ok, out, err) = run(vec![]).await.unwrap(); + assert!(!ok && err.contains("BR-402"), "{out}\n{err}"); + assert!(out.contains("releases: 3 of 3 required are signing; NO SPARE"), "{out}"); + assert!(out.contains("mints: 2 of 3 required PQ co-signers are signing (1 more unknown, not counted); BELOW QUORUM by 1"), "{out}"); + assert!(out.contains("DOWN:") && out.contains("registered guardian(s) 3") && out.contains("ALERT"), "{out}"); + // Asking for no spare still fails: the PQ side is short. + let (ok, _, _) = run(vec!["--min-spare".into(), "0".into()]).await.unwrap(); + assert!(!ok); + // Only the two that report the right key, at no spare, on a narrower question: still short of 3. + let (ok, out, _) = run(vec!["--min-spare".into(), "0".into(), "--guardian-origins".into(), origins[..2].join(",")]).await.unwrap(); + assert!(!ok && out.contains("BELOW QUORUM by 1"), "{out}"); + // The node was only read. + assert_eq!(node.rpc_methods().into_iter().collect::>().len(), 1); +} diff --git a/daemons/tests/provision_guardian_script.rs b/daemons/tests/provision_guardian_script.rs new file mode 100644 index 0000000..d0bb598 --- /dev/null +++ b/daemons/tests/provision_guardian_script.rs @@ -0,0 +1,115 @@ +//! BR-402: `mainnet/provision-guardian.sh` under `--root `. Nothing here touches a +//! real host, a real key or systemd. +use std::{fs, os::unix::fs::PermissionsExt, path::Path, process::Command}; + +use sha2::{Digest, Sha256}; + +fn script() -> String { + env!("CARGO_MANIFEST_DIR").to_string() + "/mainnet/provision-guardian.sh" +} + +struct Rig { + dir: tempfile::TempDir, + sha: String, +} + +impl Rig { + fn new() -> Rig { + let dir = tempfile::tempdir().unwrap(); + let binary = b"#!/bin/sh\necho not a real guardian\n"; + fs::write(dir.path().join("rand-guardian"), binary).unwrap(); + let config = fs::read_to_string(env!("CARGO_MANIFEST_DIR").to_string() + "/mainnet/guardian-7.toml").unwrap(); + fs::write(dir.path().join("guardian-7.toml"), config).unwrap(); + Rig { sha: hex::encode(Sha256::digest(binary)), dir } + } + fn root(&self) -> std::path::PathBuf { + self.dir.path().join("root") + } + fn run(&self, index: &str, sha: &str, config: &Path, extra: &[&str]) -> (bool, String, String) { + let o = Command::new("bash") + .arg(script()) + .args(["--index", index, "--sha256", sha, "--binary"]) + .arg(self.dir.path().join("rand-guardian")) + .arg("--config") + .arg(config) + .arg("--root") + .arg(self.root()) + .args(extra) + .output() + .unwrap(); + (o.status.success(), String::from_utf8_lossy(&o.stdout).into(), String::from_utf8_lossy(&o.stderr).into()) + } + fn cfg(&self) -> std::path::PathBuf { + self.dir.path().join("guardian-7.toml") + } +} + +fn mode(p: &Path) -> u32 { + fs::metadata(p).unwrap().permissions().mode() & 0o777 +} + +#[test] +fn it_lays_the_guardian_down_without_keys_and_without_starting_it() { + let rig = Rig::new(); + let (ok, out, err) = rig.run("7", &rig.sha, &rig.cfg(), &["--dry-run"]); + assert!(ok && out.contains("dry run: nothing written"), "{out}\n{err}"); + assert!(!rig.root().exists()); + + let (ok, out, err) = rig.run("7", &rig.sha, &rig.cfg(), &[]); + assert!(ok, "{out}\n{err}"); + let root = rig.root(); + assert_eq!(mode(&root.join("usr/local/bin/rand-guardian")), 0o755); + assert_eq!(mode(&root.join("etc/rand-guardian.toml")), 0o640); + let env_path = root.join("etc/rand-guardian/env"); + assert_eq!(mode(&env_path), 0o600); + let env = fs::read_to_string(&env_path).unwrap(); + assert!(env.contains("\nGUARDIAN7_PRIV_KEY=\n") && env.contains("\nGUARDIAN7_PQ_SEED=\n"), "names only, no values: {env}"); + let unit = fs::read_to_string(root.join("etc/systemd/system/rand-guardian.service")).unwrap(); + assert!(unit.contains("EnvironmentFile=/etc/rand-guardian/env") && unit.contains("User=rand-guardian"), "{unit}"); + assert!(root.join("var/lib/rand-guardian/data").is_dir()); + assert!(out.contains("the service is NOT started") && out.contains("--headroom"), "{out}"); + + // Run again: nothing differs, and a secrets file someone filled is left alone. + fs::write(&env_path, "GUARDIAN7_PRIV_KEY=filled-by-the-holder\n").unwrap(); + let (ok, out, err) = rig.run("7", &rig.sha, &rig.cfg(), &[]); + assert!(ok, "{out}\n{err}"); + assert_eq!(fs::read_to_string(&env_path).unwrap(), "GUARDIAN7_PRIV_KEY=filled-by-the-holder\n"); + assert!(!out.contains("filled-by-the-holder") && !err.contains("filled-by-the-holder"), "the secrets file is never read out"); +} + +#[test] +fn it_refuses_a_wrong_checksum_another_guardians_config_and_an_overwrite() { + let rig = Rig::new(); + let (ok, _, err) = rig.run("7", &"0".repeat(64), &rig.cfg(), &[]); + assert!(!ok && err.contains("do not install it"), "{err}"); + assert!(!rig.root().exists(), "nothing is written after a refusal"); + + let (ok, _, err) = rig.run("8", &rig.sha, &rig.cfg(), &[]); + assert!(!ok && err.contains("GUARDIAN8_PRIV_KEY"), "{err}"); + + // A config that listens beyond loopback, or carries a key, is refused. + let open = rig.dir.path().join("open.toml"); + fs::write(&open, fs::read_to_string(rig.cfg()).unwrap().replace("listen = \"127.0.0.1:", "listen = \"0.0.0.0:")).unwrap(); + let (ok, _, err) = rig.run("7", &rig.sha, &open, &[]); + assert!(!ok && err.contains("127.0.0.1"), "{err}"); + let keyed = rig.dir.path().join("keyed.toml"); + fs::write(&keyed, format!("{}\npriv_key = \"{}\"\n", fs::read_to_string(rig.cfg()).unwrap(), "ab".repeat(32))).unwrap(); + let (ok, _, err) = rig.run("7", &rig.sha, &keyed, &[]); + assert!(!ok && err.contains("key material"), "{err}"); + + // Installed once; a different config afterwards does not replace it. + assert!(rig.run("7", &rig.sha, &rig.cfg(), &[]).0); + let other = rig.dir.path().join("other.toml"); + fs::write(&other, fs::read_to_string(rig.cfg()).unwrap() + "\n# changed\n").unwrap(); + let (ok, _, err) = rig.run("7", &rig.sha, &other, &[]); + assert!(!ok && err.contains("does not overwrite"), "{err}"); +} + +#[test] +fn the_headroom_units_run_the_read_only_check() { + let dir = env!("CARGO_MANIFEST_DIR").to_string() + "/mainnet/systemd/"; + let service = fs::read_to_string(dir.clone() + "rand-bridge-headroom.service").unwrap(); + assert!(service.contains("rand-bridge-audit --config /etc/rand-relayer.toml --headroom --min-spare 1"), "{service}"); + assert!(service.contains("OnFailure="), "a failed check tells someone"); + assert!(fs::read_to_string(dir + "rand-bridge-headroom.timer").unwrap().contains("OnUnitActiveSec=5min")); +} diff --git a/docs/guardian-hosts.md b/docs/guardian-hosts.md index e85214c..a29d1df 100644 --- a/docs/guardian-hosts.md +++ b/docs/guardian-hosts.md @@ -44,6 +44,71 @@ Configure each guardian with more than one independent RPC per chain (`rpcs`, `r and with its peers (`guardian.peers`; BR-502). The repository's configs carry one RPC per chain and no peers; the hosts' configs are where these are set. +## Quorum headroom (BR-402) + +A set of `n` guardians needs `n * 2 / 3 + 1` signatures: 6 of 8 for set 1, and the same 6 of 8 +Dilithium2 co-signatures for a mint. With exactly six running there is no spare: one host down, +one guardian paused, or one deferral stops every transfer until it clears. + +```sh +rand-bridge-audit --config /etc/rand-relayer.toml --headroom # [--min-spare 1] [--guardian-origins a,b] +``` + +Run where the guardian APIs are reachable (the relayer host, through its forwards). It reads the +registered set from the Rand node and `/v1/health` from every origin in `[relayer].guardians`, and +prints one line per origin, then: + +``` +releases: 6 of 6 required are signing; NO SPARE: any one guardian stopping stops it +mints: 6 of 6 required PQ co-signers are signing; NO SPARE: any one guardian stopping stops it +ALERT: at least 1 spare is asked for (BR-402) +``` + +A guardian counts when it answers, the address it answers with is in the registered set, and it is +not paused; two origins answering as one guardian count once. For mints it counts only when its +health names a PQ key (`pq_key_sha256`) that is the registered one at its own index. A guardian on +a build that does not report the key is listed as unknown and not counted, so until the guardians +run a build with this change the mints line under-counts: read it as a floor. The command is +read-only and exits non-zero under `--min-spare` (default 1), so today it fails, which is the +correct reading. + +**Alert.** `daemons/mainnet/systemd/rand-bridge-headroom.{service,timer}` run it every five minutes +on the relayer host; the service's `OnFailure=` names the unit that tells someone, which you +supply (the repository ships no notifier). Install both under `/etc/systemd/system/`, set the +config path and the notifier, then `systemctl enable --now rand-bridge-headroom.timer`. + +### Bringing a guardian up on its own host + +Guardians 7 and 8 are registered and not running. Each goes on its own host, ideally under a +different hosting account from the six. On the new host, as root, after it runs a synced +`rand-node` (above): + +```sh +daemons/mainnet/provision-guardian.sh --index 7 --binary ./rand-guardian --sha256 \ + --config ./guardian-7.toml # --dry-run prints what it would write; --root rehearses +``` + +It checks the binary against the checksum, checks that the config is this guardian's (its +`key_env` carries the number, its API is on loopback, no key in it), and writes the binary, the +config, `rand-guardian.service`, an empty `/etc/rand-guardian/env` (mode 600, variable names only) +and the data directory. It reads and writes no key, overwrites nothing that differs, contacts +nothing, and does not start the service. Then: + +1. The key holder fills `GUARDIAN_PRIV_KEY` and `GUARDIAN_PQ_SEED` in `/etc/rand-guardian/env` + on that host. The key must be the one registered at that index: a guardian that was never run + from this key material is brought in by rotation (`rand-bridge-gov rotate`, + `docs/pq-rotation.md`), not by copying a key onto a new host. +2. Set the config's start blocks to recent ones the providers still serve (the "Public RPC trap" + below), so the guardian does not stall on old log ranges. +3. `systemctl enable --now rand-guardian`; its log must show the guardian's address, the PQ key + prefix, and cursors moving. +4. Give the relayer a path to its API (a forward like the others: `guardian-tunnels.sh` reads + `hosts.txt`), and list the origin in `[relayer].guardians`. +5. `rand-bridge-audit --headroom` must now count it. With both up: `2 spare`, and `OK`. + +If 7 and 8 are not coming back, the alternative is to rotate the ECDSA and the PQ set together to +the guardians that run plus new ones, so that the registered set again has spares. + **Public RPC trap.** `bsc-rpc.publicnode.com` only serves logs from about the last hour (8,000 BSC blocks), and Ethereum's window is about a day. A guardian stopped for longer than that stalls on 403 and must have its cursor moved forward: stop it, check that the endpoint's `sequence()` still equals