Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions daemons/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -205,6 +205,11 @@ cancel of a pending one. Runbook: `docs/pq-rotation.md`.

## Audit

`rand-bridge-audit --config <relayer config> --headroom` counts the guardians that are signing
against the quorum, for releases and for mints, and exits non-zero without a spare (BR-402;
`docs/guardian-hosts.md`, "Quorum headroom"). A guardian's `/v1/health` names the sha256 of the
PQ key it co-signs with (`pq_key_sha256`) for that count.

`rand-bridge-audit --config <any daemon config>` reconciles custody: for each of the seven approved
backings it reads the endpoint's `custody`, its accrued protocol fees and the token balance it really
holds, and — once the Rand node serves per-backing `locked` rows — what Rand says is outstanding. It
Expand Down
122 changes: 122 additions & 0 deletions daemons/mainnet/provision-guardian.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,122 @@
#!/usr/bin/env bash
# Lays a guardian down on a NEW host (BR-402): the binary, its config, the systemd unit, an
# empty secrets file and the data directory. Run it ON that host, as root. It contacts nothing,
# reads no key, writes no key, and does not start the service: the key holder puts the keys in
# the secrets file by hand, on that host, and starts it (docs/guardian-hosts.md, "Bringing a
# guardian up on its own host").
#
# provision-guardian.sh --index 7 --binary ./rand-guardian --sha256 <64 hex> \
# --config ./guardian-7.toml [--root <dir>] [--dry-run]
#
# --sha256 the checksum of the binary as built and recorded elsewhere; a mismatch stops here.
# --root install under <dir> instead of / (a rehearsal or a test): no user is created and
# systemd is not touched.
# It never overwrites: an existing binary, config, unit or secrets file that differs stops it.
set -euo pipefail

index="" binary="" sha="" config="" root="" dry=0
while [[ $# -gt 0 ]]; do
case "$1" in
--index) index="$2"; shift 2 ;;
--binary) binary="$2"; shift 2 ;;
--sha256) sha="$2"; shift 2 ;;
--config) config="$2"; shift 2 ;;
--root) root="${2%/}"; shift 2 ;;
--dry-run) dry=1; shift ;;
*) echo "error: unknown argument $1" >&2; exit 2 ;;
esac
done
die() { echo "error: $*" >&2; exit 1; }
[[ "$index" =~ ^[1-9][0-9]?$ ]] || die "--index <guardian number> is required"
[[ -f "$binary" ]] || die "--binary: $binary is not a file"
[[ "$sha" =~ ^[0-9a-fA-F]{64}$ ]] || die "--sha256 takes the binary's 64-hex checksum"
[[ -f "$config" ]] || die "--config: $config is not a file"

if command -v sha256sum >/dev/null 2>&1; then got=$(sha256sum "$binary" | cut -d' ' -f1)
else got=$(shasum -a 256 "$binary" | cut -d' ' -f1); fi
[[ "$(printf %s "$got" | tr A-F a-f)" == "$(printf %s "$sha" | tr A-F a-f)" ]] \
|| die "the binary's sha256 is $got, not the one given: do not install it"

# The config must be this guardian's: its key variables carry its number, and its API stays on
# loopback. Nothing in it is a secret.
key_env="GUARDIAN${index}_PRIV_KEY"
pq_env="GUARDIAN${index}_PQ_SEED"
grep -Eq "^key_env *= *\"$key_env\"" "$config" || die "the config's key_env is not \"$key_env\": wrong guardian's file?"
grep -Eq '^listen *= *"127\.0\.0\.1:[0-9]+"' "$config" || die "the config's [guardian] listen is not on 127.0.0.1"
grep -Eqi '(priv_key|pq_seed) *= *"[0-9a-f]{64}' "$config" && die "the config seems to carry key material; keys go in the secrets file only"

bin_to="$root/usr/local/bin/rand-guardian"
cfg_to="$root/etc/rand-guardian.toml"
env_to="$root/etc/rand-guardian/env"
unit_to="$root/etc/systemd/system/rand-guardian.service"
data_to="$root/var/lib/rand-guardian/data"

unit=$(cat <<UNIT
[Unit]
Description=Rand bridge guardian $index
After=network-online.target rand-node.service
Wants=network-online.target

[Service]
User=rand-guardian
Group=rand-guardian
EnvironmentFile=/etc/rand-guardian/env
ExecStart=/usr/local/bin/rand-guardian --config /etc/rand-guardian.toml
Restart=always
RestartSec=5
NoNewPrivileges=true
ProtectSystem=strict
ProtectHome=true
PrivateTmp=true
ReadWritePaths=/var/lib/rand-guardian

[Install]
WantedBy=multi-user.target
UNIT
)
envfile=$(cat <<ENV
# Secrets of guardian $index. Mode 600, root only; systemd reads it. Fill the two values ON THIS
# HOST, by the key holder; never copy this file anywhere.
$key_env=
$pq_env=
ENV
)

# Refuse to overwrite anything that is there and differs.
same_or_absent() { [[ ! -e "$1" ]] || cmp -s "$1" "$2" || die "$1 exists and differs; this script does not overwrite (move it aside after review)"; }
same_or_absent "$bin_to" "$binary"
same_or_absent "$cfg_to" "$config"
[[ ! -e "$unit_to" ]] || [[ "$(cat "$unit_to")" == "$unit" ]] || die "$unit_to exists and differs; this script does not overwrite"

echo "guardian $index: binary sha256 $got"
echo " $bin_to"
echo " $cfg_to"
echo " $unit_to"
echo " $env_to $([[ -e "$env_to" ]] && echo '(exists: left as it is)' || echo '(empty template)')"
echo " $data_to"
if [[ $dry == 1 ]]; then echo "dry run: nothing written"; exit 0; fi

if [[ -z "$root" ]]; then
[[ $EUID -eq 0 ]] || die "run as root (or pass --root <dir> for a rehearsal)"
id rand-guardian >/dev/null 2>&1 || useradd --system --home /var/lib/rand-guardian --shell /usr/sbin/nologin rand-guardian
fi
install -d -m 755 "$root/usr/local/bin" "$root/etc/systemd/system"
install -d -m 700 "$root/etc/rand-guardian" "$root/var/lib/rand-guardian" "$data_to"
install -m 755 "$binary" "$bin_to"
install -m 640 "$config" "$cfg_to"
printf '%s\n' "$unit" >"$unit_to"
if [[ ! -e "$env_to" ]]; then (umask 077; printf '%s\n' "$envfile" >"$env_to"); fi
chmod 600 "$env_to"
if [[ -z "$root" ]]; then
chown -R rand-guardian:rand-guardian /var/lib/rand-guardian
chown root:rand-guardian "$cfg_to"
systemctl daemon-reload
fi

cat <<NEXT
done; the service is NOT started. Next, on this host:
1. the key holder fills $key_env and $pq_env in /etc/rand-guardian/env (mode 600)
2. systemctl enable --now rand-guardian; journalctl -u rand-guardian -f
3. curl -s http://127.0.0.1:<listen port>/v1/health # names this guardian's address
4. from the relayer host: rand-bridge-audit --config <relayer.toml> --headroom
NEXT
12 changes: 12 additions & 0 deletions daemons/mainnet/systemd/rand-bridge-headroom.service
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
# Quorum headroom check (BR-402), for the relayer host: fails when fewer than one guardian could
# stop before signing stops. Pair it with rand-bridge-headroom.timer. OnFailure names the unit
# that tells someone; rand-bridge-alert@.service is an example to replace with your own
# (mail, a webhook) — this repository ships no notifier.
[Unit]
Description=Rand bridge guardian quorum headroom (BR-402)
OnFailure=rand-bridge-alert@%n.service

[Service]
Type=oneshot
User=rand-relayer
ExecStart=/usr/local/bin/rand-bridge-audit --config /etc/rand-relayer.toml --headroom --min-spare 1
9 changes: 9 additions & 0 deletions daemons/mainnet/systemd/rand-bridge-headroom.timer
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
[Unit]
Description=Check the Rand bridge guardian quorum headroom every five minutes (BR-402)

[Timer]
OnBootSec=2min
OnUnitActiveSec=5min

[Install]
WantedBy=timers.target
28 changes: 28 additions & 0 deletions daemons/src/api.rs
Original file line number Diff line number Diff line change
Expand Up @@ -53,17 +53,29 @@ pub struct Health {
/// Transfers waiting for room in their window, or for the peer check, `chain/sequence`.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub deferred: Vec<String>,
/// The sha256 (hex) of the Dilithium2 public key this guardian co-signs mints with;
/// absent when it holds none, and from a build before BR-402's headroom check. Public:
/// the key itself is in `rand_getBridgeState`.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub pq_key_sha256: Option<String>,
}

struct ApiState {
store: Store,
guardian: String,
pq_key_sha256: Option<String>,
}

pub fn router(store: Store, guardian: [u8; 20]) -> Router {
router_with_pq(store, guardian, None)
}

/// [`router`], naming in `/v1/health` the PQ public key the guardian co-signs with.
pub fn router_with_pq(store: Store, guardian: [u8; 20], pq_public_key: Option<&[u8]>) -> Router {
let state = Arc::new(ApiState {
store,
guardian: hex::encode(guardian),
pq_key_sha256: pq_public_key.map(crate::headroom::pq_key_id),
});
Router::new()
.route("/v1/health", get(health))
Expand All @@ -89,6 +101,7 @@ async fn health(State(state): State<Arc<ApiState>>) -> Json<Health> {
paused,
held,
deferred,
pq_key_sha256: state.pq_key_sha256.clone(),
})
}

Expand Down Expand Up @@ -132,6 +145,21 @@ impl GuardianClient {
&self.origin
}

/// `GET /v1/health`.
pub async fn health(&self) -> Result<Health> {
let url = format!("{}/v1/health", self.origin);
self.http
.get(&url)
.send()
.await
.with_context(|| format!("GET {url}"))?
.error_for_status()
.with_context(|| format!("GET {url}"))?
.json()
.await
.with_context(|| format!("GET {url}: not a health record"))
}

/// Whatever record this guardian serves for `(chain, sequence)`, unchecked:
/// the caller decides what it proves (BR-502).
pub async fn record(&self, chain: u16, sequence: u64) -> Result<Option<SignedMessage>> {
Expand Down
60 changes: 60 additions & 0 deletions daemons/src/bin/audit.rs
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,19 @@ struct Cli {
/// multisig and upgrade authority on every endpoint.
#[arg(long)]
governance: bool,
/// Check how many guardians are signing against the quorum (BR-402)
/// instead of custody: asks each guardian API for `/v1/health` and the
/// Rand node for the registered set. Read-only; exits non-zero when
/// fewer than `--min-spare` guardians could stop before signing stops.
#[arg(long, conflicts_with = "governance")]
headroom: bool,
/// The spare `--headroom` asks for, for releases and for mints.
#[arg(long, default_value_t = 1)]
min_spare: u32,
/// The guardian API origins `--headroom` asks, comma-separated; default
/// `[relayer].guardians` of the config.
#[arg(long, value_delimiter = ',')]
guardian_origins: Vec<String>,
}

/// The approved backings: (bridge chain, symbol, token, decimals).
Expand Down Expand Up @@ -640,6 +653,47 @@ async fn solana_governance(
/// can change the endpoints, not who can move custody.
const GOVERNANCE_CAVEAT: &str = "caveat: custody is still authorised by the guardian quorum, whose keys are not yet distributed (BR-4)";

// ---- --headroom --------------------------------------------------------------

/// The `--headroom` report. `Ok(false)` when the spare is under `min_spare`.
async fn headroom_report(config: &Config, origins: &[String], min_spare: i64, out: &mut impl std::io::Write) -> Result<bool> {
use bridge_daemons::headroom;
let origins: Vec<String> = if origins.is_empty() {
config.relayer.as_ref().map(|r| r.guardians.clone()).unwrap_or_default()
} else {
origins.to_vec()
};
if origins.is_empty() {
bail!("no guardian origins: pass --guardian-origins, or a config with [relayer].guardians");
}
let state = bridge_daemons::sources::rand::bridge_state(&JsonRpc::new(&config.rand.rpc))
.await
.context("reading the registered guardian set from the Rand node")?
.ok_or_else(|| anyhow!("the Rand node serves no bridge"))?;
let mut asks = tokio::task::JoinSet::new();
for (i, origin) in origins.iter().enumerate() {
let client = bridge_daemons::api::GuardianClient::new(origin);
asks.spawn(async move { (i, client.health().await.map_err(|e| format!("{e:#}"))) });
}
let mut answers: Vec<Option<Result<bridge_daemons::api::Health, String>>> = vec![None; origins.len()];
while let Some(joined) = asks.join_next().await {
let (i, health) = joined?;
answers[i] = Some(health);
}
let probes: Vec<headroom::Probe> = origins
.iter()
.zip(answers)
.map(|(origin, health)| headroom::Probe {
origin: JsonRpc::new(origin).shown(),
health: health.unwrap_or_else(|| Err("not asked".into())),
})
.collect();
let report = headroom::report(&state.guardians, &state.pq_guardians, &probes);
writeln!(out, "guardian set {} on the Rand node", state.guardian_set_index)?;
write!(out, "{}", headroom::render(&report, min_spare))?;
Ok(report.ok(min_spare))
}

/// Writes the BR-4 caveat after a `--governance` run, however it ended (an
/// early error included), and hands the verdict back unchanged.
fn with_caveat<T>(result: Result<T>, out: &mut dyn std::io::Write) -> Result<T> {
Expand Down Expand Up @@ -736,6 +790,12 @@ async fn main() -> Result<()> {
let result = governance(&config).await;
return with_caveat(result, &mut std::io::stdout());
}
if cli.headroom {
if !headroom_report(&config, &cli.guardian_origins, i64::from(cli.min_spare), &mut std::io::stdout()).await? {
bail!("guardian quorum headroom is under {} spare (BR-402)", cli.min_spare);
}
return Ok(());
}
let mut rows = evm_rows(&config).await?;
rows.extend(solana_rows(&config).await?);
let rand = rand_side(&config, &cli.token).await;
Expand Down
3 changes: 2 additions & 1 deletion daemons/src/bin/guardian.rs
Original file line number Diff line number Diff line change
Expand Up @@ -234,7 +234,8 @@ async fn main() -> Result<()> {
.await
.with_context(|| format!("binding {}", section.listen))?;
tracing::info!("signature API on {}", section.listen);
let router = api::router(store.clone(), key.address());
let pq_public = pq.as_ref().map(|p| p.key.public_key());
let router = api::router_with_pq(store.clone(), key.address(), pq_public.as_deref());
tokio::spawn(async move {
if let Err(e) = axum::serve(listener, router).await {
tracing::error!("signature API stopped: {e}");
Expand Down
Loading
Loading