From b7e4409abdb16ca5a97e951f43839641ece5230d Mon Sep 17 00:00:00 2001 From: QuicksilverSlick Date: Tue, 27 Oct 2026 08:26:05 -0500 Subject: [PATCH 01/15] feat(relay-gateway): add the crate with its config and route table Bishop (a Cloudflare Worker) cannot reach Block's hosted relay directly: Cloudflare-to-Cloudflare requests end in error 526, while a Cloudflare Container gets through. buzz-relay-gateway is the keyless forwarder that runs in that container. The Worker keeps every key and signs every event; the gateway only forwards bytes. This first part is the skeleton. - Config::from_env reads four public values (bind address, relay URL, link URL, gateway-auth public key) and never holds a secret. A missing or invalid variable names the variable, never its value. - The relay URL must be wss://host[:port] with nothing else; ws:// is accepted only on loopback, for the offline tests. - policy.rs is the anti-open-proxy table: exact method and path matches (POST /query, /events, /count; GET / for NIP-11; the socket on /), a closed request header set with no duplicates, and a response filter that keeps only content-type and retry-after. - The binary serves /healthz, installs the ring provider, logs JSON, and handles SIGTERM itself (it is PID 1 in the image). Signed-off-by: QuicksilverSlick Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 19 ++ Cargo.toml | 1 + crates/buzz-relay-gateway/Cargo.toml | 33 ++++ crates/buzz-relay-gateway/src/config.rs | 159 +++++++++++++++ crates/buzz-relay-gateway/src/lib.rs | 38 ++++ crates/buzz-relay-gateway/src/main.rs | 58 ++++++ crates/buzz-relay-gateway/src/policy.rs | 249 ++++++++++++++++++++++++ 7 files changed, 557 insertions(+) create mode 100644 crates/buzz-relay-gateway/Cargo.toml create mode 100644 crates/buzz-relay-gateway/src/config.rs create mode 100644 crates/buzz-relay-gateway/src/lib.rs create mode 100644 crates/buzz-relay-gateway/src/main.rs create mode 100644 crates/buzz-relay-gateway/src/policy.rs diff --git a/Cargo.lock b/Cargo.lock index 9206cb0c0e4..5c06f6a84a2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1505,6 +1505,25 @@ dependencies = [ "uuid", ] +[[package]] +name = "buzz-relay-gateway" +version = "0.1.0" +dependencies = [ + "anyhow", + "axum", + "nostr 0.44.7", + "rustls", + "serde", + "serde_json", + "thiserror 2.0.18", + "tokio", + "tokio-util", + "tracing", + "tracing-subscriber", + "url", + "uuid", +] + [[package]] name = "buzz-relay-mesh" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index df77ab7cb3c..51f1db14a9f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,6 +32,7 @@ members = [ "crates/buzz-dev-mcp", "crates/buzz-voice", "crates/buzz-audio-client", + "crates/buzz-relay-gateway", "crates/buzz-backend-kubernetes", "crates/buzz-datastore-tracing", "examples/countdown-bot", diff --git a/crates/buzz-relay-gateway/Cargo.toml b/crates/buzz-relay-gateway/Cargo.toml new file mode 100644 index 00000000000..9366096f7fc --- /dev/null +++ b/crates/buzz-relay-gateway/Cargo.toml @@ -0,0 +1,33 @@ +[package] +name = "buzz-relay-gateway" +description = "Keyless forwarder between one Worker link and one Buzz relay" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true + +[lib] +name = "buzz_relay_gateway" +path = "src/lib.rs" + +[[bin]] +name = "buzz-relay-gateway" +path = "src/main.rs" + +[dependencies] +anyhow = { workspace = true } +axum = { workspace = true } +nostr = { workspace = true } +# Explicit ring provider, installed in main(): a workspace-wide build unifies +# ring and aws-lc-rs, and rustls then panics on the first wss:// connect. +rustls = { version = "0.23", default-features = false, features = ["ring", "std"] } +serde = { workspace = true } +serde_json = { workspace = true } +thiserror = { workspace = true } +tokio = { workspace = true } +tokio-util = { workspace = true } +tracing = { workspace = true } +tracing-subscriber = { workspace = true } +url = { workspace = true } +uuid = { workspace = true } diff --git a/crates/buzz-relay-gateway/src/config.rs b/crates/buzz-relay-gateway/src/config.rs new file mode 100644 index 00000000000..750b5265a9b --- /dev/null +++ b/crates/buzz-relay-gateway/src/config.rs @@ -0,0 +1,159 @@ +//! Environment configuration. Every value is public. + +use std::net::SocketAddr; + +use nostr::PublicKey; +use url::Url; + +const BIND: &str = "BUZZ_RELAY_GATEWAY_BIND_ADDR"; +const RELAY: &str = "BUZZ_RELAY_GATEWAY_RELAY_URL"; +const LINK_URL: &str = "BUZZ_RELAY_GATEWAY_LINK_URL"; +const LINK_PUBKEY: &str = "BUZZ_RELAY_GATEWAY_LINK_PUBKEY"; + +/// The gateway's settings. None of them is a secret. +#[derive(Clone, Debug)] +pub struct Config { + /// Listen address; defaults to loopback. + pub bind_addr: SocketAddr, + /// The one relay the gateway reaches, `wss://host[:port]/`. + pub relay_ws: Url, + /// The same relay's HTTP origin, `https://host[:port]/`. + pub relay_http: Url, + /// The `u` tag every link bearer must carry. + pub link_url: String, + /// The gateway-auth public key that signs link bearers. + pub link_pubkey: PublicKey, +} + +/// A missing or invalid variable. It names the variable, never its value. +#[derive(Debug, thiserror::Error)] +pub enum ConfigError { + /// A required variable is not set. + #[error("{0} is not set")] + Missing(&'static str), + /// A variable is set but does not parse or is not allowed. + #[error("{0} is invalid")] + Invalid(&'static str), +} + +impl Config { + /// Reads `BUZZ_RELAY_GATEWAY_*` from the process environment. + pub fn from_env() -> Result { + Self::from_lookup(|name| std::env::var(name).ok()) + } + + /// Reads the settings through `get`, so tests need no process environment. + pub fn from_lookup(get: impl Fn(&str) -> Option) -> Result { + let required = |name: &'static str| get(name).ok_or(ConfigError::Missing(name)); + let bind_addr = get(BIND) + .as_deref() + .unwrap_or("127.0.0.1:8080") + .parse() + .map_err(|_| ConfigError::Invalid(BIND))?; + let (relay_ws, relay_http) = + relay_urls(&required(RELAY)?).ok_or(ConfigError::Invalid(RELAY))?; + let link_url = required(LINK_URL)?; + Url::parse(&link_url).map_err(|_| ConfigError::Invalid(LINK_URL))?; + let hex = required(LINK_PUBKEY)?; + let lower_hex = + hex.len() == 64 && hex.bytes().all(|b| matches!(b, b'0'..=b'9' | b'a'..=b'f')); + let link_pubkey = PublicKey::from_hex(&hex) + .ok() + .filter(|_| lower_hex) + .ok_or(ConfigError::Invalid(LINK_PUBKEY))?; + Ok(Self { + bind_addr, + relay_ws, + relay_http, + link_url, + link_pubkey, + }) + } +} + +/// `wss://host[:port]` with no userinfo, path, query or fragment; `ws://` only on loopback. +fn relay_urls(raw: &str) -> Option<(Url, Url)> { + let ws = Url::parse(raw).ok()?; + let loopback = matches!(ws.host_str()?, "127.0.0.1" | "[::1]" | "localhost"); + let scheme = match ws.scheme() { + "wss" => "https", + "ws" if loopback => "http", + _ => return None, + }; + let bare = ws.username().is_empty() + && ws.password().is_none() + && ws.path() == "/" + && ws.query().is_none() + && ws.fragment().is_none(); + let mut http = ws.clone(); + http.set_scheme(scheme).ok()?; + bare.then_some((ws, http)) +} + +#[cfg(test)] +mod tests { + use super::*; + + // The secp256k1 generator x: a valid key whose secret (1) is public anyway. + const PK: &str = "79be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798"; + + fn load(relay: &str, pubkey: &str) -> Result { + Config::from_lookup(|name| match name { + RELAY => Some(relay.to_owned()), + LINK_URL => Some("https://gateway.bishop.invalid/v1/link".to_owned()), + LINK_PUBKEY => Some(pubkey.to_owned()), + _ => None, + }) + } + + #[test] + fn accepts_wss_and_loopback_ws() { + let c = load("wss://relay.example.com", PK).unwrap(); + assert_eq!(c.relay_ws.as_str(), "wss://relay.example.com/"); + assert_eq!(c.relay_http.as_str(), "https://relay.example.com/"); + assert_eq!(c.bind_addr.to_string(), "127.0.0.1:8080"); + for ok in [ + "ws://127.0.0.1:7000", + "ws://localhost:7000/", + "ws://[::1]:7000", + "wss://r.example:444", + ] { + assert!(load(ok, PK).is_ok(), "{ok}"); + } + assert_eq!( + load("ws://127.0.0.1:7000", PK).unwrap().relay_http.as_str(), + "http://127.0.0.1:7000/" + ); + } + + #[test] + fn refuses_other_relay_urls() { + for bad in [ + "ws://example.com", + "wss://relay.example.com/path", + "wss://relay.example.com/?q=1", + "wss://relay.example.com/#f", + "wss://user:pw@relay.example.com", + "https://relay.example.com", + "http://127.0.0.1:7000", + "relay.example.com", + ] { + assert!( + matches!(load(bad, PK), Err(ConfigError::Invalid(RELAY))), + "{bad}" + ); + } + } + + #[test] + fn refuses_bad_pubkey_and_names_missing_variables() { + for bad in [&PK[..63], &PK.to_uppercase(), &format!("{}zz", &PK[..62])] { + assert!(matches!( + load("wss://r.example", bad), + Err(ConfigError::Invalid(LINK_PUBKEY)) + )); + } + let err = Config::from_lookup(|_| None).unwrap_err(); + assert_eq!(err.to_string(), "BUZZ_RELAY_GATEWAY_RELAY_URL is not set"); + } +} diff --git a/crates/buzz-relay-gateway/src/lib.rs b/crates/buzz-relay-gateway/src/lib.rs new file mode 100644 index 00000000000..5998126e196 --- /dev/null +++ b/crates/buzz-relay-gateway/src/lib.rs @@ -0,0 +1,38 @@ +//! Buzz relay gateway: a keyless forwarder between one Worker link and one Buzz relay. +//! +//! The Worker keeps every key and signs every event; the gateway checks the +//! link bearer and forwards bytes. It holds no secret. +#![deny(unsafe_code)] + +pub mod config; +pub mod policy; + +use axum::{ + extract::State, + http::StatusCode, + response::{IntoResponse, Response}, + routing::get, + Router, +}; +use tokio::net::TcpListener; +use tokio_util::sync::CancellationToken; + +pub use config::Config; + +/// Serves `/healthz` on `listener` until `shutdown` is cancelled. +pub async fn serve(listener: TcpListener, shutdown: CancellationToken) -> std::io::Result<()> { + let app = Router::new() + .route("/healthz", get(healthz)) + .with_state(shutdown.clone()); + axum::serve(listener, app) + .with_graceful_shutdown(shutdown.cancelled_owned()) + .await +} + +async fn healthz(State(shutdown): State) -> Response { + if shutdown.is_cancelled() { + StatusCode::SERVICE_UNAVAILABLE.into_response() + } else { + "ok".into_response() + } +} diff --git a/crates/buzz-relay-gateway/src/main.rs b/crates/buzz-relay-gateway/src/main.rs new file mode 100644 index 00000000000..77bf85c43b9 --- /dev/null +++ b/crates/buzz-relay-gateway/src/main.rs @@ -0,0 +1,58 @@ +use buzz_relay_gateway::Config; +use tokio::net::TcpListener; +use tokio_util::sync::CancellationToken; +use tracing_subscriber::EnvFilter; + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + // Workspace builds unify ring and aws-lc-rs; rustls then needs an explicit provider. + let _ = rustls::crypto::ring::default_provider().install_default(); + tracing_subscriber::fmt() + .json() + .with_env_filter( + EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")), + ) + .init(); + let config = match Config::from_env() { + Ok(config) => config, + Err(e) => { + tracing::error!(error = %e, "invalid configuration"); + std::process::exit(1); + } + }; + tracing::info!( + bind = %config.bind_addr, + relay = config.relay_ws.host_str().unwrap_or_default(), + link_url = %config.link_url, + link_pubkey = %config.link_pubkey, + "relay gateway starting" + ); + let listener = TcpListener::bind(config.bind_addr).await?; + let shutdown = CancellationToken::new(); + tokio::spawn({ + let shutdown = shutdown.clone(); + async move { + signal().await; + tracing::info!("shutdown signal received"); + shutdown.cancel(); + } + }); + buzz_relay_gateway::serve(listener, shutdown).await?; + Ok(()) +} + +// The binary is PID 1 in the image, which has no default signal handling. +async fn signal() { + #[cfg(unix)] + { + use tokio::signal::unix::{signal, SignalKind}; + if let Ok(mut term) = signal(SignalKind::terminate()) { + tokio::select! { + _ = term.recv() => {} + _ = tokio::signal::ctrl_c() => {} + } + return; + } + } + let _ = tokio::signal::ctrl_c().await; +} diff --git a/crates/buzz-relay-gateway/src/policy.rs b/crates/buzz-relay-gateway/src/policy.rs new file mode 100644 index 00000000000..3cfb75c2b25 --- /dev/null +++ b/crates/buzz-relay-gateway/src/policy.rs @@ -0,0 +1,249 @@ +//! What may reach the relay: an exact route table and closed header sets (design §4). + +use std::collections::BTreeMap; + +use axum::http::{header, HeaderMap, HeaderValue, Method}; +use serde::Deserialize; + +const NIP11: &str = "application/nostr+json"; + +/// A relay HTTP route. The upstream path is always one of these literals. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum HttpRoute { + /// `POST /query` + Query, + /// `POST /events` + Events, + /// `POST /count` + Count, + /// `GET /` for the NIP-11 relay information document. + Info, +} + +impl HttpRoute { + /// The exact path sent upstream. + pub fn path(self) -> &'static str { + match self { + Self::Query => "/query", + Self::Events => "/events", + Self::Count => "/count", + Self::Info => "/", + } + } + + /// The upstream method; every POST route carries a body and GET carries none. + pub fn method(self) -> Method { + if self == Self::Info { + Method::GET + } else { + Method::POST + } + } +} + +/// Why a frame may not go upstream. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Refused { + /// Not in the table, or a header that is not allowed. + NotAllowed, + /// A route reserved for a later phase (the huddle audio path). + NotSupported, +} + +/// Matches a frame's method and path exactly: no normalisation, no query string. +pub fn http_route(method: &str, path: &str) -> Result { + match (method, path) { + ("POST", "/query") => Ok(HttpRoute::Query), + ("POST", "/events") => Ok(HttpRoute::Events), + ("POST", "/count") => Ok(HttpRoute::Count), + ("GET", "/") => Ok(HttpRoute::Info), + _ => Err(Refused::NotAllowed), + } +} + +/// Relay socket paths: `/` only. The huddle audio path is reserved for Phase 2. +pub fn ws_route(path: &str) -> Result<&'static str, Refused> { + if path == "/" { + return Ok("/"); + } + let audio = path + .strip_prefix("/huddle/") + .and_then(|rest| rest.strip_suffix("/audio")) + .is_some_and(|id| id.len() == 36 && uuid::Uuid::try_parse(id).is_ok()); + Err(if audio { + Refused::NotSupported + } else { + Refused::NotAllowed + }) +} + +// A struct, not a map: serde refuses unknown and duplicated keys for it. +#[derive(Default, Deserialize)] +#[serde(deny_unknown_fields)] +struct FrameHeaders { + authorization: Option, + #[serde(rename = "content-type")] + content_type: Option, + accept: Option, +} + +/// Builds the upstream request headers from a frame's raw `headers` object. +/// POST routes need `authorization`; `GET /` needs exactly the NIP-11 `accept`. +pub fn request_headers( + route: HttpRoute, + raw: Option<&str>, + max_auth: usize, +) -> Result { + let h: FrameHeaders = match raw { + Some(raw) => serde_json::from_str(raw).map_err(|_| Refused::NotAllowed)?, + None => FrameHeaders::default(), + }; + let ok = match route { + HttpRoute::Info => { + h.authorization.is_none() + && h.content_type.is_none() + && h.accept.as_deref() == Some(NIP11) + } + _ => h + .authorization + .as_ref() + .is_some_and(|a| a.len() <= max_auth), + }; + if !ok { + return Err(Refused::NotAllowed); + } + let mut out = HeaderMap::new(); + for (name, value) in [ + (header::AUTHORIZATION, h.authorization), + (header::CONTENT_TYPE, h.content_type), + (header::ACCEPT, h.accept), + ] { + if let Some(value) = value { + out.insert( + name, + HeaderValue::from_str(&value).map_err(|_| Refused::NotAllowed)?, + ); + } + } + Ok(out) +} + +/// The relay response headers the Worker sees: `content-type` and `retry-after` only. +pub fn response_headers(headers: &HeaderMap) -> BTreeMap<&'static str, String> { + [ + ("content-type", header::CONTENT_TYPE), + ("retry-after", header::RETRY_AFTER), + ] + .into_iter() + .filter_map(|(key, name)| Some((key, headers.get(name)?.to_str().ok()?.to_owned()))) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn route_table_is_exact() { + assert_eq!(http_route("POST", "/query"), Ok(HttpRoute::Query)); + assert_eq!(http_route("POST", "/events"), Ok(HttpRoute::Events)); + assert_eq!(http_route("POST", "/count"), Ok(HttpRoute::Count)); + assert_eq!(http_route("GET", "/"), Ok(HttpRoute::Info)); + for (m, p) in [ + ("GET", "/query"), + ("PUT", "/query"), + ("post", "/query"), + ("POST", "/QUERY"), + ("POST", "/query/"), + ("POST", "//query"), + ("POST", "/query?x=1"), + ("POST", "/./query"), + ("POST", "/%71uery"), + ("POST", "/"), + ("GET", "/hooks/1"), + ] { + assert_eq!(http_route(m, p), Err(Refused::NotAllowed), "{m} {p}"); + } + assert_eq!(ws_route("/"), Ok("/")); + assert_eq!( + ws_route("/huddle/6f1c2c1e-4b9a-4c71-9a55-0f1d8c3a2b10/audio"), + Err(Refused::NotSupported) + ); + for p in [ + "", + "/query", + "//", + "/?x", + "/huddle/x/audio", + "/huddle/6f1c2c1e4b9a4c719a550f1d8c3a2b10/audio", + ] { + assert_eq!(ws_route(p), Err(Refused::NotAllowed), "{p}"); + } + } + + #[test] + fn request_headers_are_a_closed_set() { + let post = |raw: &str| request_headers(HttpRoute::Query, Some(raw), 16); + let h = post(r#"{"authorization":"Nostr abc","content-type":"application/json"}"#).unwrap(); + assert_eq!(h.len(), 2); + assert_eq!(h[header::AUTHORIZATION], "Nostr abc"); + for bad in [ + r#"{"authorization":"Nostr abc","x-pubkey":"00"}"#, + r#"{"authorization":"Nostr abc","host":"evil.example"}"#, + r#"{"authorization":"Nostr abc","x-forwarded-for":"1.2.3.4"}"#, + r#"{"authorization":"Nostr abc","accept":"a","accept":"b"}"#, + r#"{"authorization":"Nostr abc\r\nx-pubkey: 00"}"#, + r#"{"authorization":"Nostr abc","content-type":"a\nb"}"#, + r#"{"Authorization":"Nostr abc"}"#, + r#"{"authorization":"Nostr 0123456789abc"}"#, + r#"{"content-type":"application/json"}"#, + r#"["authorization"]"#, + ] { + assert_eq!(post(bad), Err(Refused::NotAllowed), "{bad}"); + } + assert_eq!( + request_headers(HttpRoute::Query, None, 16), + Err(Refused::NotAllowed) + ); + let info = |raw: &str| request_headers(HttpRoute::Info, Some(raw), 16); + assert!(info(r#"{"accept":"application/nostr+json"}"#).is_ok()); + assert_eq!( + info(r#"{"accept":"application/json"}"#), + Err(Refused::NotAllowed) + ); + assert_eq!( + info(r#"{"accept":"application/nostr+json","authorization":"x"}"#), + Err(Refused::NotAllowed) + ); + assert_eq!( + request_headers(HttpRoute::Info, None, 16), + Err(Refused::NotAllowed) + ); + } + + #[test] + fn response_headers_keep_two_names() { + let mut h = HeaderMap::new(); + for (name, value) in [ + ("content-type", "application/json"), + ("retry-after", "12"), + ("set-cookie", "a=b"), + ("location", "/elsewhere"), + ("server", "cloudflare"), + ("cf-ray", "1"), + ] { + h.insert( + header::HeaderName::from_static(name), + HeaderValue::from_static(value), + ); + } + let kept = response_headers(&h); + assert_eq!( + kept.into_iter().collect::>(), + [ + ("content-type", "application/json".into()), + ("retry-after", "12".into()) + ] + ); + } +} From c9848c6b2b0cef4d454af17177c7acd333a5df2e Mon Sep 17 00:00:00 2001 From: QuicksilverSlick Date: Tue, 27 Oct 2026 09:08:48 -0500 Subject: [PATCH 02/15] fix(relay-gateway): cap the shutdown wait and keep tungstenite out of logs Review fixes for part 1. - axum::serve sets no hyper timer, so a client that stops mid-request holds hyper's graceful shutdown open for good, and SIGTERM never leads to an exit. serve() now returns 2 s after its work is done whatever the open connections are doing. tests/serve.rs leaves a request half sent and checks that serve returns. - fmt().init() also bridges `log` records, and at trace tungstenite logs every frame it sends or receives in full. log_filter() builds the filter from RUST_LOG and always turns tungstenite and tokio_tungstenite off, so no RUST_LOG value prints link or relay frames. Signed-off-by: QuicksilverSlick Co-Authored-By: Claude Opus 5.5 --- crates/buzz-relay-gateway/src/lib.rs | 32 +++++++++++++++++++++--- crates/buzz-relay-gateway/src/main.rs | 6 ++--- crates/buzz-relay-gateway/tests/serve.rs | 26 +++++++++++++++++++ 3 files changed, 57 insertions(+), 7 deletions(-) create mode 100644 crates/buzz-relay-gateway/tests/serve.rs diff --git a/crates/buzz-relay-gateway/src/lib.rs b/crates/buzz-relay-gateway/src/lib.rs index 5998126e196..40737dcdb11 100644 --- a/crates/buzz-relay-gateway/src/lib.rs +++ b/crates/buzz-relay-gateway/src/lib.rs @@ -7,6 +7,8 @@ pub mod config; pub mod policy; +use std::{future::IntoFuture, time::Duration}; + use axum::{ extract::State, http::StatusCode, @@ -16,17 +18,41 @@ use axum::{ }; use tokio::net::TcpListener; use tokio_util::sync::CancellationToken; +use tracing_subscriber::EnvFilter; pub use config::Config; +// hyper's graceful wait never ends for a connection that stopped mid-request, +// so once the work is done the open connections get this long. +const LAST_REQUESTS: Duration = Duration::from_secs(2); + /// Serves `/healthz` on `listener` until `shutdown` is cancelled. pub async fn serve(listener: TcpListener, shutdown: CancellationToken) -> std::io::Result<()> { let app = Router::new() .route("/healthz", get(healthz)) .with_state(shutdown.clone()); - axum::serve(listener, app) - .with_graceful_shutdown(shutdown.cancelled_owned()) - .await + let server = + axum::serve(listener, app).with_graceful_shutdown(shutdown.clone().cancelled_owned()); + tokio::select! { + served = server.into_future() => served, + () = async { + shutdown.cancelled().await; + tokio::time::sleep(LAST_REQUESTS).await; + } => Ok(()), + } +} + +/// The log filter: `directives` in `RUST_LOG` syntax, or `info` if they do not +/// parse. The WebSocket crates stay off at any level, because at trace they +/// print whole frames: bearers, NIP-98 headers, bodies and relay messages. +pub fn log_filter(directives: &str) -> EnvFilter { + let filter = EnvFilter::try_new(directives).unwrap_or_else(|_| EnvFilter::new("info")); + // shortcut: a more specific target (tungstenite::protocol=trace) still wins; + // whoever sets RUST_LOG also picks the image, so pinning harder buys nothing. + ["tungstenite=off", "tokio_tungstenite=off"] + .into_iter() + .filter_map(|d| d.parse().ok()) + .fold(filter, EnvFilter::add_directive) } async fn healthz(State(shutdown): State) -> Response { diff --git a/crates/buzz-relay-gateway/src/main.rs b/crates/buzz-relay-gateway/src/main.rs index 77bf85c43b9..d6dbe99b8cf 100644 --- a/crates/buzz-relay-gateway/src/main.rs +++ b/crates/buzz-relay-gateway/src/main.rs @@ -1,17 +1,15 @@ use buzz_relay_gateway::Config; use tokio::net::TcpListener; use tokio_util::sync::CancellationToken; -use tracing_subscriber::EnvFilter; #[tokio::main] async fn main() -> anyhow::Result<()> { // Workspace builds unify ring and aws-lc-rs; rustls then needs an explicit provider. let _ = rustls::crypto::ring::default_provider().install_default(); + let directives = std::env::var("RUST_LOG").unwrap_or_else(|_| "info".to_owned()); tracing_subscriber::fmt() .json() - .with_env_filter( - EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")), - ) + .with_env_filter(buzz_relay_gateway::log_filter(&directives)) .init(); let config = match Config::from_env() { Ok(config) => config, diff --git a/crates/buzz-relay-gateway/tests/serve.rs b/crates/buzz-relay-gateway/tests/serve.rs new file mode 100644 index 00000000000..e813b1ea13f --- /dev/null +++ b/crates/buzz-relay-gateway/tests/serve.rs @@ -0,0 +1,26 @@ +use std::time::Duration; + +use tokio::{ + io::AsyncWriteExt, + net::{TcpListener, TcpStream}, + time::timeout, +}; +use tokio_util::sync::CancellationToken; + +#[tokio::test] +async fn a_half_sent_request_does_not_hold_up_shutdown() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let mut tcp = TcpStream::connect(listener.local_addr().unwrap()) + .await + .unwrap(); + let shutdown = CancellationToken::new(); + let server = tokio::spawn(buzz_relay_gateway::serve(listener, shutdown.clone())); + // No blank line after the headers: hyper keeps waiting for the rest. + tcp.write_all(b"GET /healthz HTTP/1.1\r\nhost: x\r\n") + .await + .unwrap(); + tokio::time::sleep(Duration::from_millis(100)).await; + shutdown.cancel(); + let served = timeout(Duration::from_secs(5), server).await; + assert!(matches!(served, Ok(Ok(Ok(())))), "serve did not return"); +} From 9111346fe910c463c2742607ef2007fb1bf324be Mon Sep 17 00:00:00 2001 From: QuicksilverSlick Date: Tue, 27 Oct 2026 08:27:25 -0500 Subject: [PATCH 03/15] feat(relay-gateway): gate /v1/link with a single-use NIP-98 bearer The Worker opens a link with Authorization: Nostr , a kind 27235 event signed by a dedicated gateway-auth key that is never a relay member. buzz_auth::verify_nip98_event checks the kind, the signature, the 60 s window, one u (the configured link URL) and one method (GET); the gateway then requires the pinned pubkey and exactly one nonce of 16 to 128 characters. - Each bearer works once: its event id is claimed in an in-memory replay cache for the relay's 120 s TTL. Only a bearer that passed every other check uses a slot, and a full cache refuses new bearers with 503 rather than forgetting old ones. - Refusals answer 401 {"error":"unauthorized"} and log only a reason code (missing, malformed, invalid, wrong_key, no_nonce, replayed), never the header or the verifier's text, which quotes bearer content. - At most four links at once; the frame loop lands in a later part, so a good bearer gets 501 for now. Signed-off-by: QuicksilverSlick Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 3 + crates/buzz-relay-gateway/Cargo.toml | 5 + crates/buzz-relay-gateway/src/auth.rs | 251 ++++++++++++++++++ crates/buzz-relay-gateway/src/config.rs | 23 +- crates/buzz-relay-gateway/src/lib.rs | 74 +++++- crates/buzz-relay-gateway/src/main.rs | 4 +- crates/buzz-relay-gateway/tests/common/mod.rs | 75 ++++++ crates/buzz-relay-gateway/tests/link.rs | 46 ++++ crates/buzz-relay-gateway/tests/serve.rs | 23 +- 9 files changed, 477 insertions(+), 27 deletions(-) create mode 100644 crates/buzz-relay-gateway/src/auth.rs create mode 100644 crates/buzz-relay-gateway/tests/common/mod.rs create mode 100644 crates/buzz-relay-gateway/tests/link.rs diff --git a/Cargo.lock b/Cargo.lock index 5c06f6a84a2..aafb5d50b5d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1511,7 +1511,10 @@ version = "0.1.0" dependencies = [ "anyhow", "axum", + "base64 0.22.1", + "buzz-auth", "nostr 0.44.7", + "reqwest 0.13.4", "rustls", "serde", "serde_json", diff --git a/crates/buzz-relay-gateway/Cargo.toml b/crates/buzz-relay-gateway/Cargo.toml index 9366096f7fc..e2a342853c2 100644 --- a/crates/buzz-relay-gateway/Cargo.toml +++ b/crates/buzz-relay-gateway/Cargo.toml @@ -18,6 +18,8 @@ path = "src/main.rs" [dependencies] anyhow = { workspace = true } axum = { workspace = true } +base64 = { workspace = true } +buzz-auth = { workspace = true } nostr = { workspace = true } # Explicit ring provider, installed in main(): a workspace-wide build unifies # ring and aws-lc-rs, and rustls then panics on the first wss:// connect. @@ -31,3 +33,6 @@ tracing = { workspace = true } tracing-subscriber = { workspace = true } url = { workspace = true } uuid = { workspace = true } + +[dev-dependencies] +reqwest = { workspace = true } diff --git a/crates/buzz-relay-gateway/src/auth.rs b/crates/buzz-relay-gateway/src/auth.rs new file mode 100644 index 00000000000..ccfa288fc52 --- /dev/null +++ b/crates/buzz-relay-gateway/src/auth.rs @@ -0,0 +1,251 @@ +//! The link bearer: a single-use NIP-98 event signed by the gateway-auth key. + +use std::{ + collections::HashMap, + sync::{Mutex, PoisonError}, + time::{Duration, Instant}, +}; + +use axum::http::{header::AUTHORIZATION, HeaderMap}; +use base64::{engine::general_purpose::STANDARD, Engine as _}; +use buzz_auth::{nip98_replay::DEFAULT_REPLAY_TTL_SECS, verify_nip98_event}; +use nostr::{Event, EventId, PublicKey}; + +/// Why a bearer was refused. Only [`Reject::code`] is logged: the underlying +/// error text quotes bearer content. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Reject { + /// No `Authorization` header. + Missing, + /// Repeated, oversized, not `Nostr `, or not UTF-8. + Malformed, + /// Failed NIP-98: kind, signature, time window, `u` or `method`. + Invalid, + /// A valid signature by a key other than the pinned one. + WrongKey, + /// Not exactly one `nonce` tag of 16 to 128 characters. + NoNonce, + /// The event id was already used. + Replayed, + /// The replay cache is full; the gateway fails closed. + CacheFull, +} + +impl Reject { + /// The fixed reason code for logs. + pub fn code(self) -> &'static str { + match self { + Self::Missing => "missing", + Self::Malformed => "malformed", + Self::Invalid => "invalid", + Self::WrongKey => "wrong_key", + Self::NoNonce => "no_nonce", + Self::Replayed => "replayed", + Self::CacheFull => "cache_full", + } + } +} + +/// Checks the link bearer on an upgrade request and returns its event id, +/// which the caller then claims in the [`ReplayCache`]. +pub fn verify_link_bearer( + headers: &HeaderMap, + link_url: &str, + pubkey: &PublicKey, + max_len: usize, +) -> Result { + let mut values = headers.get_all(AUTHORIZATION).iter(); + let (value, None) = (values.next().ok_or(Reject::Missing)?, values.next()) else { + return Err(Reject::Malformed); + }; + let json = (value.len() <= max_len) + .then(|| value.to_str().ok()?.strip_prefix("Nostr ")) + .flatten() + .and_then(|b64| STANDARD.decode(b64).ok()) + .and_then(|bytes| String::from_utf8(bytes).ok()) + .ok_or(Reject::Malformed)?; + verify_nip98_event(&json, link_url, "GET", None).map_err(|_| Reject::Invalid)?; + let event: Event = serde_json::from_str(&json).map_err(|_| Reject::Invalid)?; + if event.pubkey != *pubkey { + return Err(Reject::WrongKey); + } + let mut nonces = event + .tags + .iter() + .map(|t| t.as_slice()) + .filter(|t| t.first().is_some_and(|n| n == "nonce")); + match (nonces.next(), nonces.next()) { + (Some([_, n, ..]), None) if (16..=128).contains(&n.chars().count()) => Ok(event.id), + _ => Err(Reject::NoNonce), + } +} + +/// Event ids of accepted bearers, each kept for the relay's replay TTL +/// (twice the ±60 s window). A restart empties it. +pub struct ReplayCache { + cap: usize, + seen: Mutex>, +} + +impl ReplayCache { + /// A cache that holds at most `cap` live ids. + pub fn new(cap: usize) -> Self { + Self { + cap, + seen: Mutex::new(HashMap::new()), + } + } + + /// Claims `id` once. Refuses a replay, and refuses every id while full. + pub fn claim(&self, id: EventId, now: Instant) -> Result<(), Reject> { + let ttl = Duration::from_secs(DEFAULT_REPLAY_TTL_SECS); + let mut seen = self.seen.lock().unwrap_or_else(PoisonError::into_inner); + seen.retain(|_, at| now.saturating_duration_since(*at) < ttl); + if seen.contains_key(&id) { + return Err(Reject::Replayed); + } + if seen.len() >= self.cap { + return Err(Reject::CacheFull); + } + seen.insert(id, now); + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use axum::http::HeaderValue; + use nostr::{EventBuilder, Keys, Kind, Tag, Timestamp}; + + const URL: &str = "https://gateway.bishop.invalid/v1/link"; + const NONCE: &str = "6f1c2c1e-4b9a-4c71-9a55-0f1d8c3a2b10"; + + fn event(keys: &Keys, kind: u16, tags: &[&[&str]], age: i64) -> Event { + let tags = tags.iter().map(|t| Tag::parse(t.iter().copied()).unwrap()); + let at = Timestamp::from((Timestamp::now().as_secs() as i64 - age) as u64); + EventBuilder::new(Kind::from(kind), "") + .tags(tags) + .custom_created_at(at) + .sign_with_keys(keys) + .unwrap() + } + + fn header(event: &Event) -> String { + format!( + "Nostr {}", + STANDARD.encode(serde_json::to_string(event).unwrap()) + ) + } + + fn good(keys: &Keys) -> String { + let nonce = uuid::Uuid::new_v4().to_string(); + header(&event( + keys, + 27235, + &[&["u", URL], &["method", "GET"], &["nonce", &nonce]], + 0, + )) + } + + fn check(keys: &Keys, values: &[&str]) -> Result { + let mut headers = HeaderMap::new(); + for v in values { + headers.append(AUTHORIZATION, HeaderValue::from_str(v).unwrap()); + } + verify_link_bearer(&headers, URL, &keys.public_key(), 8 << 10) + } + + #[test] + fn accepts_a_good_bearer_and_url_normalisation() { + let k = Keys::generate(); + assert!(check(&k, &[&good(&k)]).is_ok()); + for u in [ + "https://gateway.bishop.invalid/v1/link/", + "HTTPS://Gateway.Bishop.Invalid/v1/link", + ] { + let b = header(&event( + &k, + 27235, + &[&["u", u], &["method", "get"], &["nonce", NONCE]], + 0, + )); + assert!(check(&k, &[&b]).is_ok(), "{u}"); + } + } + + #[test] + fn refuses_bad_bearers() { + let k = Keys::generate(); + let with = |kind, tags: &[&[&str]], age| header(&event(&k, kind, tags, age)); + let (u, get, nonce) = ( + &["u", URL][..], + &["method", "GET"][..], + &["nonce", NONCE][..], + ); + let mut tampered: serde_json::Value = + serde_json::to_value(event(&k, 27235, &[u, get, nonce], 0)).unwrap(); + tampered["content"] = "x".into(); + let tampered = format!("Nostr {}", STANDARD.encode(tampered.to_string())); + let cases = [ + (good(&Keys::generate()), Reject::WrongKey), + (with(27235, &[u, get, nonce], 120), Reject::Invalid), + (with(27235, &[u, get, nonce], -120), Reject::Invalid), + ( + with( + 27235, + &[&["u", "https://other.invalid/v1/link"], get, nonce], + 0, + ), + Reject::Invalid, + ), + ( + with(27235, &[u, &["method", "POST"], nonce], 0), + Reject::Invalid, + ), + (with(22242, &[u, get, nonce], 0), Reject::Invalid), + (tampered, Reject::Invalid), + (with(27235, &[u, get], 0), Reject::NoNonce), + ( + with( + 27235, + &[u, get, nonce, &["nonce", "a-second-nonce-value"]], + 0, + ), + Reject::NoNonce, + ), + ( + with(27235, &[u, get, &["nonce", "short"]], 0), + Reject::NoNonce, + ), + (good(&k).replacen("Nostr ", "Bearer ", 1), Reject::Malformed), + ("Nostr !!!".to_owned(), Reject::Malformed), + ( + format!("Nostr {}", STANDARD.encode([0xff, 0xfe])), + Reject::Malformed, + ), + (format!("Nostr {}", "A".repeat(8 << 10)), Reject::Malformed), + ]; + for (i, (bearer, want)) in cases.iter().enumerate() { + assert_eq!(check(&k, &[bearer]), Err(*want), "case {i}"); + } + assert_eq!(check(&k, &[]), Err(Reject::Missing)); + assert_eq!(check(&k, &[&good(&k), &good(&k)]), Err(Reject::Malformed)); + } + + #[test] + fn replay_cache_is_single_use_prunes_and_fails_closed() { + let k = Keys::generate(); + let (a, b) = ( + check(&k, &[&good(&k)]).unwrap(), + check(&k, &[&good(&k)]).unwrap(), + ); + let cache = ReplayCache::new(1); + let now = Instant::now(); + assert_eq!(cache.claim(a, now), Ok(())); + assert_eq!(cache.claim(a, now), Err(Reject::Replayed)); + assert_eq!(cache.claim(b, now), Err(Reject::CacheFull)); + let later = now + Duration::from_secs(DEFAULT_REPLAY_TTL_SECS); + assert_eq!(cache.claim(b, later), Ok(())); + } +} diff --git a/crates/buzz-relay-gateway/src/config.rs b/crates/buzz-relay-gateway/src/config.rs index 750b5265a9b..2e0178b032c 100644 --- a/crates/buzz-relay-gateway/src/config.rs +++ b/crates/buzz-relay-gateway/src/config.rs @@ -1,4 +1,4 @@ -//! Environment configuration. Every value is public. +//! Environment configuration (all public values) and the fixed limits. use std::net::SocketAddr; @@ -90,6 +90,27 @@ fn relay_urls(raw: &str) -> Option<(Url, Url)> { bare.then_some((ws, http)) } +/// Fixed limits (design §5). Not configurable by env; tests pass smaller values. +#[derive(Clone, Debug)] +pub struct Limits { + /// Links open at once; the next upgrade gets 503. + pub max_links: usize, + /// Longest `Authorization` value, on the link bearer and on forwarded requests. + pub max_auth_header: usize, + /// Bearer ids remembered for replay protection; when full, new bearers get 503. + pub replay_entries: usize, +} + +impl Default for Limits { + fn default() -> Self { + Self { + max_links: 4, + max_auth_header: 8 << 10, + replay_entries: 1024, + } + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/buzz-relay-gateway/src/lib.rs b/crates/buzz-relay-gateway/src/lib.rs index 40737dcdb11..ef93a05cde1 100644 --- a/crates/buzz-relay-gateway/src/lib.rs +++ b/crates/buzz-relay-gateway/src/lib.rs @@ -4,33 +4,59 @@ //! link bearer and forwards bytes. It holds no secret. #![deny(unsafe_code)] +pub mod auth; pub mod config; pub mod policy; -use std::{future::IntoFuture, time::Duration}; +use std::{ + future::IntoFuture, + sync::Arc, + time::{Duration, Instant}, +}; use axum::{ extract::State, - http::StatusCode, + http::{HeaderMap, StatusCode}, response::{IntoResponse, Response}, routing::get, - Router, + Json, Router, }; -use tokio::net::TcpListener; +use tokio::{net::TcpListener, sync::Semaphore}; use tokio_util::sync::CancellationToken; use tracing_subscriber::EnvFilter; -pub use config::Config; +pub use config::{Config, Limits}; // hyper's graceful wait never ends for a connection that stopped mid-request, // so once the work is done the open connections get this long. const LAST_REQUESTS: Duration = Duration::from_secs(2); -/// Serves `/healthz` on `listener` until `shutdown` is cancelled. -pub async fn serve(listener: TcpListener, shutdown: CancellationToken) -> std::io::Result<()> { +struct AppState { + config: Config, + limits: Limits, + replay: auth::ReplayCache, + links: Arc, + shutdown: CancellationToken, +} + +/// Serves `/healthz` and `/v1/link` on `listener` until `shutdown` is cancelled. +pub async fn serve( + listener: TcpListener, + config: Config, + limits: Limits, + shutdown: CancellationToken, +) -> std::io::Result<()> { + let state = Arc::new(AppState { + replay: auth::ReplayCache::new(limits.replay_entries), + links: Arc::new(Semaphore::new(limits.max_links)), + shutdown: shutdown.clone(), + config, + limits, + }); let app = Router::new() .route("/healthz", get(healthz)) - .with_state(shutdown.clone()); + .route("/v1/link", get(link)) + .with_state(state); let server = axum::serve(listener, app).with_graceful_shutdown(shutdown.clone().cancelled_owned()); tokio::select! { @@ -55,10 +81,38 @@ pub fn log_filter(directives: &str) -> EnvFilter { .fold(filter, EnvFilter::add_directive) } -async fn healthz(State(shutdown): State) -> Response { - if shutdown.is_cancelled() { +async fn healthz(State(s): State>) -> Response { + if s.shutdown.is_cancelled() { StatusCode::SERVICE_UNAVAILABLE.into_response() } else { "ok".into_response() } } + +async fn link(State(s): State>, headers: HeaderMap) -> Response { + if s.shutdown.is_cancelled() { + return StatusCode::SERVICE_UNAVAILABLE.into_response(); + } + let Ok(_permit) = s.links.clone().try_acquire_owned() else { + return StatusCode::SERVICE_UNAVAILABLE.into_response(); + }; + let claimed = auth::verify_link_bearer( + &headers, + &s.config.link_url, + &s.config.link_pubkey, + s.limits.max_auth_header, + ) + .and_then(|id| s.replay.claim(id, Instant::now())); + match claimed { + Err(reject) => { + tracing::warn!(reason = reject.code(), "link bearer refused"); + if reject == auth::Reject::CacheFull { + return StatusCode::SERVICE_UNAVAILABLE.into_response(); + } + let body = Json(serde_json::json!({ "error": "unauthorized" })); + (StatusCode::UNAUTHORIZED, body).into_response() + } + // The link's frame loop lands in a later change. + Ok(()) => StatusCode::NOT_IMPLEMENTED.into_response(), + } +} diff --git a/crates/buzz-relay-gateway/src/main.rs b/crates/buzz-relay-gateway/src/main.rs index d6dbe99b8cf..792d4206d03 100644 --- a/crates/buzz-relay-gateway/src/main.rs +++ b/crates/buzz-relay-gateway/src/main.rs @@ -1,4 +1,4 @@ -use buzz_relay_gateway::Config; +use buzz_relay_gateway::{Config, Limits}; use tokio::net::TcpListener; use tokio_util::sync::CancellationToken; @@ -35,7 +35,7 @@ async fn main() -> anyhow::Result<()> { shutdown.cancel(); } }); - buzz_relay_gateway::serve(listener, shutdown).await?; + buzz_relay_gateway::serve(listener, config, Limits::default(), shutdown).await?; Ok(()) } diff --git a/crates/buzz-relay-gateway/tests/common/mod.rs b/crates/buzz-relay-gateway/tests/common/mod.rs new file mode 100644 index 00000000000..8a3dd608a30 --- /dev/null +++ b/crates/buzz-relay-gateway/tests/common/mod.rs @@ -0,0 +1,75 @@ +//! Shared harness: the gateway in-process and a bearer signer. +#![allow(dead_code)] + +use std::net::SocketAddr; + +use base64::{engine::general_purpose::STANDARD, Engine as _}; +use buzz_relay_gateway::{Config, Limits}; +use nostr::{EventBuilder, Keys, Kind, Tag}; +use tokio::{net::TcpListener, task::JoinHandle}; +use tokio_util::sync::CancellationToken; + +pub const LINK_URL: &str = "http://gateway.test/v1/link"; + +/// A fresh link bearer signed by `keys`, carrying `content`. +pub fn bearer_with(keys: &Keys, content: &str) -> String { + let nonce = uuid::Uuid::new_v4().to_string(); + let tags = [ + ["u", LINK_URL], + ["method", "GET"], + ["nonce", nonce.as_str()], + ] + .map(|t| Tag::parse(t).unwrap()); + let event = EventBuilder::new(Kind::HttpAuth, content) + .tags(tags) + .sign_with_keys(keys) + .unwrap(); + format!( + "Nostr {}", + STANDARD.encode(serde_json::to_string(&event).unwrap()) + ) +} + +pub fn bearer(keys: &Keys) -> String { + bearer_with(keys, "") +} + +pub struct Gateway { + pub addr: SocketAddr, + pub keys: Keys, + pub shutdown: CancellationToken, + pub task: JoinHandle>, +} + +impl Gateway { + pub async fn start(relay_ws: &str, limits: Limits) -> Self { + let keys = Keys::generate(); + let pubkey = keys.public_key().to_hex(); + let config = Config::from_lookup(|name| match name { + "BUZZ_RELAY_GATEWAY_RELAY_URL" => Some(relay_ws.to_owned()), + "BUZZ_RELAY_GATEWAY_LINK_URL" => Some(LINK_URL.to_owned()), + "BUZZ_RELAY_GATEWAY_LINK_PUBKEY" => Some(pubkey.clone()), + _ => None, + }) + .unwrap(); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let shutdown = CancellationToken::new(); + let task = tokio::spawn(buzz_relay_gateway::serve( + listener, + config, + limits, + shutdown.clone(), + )); + Self { + addr, + keys, + shutdown, + task, + } + } + + pub fn url(&self, path: &str) -> String { + format!("http://{}{path}", self.addr) + } +} diff --git a/crates/buzz-relay-gateway/tests/link.rs b/crates/buzz-relay-gateway/tests/link.rs new file mode 100644 index 00000000000..bf1bab88483 --- /dev/null +++ b/crates/buzz-relay-gateway/tests/link.rs @@ -0,0 +1,46 @@ +mod common; + +use buzz_relay_gateway::Limits; +use common::{bearer, Gateway}; +use nostr::Keys; + +#[tokio::test] +async fn bearer_gate_is_single_use_and_fails_closed() { + let gw = Gateway::start( + "ws://127.0.0.1:9", + Limits { + replay_entries: 1, + ..Limits::default() + }, + ) + .await; + let http = reqwest::Client::new(); + let link = |auth: Option<&str>| { + let req = http.get(gw.url("/v1/link")); + let req = match auth { + Some(auth) => req.header("authorization", auth), + None => req, + }; + async move { req.send().await.unwrap() } + }; + let none = link(None).await; + assert_eq!(none.status(), 401); + assert_eq!(none.text().await.unwrap(), r#"{"error":"unauthorized"}"#); + assert_eq!(link(Some(&bearer(&Keys::generate()))).await.status(), 401); + // Refused bearers used no replay slot, so the one-slot cache still admits this one. + let good = bearer(&gw.keys); + assert_eq!(link(Some(&good)).await.status(), 501); + assert_eq!(link(Some(&good)).await.status(), 401); + assert_eq!(link(Some(&bearer(&gw.keys))).await.status(), 503); + + let health = http.get(gw.url("/healthz")).send().await.unwrap(); + assert_eq!(health.status(), 200); + assert_eq!(health.text().await.unwrap(), "ok"); + assert_eq!( + http.get(gw.url("/v1/other")).send().await.unwrap().status(), + 404 + ); + + gw.shutdown.cancel(); + gw.task.await.unwrap().unwrap(); +} diff --git a/crates/buzz-relay-gateway/tests/serve.rs b/crates/buzz-relay-gateway/tests/serve.rs index e813b1ea13f..f682f365bc0 100644 --- a/crates/buzz-relay-gateway/tests/serve.rs +++ b/crates/buzz-relay-gateway/tests/serve.rs @@ -1,26 +1,21 @@ +mod common; + use std::time::Duration; -use tokio::{ - io::AsyncWriteExt, - net::{TcpListener, TcpStream}, - time::timeout, -}; -use tokio_util::sync::CancellationToken; +use buzz_relay_gateway::Limits; +use common::Gateway; +use tokio::{io::AsyncWriteExt, net::TcpStream, time::timeout}; #[tokio::test] async fn a_half_sent_request_does_not_hold_up_shutdown() { - let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); - let mut tcp = TcpStream::connect(listener.local_addr().unwrap()) - .await - .unwrap(); - let shutdown = CancellationToken::new(); - let server = tokio::spawn(buzz_relay_gateway::serve(listener, shutdown.clone())); + let gw = Gateway::start("ws://127.0.0.1:9", Limits::default()).await; + let mut tcp = TcpStream::connect(gw.addr).await.unwrap(); // No blank line after the headers: hyper keeps waiting for the rest. tcp.write_all(b"GET /healthz HTTP/1.1\r\nhost: x\r\n") .await .unwrap(); tokio::time::sleep(Duration::from_millis(100)).await; - shutdown.cancel(); - let served = timeout(Duration::from_secs(5), server).await; + gw.shutdown.cancel(); + let served = timeout(Duration::from_secs(5), gw.task).await; assert!(matches!(served, Ok(Ok(Ok(())))), "serve did not return"); } From f63f9865717b0c95296ae2f296098006f72e03d5 Mon Sep 17 00:00:00 2001 From: QuicksilverSlick Date: Tue, 27 Oct 2026 09:12:26 -0500 Subject: [PATCH 04/15] fix(relay-gateway): keep claimed bearer ids 2 s past the relay's TTL Review fix for part 2. The bearer window is inclusive and counted in whole seconds, so a bearer dated 60 s ahead is valid for up to 121 s, while the cache let its id go after exactly 120 s. Ids are now kept for 122 s, and the unit test checks that an id is still refused at 120 s. Signed-off-by: QuicksilverSlick Co-Authored-By: Claude Opus 5.5 --- crates/buzz-relay-gateway/src/auth.rs | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/crates/buzz-relay-gateway/src/auth.rs b/crates/buzz-relay-gateway/src/auth.rs index ccfa288fc52..bb83049ebe1 100644 --- a/crates/buzz-relay-gateway/src/auth.rs +++ b/crates/buzz-relay-gateway/src/auth.rs @@ -80,8 +80,12 @@ pub fn verify_link_bearer( } } -/// Event ids of accepted bearers, each kept for the relay's replay TTL -/// (twice the ±60 s window). A restart empties it. +/// How long a claimed id is kept: the relay's replay TTL (twice the ±60 s +/// window) plus 2 s, because the window is inclusive and counts whole seconds, +/// so a bearer dated 60 s ahead stays valid for up to 121 s. +const REPLAY_TTL: Duration = Duration::from_secs(DEFAULT_REPLAY_TTL_SECS + 2); + +/// Event ids of accepted bearers, each kept for `REPLAY_TTL`. A restart empties it. pub struct ReplayCache { cap: usize, seen: Mutex>, @@ -98,9 +102,8 @@ impl ReplayCache { /// Claims `id` once. Refuses a replay, and refuses every id while full. pub fn claim(&self, id: EventId, now: Instant) -> Result<(), Reject> { - let ttl = Duration::from_secs(DEFAULT_REPLAY_TTL_SECS); let mut seen = self.seen.lock().unwrap_or_else(PoisonError::into_inner); - seen.retain(|_, at| now.saturating_duration_since(*at) < ttl); + seen.retain(|_, at| now.saturating_duration_since(*at) < REPLAY_TTL); if seen.contains_key(&id) { return Err(Reject::Replayed); } @@ -245,7 +248,9 @@ mod tests { assert_eq!(cache.claim(a, now), Ok(())); assert_eq!(cache.claim(a, now), Err(Reject::Replayed)); assert_eq!(cache.claim(b, now), Err(Reject::CacheFull)); - let later = now + Duration::from_secs(DEFAULT_REPLAY_TTL_SECS); - assert_eq!(cache.claim(b, later), Ok(())); + // Still held when the relay's own 120 s would have let it go. + let relay_ttl = now + Duration::from_secs(DEFAULT_REPLAY_TTL_SECS); + assert_eq!(cache.claim(a, relay_ttl), Err(Reject::Replayed)); + assert_eq!(cache.claim(b, now + REPLAY_TTL), Ok(())); } } From 5abb8ce7d1d8c585dee903875c3d16fd9a21023f Mon Sep 17 00:00:00 2001 From: QuicksilverSlick Date: Tue, 27 Oct 2026 09:34:59 -0500 Subject: [PATCH 05/15] fix(relay-gateway): keep a bearer id while the window could still accept it The 122 s monotonic hold covers the inclusive whole-second window, but a backward wall-clock step after the claim could make the verifier accept a bearer the cache had already forgotten. Each entry now also stays until the wall clock is past its created_at plus the window. Signed-off-by: QuicksilverSlick Co-Authored-By: Claude Opus 5.5 --- crates/buzz-relay-gateway/src/auth.rs | 67 ++++++++++++++++++++------- crates/buzz-relay-gateway/src/lib.rs | 6 ++- 2 files changed, 55 insertions(+), 18 deletions(-) diff --git a/crates/buzz-relay-gateway/src/auth.rs b/crates/buzz-relay-gateway/src/auth.rs index bb83049ebe1..ae8206ca3bb 100644 --- a/crates/buzz-relay-gateway/src/auth.rs +++ b/crates/buzz-relay-gateway/src/auth.rs @@ -46,14 +46,14 @@ impl Reject { } } -/// Checks the link bearer on an upgrade request and returns its event id, -/// which the caller then claims in the [`ReplayCache`]. +/// Checks the link bearer on an upgrade request and returns its event id and +/// `created_at` (unix seconds), which the caller then claims in the [`ReplayCache`]. pub fn verify_link_bearer( headers: &HeaderMap, link_url: &str, pubkey: &PublicKey, max_len: usize, -) -> Result { +) -> Result<(EventId, u64), Reject> { let mut values = headers.get_all(AUTHORIZATION).iter(); let (value, None) = (values.next().ok_or(Reject::Missing)?, values.next()) else { return Err(Reject::Malformed); @@ -75,7 +75,9 @@ pub fn verify_link_bearer( .map(|t| t.as_slice()) .filter(|t| t.first().is_some_and(|n| n == "nonce")); match (nonces.next(), nonces.next()) { - (Some([_, n, ..]), None) if (16..=128).contains(&n.chars().count()) => Ok(event.id), + (Some([_, n, ..]), None) if (16..=128).contains(&n.chars().count()) => { + Ok((event.id, event.created_at.as_secs())) + } _ => Err(Reject::NoNonce), } } @@ -85,10 +87,15 @@ pub fn verify_link_bearer( /// so a bearer dated 60 s ahead stays valid for up to 121 s. const REPLAY_TTL: Duration = Duration::from_secs(DEFAULT_REPLAY_TTL_SECS + 2); -/// Event ids of accepted bearers, each kept for `REPLAY_TTL`. A restart empties it. +/// `verify_nip98_event`'s ±60 s window on `created_at` (the relay's TTL is twice it). +const WINDOW_SECS: u64 = DEFAULT_REPLAY_TTL_SECS / 2; + +/// Event ids of accepted bearers, each kept for `REPLAY_TTL` of monotonic time +/// and, if the wall clock steps back, for as long as the window could still +/// accept its `created_at`. A restart empties it. pub struct ReplayCache { cap: usize, - seen: Mutex>, + seen: Mutex>, } impl ReplayCache { @@ -100,17 +107,26 @@ impl ReplayCache { } } - /// Claims `id` once. Refuses a replay, and refuses every id while full. - pub fn claim(&self, id: EventId, now: Instant) -> Result<(), Reject> { + /// Claims a bearer `(id, created_at)` once, `now` being monotonic and `wall` + /// unix seconds. Refuses a replay, and refuses every id while full. + pub fn claim( + &self, + (id, created_at): (EventId, u64), + now: Instant, + wall: u64, + ) -> Result<(), Reject> { let mut seen = self.seen.lock().unwrap_or_else(PoisonError::into_inner); - seen.retain(|_, at| now.saturating_duration_since(*at) < REPLAY_TTL); + seen.retain(|_, (at, created)| { + now.saturating_duration_since(*at) < REPLAY_TTL + || wall <= created.saturating_add(WINDOW_SECS + 1) + }); if seen.contains_key(&id) { return Err(Reject::Replayed); } if seen.len() >= self.cap { return Err(Reject::CacheFull); } - seen.insert(id, now); + seen.insert(id, (now, created_at)); Ok(()) } } @@ -151,7 +167,7 @@ mod tests { )) } - fn check(keys: &Keys, values: &[&str]) -> Result { + fn check(keys: &Keys, values: &[&str]) -> Result<(EventId, u64), Reject> { let mut headers = HeaderMap::new(); for v in values { headers.append(AUTHORIZATION, HeaderValue::from_str(v).unwrap()); @@ -244,13 +260,30 @@ mod tests { check(&k, &[&good(&k)]).unwrap(), ); let cache = ReplayCache::new(1); - let now = Instant::now(); - assert_eq!(cache.claim(a, now), Ok(())); - assert_eq!(cache.claim(a, now), Err(Reject::Replayed)); - assert_eq!(cache.claim(b, now), Err(Reject::CacheFull)); + let (now, wall) = (Instant::now(), a.1); + assert_eq!(cache.claim(a, now, wall), Ok(())); + assert_eq!(cache.claim(a, now, wall), Err(Reject::Replayed)); + assert_eq!(cache.claim(b, now, wall), Err(Reject::CacheFull)); // Still held when the relay's own 120 s would have let it go. let relay_ttl = now + Duration::from_secs(DEFAULT_REPLAY_TTL_SECS); - assert_eq!(cache.claim(a, relay_ttl), Err(Reject::Replayed)); - assert_eq!(cache.claim(b, now + REPLAY_TTL), Ok(())); + assert_eq!(cache.claim(a, relay_ttl, wall + 120), Err(Reject::Replayed)); + assert_eq!(cache.claim(b, now + REPLAY_TTL, wall + 122), Ok(())); + } + + #[test] + fn a_wall_clock_step_back_keeps_the_id_while_the_window_accepts_it() { + let k = Keys::generate(); + let a = check(&k, &[&good(&k)]).unwrap(); + let cache = ReplayCache::new(4); + let now = Instant::now(); + assert_eq!(cache.claim(a, now, a.1), Ok(())); + // Monotonic time is past REPLAY_TTL, but the wall clock reads + // created_at + 60 s, which the verifier still accepts. + let later = now + REPLAY_TTL; + assert_eq!( + cache.claim(a, later, a.1 + WINDOW_SECS), + Err(Reject::Replayed) + ); + assert_eq!(cache.claim(a, later, a.1 + WINDOW_SECS + 2), Ok(())); } } diff --git a/crates/buzz-relay-gateway/src/lib.rs b/crates/buzz-relay-gateway/src/lib.rs index ef93a05cde1..09bcf5fb695 100644 --- a/crates/buzz-relay-gateway/src/lib.rs +++ b/crates/buzz-relay-gateway/src/lib.rs @@ -21,6 +21,7 @@ use axum::{ routing::get, Json, Router, }; +use nostr::Timestamp; use tokio::{net::TcpListener, sync::Semaphore}; use tokio_util::sync::CancellationToken; use tracing_subscriber::EnvFilter; @@ -102,7 +103,10 @@ async fn link(State(s): State>, headers: HeaderMap) -> Response { &s.config.link_pubkey, s.limits.max_auth_header, ) - .and_then(|id| s.replay.claim(id, Instant::now())); + .and_then(|bearer| { + s.replay + .claim(bearer, Instant::now(), Timestamp::now().as_secs()) + }); match claimed { Err(reject) => { tracing::warn!(reason = reject.code(), "link bearer refused"); From 948cdde8bad401b19f4ebd671f0456aa1667cf8b Mon Sep 17 00:00:00 2001 From: QuicksilverSlick Date: Tue, 27 Oct 2026 08:27:55 -0500 Subject: [PATCH 06/15] feat(relay-gateway): define the v1 link frames Every frame is one JSON object in a WebSocket text frame, with a type field. Worker to gateway: http, ws_open, ws_send, ws_close, ping, pong. Gateway to Worker: hello, http_res, ws_opened, ws_msg, ws_closed, error, rotate, ping, pong. Bodies travel as standard base64 so they arrive byte for byte, and relay socket text travels as a JSON string, sent as is. - Inbound frames parse through one flat struct, because serde_json's RawValue does not survive internally tagged enums. That keeps the raw headers object, so the policy still sees a duplicated key. - An unknown type maps to Unknown (the link stays open), even when it reuses a known field name with another type. A known type with a missing or mistyped field is BadFrame. - Request and socket ids are 1 to 64 of [A-Za-z0-9_-], so they are safe to log. - The frame types that carry headers, bodies or relay text do not derive Debug, so they cannot reach a log line by accident. - Every error code has a fixed message that never quotes frame content. Signed-off-by: QuicksilverSlick Co-Authored-By: Claude Opus 5.5 --- crates/buzz-relay-gateway/src/frame.rs | 440 +++++++++++++++++++++++++ crates/buzz-relay-gateway/src/lib.rs | 1 + 2 files changed, 441 insertions(+) create mode 100644 crates/buzz-relay-gateway/src/frame.rs diff --git a/crates/buzz-relay-gateway/src/frame.rs b/crates/buzz-relay-gateway/src/frame.rs new file mode 100644 index 00000000000..bdb074f5f75 --- /dev/null +++ b/crates/buzz-relay-gateway/src/frame.rs @@ -0,0 +1,440 @@ +//! Link frames, version 1 (design §3): one JSON object per WebSocket text frame. +//! +//! The frame types that carry headers, bodies or relay text do not derive +//! `Debug`, so none of them can reach a log line by accident. + +use std::collections::BTreeMap; + +use serde::{Deserialize, Serialize}; +use serde_json::{value::RawValue, Number}; + +use crate::policy::Refused; + +/// A Worker-chosen request or socket id: 1 to 64 of `[A-Za-z0-9_-]`, so it is safe to log. +#[derive(Clone, Debug, PartialEq, Eq, Hash, Deserialize, Serialize)] +#[serde(try_from = "String")] +pub struct Id(String); + +impl TryFrom for Id { + type Error = &'static str; + + fn try_from(s: String) -> Result { + let valid = (1..=64).contains(&s.len()) + && s.bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'_' || b == b'-'); + valid.then_some(Self(s)).ok_or("invalid id") + } +} + +impl Id { + /// The id, for logs. + pub fn as_str(&self) -> &str { + &self.0 + } +} + +/// An `http` frame: one request to forward to the relay. +pub struct HttpFrame { + /// Unique among this link's requests in flight. + pub id: Id, + /// `GET` or `POST`, matched exactly by [`crate::policy::http_route`]. + pub method: String, + /// Matched exactly by [`crate::policy::http_route`]. + pub path: String, + /// The raw `headers` object, checked by [`crate::policy::request_headers`]. + pub headers: Option>, + /// The request body, standard base64 (POST only). + pub body_b64: Option, +} + +/// A frame from the Worker. +pub enum Inbound { + /// Forward one HTTP request. + Http(HttpFrame), + /// Open a relay socket on `path`. + WsOpen { sid: Id, path: String }, + /// Send `text` on a relay socket, as is. + WsSend { sid: Id, text: String }, + /// Close a relay socket. + WsClose { + sid: Id, + code: Option, + reason: Option, + }, + /// Liveness; answered with a `pong` carrying the same `ts`. + Ping { ts: Option }, + /// The answer to a gateway `ping`. + Pong, + /// A `type` this version does not know; the link stays open. + Unknown, +} + +/// Not JSON, no `type`, or a known type with a missing or mistyped field. +#[derive(Debug, PartialEq, Eq)] +pub struct BadFrame; + +// Flat on purpose: `RawValue` does not survive serde's internally tagged enums. +#[derive(Deserialize)] +struct Raw { + #[serde(rename = "type")] + kind: String, + id: Option, + sid: Option, + method: Option, + path: Option, + headers: Option>, + body_b64: Option, + text: Option, + code: Option, + reason: Option, + ts: Option, +} + +#[derive(Deserialize)] +struct TypeOnly { + #[serde(rename = "type")] + kind: String, +} + +const KNOWN: [&str; 6] = ["http", "ws_open", "ws_send", "ws_close", "ping", "pong"]; + +impl Inbound { + /// Parses one text frame. + pub fn parse(text: &str) -> Result { + let raw: Raw = match serde_json::from_str(text) { + Ok(raw) => raw, + // A newer frame type may use a known field name with another type. + Err(_) => { + return match serde_json::from_str::(text) { + Ok(t) if !KNOWN.contains(&t.kind.as_str()) => Ok(Self::Unknown), + _ => Err(BadFrame), + } + } + }; + let sid = || raw.sid.clone().ok_or(BadFrame); + Ok(match raw.kind.as_str() { + "http" => Self::Http(HttpFrame { + id: raw.id.ok_or(BadFrame)?, + method: raw.method.ok_or(BadFrame)?, + path: raw.path.ok_or(BadFrame)?, + headers: raw.headers, + body_b64: raw.body_b64, + }), + "ws_open" => Self::WsOpen { + sid: sid()?, + path: raw.path.ok_or(BadFrame)?, + }, + "ws_send" => Self::WsSend { + sid: sid()?, + text: raw.text.ok_or(BadFrame)?, + }, + "ws_close" => Self::WsClose { + sid: sid()?, + code: raw.code, + reason: raw.reason, + }, + "ping" => Self::Ping { ts: raw.ts }, + "pong" => Self::Pong, + _ => Self::Unknown, + }) + } +} + +/// Which side ended a relay socket. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum Side { + /// The relay closed it or the connection dropped. + Relay, + /// The gateway closed it (limits, link end, shutdown, failed open). + Gateway, + /// The Worker asked with `ws_close`. + Worker, +} + +/// The limits announced in `hello`. +#[derive(Clone, Copy, Debug, Serialize)] +pub struct HelloLimits { + pub max_request_body: usize, + pub max_response_body: usize, + pub max_ws_text: usize, + pub max_http_inflight: usize, + pub max_sockets: usize, +} + +/// A frame to the Worker. +#[derive(Serialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum Out { + /// The first frame on every link. + Hello { + v: u8, + link_id: String, + expires_at: u64, + rotate_at: u64, + limits: HelloLimits, + }, + /// The relay's answer, whatever its status. + HttpRes { + id: Id, + status: u16, + headers: BTreeMap<&'static str, String>, + body_b64: String, + ms: u64, + }, + /// The relay accepted the socket upgrade. + WsOpened { sid: Id, ms: u64 }, + /// One text frame from the relay, as is. + WsMsg { sid: Id, text: String }, + /// The end of a socket, sent exactly once per `sid`. + WsClosed { + sid: Id, + by: Side, + code: Option, + reason: String, + #[serde(skip_serializing_if = "Option::is_none")] + http_status: Option, + }, + /// A frame the gateway refused or could not perform. + Error { + #[serde(skip_serializing_if = "Option::is_none")] + id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + sid: Option, + code: ErrorCode, + message: &'static str, + }, + /// The link closes with 4001 at `close_at` (unix seconds). + Rotate { close_at: u64 }, + /// Liveness; the Worker answers `pong`. + Ping { ts: Number }, + /// The answer to a Worker `ping`. + Pong { + #[serde(skip_serializing_if = "Option::is_none")] + ts: Option, + }, +} + +impl Out { + /// An `error` frame with the code's fixed message. + pub fn error(code: ErrorCode, id: Option, sid: Option) -> Self { + Self::Error { + id, + sid, + code, + message: code.message(), + } + } +} + +/// `error.code`. Each code has a fixed message that never quotes frame content. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum ErrorCode { + BadFrame, + UnknownType, + NotAllowed, + NotSupported, + BadBase64, + BodyTooLarge, + FrameTooLarge, + ResponseTooLarge, + DuplicateId, + UnknownSocket, + Busy, + UpstreamConnect, + UpstreamTimeout, + ShuttingDown, +} + +impl ErrorCode { + /// The fixed English phrase for `message`. + pub fn message(self) -> &'static str { + match self { + Self::BadFrame => "malformed frame", + Self::UnknownType => "unknown frame type", + Self::NotAllowed => "not allowed", + Self::NotSupported => "not supported yet", + Self::BadBase64 => "body_b64 is not valid base64", + Self::BodyTooLarge => "request body too large", + Self::FrameTooLarge => "text frame too large", + Self::ResponseTooLarge => "relay response too large", + Self::DuplicateId => "id already in use", + Self::UnknownSocket => "no such socket", + Self::Busy => "too many requests or sockets in flight", + Self::UpstreamConnect => "could not reach the relay", + Self::UpstreamTimeout => "the relay did not answer in time", + Self::ShuttingDown => "gateway shutting down", + } + } +} + +impl From for ErrorCode { + fn from(refused: Refused) -> Self { + match refused { + Refused::NotAllowed => Self::NotAllowed, + Refused::NotSupported => Self::NotSupported, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn id(s: &str) -> Id { + Id::try_from(s.to_owned()).unwrap() + } + + #[test] + fn parses_every_inbound_type() { + let http = r#"{"type":"http","id":"q-1","method":"POST","path":"/query", + "headers":{"accept":"a","accept":"b"},"body_b64":"e30=","extra":1}"#; + let Ok(Inbound::Http(f)) = Inbound::parse(http) else { + panic!("http") + }; + assert_eq!( + (f.id, f.method.as_str(), f.path.as_str()), + (id("q-1"), "POST", "/query") + ); + // Kept raw, so the policy can still see the duplicated key. + assert_eq!(f.headers.unwrap().get(), r#"{"accept":"a","accept":"b"}"#); + assert_eq!(f.body_b64.as_deref(), Some("e30=")); + let open = Inbound::parse(r#"{"type":"ws_open","sid":"s1","path":"/"}"#); + assert!( + matches!(open, Ok(Inbound::WsOpen { sid, path }) if sid == id("s1") && path == "/") + ); + let send = Inbound::parse(r#"{"type":"ws_send","sid":"s1","text":"[\"AUTH\",{}]"}"#); + assert!(matches!(send, Ok(Inbound::WsSend { text, .. }) if text == r#"["AUTH",{}]"#)); + let close = Inbound::parse(r#"{"type":"ws_close","sid":"s1","code":4000,"reason":"bye"}"#); + assert!( + matches!(close, Ok(Inbound::WsClose { code: Some(4000), reason: Some(r), .. }) if r == "bye") + ); + let ping = Inbound::parse(r#"{"type":"ping","ts":1793149200123}"#); + assert!( + matches!(ping, Ok(Inbound::Ping { ts: Some(ts) }) if ts.to_string() == "1793149200123") + ); + assert!(matches!( + Inbound::parse(r#"{"type":"pong","ts":1}"#), + Ok(Inbound::Pong) + )); + assert!(matches!( + Inbound::parse(r#"{"type":"later"}"#), + Ok(Inbound::Unknown) + )); + assert!(matches!( + Inbound::parse(r#"{"type":"later","code":"x"}"#), + Ok(Inbound::Unknown) + )); + } + + #[test] + fn refuses_bad_frames_and_ids() { + for bad in [ + "not json", + "[]", + r#"{"id":"x"}"#, + r#"{"type":"http","id":"q","method":"POST"}"#, + r#"{"type":"ws_send","sid":"s1"}"#, + r#"{"type":"ws_open","sid":"has space","path":"/"}"#, + r#"{"type":"ws_open","sid":"","path":"/"}"#, + r#"{"type":"ping","ts":"soon"}"#, + ] { + assert!(Inbound::parse(bad).is_err(), "{bad}"); + } + let long = format!( + r#"{{"type":"ws_open","sid":"{}","path":"/"}}"#, + "a".repeat(65) + ); + assert_eq!(Inbound::parse(&long).err(), Some(BadFrame)); + assert!(Id::try_from("A-z_09".to_owned()).is_ok()); + assert!(Id::try_from("a".repeat(64)).is_ok()); + } + + #[test] + fn serializes_every_outbound_type() { + let limits = HelloLimits { + max_request_body: 1, + max_response_body: 2, + max_ws_text: 3, + max_http_inflight: 4, + max_sockets: 5, + }; + let headers = BTreeMap::from([("content-type", "application/json".to_owned())]); + let cases = [ + ( + Out::Hello { + v: 1, + link_id: "l".into(), + expires_at: 10, + rotate_at: 9, + limits, + }, + json!({"type":"hello","v":1,"link_id":"l","expires_at":10,"rotate_at":9,"limits": + {"max_request_body":1,"max_response_body":2,"max_ws_text":3,"max_http_inflight":4,"max_sockets":5}}), + ), + ( + Out::HttpRes { + id: id("q"), + status: 429, + headers, + body_b64: "e30=".into(), + ms: 7, + }, + json!({"type":"http_res","id":"q","status":429,"headers":{"content-type":"application/json"},"body_b64":"e30=","ms":7}), + ), + ( + Out::WsOpened { + sid: id("s"), + ms: 3, + }, + json!({"type":"ws_opened","sid":"s","ms":3}), + ), + ( + Out::WsMsg { + sid: id("s"), + text: "[1]".into(), + }, + json!({"type":"ws_msg","sid":"s","text":"[1]"}), + ), + ( + Out::WsClosed { + sid: id("s"), + by: Side::Gateway, + code: None, + reason: "upgrade_refused".into(), + http_status: Some(503), + }, + json!({"type":"ws_closed","sid":"s","by":"gateway","code":null,"reason":"upgrade_refused","http_status":503}), + ), + ( + Out::WsClosed { + sid: id("s"), + by: Side::Relay, + code: Some(1012), + reason: "r".into(), + http_status: None, + }, + json!({"type":"ws_closed","sid":"s","by":"relay","code":1012,"reason":"r"}), + ), + ( + Out::error(ErrorCode::DuplicateId, Some(id("q")), None), + json!({"type":"error","id":"q","code":"duplicate_id","message":"id already in use"}), + ), + ( + Out::error(ErrorCode::UnknownType, None, None), + json!({"type":"error","code":"unknown_type","message":"unknown frame type"}), + ), + ( + Out::Rotate { close_at: 5 }, + json!({"type":"rotate","close_at":5}), + ), + (Out::Ping { ts: 6.into() }, json!({"type":"ping","ts":6})), + (Out::Pong { ts: None }, json!({"type":"pong"})), + ]; + for (frame, want) in cases { + assert_eq!(serde_json::to_value(&frame).unwrap(), want); + } + } +} diff --git a/crates/buzz-relay-gateway/src/lib.rs b/crates/buzz-relay-gateway/src/lib.rs index 09bcf5fb695..b3f3787359c 100644 --- a/crates/buzz-relay-gateway/src/lib.rs +++ b/crates/buzz-relay-gateway/src/lib.rs @@ -6,6 +6,7 @@ pub mod auth; pub mod config; +pub mod frame; pub mod policy; use std::{ From 5fc3a0bfaf3b5b75b8d9f7444418796d182ef8b2 Mon Sep 17 00:00:00 2001 From: QuicksilverSlick Date: Tue, 27 Oct 2026 09:13:51 -0500 Subject: [PATCH 07/15] fix(relay-gateway): parse each frame type with only its own fields Review fix for part 3. Inbound::parse read every frame into one flat struct, so each field name had one type for all frame types. A ping carrying "code":"later" (a ws_close field) failed to parse, got bad_frame, and closed the link with 1008, although the protocol says unknown fields are ignored and new optional fields stay v1. parse() now reads `type` first, then deserializes the text as that type alone: HttpFrame for http (RawValue needs a plain struct), and a serde-tagged Inbound for the rest, with #[serde(other)] for unknown types. A duplicated top-level key is still bad_frame. The unit tests gain foreign-field cases for ping, pong, ws_send and http. Signed-off-by: QuicksilverSlick Co-Authored-By: Claude Opus 5.5 --- crates/buzz-relay-gateway/src/frame.rs | 83 +++++++++----------------- 1 file changed, 27 insertions(+), 56 deletions(-) diff --git a/crates/buzz-relay-gateway/src/frame.rs b/crates/buzz-relay-gateway/src/frame.rs index bdb074f5f75..0486c19ef05 100644 --- a/crates/buzz-relay-gateway/src/frame.rs +++ b/crates/buzz-relay-gateway/src/frame.rs @@ -34,6 +34,7 @@ impl Id { } /// An `http` frame: one request to forward to the relay. +#[derive(Deserialize)] pub struct HttpFrame { /// Unique among this link's requests in flight. pub id: Id, @@ -48,8 +49,12 @@ pub struct HttpFrame { } /// A frame from the Worker. +#[derive(Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] pub enum Inbound { - /// Forward one HTTP request. + /// Forward one HTTP request. Parsed on its own: `RawValue` does not + /// survive serde's internally tagged enums. + #[serde(skip)] Http(HttpFrame), /// Open a relay socket on `path`. WsOpen { sid: Id, path: String }, @@ -66,6 +71,7 @@ pub enum Inbound { /// The answer to a gateway `ping`. Pong, /// A `type` this version does not know; the link stays open. + #[serde(other)] Unknown, } @@ -73,70 +79,23 @@ pub enum Inbound { #[derive(Debug, PartialEq, Eq)] pub struct BadFrame; -// Flat on purpose: `RawValue` does not survive serde's internally tagged enums. -#[derive(Deserialize)] -struct Raw { - #[serde(rename = "type")] - kind: String, - id: Option, - sid: Option, - method: Option, - path: Option, - headers: Option>, - body_b64: Option, - text: Option, - code: Option, - reason: Option, - ts: Option, -} - #[derive(Deserialize)] struct TypeOnly { #[serde(rename = "type")] kind: String, } -const KNOWN: [&str; 6] = ["http", "ws_open", "ws_send", "ws_close", "ping", "pong"]; - impl Inbound { - /// Parses one text frame. + /// Parses one text frame. Each type reads only its own fields, so a field + /// that belongs to another type is ignored like any unknown field. pub fn parse(text: &str) -> Result { - let raw: Raw = match serde_json::from_str(text) { - Ok(raw) => raw, - // A newer frame type may use a known field name with another type. - Err(_) => { - return match serde_json::from_str::(text) { - Ok(t) if !KNOWN.contains(&t.kind.as_str()) => Ok(Self::Unknown), - _ => Err(BadFrame), - } - } + let TypeOnly { kind } = serde_json::from_str(text).map_err(|_| BadFrame)?; + let frame = if kind == "http" { + serde_json::from_str(text).map(Self::Http) + } else { + serde_json::from_str(text) }; - let sid = || raw.sid.clone().ok_or(BadFrame); - Ok(match raw.kind.as_str() { - "http" => Self::Http(HttpFrame { - id: raw.id.ok_or(BadFrame)?, - method: raw.method.ok_or(BadFrame)?, - path: raw.path.ok_or(BadFrame)?, - headers: raw.headers, - body_b64: raw.body_b64, - }), - "ws_open" => Self::WsOpen { - sid: sid()?, - path: raw.path.ok_or(BadFrame)?, - }, - "ws_send" => Self::WsSend { - sid: sid()?, - text: raw.text.ok_or(BadFrame)?, - }, - "ws_close" => Self::WsClose { - sid: sid()?, - code: raw.code, - reason: raw.reason, - }, - "ping" => Self::Ping { ts: raw.ts }, - "pong" => Self::Pong, - _ => Self::Unknown, - }) + frame.map_err(|_| BadFrame) } } @@ -327,6 +286,16 @@ mod tests { Inbound::parse(r#"{"type":"later","code":"x"}"#), Ok(Inbound::Unknown) )); + // Another frame type's field, of another type, is still an unknown field here. + for foreign in [ + r#"{"type":"ping","ts":1,"code":"later"}"#, + r#"{"type":"ping","ts":1,"id":7}"#, + r#"{"type":"pong","sid":[]}"#, + r#"{"type":"ws_send","sid":"s1","text":"x","path":5}"#, + r#"{"type":"http","id":"q","method":"GET","path":"/","ts":"x","text":1}"#, + ] { + assert!(Inbound::parse(foreign).is_ok(), "{foreign}"); + } } #[test] @@ -340,6 +309,8 @@ mod tests { r#"{"type":"ws_open","sid":"has space","path":"/"}"#, r#"{"type":"ws_open","sid":"","path":"/"}"#, r#"{"type":"ping","ts":"soon"}"#, + r#"{"type":5}"#, + r#"{"type":"ws_open","sid":"s1","sid":"s2","path":"/"}"#, ] { assert!(Inbound::parse(bad).is_err(), "{bad}"); } From 9a2eabbd157409b32736e0c2fb09e567ee6a491e Mon Sep 17 00:00:00 2001 From: QuicksilverSlick Date: Tue, 27 Oct 2026 08:29:30 -0500 Subject: [PATCH 08/15] feat(relay-gateway): run each link as a task with bounded output A good bearer now upgrades to the link WebSocket. The link sends hello (version 1, link_id, expiry and the limits), answers ping with pong, and pings the Worker every 20 s. HTTP and relay socket frames answer not_supported until the next parts land. - Lifetime: rotate 5 minutes before the hour, then close 4001, so the Worker's reconnect path runs every hour and one bearer never lasts long. - Idle: no Worker frame for 60 s closes 4002. - Backpressure: frames for the Worker go into a queue bounded at 1,024 frames and 16 MiB of JSON, with try_send only. Nothing waits on it; when it fills, the Worker is not reading and the link closes 4003. - Protocol: a binary frame closes 1003 (reserved for the Phase 2 audio bridge), a malformed frame gets bad_frame then 1008, a frame over 2 MiB closes 1009, and an unknown type gets unknown_type with the link open. - Shutdown: /v1/link and /healthz answer 503, links close 1012, and serve keeps accepting until every tracked link has closed. Signed-off-by: QuicksilverSlick Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 2 + crates/buzz-relay-gateway/Cargo.toml | 2 + crates/buzz-relay-gateway/src/config.rs | 43 +++- crates/buzz-relay-gateway/src/lib.rs | 48 ++-- crates/buzz-relay-gateway/src/link.rs | 214 ++++++++++++++++++ crates/buzz-relay-gateway/tests/common/mod.rs | 102 ++++++++- crates/buzz-relay-gateway/tests/link.rs | 149 +++++++++++- 7 files changed, 537 insertions(+), 23 deletions(-) create mode 100644 crates/buzz-relay-gateway/src/link.rs diff --git a/Cargo.lock b/Cargo.lock index aafb5d50b5d..e2e598ae4d3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1513,6 +1513,7 @@ dependencies = [ "axum", "base64 0.22.1", "buzz-auth", + "futures-util", "nostr 0.44.7", "reqwest 0.13.4", "rustls", @@ -1520,6 +1521,7 @@ dependencies = [ "serde_json", "thiserror 2.0.18", "tokio", + "tokio-tungstenite 0.29.0", "tokio-util", "tracing", "tracing-subscriber", diff --git a/crates/buzz-relay-gateway/Cargo.toml b/crates/buzz-relay-gateway/Cargo.toml index e2a342853c2..1ae3392df7b 100644 --- a/crates/buzz-relay-gateway/Cargo.toml +++ b/crates/buzz-relay-gateway/Cargo.toml @@ -20,6 +20,7 @@ anyhow = { workspace = true } axum = { workspace = true } base64 = { workspace = true } buzz-auth = { workspace = true } +futures-util = { workspace = true } nostr = { workspace = true } # Explicit ring provider, installed in main(): a workspace-wide build unifies # ring and aws-lc-rs, and rustls then panics on the first wss:// connect. @@ -36,3 +37,4 @@ uuid = { workspace = true } [dev-dependencies] reqwest = { workspace = true } +tokio-tungstenite = { workspace = true } diff --git a/crates/buzz-relay-gateway/src/config.rs b/crates/buzz-relay-gateway/src/config.rs index 2e0178b032c..ed8c8c408e0 100644 --- a/crates/buzz-relay-gateway/src/config.rs +++ b/crates/buzz-relay-gateway/src/config.rs @@ -1,6 +1,6 @@ //! Environment configuration (all public values) and the fixed limits. -use std::net::SocketAddr; +use std::{net::SocketAddr, time::Duration}; use nostr::PublicKey; use url::Url; @@ -90,7 +90,7 @@ fn relay_urls(raw: &str) -> Option<(Url, Url)> { bare.then_some((ws, http)) } -/// Fixed limits (design §5). Not configurable by env; tests pass smaller values. +/// Fixed limits and timeouts (design §5). Not configurable by env; tests pass smaller values. #[derive(Clone, Debug)] pub struct Limits { /// Links open at once; the next upgrade gets 503. @@ -99,6 +99,32 @@ pub struct Limits { pub max_auth_header: usize, /// Bearer ids remembered for replay protection; when full, new bearers get 503. pub replay_entries: usize, + /// Largest Worker frame; a bigger one closes the link with 1009. + pub max_frame: usize, + /// Largest decoded request body (the relay's own API limit). + pub max_request_body: usize, + /// Largest relay response body; the read stops there. + pub max_response_body: usize, + /// Largest `ws_send` text (the relay's frame limit). + pub max_ws_text: usize, + /// HTTP requests in flight per link. + pub max_http_inflight: usize, + /// Relay sockets per link. + pub max_sockets: usize, + /// Frames queued for the Worker; a full queue closes the link with 4003. + pub out_frames: usize, + /// Serialized bytes queued for the Worker, with the same rule. + pub out_bytes: usize, + /// How long a link lives before it closes with 4001. + pub link_lifetime: Duration, + /// How long before that the gateway sends `rotate`. + pub rotate_before: Duration, + /// Gateway `ping` interval. + pub ping_every: Duration, + /// A link with no Worker frame for this long closes with 4002. + pub idle: Duration, + /// How long shutdown waits for in-flight HTTP requests. + pub shutdown_grace: Duration, } impl Default for Limits { @@ -107,6 +133,19 @@ impl Default for Limits { max_links: 4, max_auth_header: 8 << 10, replay_entries: 1024, + max_frame: 2 << 20, + max_request_body: 1 << 20, + max_response_body: 4 << 20, + max_ws_text: 512 << 10, + max_http_inflight: 16, + max_sockets: 8, + out_frames: 1024, + out_bytes: 16 << 20, + link_lifetime: Duration::from_secs(3600), + rotate_before: Duration::from_secs(300), + ping_every: Duration::from_secs(20), + idle: Duration::from_secs(60), + shutdown_grace: Duration::from_secs(10), } } } diff --git a/crates/buzz-relay-gateway/src/lib.rs b/crates/buzz-relay-gateway/src/lib.rs index b3f3787359c..a26d5bfd3f1 100644 --- a/crates/buzz-relay-gateway/src/lib.rs +++ b/crates/buzz-relay-gateway/src/lib.rs @@ -7,6 +7,7 @@ pub mod auth; pub mod config; pub mod frame; +mod link; pub mod policy; use std::{ @@ -16,7 +17,7 @@ use std::{ }; use axum::{ - extract::State, + extract::{ws::rejection::WebSocketUpgradeRejection, State, WebSocketUpgrade}, http::{HeaderMap, StatusCode}, response::{IntoResponse, Response}, routing::get, @@ -24,7 +25,7 @@ use axum::{ }; use nostr::Timestamp; use tokio::{net::TcpListener, sync::Semaphore}; -use tokio_util::sync::CancellationToken; +use tokio_util::{sync::CancellationToken, task::TaskTracker}; use tracing_subscriber::EnvFilter; pub use config::{Config, Limits}; @@ -39,9 +40,11 @@ struct AppState { replay: auth::ReplayCache, links: Arc, shutdown: CancellationToken, + tracker: TaskTracker, } -/// Serves `/healthz` and `/v1/link` on `listener` until `shutdown` is cancelled. +/// Serves `/healthz` and `/v1/link` on `listener`. Once `shutdown` is +/// cancelled both answer 503, every link drains and closes, and this returns. pub async fn serve( listener: TcpListener, config: Config, @@ -52,19 +55,27 @@ pub async fn serve( replay: auth::ReplayCache::new(limits.replay_entries), links: Arc::new(Semaphore::new(limits.max_links)), shutdown: shutdown.clone(), + tracker: TaskTracker::new(), config, limits, }); let app = Router::new() .route("/healthz", get(healthz)) .route("/v1/link", get(link)) - .with_state(state); - let server = - axum::serve(listener, app).with_graceful_shutdown(shutdown.clone().cancelled_owned()); + .with_state(state.clone()); + // Keep accepting (and answering 503) until every link has closed. + let tracker = state.tracker.clone(); + let drained = async move { + shutdown.cancelled().await; + state.tracker.close(); + state.tracker.wait().await; + }; + let server = axum::serve(listener, app).with_graceful_shutdown(drained); tokio::select! { served = server.into_future() => served, + // wait() returns only after `drained` has closed the tracker and every link has ended. () = async { - shutdown.cancelled().await; + tracker.wait().await; tokio::time::sleep(LAST_REQUESTS).await; } => Ok(()), } @@ -91,11 +102,15 @@ async fn healthz(State(s): State>) -> Response { } } -async fn link(State(s): State>, headers: HeaderMap) -> Response { +async fn link( + State(s): State>, + headers: HeaderMap, + upgrade: Result, +) -> Response { if s.shutdown.is_cancelled() { return StatusCode::SERVICE_UNAVAILABLE.into_response(); } - let Ok(_permit) = s.links.clone().try_acquire_owned() else { + let Ok(permit) = s.links.clone().try_acquire_owned() else { return StatusCode::SERVICE_UNAVAILABLE.into_response(); }; let claimed = auth::verify_link_bearer( @@ -108,8 +123,8 @@ async fn link(State(s): State>, headers: HeaderMap) -> Response { s.replay .claim(bearer, Instant::now(), Timestamp::now().as_secs()) }); - match claimed { - Err(reject) => { + match (claimed, upgrade) { + (Err(reject), _) => { tracing::warn!(reason = reject.code(), "link bearer refused"); if reject == auth::Reject::CacheFull { return StatusCode::SERVICE_UNAVAILABLE.into_response(); @@ -117,7 +132,14 @@ async fn link(State(s): State>, headers: HeaderMap) -> Response { let body = Json(serde_json::json!({ "error": "unauthorized" })); (StatusCode::UNAUTHORIZED, body).into_response() } - // The link's frame loop lands in a later change. - Ok(()) => StatusCode::NOT_IMPLEMENTED.into_response(), + (Ok(()), Err(rejection)) => rejection.into_response(), + (Ok(()), Ok(upgrade)) => { + let max = s.limits.max_frame; + let tracked = s.tracker.clone(); + upgrade + .max_message_size(max) + .max_frame_size(max) + .on_upgrade(move |socket| tracked.track_future(link::run(socket, s, permit))) + } } } diff --git a/crates/buzz-relay-gateway/src/link.rs b/crates/buzz-relay-gateway/src/link.rs new file mode 100644 index 00000000000..a1d58ea860d --- /dev/null +++ b/crates/buzz-relay-gateway/src/link.rs @@ -0,0 +1,214 @@ +//! One task per link: `hello`, frame dispatch, pings, rotation, idle and +//! backpressure, and the link's close. + +use std::{ + sync::Arc, + time::{Duration, SystemTime, UNIX_EPOCH}, +}; + +use axum::extract::ws::{CloseFrame, Message, WebSocket}; +use futures_util::{stream::SplitSink, SinkExt, StreamExt}; +use tokio::{ + sync::{mpsc, mpsc::error::TrySendError, OwnedSemaphorePermit, Semaphore}, + time::{interval_at, sleep_until, timeout, Instant}, +}; +use tokio_util::sync::CancellationToken; + +use crate::{ + frame::{ErrorCode, HelloLimits, Id, Inbound, Out}, + AppState, +}; + +// Teardown waits this long for the Worker to take the close frame. +const CLOSE_WAIT: Duration = Duration::from_secs(2); +const SHUTDOWN: (u16, &str) = (1012, "gateway restarting"); + +type Queued = (Message, Option); + +/// Milliseconds since `since`, for logs and frames. +pub(crate) fn ms(since: Instant) -> u64 { + u64::try_from(since.elapsed().as_millis()).unwrap_or(u64::MAX) +} + +fn unix_now() -> Duration { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() +} + +/// The link's bounded outbound queue. Nothing waits on it: when it is full the +/// Worker is not reading, and the link closes with 4003. +#[derive(Clone)] +pub(crate) struct Outbox { + tx: mpsc::Sender, + bytes: Arc, + overflow: CancellationToken, +} + +impl Outbox { + pub(crate) fn send(&self, frame: &Out) { + let Ok(text) = serde_json::to_string(frame) else { + return; + }; + let permit = u32::try_from(text.len()) + .ok() + .and_then(|n| self.bytes.clone().try_acquire_many_owned(n).ok()); + let full = match permit { + Some(permit) => matches!( + self.tx.try_send((Message::Text(text.into()), Some(permit))), + Err(TrySendError::Full(_)) + ), + None => true, + }; + if full { + self.overflow.cancel(); + } + } +} + +struct Link { + state: Arc, + id: String, + out: Outbox, +} + +/// Runs one link until it closes. `_permit` holds its slot under the link cap. +pub(crate) async fn run(socket: WebSocket, state: Arc, _permit: OwnedSemaphorePermit) { + let lim = &state.limits; + let (sink, mut stream) = socket.split(); + let (tx, rx) = mpsc::channel(lim.out_frames); + let out = Outbox { + tx, + bytes: Arc::new(Semaphore::new(lim.out_bytes)), + overflow: CancellationToken::new(), + }; + let mut writer = tokio::spawn(write(sink, rx)); + let now = unix_now().as_secs(); + let rotate_in = lim.link_lifetime.saturating_sub(lim.rotate_before); + let id = uuid::Uuid::new_v4().to_string(); + out.send(&Out::Hello { + v: 1, + link_id: id.clone(), + expires_at: now + lim.link_lifetime.as_secs(), + rotate_at: now + rotate_in.as_secs(), + limits: HelloLimits { + max_request_body: lim.max_request_body, + max_response_body: lim.max_response_body, + max_ws_text: lim.max_ws_text, + max_http_inflight: lim.max_http_inflight, + max_sockets: lim.max_sockets, + }, + }); + tracing::info!(link_id = id, "link opened"); + let opened = Instant::now(); + let mut link = Link { + state: state.clone(), + id, + out, + }; + let (code, reason) = link + .serve(&mut stream, opened + lim.link_lifetime, opened + rotate_in) + .await; + tracing::info!( + link_id = link.id, + code, + reason, + ms = ms(opened), + "link closing" + ); + let close = Message::Close(Some(CloseFrame { + code, + reason: reason.into(), + })); + let _ = timeout(CLOSE_WAIT, link.out.tx.send((close, None))).await; + if timeout(CLOSE_WAIT, &mut writer).await.is_err() { + writer.abort(); + } +} + +async fn write(mut sink: SplitSink, mut rx: mpsc::Receiver) { + while let Some((msg, _bytes)) = rx.recv().await { + let close = matches!(msg, Message::Close(_)); + if sink.send(msg).await.is_err() || close { + break; + } + } +} + +impl Link { + async fn serve( + &mut self, + stream: &mut futures_util::stream::SplitStream, + expires: Instant, + rotate: Instant, + ) -> (u16, &'static str) { + let lim = self.state.limits.clone(); + let mut ping = interval_at(Instant::now() + lim.ping_every, lim.ping_every); + let mut idle = Instant::now() + lim.idle; + let mut rotated = false; + loop { + tokio::select! { + msg = stream.next() => { + idle = Instant::now() + lim.idle; + match msg { + Some(Ok(Message::Text(text))) => { + if let Err(close) = self.on_text(text.as_str()) { + return close; + } + } + Some(Ok(Message::Binary(_))) => return (1003, "binary frames are reserved"), + Some(Ok(Message::Close(_))) | None => return (1000, "closed by worker"), + Some(Ok(_)) => {} + // axum's tungstenite is not ours to downcast; its capacity error reads this way. + Some(Err(e)) if e.to_string().contains("Space limit exceeded") => return (1009, "frame too large"), + Some(Err(_)) => return (1002, "protocol error"), + } + } + _ = ping.tick() => { + let ts = u64::try_from(unix_now().as_millis()).unwrap_or(u64::MAX); + self.out.send(&Out::Ping { ts: ts.into() }); + } + _ = sleep_until(rotate), if !rotated => { + rotated = true; + let close_at = (unix_now() + expires.saturating_duration_since(Instant::now())).as_secs(); + self.out.send(&Out::Rotate { close_at }); + } + _ = sleep_until(expires) => return (4001, "rotate"), + _ = sleep_until(idle) => return (4002, "idle"), + _ = self.out.overflow.cancelled() => return (4003, "backpressure"), + _ = self.state.shutdown.cancelled() => return SHUTDOWN, + } + } + } + + /// Handles one Worker frame. `Err` closes the link with that code. + fn on_text(&mut self, text: &str) -> Result<(), (u16, &'static str)> { + let Ok(frame) = Inbound::parse(text) else { + self.refuse(ErrorCode::BadFrame, None, None); + return Err((1008, "bad frame")); + }; + match frame { + // HTTP forwarding and relay sockets land in later changes. + Inbound::Http(f) => self.refuse(ErrorCode::NotSupported, Some(f.id), None), + Inbound::WsOpen { sid, .. } => self.refuse(ErrorCode::NotSupported, None, Some(sid)), + Inbound::WsSend { sid, .. } | Inbound::WsClose { sid, .. } => { + self.refuse(ErrorCode::UnknownSocket, None, Some(sid)) + } + Inbound::Ping { ts } => self.out.send(&Out::Pong { ts }), + Inbound::Pong => {} + Inbound::Unknown => self.refuse(ErrorCode::UnknownType, None, None), + } + Ok(()) + } + + fn refuse(&self, code: ErrorCode, id: Option, sid: Option) { + tracing::info!( + link_id = self.id, + id = id.as_ref().map(Id::as_str), + sid = sid.as_ref().map(Id::as_str), + code = ?code, + "frame refused" + ); + self.out.send(&Out::error(code, id, sid)); + } +} diff --git a/crates/buzz-relay-gateway/tests/common/mod.rs b/crates/buzz-relay-gateway/tests/common/mod.rs index 8a3dd608a30..9b497fcf88e 100644 --- a/crates/buzz-relay-gateway/tests/common/mod.rs +++ b/crates/buzz-relay-gateway/tests/common/mod.rs @@ -1,12 +1,19 @@ -//! Shared harness: the gateway in-process and a bearer signer. +//! Shared harness: the gateway in-process, a mock Worker and a bearer signer. #![allow(dead_code)] -use std::net::SocketAddr; +use std::{net::SocketAddr, time::Duration}; use base64::{engine::general_purpose::STANDARD, Engine as _}; use buzz_relay_gateway::{Config, Limits}; +use futures_util::{SinkExt, StreamExt}; use nostr::{EventBuilder, Keys, Kind, Tag}; -use tokio::{net::TcpListener, task::JoinHandle}; +use serde_json::Value; +use tokio::{net::TcpListener, task::JoinHandle, time::timeout}; +use tokio_tungstenite::{ + connect_async, + tungstenite::{client::IntoClientRequest, Message}, + MaybeTlsStream, WebSocketStream, +}; use tokio_util::sync::CancellationToken; pub const LINK_URL: &str = "http://gateway.test/v1/link"; @@ -34,6 +41,14 @@ pub fn bearer(keys: &Keys) -> String { bearer_with(keys, "") } +/// Short timers for tests; every other limit stays at its default. +pub fn limits() -> Limits { + Limits { + shutdown_grace: Duration::from_millis(500), + ..Limits::default() + } +} + pub struct Gateway { pub addr: SocketAddr, pub keys: Keys, @@ -72,4 +87,85 @@ impl Gateway { pub fn url(&self, path: &str) -> String { format!("http://{}{path}", self.addr) } + + /// A Worker on a fresh link, past `hello`. + pub async fn worker(&self) -> Worker { + Worker::connect(self, &bearer(&self.keys)).await.unwrap() + } +} + +/// The mock Worker: one link, frames as JSON values. +pub struct Worker { + pub ws: WebSocketStream>, + pub hello: Value, +} + +pub enum Next { + Frame(Value), + Closed(Option), +} + +impl Worker { + pub async fn connect( + gw: &Gateway, + bearer: &str, + ) -> Result { + let mut request = format!("ws://{}/v1/link", gw.addr).into_client_request()?; + request + .headers_mut() + .insert("authorization", bearer.parse().unwrap()); + let (ws, _) = connect_async(request).await?; + let mut worker = Self { + ws, + hello: Value::Null, + }; + worker.hello = worker.frame().await; + assert_eq!(worker.hello["type"], "hello"); + Ok(worker) + } + + pub async fn send(&mut self, frame: Value) { + self.ws + .send(Message::text(frame.to_string())) + .await + .unwrap(); + } + + /// The next frame other than a gateway `ping`. + pub async fn next(&mut self) -> Next { + loop { + let msg = timeout(Duration::from_secs(10), self.ws.next()) + .await + .expect("no frame within 10 s"); + match msg { + Some(Ok(Message::Text(text))) => { + let frame: Value = serde_json::from_str(&text).unwrap(); + if frame["type"] != "ping" { + return Next::Frame(frame); + } + } + Some(Ok(Message::Close(frame))) => { + return Next::Closed(frame.map(|f| f.code.into())) + } + Some(Ok(_)) => {} + Some(Err(_)) | None => return Next::Closed(None), + } + } + } + + pub async fn frame(&mut self) -> Value { + match self.next().await { + Next::Frame(frame) => frame, + Next::Closed(code) => panic!("link closed: {code:?}"), + } + } + + /// Reads until the link closes; returns the close code. + pub async fn closed(&mut self) -> Option { + loop { + if let Next::Closed(code) = self.next().await { + return code; + } + } + } } diff --git a/crates/buzz-relay-gateway/tests/link.rs b/crates/buzz-relay-gateway/tests/link.rs index bf1bab88483..4b5a699bc12 100644 --- a/crates/buzz-relay-gateway/tests/link.rs +++ b/crates/buzz-relay-gateway/tests/link.rs @@ -1,16 +1,24 @@ mod common; +use std::time::Duration; + use buzz_relay_gateway::Limits; -use common::{bearer, Gateway}; +use common::{bearer, limits, Gateway, Worker}; +use futures_util::SinkExt; use nostr::Keys; +use serde_json::json; +use tokio_tungstenite::tungstenite::{Error, Message}; + +// Nothing listens on port 9: link control never reaches the relay. +const NO_RELAY: &str = "ws://127.0.0.1:9"; #[tokio::test] async fn bearer_gate_is_single_use_and_fails_closed() { let gw = Gateway::start( - "ws://127.0.0.1:9", + NO_RELAY, Limits { replay_entries: 1, - ..Limits::default() + ..limits() }, ) .await; @@ -27,9 +35,10 @@ async fn bearer_gate_is_single_use_and_fails_closed() { assert_eq!(none.status(), 401); assert_eq!(none.text().await.unwrap(), r#"{"error":"unauthorized"}"#); assert_eq!(link(Some(&bearer(&Keys::generate()))).await.status(), 401); - // Refused bearers used no replay slot, so the one-slot cache still admits this one. + // Refused bearers used no replay slot, so the one-slot cache still admits this one; + // it then fails only the upgrade, because this is a plain GET. let good = bearer(&gw.keys); - assert_eq!(link(Some(&good)).await.status(), 501); + assert_eq!(link(Some(&good)).await.status(), 400); assert_eq!(link(Some(&good)).await.status(), 401); assert_eq!(link(Some(&bearer(&gw.keys))).await.status(), 503); @@ -44,3 +53,133 @@ async fn bearer_gate_is_single_use_and_fails_closed() { gw.shutdown.cancel(); gw.task.await.unwrap().unwrap(); } + +#[tokio::test] +async fn bad_bearers_get_no_upgrade() { + let gw = Gateway::start(NO_RELAY, limits()).await; + for bearer in [bearer(&Keys::generate()), "Nostr x".to_owned()] { + let Err(Error::Http(response)) = Worker::connect(&gw, &bearer).await else { + panic!("upgraded with a bad bearer"); + }; + assert_eq!(response.status(), 401); + } +} + +#[tokio::test] +async fn hello_then_ping_pong() { + let gw = Gateway::start(NO_RELAY, limits()).await; + let mut w = gw.worker().await; + let hello = &w.hello; + assert_eq!(hello["v"], 1); + assert_eq!( + hello["expires_at"].as_u64().unwrap() - hello["rotate_at"].as_u64().unwrap(), + 300 + ); + assert_eq!( + hello["limits"], + json!({"max_request_body":1048576,"max_response_body":4194304,"max_ws_text":524288, + "max_http_inflight":16,"max_sockets":8}) + ); + w.send(json!({"type":"ping","ts":1793149200123u64})).await; + assert_eq!( + w.frame().await, + json!({"type":"pong","ts":1793149200123u64}) + ); + w.send(json!({"type":"later","x":1})).await; + assert_eq!(w.frame().await["code"], "unknown_type"); + w.send(json!({"type":"ping"})).await; + assert_eq!(w.frame().await, json!({"type":"pong"})); +} + +#[tokio::test] +async fn gateway_pings_and_silence_closes_4002() { + let quick = Limits { + ping_every: Duration::from_millis(100), + idle: Duration::from_millis(600), + ..limits() + }; + let gw = Gateway::start(NO_RELAY, quick).await; + let mut w = gw.worker().await; + let ping = tokio::time::timeout(Duration::from_secs(5), async { + loop { + if let Some(Ok(Message::Text(text))) = futures_util::StreamExt::next(&mut w.ws).await { + if text.contains(r#""type":"ping""#) { + return; + } + } + } + }); + ping.await.unwrap(); + assert_eq!(w.closed().await, Some(4002)); +} + +#[tokio::test] +async fn rotate_then_close_4001() { + let short = Limits { + link_lifetime: Duration::from_secs(2), + rotate_before: Duration::from_secs(1), + ..limits() + }; + let gw = Gateway::start(NO_RELAY, short).await; + let mut w = gw.worker().await; + let rotate = w.frame().await; + assert_eq!(rotate["type"], "rotate"); + assert_eq!(rotate["close_at"], w.hello["expires_at"]); + assert_eq!(w.closed().await, Some(4001)); +} + +#[tokio::test] +async fn fifth_link_gets_503() { + let gw = Gateway::start(NO_RELAY, limits()).await; + let mut links = Vec::new(); + for _ in 0..4 { + links.push(gw.worker().await); + } + let Err(Error::Http(response)) = Worker::connect(&gw, &bearer(&gw.keys)).await else { + panic!("a fifth link opened"); + }; + assert_eq!(response.status(), 503); + links.pop().unwrap().ws.close(None).await.unwrap(); + // The freed slot is usable again once that link has torn down. + tokio::time::sleep(Duration::from_millis(300)).await; + gw.worker().await; +} + +#[tokio::test] +async fn protocol_violations_close_the_link() { + let gw = Gateway::start( + NO_RELAY, + Limits { + max_frame: 1024, + ..limits() + }, + ) + .await; + let mut w = gw.worker().await; + w.ws.send(Message::Binary(vec![1, 2, 3].into())) + .await + .unwrap(); + assert_eq!(w.closed().await, Some(1003)); + + let mut w = gw.worker().await; + w.ws.send(Message::text("not json")).await.unwrap(); + assert_eq!(w.frame().await["code"], "bad_frame"); + assert_eq!(w.closed().await, Some(1008)); + + let mut w = gw.worker().await; + w.ws.send(Message::text("x".repeat(2048))).await.unwrap(); + assert_eq!(w.closed().await, Some(1009)); +} + +#[tokio::test] +async fn shutdown_closes_links_1012_and_serve_returns() { + let gw = Gateway::start(NO_RELAY, limits()).await; + let mut w = gw.worker().await; + gw.shutdown.cancel(); + assert_eq!(w.closed().await, Some(1012)); + tokio::time::timeout(Duration::from_secs(5), gw.task) + .await + .unwrap() + .unwrap() + .unwrap(); +} From 723226e8c086f9923e82dabe09fbd7407359c9a1 Mon Sep 17 00:00:00 2001 From: QuicksilverSlick Date: Tue, 27 Oct 2026 08:32:02 -0500 Subject: [PATCH 09/15] feat(relay-gateway): forward http frames to the relay byte for byte An http frame now becomes exactly one request to the pinned relay origin, on a path taken from the route table, never from frame text. The decoded body_b64 bytes go out as the body and the response bytes come back base64-encoded as they arrived, so a NIP-98 payload tag the Worker signed still verifies at the relay. - The client never follows redirects (a 3xx comes back as a status, its location dropped), ignores proxy env, never decompresses, and is https-only for a wss relay. It sets only user-agent; nothing can add X-Forwarded-*, X-Pubkey, cookies, Origin or Accept-Encoding, or move Host. - No retries, ever: a retry would burn the NIP-98 event id. A 429 and its body pass through untouched. - Limits: 16 requests in flight per link (busy beyond), a 1 MiB request body, a 4 MiB response read in chunks, 10 s to connect and 30 s per request (upstream_connect, upstream_timeout). - Shutdown gives in-flight requests up to 10 s while new ones get shutting_down, then closes the link 1012. Signed-off-by: QuicksilverSlick Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 2 + crates/buzz-relay-gateway/Cargo.toml | 4 +- crates/buzz-relay-gateway/src/config.rs | 6 + crates/buzz-relay-gateway/src/http_forward.rs | 71 ++++++ crates/buzz-relay-gateway/src/lib.rs | 3 + crates/buzz-relay-gateway/src/link.rs | 104 +++++++- crates/buzz-relay-gateway/tests/common/mod.rs | 182 +++++++++++++- crates/buzz-relay-gateway/tests/http.rs | 225 ++++++++++++++++++ 8 files changed, 588 insertions(+), 9 deletions(-) create mode 100644 crates/buzz-relay-gateway/src/http_forward.rs create mode 100644 crates/buzz-relay-gateway/tests/http.rs diff --git a/Cargo.lock b/Cargo.lock index e2e598ae4d3..648b5a84012 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1514,11 +1514,13 @@ dependencies = [ "base64 0.22.1", "buzz-auth", "futures-util", + "hex", "nostr 0.44.7", "reqwest 0.13.4", "rustls", "serde", "serde_json", + "sha2 0.11.0", "thiserror 2.0.18", "tokio", "tokio-tungstenite 0.29.0", diff --git a/crates/buzz-relay-gateway/Cargo.toml b/crates/buzz-relay-gateway/Cargo.toml index 1ae3392df7b..1503f35d8e6 100644 --- a/crates/buzz-relay-gateway/Cargo.toml +++ b/crates/buzz-relay-gateway/Cargo.toml @@ -22,6 +22,7 @@ base64 = { workspace = true } buzz-auth = { workspace = true } futures-util = { workspace = true } nostr = { workspace = true } +reqwest = { workspace = true } # Explicit ring provider, installed in main(): a workspace-wide build unifies # ring and aws-lc-rs, and rustls then panics on the first wss:// connect. rustls = { version = "0.23", default-features = false, features = ["ring", "std"] } @@ -36,5 +37,6 @@ url = { workspace = true } uuid = { workspace = true } [dev-dependencies] -reqwest = { workspace = true } +hex = { workspace = true } +sha2 = { workspace = true } tokio-tungstenite = { workspace = true } diff --git a/crates/buzz-relay-gateway/src/config.rs b/crates/buzz-relay-gateway/src/config.rs index ed8c8c408e0..d56ecbecd14 100644 --- a/crates/buzz-relay-gateway/src/config.rs +++ b/crates/buzz-relay-gateway/src/config.rs @@ -123,6 +123,10 @@ pub struct Limits { pub ping_every: Duration, /// A link with no Worker frame for this long closes with 4002. pub idle: Duration, + /// Reaching the relay: TCP and TLS. + pub connect_timeout: Duration, + /// One whole HTTP exchange with the relay. + pub request_timeout: Duration, /// How long shutdown waits for in-flight HTTP requests. pub shutdown_grace: Duration, } @@ -145,6 +149,8 @@ impl Default for Limits { rotate_before: Duration::from_secs(300), ping_every: Duration::from_secs(20), idle: Duration::from_secs(60), + connect_timeout: Duration::from_secs(10), + request_timeout: Duration::from_secs(30), shutdown_grace: Duration::from_secs(10), } } diff --git a/crates/buzz-relay-gateway/src/http_forward.rs b/crates/buzz-relay-gateway/src/http_forward.rs new file mode 100644 index 00000000000..db708964dd1 --- /dev/null +++ b/crates/buzz-relay-gateway/src/http_forward.rs @@ -0,0 +1,71 @@ +//! HTTP forwarding: one upstream request per `http` frame, never retried. + +use std::collections::BTreeMap; + +use axum::http::{HeaderMap, Method}; +use reqwest::{redirect::Policy, Client}; +use url::Url; + +use crate::{frame::ErrorCode, policy, Config, Limits}; + +/// The relay's answer, read up to the response cap. +pub(crate) struct Answer { + pub status: u16, + pub headers: BTreeMap<&'static str, String>, + pub body: Vec, +} + +/// The one client for the relay: no redirects, no proxy, no decompression. +pub(crate) fn client(config: &Config, limits: &Limits) -> reqwest::Result { + // The no_* calls also hold if workspace feature unification turns a codec on. + Client::builder() + .redirect(Policy::none()) + .no_proxy() + .no_gzip() + .no_brotli() + .no_deflate() + .no_zstd() + .https_only(config.relay_http.scheme() == "https") + .connect_timeout(limits.connect_timeout) + .timeout(limits.request_timeout) + .user_agent(concat!("buzz-relay-gateway/", env!("CARGO_PKG_VERSION"))) + .build() +} + +/// Sends one request and reads the whole answer, whatever its status. +pub(crate) async fn forward( + client: &Client, + method: Method, + url: Url, + headers: HeaderMap, + body: Option>, + max_body: usize, +) -> Result { + let mut request = client.request(method, url).headers(headers); + if let Some(body) = body { + request = request.body(body); + } + let mut response = request.send().await.map_err(upstream)?; + let status = response.status().as_u16(); + let headers = policy::response_headers(response.headers()); + let mut body = Vec::new(); + while let Some(chunk) = response.chunk().await.map_err(upstream)? { + if body.len() + chunk.len() > max_body { + return Err(ErrorCode::ResponseTooLarge); + } + body.extend_from_slice(&chunk); + } + Ok(Answer { + status, + headers, + body, + }) +} + +fn upstream(e: reqwest::Error) -> ErrorCode { + if e.is_timeout() { + ErrorCode::UpstreamTimeout + } else { + ErrorCode::UpstreamConnect + } +} diff --git a/crates/buzz-relay-gateway/src/lib.rs b/crates/buzz-relay-gateway/src/lib.rs index a26d5bfd3f1..0188703ff66 100644 --- a/crates/buzz-relay-gateway/src/lib.rs +++ b/crates/buzz-relay-gateway/src/lib.rs @@ -7,6 +7,7 @@ pub mod auth; pub mod config; pub mod frame; +mod http_forward; mod link; pub mod policy; @@ -41,6 +42,7 @@ struct AppState { links: Arc, shutdown: CancellationToken, tracker: TaskTracker, + client: reqwest::Client, } /// Serves `/healthz` and `/v1/link` on `listener`. Once `shutdown` is @@ -52,6 +54,7 @@ pub async fn serve( shutdown: CancellationToken, ) -> std::io::Result<()> { let state = Arc::new(AppState { + client: http_forward::client(&config, &limits).map_err(std::io::Error::other)?, replay: auth::ReplayCache::new(limits.replay_entries), links: Arc::new(Semaphore::new(limits.max_links)), shutdown: shutdown.clone(), diff --git a/crates/buzz-relay-gateway/src/link.rs b/crates/buzz-relay-gateway/src/link.rs index a1d58ea860d..4e2f85f04ac 100644 --- a/crates/buzz-relay-gateway/src/link.rs +++ b/crates/buzz-relay-gateway/src/link.rs @@ -2,21 +2,24 @@ //! backpressure, and the link's close. use std::{ + collections::HashMap, sync::Arc, time::{Duration, SystemTime, UNIX_EPOCH}, }; use axum::extract::ws::{CloseFrame, Message, WebSocket}; +use base64::{engine::general_purpose::STANDARD, Engine as _}; use futures_util::{stream::SplitSink, SinkExt, StreamExt}; use tokio::{ sync::{mpsc, mpsc::error::TrySendError, OwnedSemaphorePermit, Semaphore}, + task::{AbortHandle, JoinSet}, time::{interval_at, sleep_until, timeout, Instant}, }; use tokio_util::sync::CancellationToken; use crate::{ - frame::{ErrorCode, HelloLimits, Id, Inbound, Out}, - AppState, + frame::{ErrorCode, HelloLimits, HttpFrame, Id, Inbound, Out}, + http_forward, policy, AppState, }; // Teardown waits this long for the Worker to take the close frame. @@ -70,6 +73,9 @@ struct Link { state: Arc, id: String, out: Outbox, + draining: bool, + http: JoinSet<()>, + inflight: HashMap, } /// Runs one link until it closes. `_permit` holds its slot under the link cap. @@ -105,6 +111,9 @@ pub(crate) async fn run(socket: WebSocket, state: Arc, _permit: OwnedS state: state.clone(), id, out, + draining: false, + http: JoinSet::new(), + inflight: HashMap::new(), }; let (code, reason) = link .serve(&mut stream, opened + lim.link_lifetime, opened + rotate_in) @@ -116,6 +125,7 @@ pub(crate) async fn run(socket: WebSocket, state: Arc, _permit: OwnedS ms = ms(opened), "link closing" ); + link.http.abort_all(); let close = Message::Close(Some(CloseFrame { code, reason: reason.into(), @@ -146,6 +156,7 @@ impl Link { let mut ping = interval_at(Instant::now() + lim.ping_every, lim.ping_every); let mut idle = Instant::now() + lim.idle; let mut rotated = false; + let mut drain_until = None; loop { tokio::select! { msg = stream.next() => { @@ -176,7 +187,20 @@ impl Link { _ = sleep_until(expires) => return (4001, "rotate"), _ = sleep_until(idle) => return (4002, "idle"), _ = self.out.overflow.cancelled() => return (4003, "backpressure"), - _ = self.state.shutdown.cancelled() => return SHUTDOWN, + _ = self.state.shutdown.cancelled(), if !self.draining => { + self.draining = true; + drain_until = Some(Instant::now() + lim.shutdown_grace); + if self.http.is_empty() { + return SHUTDOWN; + } + } + _ = sleep_until(drain_until.unwrap_or(expires)), if drain_until.is_some() => return SHUTDOWN, + Some(_) = self.http.join_next(), if !self.http.is_empty() => { + self.inflight.retain(|_, task| !task.is_finished()); + if self.draining && self.http.is_empty() { + return SHUTDOWN; + } + } } } } @@ -188,8 +212,13 @@ impl Link { return Err((1008, "bad frame")); }; match frame { - // HTTP forwarding and relay sockets land in later changes. - Inbound::Http(f) => self.refuse(ErrorCode::NotSupported, Some(f.id), None), + Inbound::Http(f) => { + let id = f.id.clone(); + if let Err(code) = self.http(f) { + self.refuse(code, Some(id), None); + } + } + // Relay sockets land in the next change. Inbound::WsOpen { sid, .. } => self.refuse(ErrorCode::NotSupported, None, Some(sid)), Inbound::WsSend { sid, .. } | Inbound::WsClose { sid, .. } => { self.refuse(ErrorCode::UnknownSocket, None, Some(sid)) @@ -211,4 +240,69 @@ impl Link { ); self.out.send(&Out::error(code, id, sid)); } + + fn http(&mut self, f: HttpFrame) -> Result<(), ErrorCode> { + let lim = &self.state.limits; + if self.draining { + return Err(ErrorCode::ShuttingDown); + } + if self.inflight.contains_key(&f.id) { + return Err(ErrorCode::DuplicateId); + } + if self.inflight.len() >= lim.max_http_inflight { + return Err(ErrorCode::Busy); + } + let route = policy::http_route(&f.method, &f.path)?; + let raw_headers = f.headers.as_deref().map(|raw| raw.get()); + let headers = policy::request_headers(route, raw_headers, lim.max_auth_header)?; + let body = match (route.method() == axum::http::Method::POST, f.body_b64) { + (true, Some(b64)) => { + let body = STANDARD.decode(b64).map_err(|_| ErrorCode::BadBase64)?; + if body.len() > lim.max_request_body { + return Err(ErrorCode::BodyTooLarge); + } + Some(body) + } + (false, None) => None, + _ => return Err(ErrorCode::NotAllowed), + }; + let url = self + .state + .config + .relay_http + .join(route.path()) + .map_err(|_| ErrorCode::NotAllowed)?; + let (state, out, link_id, id) = ( + self.state.clone(), + self.out.clone(), + self.id.clone(), + f.id.clone(), + ); + let task = self.http.spawn(async move { + let started = Instant::now(); + let result = http_forward::forward( + &state.client, + route.method(), + url, + headers, + body, + state.limits.max_response_body, + ) + .await; + let (link_id, path, ms) = (link_id.as_str(), route.path(), ms(started)); + out.send(&match result { + Ok(answer) => { + tracing::info!(link_id, id = id.as_str(), path, status = answer.status, ms, "http forwarded"); + let body_b64 = STANDARD.encode(answer.body); + Out::HttpRes { id, status: answer.status, headers: answer.headers, body_b64, ms } + } + Err(code) => { + tracing::info!(link_id, id = id.as_str(), path, code = ?code, ms, "http failed"); + Out::error(code, Some(id), None) + } + }); + }); + self.inflight.insert(f.id, task); + Ok(()) + } } diff --git a/crates/buzz-relay-gateway/tests/common/mod.rs b/crates/buzz-relay-gateway/tests/common/mod.rs index 9b497fcf88e..d84b7b44b8b 100644 --- a/crates/buzz-relay-gateway/tests/common/mod.rs +++ b/crates/buzz-relay-gateway/tests/common/mod.rs @@ -1,13 +1,27 @@ -//! Shared harness: the gateway in-process, a mock Worker and a bearer signer. +//! Shared harness: the gateway in-process, a mock relay (HTTP side), a mock Worker +//! and a bearer signer. #![allow(dead_code)] -use std::{net::SocketAddr, time::Duration}; +use std::{ + collections::HashMap, + net::SocketAddr, + sync::{Arc, Mutex}, + time::Duration, +}; +use axum::{ + body::{Body, Bytes}, + extract::State, + http::{HeaderMap, Method, StatusCode, Uri}, + response::{IntoResponse, Response}, + Router, +}; use base64::{engine::general_purpose::STANDARD, Engine as _}; use buzz_relay_gateway::{Config, Limits}; use futures_util::{SinkExt, StreamExt}; use nostr::{EventBuilder, Keys, Kind, Tag}; -use serde_json::Value; +use serde_json::{json, Value}; +use sha2::{Digest, Sha256}; use tokio::{net::TcpListener, task::JoinHandle, time::timeout}; use tokio_tungstenite::{ connect_async, @@ -44,6 +58,7 @@ pub fn bearer(keys: &Keys) -> String { /// Short timers for tests; every other limit stays at its default. pub fn limits() -> Limits { Limits { + request_timeout: Duration::from_secs(2), shutdown_grace: Duration::from_millis(500), ..Limits::default() } @@ -168,4 +183,165 @@ impl Worker { } } } + + pub async fn http( + &mut self, + id: &str, + method: &str, + path: &str, + headers: Value, + body: Option<&[u8]>, + ) -> Value { + let mut frame = + json!({ "type": "http", "id": id, "method": method, "path": path, "headers": headers }); + if let Some(body) = body { + frame["body_b64"] = STANDARD.encode(body).into(); + } + self.send(frame).await; + self.frame().await + } + + pub async fn post(&mut self, id: &str, path: &str, body: &[u8]) -> Value { + self.http( + id, + "POST", + path, + json!({ "authorization": "Nostr test" }), + Some(body), + ) + .await + } +} + +/// A mock relay, a gateway pointed at it, and a Worker on a fresh link. +pub async fn setup(limits: Limits) -> (Relay, Gateway, Worker) { + let relay = Relay::start().await; + let gw = Gateway::start(&relay.ws_url(), limits).await; + let w = gw.worker().await; + (relay, gw, w) +} + +pub fn body_of(frame: &Value) -> Vec { + STANDARD + .decode(frame["body_b64"].as_str().unwrap()) + .unwrap() +} + +pub fn sha256_hex(bytes: &[u8]) -> String { + hex::encode(Sha256::digest(bytes)) +} + +/// How the mock relay answers HTTP. +#[derive(Clone, Default)] +pub enum Mode { + /// Report the hash, length and headers of what arrived. + #[default] + Echo, + /// This status and body. + Reply(u16, Vec), + /// 302 to `/elsewhere`. + Redirect, + /// Never answer. + Stall, + /// Send part of a body, then drop the connection. + CloseMid, +} + +#[derive(Default)] +pub struct RelayState { + pub mode: Mutex, + pub hits: Mutex>, + pub url: Mutex, +} + +/// The mock relay: HTTP routes on every path. +pub struct Relay { + pub addr: SocketAddr, + pub state: Arc, +} + +impl Relay { + pub async fn start() -> Self { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let state = Arc::new(RelayState::default()); + *state.url.lock().unwrap() = format!("ws://{addr}"); + let app = Router::new().fallback(handle).with_state(state.clone()); + tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + Self { addr, state } + } + + pub fn ws_url(&self) -> String { + format!("ws://{}", self.addr) + } + + pub fn set(&self, mode: Mode) { + *self.state.mode.lock().unwrap() = mode; + } + + pub fn hits(&self, path: &str) -> usize { + self.state + .hits + .lock() + .unwrap() + .get(path) + .copied() + .unwrap_or(0) + } +} + +async fn handle( + State(s): State>, + method: Method, + uri: Uri, + headers: HeaderMap, + body: Bytes, +) -> Response { + let path = uri.path().to_owned(); + *s.hits.lock().unwrap().entry(path.clone()).or_default() += 1; + let mode = s.mode.lock().unwrap().clone(); + match mode { + Mode::Echo if method == Method::GET => { + json!({ "name": "mock relay", "accept": headers["accept"].to_str().unwrap() }) + .to_string() + .into_response() + } + Mode::Echo => { + let seen: serde_json::Map = headers + .iter() + .map(|(k, v)| (k.to_string(), v.to_str().unwrap_or("").into())) + .collect(); + let url = format!( + "http://{}{path}", + s.url.lock().unwrap().trim_start_matches("ws://") + ); + let nip98 = headers + .get("authorization") + .and_then(|v| v.to_str().ok()?.strip_prefix("Nostr ").map(str::to_owned)) + .and_then(|b64| String::from_utf8(STANDARD.decode(b64).ok()?).ok()) + .is_some_and(|event| { + buzz_auth::verify_nip98_event(&event, &url, "POST", Some(&body)).is_ok() + }); + let echo = json!({ "sha256": sha256_hex(&body), "len": body.len(), "headers": seen, "nip98": nip98 }); + ([("content-type", "application/json")], echo.to_string()).into_response() + } + Mode::Reply(status, body) => ( + StatusCode::from_u16(status).unwrap(), + [("set-cookie", "a=b")], + body, + ) + .into_response(), + Mode::Redirect => (StatusCode::FOUND, [("location", "/elsewhere")]).into_response(), + Mode::Stall => { + tokio::time::sleep(Duration::from_secs(30)).await; + StatusCode::OK.into_response() + } + Mode::CloseMid => { + let chunks = [ + Ok(Bytes::from_static(b"partial")), + Err(std::io::Error::other("cut")), + ]; + Body::from_stream(futures_util::stream::iter(chunks)).into_response() + } + } } diff --git a/crates/buzz-relay-gateway/tests/http.rs b/crates/buzz-relay-gateway/tests/http.rs new file mode 100644 index 00000000000..8bf16699d10 --- /dev/null +++ b/crates/buzz-relay-gateway/tests/http.rs @@ -0,0 +1,225 @@ +mod common; + +use std::time::Duration; + +use base64::{engine::general_purpose::STANDARD, Engine as _}; +use buzz_relay_gateway::Limits; +use common::{body_of, limits, setup, sha256_hex, Mode}; +use nostr::{EventBuilder, Keys, Kind, Tag}; +use serde_json::{json, Value}; + +/// A POST /query frame, for requests whose answer the test reads later. +fn query(id: &str) -> Value { + json!({"type":"http","id":id,"method":"POST","path":"/query","headers":{"authorization":"x"},"body_b64":""}) +} + +/// Deterministic bytes that are not valid UTF-8. +fn noise(len: usize) -> Vec { + let mut x: u32 = 0x9e37_79b9; + (0..len) + .map(|_| { + x ^= x << 13; + x ^= x >> 17; + x ^= x << 5; + (x >> 24) as u8 | 0x80 + }) + .collect() +} + +#[tokio::test] +async fn bodies_and_headers_arrive_untouched() { + let (relay, _gw, mut w) = setup(limits()).await; + let odd_json = b" [ {\"kinds\" :[0] ,\n\t\"limit\":1 } ] "; + for (i, body) in [noise(64 << 10), odd_json.to_vec()].iter().enumerate() { + let res = w.post(&format!("b{i}"), "/events", body).await; + assert_eq!( + (res["type"].as_str(), res["status"].as_u64()), + (Some("http_res"), Some(200)) + ); + let echo: Value = serde_json::from_slice(&body_of(&res)).unwrap(); + assert_eq!(echo["sha256"], sha256_hex(body)); + assert_eq!(echo["len"], body.len()); + let seen = echo["headers"].as_object().unwrap(); + assert_eq!(seen["host"], relay.addr.to_string()); + assert_eq!( + seen["user-agent"], + concat!("buzz-relay-gateway/", env!("CARGO_PKG_VERSION")) + ); + assert_eq!(seen["authorization"], "Nostr test"); + for name in seen.keys() { + let banned = name.starts_with("x-forwarded") + || [ + "forwarded", + "x-real-ip", + "x-pubkey", + "accept-encoding", + "cookie", + "origin", + ] + .contains(&name.as_str()); + assert!(!banned, "{name} reached the relay"); + } + assert_eq!(res["headers"], json!({"content-type": "application/json"})); + } + assert_eq!(relay.hits("/events"), 2); +} + +#[tokio::test] +async fn response_bytes_come_back_identical() { + let (relay, _gw, mut w) = setup(limits()).await; + let bytes = noise(100_000); + relay.set(Mode::Reply(200, bytes.clone())); + let res = w.post("r", "/query", b"[]").await; + assert_eq!(body_of(&res), bytes); + // Only content-type and retry-after come back; set-cookie stays at the gateway. + assert_eq!( + res["headers"], + json!({"content-type": "application/octet-stream"}) + ); +} + +#[tokio::test] +async fn a_signed_nip98_request_still_verifies_at_the_relay() { + let (relay, _gw, mut w) = setup(limits()).await; + let keys = Keys::generate(); + let body = br#"[{"kinds":[9],"limit":5}]"#; + let url = format!("http://{}/query", relay.addr); + let tags = [ + ["u", url.as_str()], + ["method", "POST"], + ["payload", &sha256_hex(body)], + ] + .map(|t| Tag::parse(t).unwrap()); + let event = EventBuilder::new(Kind::HttpAuth, "") + .tags(tags) + .sign_with_keys(&keys) + .unwrap(); + let auth = format!( + "Nostr {}", + STANDARD.encode(serde_json::to_string(&event).unwrap()) + ); + let headers = json!({"authorization": auth, "content-type": "application/json", "accept": "application/json"}); + let res = w.http("n", "POST", "/query", headers, Some(body)).await; + let echo: Value = serde_json::from_slice(&body_of(&res)).unwrap(); + assert_eq!(echo["nip98"], true); +} + +#[tokio::test] +async fn relay_answers_pass_through_once() { + let (relay, _gw, mut w) = setup(limits()).await; + let limited = br#"{"error":"rate-limited: quota exceeded; retry in 12s"}"#.to_vec(); + relay.set(Mode::Reply(429, limited.clone())); + let res = w.post("a", "/query", b"[]").await; + assert_eq!(res["status"], 429); + assert_eq!(body_of(&res), limited); + relay.set(Mode::Reply(500, b"boom".to_vec())); + assert_eq!(w.post("b", "/count", b"[]").await["status"], 500); + relay.set(Mode::CloseMid); + let res = w.post("c", "/events", b"{}").await; + assert_eq!(res["code"], "upstream_connect"); + relay.set(Mode::Redirect); + let res = w.post("d", "/events", b"{}").await; + assert_eq!( + (res["status"].as_u64(), &res["headers"]), + (Some(302), &json!({})) + ); + // Every request reached the relay exactly once, and the redirect was not followed. + assert_eq!(relay.hits("/query"), 1); + assert_eq!(relay.hits("/count"), 1); + assert_eq!(relay.hits("/events"), 2); + assert_eq!(relay.hits("/elsewhere"), 0); +} + +#[tokio::test] +async fn size_limits_hold_both_ways() { + let (relay, _gw, mut w) = setup(limits()).await; + let res = w.post("big", "/events", &vec![b'a'; (1 << 20) + 1]).await; + assert_eq!(res["code"], "body_too_large"); + assert_eq!(relay.hits("/events"), 0); + relay.set(Mode::Reply(200, vec![b'a'; (4 << 20) + 1])); + assert_eq!( + w.post("huge", "/query", b"[]").await["code"], + "response_too_large" + ); +} + +#[tokio::test] +async fn frames_outside_the_table_are_refused() { + let (relay, _gw, mut w) = setup(limits()).await; + let auth = json!({"authorization": "Nostr test"}); + let cases = [ + ( + json!({"type":"http","id":"1","method":"POST","path":"/query?x=1","headers":auth,"body_b64":""}), + "not_allowed", + ), + ( + json!({"type":"http","id":"2","method":"POST","path":"/query","headers":{"authorization":"x","x-pubkey":"00"},"body_b64":""}), + "not_allowed", + ), + ( + json!({"type":"http","id":"3","method":"POST","path":"/query","headers":auth}), + "not_allowed", + ), + ( + json!({"type":"http","id":"4","method":"POST","path":"/query","headers":auth,"body_b64":"!!"}), + "bad_base64", + ), + ( + json!({"type":"http","id":"5","method":"GET","path":"/","headers":{"accept":"text/html"}}), + "not_allowed", + ), + ]; + for (frame, code) in cases { + let id = frame["id"].clone(); + w.send(frame).await; + let res = w.frame().await; + assert_eq!((&res["id"], res["code"].as_str()), (&id, Some(code))); + } + assert_eq!(relay.hits("/query") + relay.hits("/"), 0); + let nip11 = json!({"accept": "application/nostr+json"}); + let info = w.http("i", "GET", "/", nip11, None).await; + let doc: Value = serde_json::from_slice(&body_of(&info)).unwrap(); + assert_eq!(doc["accept"], "application/nostr+json"); +} + +#[tokio::test] +async fn in_flight_cap_duplicates_and_timeouts() { + let quick = Limits { + request_timeout: Duration::from_millis(800), + ..limits() + }; + let (relay, _gw, mut w) = setup(quick).await; + relay.set(Mode::Stall); + for i in 0..16 { + w.send(query(&format!("s{i}"))).await; + } + w.send(query("s0")).await; + assert_eq!(w.frame().await["code"], "duplicate_id"); + assert_eq!(w.post("s16", "/query", b"").await["code"], "busy"); + for _ in 0..16 { + assert_eq!(w.frame().await["code"], "upstream_timeout"); + } + relay.set(Mode::Echo); + assert_eq!(w.post("after", "/query", b"[]").await["status"], 200); +} + +#[tokio::test] +async fn shutdown_drains_in_flight_requests() { + let (relay, gw, mut w) = setup(limits()).await; + relay.set(Mode::Stall); + w.send(query("slow")).await; + tokio::time::sleep(Duration::from_millis(100)).await; + gw.shutdown.cancel(); + let health = reqwest::get(gw.url("/healthz")).await.unwrap(); + assert_eq!(health.status(), 503); + assert_eq!( + w.post("late", "/query", b"[]").await["code"], + "shutting_down" + ); + assert_eq!(w.closed().await, Some(1012)); + tokio::time::timeout(Duration::from_secs(5), gw.task) + .await + .unwrap() + .unwrap() + .unwrap(); +} From 3b7159d59295f193cca3333ec322252a8016ec54 Mon Sep 17 00:00:00 2001 From: QuicksilverSlick Date: Tue, 27 Oct 2026 09:15:37 -0500 Subject: [PATCH 10/15] fix(relay-gateway): report an expired connect timeout as upstream_connect Review fix for part 5. reqwest marks an expired connect_timeout as a timeout too, so a relay that never answered the TCP or TLS handshake was reported as upstream_timeout. That code means a request may have reached the relay; upstream_connect means it did not. A connect error is now upstream_connect whatever its cause. The test for it lands in part 7, which has room for it. Signed-off-by: QuicksilverSlick Co-Authored-By: Claude Opus 5.5 --- crates/buzz-relay-gateway/src/http_forward.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/buzz-relay-gateway/src/http_forward.rs b/crates/buzz-relay-gateway/src/http_forward.rs index db708964dd1..72e8e037940 100644 --- a/crates/buzz-relay-gateway/src/http_forward.rs +++ b/crates/buzz-relay-gateway/src/http_forward.rs @@ -63,7 +63,8 @@ pub(crate) async fn forward( } fn upstream(e: reqwest::Error) -> ErrorCode { - if e.is_timeout() { + // An expired connect_timeout is also a timeout, but then nothing reached the relay. + if e.is_timeout() && !e.is_connect() { ErrorCode::UpstreamTimeout } else { ErrorCode::UpstreamConnect From d31598b9c3e684e789609ca24e3d1d44f73ca6be Mon Sep 17 00:00:00 2001 From: QuicksilverSlick Date: Tue, 27 Oct 2026 08:34:30 -0500 Subject: [PATCH 11/15] feat(relay-gateway): hold NIP-42 relay sockets for the Worker ws_open now opens one relay WebSocket on the pinned wss origin plus /, with only a user-agent header; no Worker header is forwarded, because NIP-42 runs in-band. The relay's ["AUTH", challenge] reaches the Worker as an ordinary ws_msg, and the Worker's signed reply goes back with ws_send. The gateway parses no Nostr frames: it only times the AUTH exchange by prefix (auth_challenge_ms, auth_reply_ms), since the relay closes a socket 5 s after its challenge. - Every socket reports exactly one ws_closed, with by (relay, gateway or worker), the close code, and http_status when the upgrade was refused. - The gateway keeps reading every relay socket, so the relay's pings are answered at once. A relay frame over 1 MiB, 90 s of relay silence, or 64 Worker frames the relay is not taking close that one socket. - ws_send text over 512 KiB gets frame_too_large and the socket stays open, so one bad frame cannot drop every subscription on it. - At most 8 sockets per link. The huddle audio path is reserved for the Phase 2 bridge and answers not_supported. - No relay socket outlives its link: on any link end each socket closes (1001 on shutdown, after in-flight HTTP drains), reports ws_closed, and anything still running after 2 s is aborted. The gateway never reopens a socket or replays a REQ; reconnects are the Worker's job. Signed-off-by: QuicksilverSlick Co-Authored-By: Claude Opus 5.5 --- crates/buzz-relay-gateway/Cargo.toml | 2 +- crates/buzz-relay-gateway/src/config.rs | 11 +- crates/buzz-relay-gateway/src/lib.rs | 1 + crates/buzz-relay-gateway/src/link.rs | 111 +++++++++- crates/buzz-relay-gateway/src/relay_ws.rs | 197 ++++++++++++++++++ crates/buzz-relay-gateway/tests/common/mod.rs | 109 +++++++++- crates/buzz-relay-gateway/tests/ws.rs | 165 +++++++++++++++ 7 files changed, 582 insertions(+), 14 deletions(-) create mode 100644 crates/buzz-relay-gateway/src/relay_ws.rs create mode 100644 crates/buzz-relay-gateway/tests/ws.rs diff --git a/crates/buzz-relay-gateway/Cargo.toml b/crates/buzz-relay-gateway/Cargo.toml index 1503f35d8e6..33243ae84cc 100644 --- a/crates/buzz-relay-gateway/Cargo.toml +++ b/crates/buzz-relay-gateway/Cargo.toml @@ -30,6 +30,7 @@ serde = { workspace = true } serde_json = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true } +tokio-tungstenite = { workspace = true } tokio-util = { workspace = true } tracing = { workspace = true } tracing-subscriber = { workspace = true } @@ -39,4 +40,3 @@ uuid = { workspace = true } [dev-dependencies] hex = { workspace = true } sha2 = { workspace = true } -tokio-tungstenite = { workspace = true } diff --git a/crates/buzz-relay-gateway/src/config.rs b/crates/buzz-relay-gateway/src/config.rs index d56ecbecd14..57f4df543e7 100644 --- a/crates/buzz-relay-gateway/src/config.rs +++ b/crates/buzz-relay-gateway/src/config.rs @@ -107,6 +107,8 @@ pub struct Limits { pub max_response_body: usize, /// Largest `ws_send` text (the relay's frame limit). pub max_ws_text: usize, + /// Largest frame the gateway accepts from the relay. + pub max_relay_frame: usize, /// HTTP requests in flight per link. pub max_http_inflight: usize, /// Relay sockets per link. @@ -115,6 +117,8 @@ pub struct Limits { pub out_frames: usize, /// Serialized bytes queued for the Worker, with the same rule. pub out_bytes: usize, + /// Worker frames queued for one relay socket; a full queue closes that socket. + pub relay_queue: usize, /// How long a link lives before it closes with 4001. pub link_lifetime: Duration, /// How long before that the gateway sends `rotate`. @@ -123,10 +127,12 @@ pub struct Limits { pub ping_every: Duration, /// A link with no Worker frame for this long closes with 4002. pub idle: Duration, - /// Reaching the relay: TCP and TLS. + /// Reaching the relay: TCP, TLS and, for sockets, the upgrade. pub connect_timeout: Duration, /// One whole HTTP exchange with the relay. pub request_timeout: Duration, + /// A relay socket with no frame (pings included) for this long is closed. + pub relay_silence: Duration, /// How long shutdown waits for in-flight HTTP requests. pub shutdown_grace: Duration, } @@ -141,16 +147,19 @@ impl Default for Limits { max_request_body: 1 << 20, max_response_body: 4 << 20, max_ws_text: 512 << 10, + max_relay_frame: 1 << 20, max_http_inflight: 16, max_sockets: 8, out_frames: 1024, out_bytes: 16 << 20, + relay_queue: 64, link_lifetime: Duration::from_secs(3600), rotate_before: Duration::from_secs(300), ping_every: Duration::from_secs(20), idle: Duration::from_secs(60), connect_timeout: Duration::from_secs(10), request_timeout: Duration::from_secs(30), + relay_silence: Duration::from_secs(90), shutdown_grace: Duration::from_secs(10), } } diff --git a/crates/buzz-relay-gateway/src/lib.rs b/crates/buzz-relay-gateway/src/lib.rs index 0188703ff66..1818516ca0f 100644 --- a/crates/buzz-relay-gateway/src/lib.rs +++ b/crates/buzz-relay-gateway/src/lib.rs @@ -10,6 +10,7 @@ pub mod frame; mod http_forward; mod link; pub mod policy; +mod relay_ws; use std::{ future::IntoFuture, diff --git a/crates/buzz-relay-gateway/src/link.rs b/crates/buzz-relay-gateway/src/link.rs index 4e2f85f04ac..c0036cfe7c3 100644 --- a/crates/buzz-relay-gateway/src/link.rs +++ b/crates/buzz-relay-gateway/src/link.rs @@ -1,5 +1,5 @@ //! One task per link: `hello`, frame dispatch, pings, rotation, idle and -//! backpressure, and the link's close. +//! backpressure, and the teardown of everything the link opened. use std::{ collections::HashMap, @@ -19,10 +19,12 @@ use tokio_util::sync::CancellationToken; use crate::{ frame::{ErrorCode, HelloLimits, HttpFrame, Id, Inbound, Out}, - http_forward, policy, AppState, + http_forward, policy, + relay_ws::{Cmd, Socket}, + AppState, }; -// Teardown waits this long for the Worker to take the close frame. +// Teardown waits this long for the relay sockets, then for the Worker to take the close frame. const CLOSE_WAIT: Duration = Duration::from_secs(2); const SHUTDOWN: (u16, &str) = (1012, "gateway restarting"); @@ -69,6 +71,11 @@ impl Outbox { } } +struct SocketHandle { + tx: mpsc::Sender, + overflow: CancellationToken, +} + struct Link { state: Arc, id: String, @@ -76,6 +83,9 @@ struct Link { draining: bool, http: JoinSet<()>, inflight: HashMap, + sockets: HashMap, + socket_tasks: JoinSet<()>, + sockets_closed: CancellationToken, } /// Runs one link until it closes. `_permit` holds its slot under the link cap. @@ -114,6 +124,9 @@ pub(crate) async fn run(socket: WebSocket, state: Arc, _permit: OwnedS draining: false, http: JoinSet::new(), inflight: HashMap::new(), + sockets: HashMap::new(), + socket_tasks: JoinSet::new(), + sockets_closed: CancellationToken::new(), }; let (code, reason) = link .serve(&mut stream, opened + lim.link_lifetime, opened + rotate_in) @@ -125,7 +138,14 @@ pub(crate) async fn run(socket: WebSocket, state: Arc, _permit: OwnedS ms = ms(opened), "link closing" ); + // No relay socket outlives its link. link.http.abort_all(); + link.sockets_closed.cancel(); + let _ = timeout(CLOSE_WAIT, async { + while link.socket_tasks.join_next().await.is_some() {} + }) + .await; + link.socket_tasks.abort_all(); let close = Message::Close(Some(CloseFrame { code, reason: reason.into(), @@ -201,6 +221,9 @@ impl Link { return SHUTDOWN; } } + Some(_) = self.socket_tasks.join_next(), if !self.socket_tasks.is_empty() => { + self.sockets.retain(|_, socket| !socket.tx.is_closed()); + } } } } @@ -218,10 +241,34 @@ impl Link { self.refuse(code, Some(id), None); } } - // Relay sockets land in the next change. - Inbound::WsOpen { sid, .. } => self.refuse(ErrorCode::NotSupported, None, Some(sid)), - Inbound::WsSend { sid, .. } | Inbound::WsClose { sid, .. } => { - self.refuse(ErrorCode::UnknownSocket, None, Some(sid)) + Inbound::WsOpen { sid, path } => { + if let Err(code) = self.ws_open(sid.clone(), &path) { + self.refuse(code, None, Some(sid)); + } + } + Inbound::WsSend { sid, text } => { + let result = if text.len() > self.state.limits.max_ws_text { + Err(ErrorCode::FrameTooLarge) + } else { + self.to_socket(&sid, Cmd::Send(text)) + }; + if let Err(code) = result { + self.refuse(code, None, Some(sid)); + } + } + Inbound::WsClose { sid, code, reason } => { + let code = code.unwrap_or(1000); + let reason = reason.unwrap_or_default(); + // A close frame allows 1000 or an application code, and a 123-byte reason. + let result = + if (code == 1000 || (3000..=4999).contains(&code)) && reason.len() <= 123 { + self.to_socket(&sid, Cmd::Close(code, reason)) + } else { + Err(ErrorCode::NotAllowed) + }; + if let Err(code) = result { + self.refuse(code, None, Some(sid)); + } } Inbound::Ping { ts } => self.out.send(&Out::Pong { ts }), Inbound::Pong => {} @@ -305,4 +352,54 @@ impl Link { self.inflight.insert(f.id, task); Ok(()) } + + fn ws_open(&mut self, sid: Id, path: &str) -> Result<(), ErrorCode> { + let lim = &self.state.limits; + if self.draining { + return Err(ErrorCode::ShuttingDown); + } + if self.sockets.contains_key(&sid) { + return Err(ErrorCode::DuplicateId); + } + if self.sockets.len() >= lim.max_sockets { + return Err(ErrorCode::Busy); + } + let url = self + .state + .config + .relay_ws + .join(policy::ws_route(path)?) + .map_err(|_| ErrorCode::NotAllowed)?; + let (tx, cmds) = mpsc::channel(lim.relay_queue); + let overflow = CancellationToken::new(); + let socket = Socket { + link_id: self.id.clone(), + sid: sid.clone(), + url, + out: self.out.clone(), + cmds, + overflow: overflow.clone(), + link_closed: self.sockets_closed.clone(), + shutdown: self.state.shutdown.clone(), + connect_timeout: lim.connect_timeout, + silence: lim.relay_silence, + max_frame: lim.max_relay_frame, + }; + self.socket_tasks.spawn(socket.run()); + self.sockets.insert(sid, SocketHandle { tx, overflow }); + Ok(()) + } + + fn to_socket(&self, sid: &Id, cmd: Cmd) -> Result<(), ErrorCode> { + let socket = self.sockets.get(sid).ok_or(ErrorCode::UnknownSocket)?; + match socket.tx.try_send(cmd) { + Ok(()) => Ok(()), + Err(TrySendError::Closed(_)) => Err(ErrorCode::UnknownSocket), + // The relay is not taking frames: close that socket, not the link. + Err(TrySendError::Full(_)) => { + socket.overflow.cancel(); + Ok(()) + } + } + } } diff --git a/crates/buzz-relay-gateway/src/relay_ws.rs b/crates/buzz-relay-gateway/src/relay_ws.rs new file mode 100644 index 00000000000..6be7e13c08b --- /dev/null +++ b/crates/buzz-relay-gateway/src/relay_ws.rs @@ -0,0 +1,197 @@ +//! One task per relay socket. It keeps reading the relay, so the relay's pings +//! are answered at once, and it reports exactly one `ws_closed`. + +use std::time::Duration; + +use futures_util::{SinkExt, StreamExt}; +use tokio::{ + sync::mpsc, + time::{sleep_until, timeout, Instant}, +}; +use tokio_tungstenite::{ + connect_async_with_config, + tungstenite::{ + client::IntoClientRequest, + http::{header::USER_AGENT, HeaderValue}, + protocol::{CloseFrame, WebSocketConfig}, + Error, Message, + }, +}; +use tokio_util::sync::CancellationToken; +use url::Url; + +use crate::{ + frame::{Id, Out, Side}, + link::{ms, Outbox}, +}; + +const AUTH_PREFIX: &str = r#"["AUTH","#; +const CLOSE_CAP: Duration = Duration::from_secs(1); + +/// A Worker frame for one relay socket. +pub(crate) enum Cmd { + Send(String), + Close(u16, String), +} + +/// One relay socket. The link keeps the sending half of `cmds`. +pub(crate) struct Socket { + pub link_id: String, + pub sid: Id, + pub url: Url, + pub out: Outbox, + pub cmds: mpsc::Receiver, + /// Cancelled when the link could not queue a Worker frame for this socket. + pub overflow: CancellationToken, + /// Cancelled when the link ends. + pub link_closed: CancellationToken, + pub shutdown: CancellationToken, + pub connect_timeout: Duration, + pub silence: Duration, + pub max_frame: usize, +} + +struct Closed { + by: Side, + code: Option, + reason: String, + http_status: Option, +} + +fn by_gateway(code: Option, reason: &str) -> Closed { + Closed { + by: Side::Gateway, + code, + reason: reason.to_owned(), + http_status: None, + } +} + +impl Socket { + pub(crate) async fn run(mut self) { + let opened = Instant::now(); + let closed = self.serve(opened).await; + // Relay and Worker close reasons are frame content, so only gateway reasons are logged. + let reason = (closed.by == Side::Gateway).then_some(closed.reason.as_str()); + tracing::info!( + link_id = self.link_id, + sid = self.sid.as_str(), + by = ?closed.by, + code = closed.code, + reason, + http_status = closed.http_status, + ms = ms(opened), + "relay socket closed" + ); + self.out.send(&Out::WsClosed { + sid: self.sid, + by: closed.by, + code: closed.code, + reason: closed.reason, + http_status: closed.http_status, + }); + } + + fn end(&self) -> (u16, &'static str) { + if self.shutdown.is_cancelled() { + (1001, "shutdown") + } else { + (1000, "link_closed") + } + } + + async fn serve(&mut self, opened: Instant) -> Closed { + let Ok(mut request) = self.url.as_str().into_client_request() else { + return by_gateway(None, "connect_failed"); + }; + let agent = concat!("buzz-relay-gateway/", env!("CARGO_PKG_VERSION")); + request + .headers_mut() + .insert(USER_AGENT, HeaderValue::from_static(agent)); + let config = WebSocketConfig::default() + .max_message_size(Some(self.max_frame)) + .max_frame_size(Some(self.max_frame)); + let connecting = connect_async_with_config(request, Some(config), false); + let connect = timeout(self.connect_timeout, connecting); + let mut ws = tokio::select! { + result = connect => match result { + Err(_) => return by_gateway(None, "connect_timeout"), + Ok(Err(Error::Http(response))) => { + let status = Some(response.status().as_u16()); + return Closed { http_status: status, ..by_gateway(None, "upgrade_refused") }; + } + Ok(Err(_)) => return by_gateway(None, "connect_failed"), + Ok(Ok((ws, _))) => ws, + }, + _ = self.link_closed.cancelled() => return by_gateway(None, self.end().1), + }; + self.out.send(&Out::WsOpened { + sid: self.sid.clone(), + ms: ms(opened), + }); + let (link_id, sid) = (self.link_id.as_str(), self.sid.as_str()); + let mut challenge = None; + let mut silent_at = Instant::now() + self.silence; + let (frame, closed) = loop { + let (code, reason) = tokio::select! { + msg = ws.next() => { + silent_at = Instant::now() + self.silence; + match msg { + Some(Ok(Message::Text(text))) => { + if text.starts_with(AUTH_PREFIX) { + tracing::info!(link_id, sid, auth_challenge_ms = ms(opened), "relay sent AUTH"); + challenge = Some(Instant::now()); + } + self.out.send(&Out::WsMsg { sid: self.sid.clone(), text: text.as_str().to_owned() }); + continue; + } + Some(Ok(Message::Close(frame))) => { + let (code, reason) = frame.map_or((None, String::new()), |f| (Some(f.code.into()), f.reason.as_str().to_owned())); + return Closed { by: Side::Relay, code, reason, http_status: None }; + } + Some(Ok(_)) => continue, + Some(Err(Error::Capacity(_))) => (1009, "frame_too_large"), + Some(Err(_)) | None => { + return Closed { by: Side::Relay, code: None, reason: "connection_lost".into(), http_status: None }; + } + } + } + cmd = self.cmds.recv() => match cmd { + Some(Cmd::Send(text)) => { + if let Some(at) = challenge.filter(|_| text.starts_with(AUTH_PREFIX)) { + tracing::info!(link_id, sid, auth_reply_ms = ms(at), "worker answered AUTH"); + challenge = None; + } + if ws.send(Message::Text(text.into())).await.is_err() { + return Closed { by: Side::Relay, code: None, reason: "connection_lost".into(), http_status: None }; + } + continue; + } + Some(Cmd::Close(code, reason)) => { + let frame = CloseFrame { code: code.into(), reason: reason.clone().into() }; + break (frame, Closed { by: Side::Worker, code: Some(code), reason, http_status: None }); + } + None => self.end(), + }, + _ = self.overflow.cancelled() => (1000, "backpressure"), + _ = self.link_closed.cancelled() => self.end(), + _ = sleep_until(silent_at) => (1001, "relay_silent"), + }; + break ( + CloseFrame { + code: code.into(), + reason: reason.into(), + }, + by_gateway(Some(code), reason), + ); + }; + // The relay's close reply ends the stream; capped so teardown stays bounded. + let _ = timeout(CLOSE_CAP, async { + if ws.close(Some(frame)).await.is_ok() { + while let Some(Ok(_)) = ws.next().await {} + } + }) + .await; + closed + } +} diff --git a/crates/buzz-relay-gateway/tests/common/mod.rs b/crates/buzz-relay-gateway/tests/common/mod.rs index d84b7b44b8b..4397aa0d491 100644 --- a/crates/buzz-relay-gateway/tests/common/mod.rs +++ b/crates/buzz-relay-gateway/tests/common/mod.rs @@ -1,5 +1,4 @@ -//! Shared harness: the gateway in-process, a mock relay (HTTP side), a mock Worker -//! and a bearer signer. +//! Shared harness: the gateway in-process, a mock relay, a mock Worker and a bearer signer. #![allow(dead_code)] use std::{ @@ -11,7 +10,10 @@ use std::{ use axum::{ body::{Body, Bytes}, - extract::State, + extract::{ + ws::{rejection::WebSocketUpgradeRejection, Message as AxMessage, WebSocket}, + State, WebSocketUpgrade, + }, http::{HeaderMap, Method, StatusCode, Uri}, response::{IntoResponse, Response}, Router, @@ -211,6 +213,39 @@ impl Worker { ) .await } + + /// Opens relay socket `sid` and returns the relay's AUTH challenge. + pub async fn open(&mut self, sid: &str) -> String { + self.send(json!({"type": "ws_open", "sid": sid, "path": "/"})) + .await; + let opened = self.frame().await; + assert_eq!( + (opened["type"].as_str(), opened["sid"].as_str()), + (Some("ws_opened"), Some(sid)) + ); + let msg = self.frame().await; + assert_eq!( + (msg["type"].as_str(), msg["sid"].as_str()), + (Some("ws_msg"), Some(sid)) + ); + let auth: Value = serde_json::from_str(msg["text"].as_str().unwrap()).unwrap(); + assert_eq!(auth[0], "AUTH"); + auth[1].as_str().unwrap().to_owned() + } + + /// Sends `text` on relay socket `sid` without waiting for an answer. + pub async fn ws_send(&mut self, sid: &str, text: &str) { + self.send(json!({"type": "ws_send", "sid": sid, "text": text})) + .await; + } + + /// Sends `text` on socket `sid` and returns the relay's next frame on the link. + pub async fn say(&mut self, sid: &str, text: Value) -> Value { + self.ws_send(sid, &text.to_string()).await; + let msg = self.frame().await; + assert_eq!(msg["type"], "ws_msg", "{msg}"); + serde_json::from_str(msg["text"].as_str().unwrap()).unwrap() + } } /// A mock relay, a gateway pointed at it, and a Worker on a fresh link. @@ -231,7 +266,7 @@ pub fn sha256_hex(bytes: &[u8]) -> String { hex::encode(Sha256::digest(bytes)) } -/// How the mock relay answers HTTP. +/// How the mock relay answers HTTP (and socket upgrades, for `Reply`). #[derive(Clone, Default)] pub enum Mode { /// Report the hash, length and headers of what arrived. @@ -251,10 +286,12 @@ pub enum Mode { pub struct RelayState { pub mode: Mutex, pub hits: Mutex>, + pub open_sockets: Mutex, + pub socket_closes: Mutex>>, pub url: Mutex, } -/// The mock relay: HTTP routes on every path. +/// The mock relay: HTTP routes on every path and NIP-42 sockets on `/`. pub struct Relay { pub addr: SocketAddr, pub state: Arc, @@ -288,6 +325,14 @@ impl Relay { .copied() .unwrap_or(0) } + + pub fn open_sockets(&self) -> usize { + *self.state.open_sockets.lock().unwrap() + } + + pub fn socket_closes(&self) -> Vec> { + self.state.socket_closes.lock().unwrap().clone() + } } async fn handle( @@ -295,11 +340,15 @@ async fn handle( method: Method, uri: Uri, headers: HeaderMap, + upgrade: Result, body: Bytes, ) -> Response { let path = uri.path().to_owned(); *s.hits.lock().unwrap().entry(path.clone()).or_default() += 1; let mode = s.mode.lock().unwrap().clone(); + if let (Ok(upgrade), Mode::Echo) = (upgrade, &mode) { + return upgrade.on_upgrade(move |ws| relay_socket(ws, s)); + } match mode { Mode::Echo if method == Method::GET => { json!({ "name": "mock relay", "accept": headers["accept"].to_str().unwrap() }) @@ -345,3 +394,53 @@ async fn handle( } } } + +/// A relay socket: AUTH challenge first, then commands from the test. +async fn relay_socket(mut ws: WebSocket, s: Arc) { + *s.open_sockets.lock().unwrap() += 1; + let challenge = buzz_auth::generate_challenge(); + let relay_url = s.url.lock().unwrap().clone(); + let reply = |v: Value| AxMessage::text(v.to_string()); + let _ = ws.send(reply(json!(["AUTH", challenge]))).await; + while let Some(Ok(msg)) = ws.recv().await { + let AxMessage::Text(text) = msg else { + if let AxMessage::Close(frame) = msg { + s.socket_closes.lock().unwrap().push(frame.map(|f| f.code)); + break; + } + continue; + }; + let cmd: Value = serde_json::from_str(&text).unwrap_or_default(); + let out = match cmd[0].as_str() { + Some("AUTH") => { + let event: nostr::Event = serde_json::from_value(cmd[1].clone()).unwrap(); + let ok = buzz_auth::verify_nip42_event(&event, &challenge, &relay_url).is_ok(); + vec![reply(json!(["OK", event.id, ok, ""]))] + } + Some("REQ") => vec![reply(json!(["EOSE", cmd[1]]))], + Some("CLOSE_WITH") => { + let code = cmd[1].as_u64().unwrap() as u16; + let frame = axum::extract::ws::CloseFrame { + code, + reason: "relay restarting".into(), + }; + vec![AxMessage::Close(Some(frame))] + } + Some("SEND") => { + let (n, size) = (cmd[1].as_u64().unwrap(), cmd[2].as_u64().unwrap() as usize); + (0..n).map(|_| AxMessage::text("x".repeat(size))).collect() + } + Some("SILENT") => { + tokio::time::sleep(Duration::from_secs(30)).await; + vec![] + } + _ => vec![], + }; + for msg in out { + if ws.send(msg).await.is_err() { + break; + } + } + } + *s.open_sockets.lock().unwrap() -= 1; +} diff --git a/crates/buzz-relay-gateway/tests/ws.rs b/crates/buzz-relay-gateway/tests/ws.rs new file mode 100644 index 00000000000..ea542a0e076 --- /dev/null +++ b/crates/buzz-relay-gateway/tests/ws.rs @@ -0,0 +1,165 @@ +mod common; + +use std::time::Duration; + +use buzz_relay_gateway::Limits; +use common::{limits, setup, Relay, Worker}; +use nostr::{EventBuilder, Keys, RelayUrl}; +use serde_json::json; + +const AUDIO: &str = "/huddle/6f1c2c1e-4b9a-4c71-9a55-0f1d8c3a2b10/audio"; + +async fn authenticate(w: &mut Worker, relay: &Relay, sid: &str) { + let challenge = w.open(sid).await; + let url = RelayUrl::parse(&relay.ws_url()).unwrap(); + let event = EventBuilder::auth(challenge, url) + .sign_with_keys(&Keys::generate()) + .unwrap(); + let ok = w.say(sid, json!(["AUTH", event])).await; + assert_eq!(ok, json!(["OK", event.id, true, ""])); +} + +async fn wait_for(what: &str, check: impl Fn() -> bool) { + for _ in 0..100 { + if check() { + return; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + panic!("timed out waiting for {what}"); +} + +#[tokio::test] +async fn nip42_auth_and_subscriptions_pass_through() { + let (relay, _gw, mut w) = setup(limits()).await; + authenticate(&mut w, &relay, "s1").await; + let eose = w.say("s1", json!(["REQ", "sub-1", {"kinds": [9]}])).await; + assert_eq!(eose, json!(["EOSE", "sub-1"])); +} + +#[tokio::test] +async fn closes_reach_the_other_side() { + let (relay, _gw, mut w) = setup(limits()).await; + w.open("r").await; + w.ws_send("r", r#"["CLOSE_WITH",1012]"#).await; + assert_eq!( + w.frame().await, + json!({"type":"ws_closed","sid":"r","by":"relay","code":1012,"reason":"relay restarting"}) + ); + w.open("w").await; + w.send(json!({"type":"ws_close","sid":"w","code":4000,"reason":"done"})) + .await; + assert_eq!( + w.frame().await, + json!({"type":"ws_closed","sid":"w","by":"worker","code":4000,"reason":"done"}) + ); + wait_for("close 4000", || relay.socket_closes().contains(&Some(4000))).await; + // A closed socket is gone, and its sid can be opened again. + w.ws_send("w", "[]").await; + assert_eq!(w.frame().await["code"], "unknown_socket"); + tokio::time::sleep(Duration::from_millis(100)).await; + w.open("w").await; +} + +#[tokio::test] +async fn closing_the_link_closes_every_relay_socket() { + let (relay, _gw, mut w) = setup(limits()).await; + for sid in ["a", "b", "c"] { + w.open(sid).await; + } + assert_eq!(relay.open_sockets(), 3); + w.ws.close(None).await.unwrap(); + wait_for("every relay socket to close", || relay.open_sockets() == 0).await; +} + +#[tokio::test] +async fn socket_limits() { + let (relay, _gw, mut w) = setup(Limits { + max_ws_text: 64, + ..limits() + }) + .await; + for i in 0..8 { + w.open(&format!("s{i}")).await; + } + w.send(json!({"type":"ws_open","sid":"s8","path":"/"})) + .await; + assert_eq!( + w.frame().await, + json!({"type":"error","sid":"s8","code":"busy","message":"too many requests or sockets in flight"}) + ); + w.send(json!({"type":"ws_open","sid":"s0","path":"/"})) + .await; + assert_eq!(w.frame().await["code"], "duplicate_id"); + w.ws_send("s0", &"x".repeat(65)).await; + assert_eq!(w.frame().await["code"], "frame_too_large"); + // The socket stays usable after a refused frame. + assert_eq!(w.say("s0", json!(["REQ", "q"])).await, json!(["EOSE", "q"])); + for (path, code) in [(AUDIO, "not_supported"), ("/x", "not_allowed")] { + w.send(json!({"type":"ws_close","sid":"s7"})).await; + assert_eq!(w.frame().await["by"], "worker"); + tokio::time::sleep(Duration::from_millis(100)).await; + w.send(json!({"type":"ws_open","sid":"s7","path":path})) + .await; + assert_eq!(w.frame().await["code"], code); + w.open("s7").await; + } + assert_eq!(relay.hits("/x"), 0); +} + +#[tokio::test] +async fn refused_and_failed_opens_report_ws_closed() { + let (relay, _gw, mut w) = setup(limits()).await; + relay.set(common::Mode::Reply(503, b"busy".to_vec())); + w.send(json!({"type":"ws_open","sid":"x","path":"/"})).await; + assert_eq!( + w.frame().await, + json!({"type":"ws_closed","sid":"x","by":"gateway","code":null,"reason":"upgrade_refused","http_status":503}) + ); +} + +#[tokio::test] +async fn relay_side_limits_close_one_socket() { + let quick = Limits { + max_relay_frame: 1024, + relay_silence: Duration::from_millis(500), + ..limits() + }; + let (_relay, _gw, mut w) = setup(quick).await; + w.open("big").await; + w.ws_send("big", r#"["SEND",1,2048]"#).await; + let closed = w.frame().await; + assert_eq!(closed["by"], "gateway"); + assert_eq!(closed["reason"], "frame_too_large"); + w.open("quiet").await; + w.ws_send("quiet", r#"["SILENT"]"#).await; + assert_eq!(w.frame().await["reason"], "relay_silent"); +} + +#[tokio::test] +async fn a_worker_that_stops_reading_loses_the_link() { + let slow = Limits { + out_frames: 16, + out_bytes: 64 << 10, + ..limits() + }; + let (relay, _gw, mut w) = setup(slow).await; + w.open("f").await; + w.ws_send("f", r#"["SEND",4000,4096]"#).await; + // Stop reading while the relay floods; the queue fills behind the full TCP buffers. + tokio::time::sleep(Duration::from_millis(500)).await; + assert_eq!(w.closed().await, Some(4003)); + wait_for("the relay socket to close", || relay.open_sockets() == 0).await; +} + +#[tokio::test] +async fn shutdown_closes_sockets_then_the_link() { + let (relay, gw, mut w) = setup(limits()).await; + w.open("s").await; + gw.shutdown.cancel(); + let closed = w.frame().await; + assert_eq!(closed["by"], "gateway"); + assert_eq!(closed["reason"], "shutdown"); + assert_eq!(w.closed().await, Some(1012)); + wait_for("close 1001", || relay.socket_closes().contains(&Some(1001))).await; +} From 59f315ea5b121b9567d62abb55edc70a8cd899e6 Mon Sep 17 00:00:00 2001 From: QuicksilverSlick Date: Tue, 27 Oct 2026 08:36:29 -0500 Subject: [PATCH 12/15] feat(relay-gateway): package the gateway as a container image Dockerfile.relay-gateway builds a static musl binary on Alpine and runs it on distroless static as the nonroot user, on port 8080, which is what a Cloudflare Container needs (linux/amd64). Both base images are pinned by digest. The image env holds only the bind address and RUST_LOG (plus the base image's PATH and SSL_CERT_FILE); the three relay and link values come from the Durable Object's start() env, and none of them is secret. The binary is PID 1, so it handles SIGTERM itself. The image is about 6 MB compressed. tests/logs.rs runs one HTTP forward and one socket AUTH whose bearer, body and frames all carry a marker, at TRACE level, and checks that the captured log holds neither the marker nor "Nostr ". It is its own test binary: tracing caches callsite interest process-wide, so a scoped subscriber misses events while other tests run on other threads. The README states the trust boundary, the environment and how to test. The tests are not in the Justfile unit-test list yet, so CI does not run them. Signed-off-by: QuicksilverSlick Co-Authored-By: Claude Opus 5.5 --- Dockerfile.relay-gateway | 28 ++++++++++ crates/buzz-relay-gateway/README.md | 62 ++++++++++++++++++++++ crates/buzz-relay-gateway/tests/logs.rs | 70 +++++++++++++++++++++++++ 3 files changed, 160 insertions(+) create mode 100644 Dockerfile.relay-gateway create mode 100644 crates/buzz-relay-gateway/README.md create mode 100644 crates/buzz-relay-gateway/tests/logs.rs diff --git a/Dockerfile.relay-gateway b/Dockerfile.relay-gateway new file mode 100644 index 00000000000..ae29eedb94c --- /dev/null +++ b/Dockerfile.relay-gateway @@ -0,0 +1,28 @@ +# syntax=docker/dockerfile:1.7 +# Buzz relay gateway: a keyless forwarder between one Worker link and one Buzz relay. +# The build context is the repo root (cargo needs the workspace manifests and Cargo.lock): +# docker build -f Dockerfile.relay-gateway --platform linux/amd64 -t buzz-relay-gateway:dev . +# Base images pinned by digest (first build, 27 Oct 2026). +FROM --platform=linux/amd64 rust:1.95.0-alpine@sha256:606fd313a0f49743ee2a7bd49a0914bab7deedb12791f3a846a34a4711db7ed2 AS builder +# build-base for ring and aws-lc-sys (C); cmake in case aws-lc-sys picks its cmake builder. +RUN apk add --no-cache build-base cmake +WORKDIR /build +COPY . . +# The image already has 1.95.0; this skips the components rust-toolchain.toml would install. +ENV RUSTUP_TOOLCHAIN=1.95.0 +RUN --mount=type=cache,target=/usr/local/cargo/registry \ + --mount=type=cache,target=/usr/local/cargo/git \ + --mount=type=cache,target=/build/target \ + cargo build --release --locked -p buzz-relay-gateway --bin buzz-relay-gateway \ + && install -D -m 0755 target/release/buzz-relay-gateway /out/buzz-relay-gateway \ + && strip /out/buzz-relay-gateway + +FROM gcr.io/distroless/static-debian12:nonroot@sha256:afa5c872c891853ca7fcf1f12c3edb23f7eeef36189728842dd51042ff57f7ab +LABEL org.opencontainers.image.title="Buzz relay gateway" \ + org.opencontainers.image.description="Keyless forwarder from a Worker link to one Buzz relay" \ + org.opencontainers.image.licenses="Apache-2.0" +COPY --from=builder /out/buzz-relay-gateway /usr/local/bin/buzz-relay-gateway +ENV BUZZ_RELAY_GATEWAY_BIND_ADDR=0.0.0.0:8080 RUST_LOG=info +EXPOSE 8080 +USER nonroot:nonroot +ENTRYPOINT ["/usr/local/bin/buzz-relay-gateway"] diff --git a/crates/buzz-relay-gateway/README.md b/crates/buzz-relay-gateway/README.md new file mode 100644 index 00000000000..d44e351de14 --- /dev/null +++ b/crates/buzz-relay-gateway/README.md @@ -0,0 +1,62 @@ +# Buzz relay gateway + +`buzz-relay-gateway` is a keyless forwarder between one Worker link and one +Buzz relay. It exists because a Cloudflare Worker cannot reach a relay that +sits behind Cloudflare itself (error 526), while a Cloudflare Container can. +The Worker keeps every key and signs every event; the gateway only checks the +link bearer and moves bytes. + +## Trust boundary + +- The gateway holds no secret. Its environment is four public values. +- The Worker opens `GET /v1/link` with `Authorization: Nostr `: + a kind 27235 event signed by a dedicated gateway-auth key (never a relay + member), `u` = the configured link URL, `method` = `GET`, within ±60 s, with + one `nonce`. Each bearer works once, and a link lives at most one hour. +- The gateway authenticates the Worker by signature. The Worker authenticates + the gateway only by transport (the Durable Object to container channel, or + TLS on a VM). +- Only one relay origin is reachable, through an exact route table: + `POST /query`, `/events`, `/count`, `GET /` (NIP-11) and the WebSocket on + `/`. Request and response headers are allowlisted, redirects are not + followed, nothing is retried, and bodies are never re-encoded, so NIP-98 + `payload` tags still verify at the relay. +- NIP-42 is transparent: the relay's `AUTH` challenge reaches the Worker as an + ordinary `ws_msg`, and the Worker signs the reply. +- Logs carry ids, status codes, close codes and timings. They never carry + bearers, bodies, relay frames or close-reason text. + +The frames are in `src/frame.rs`, the route table in `src/policy.rs`, the +limits in `src/config.rs`, and the link close codes in `src/link.rs`. + +## Environment + +| Variable | Required | Value | +|---|---|---| +| `BUZZ_RELAY_GATEWAY_BIND_ADDR` | no | default `127.0.0.1:8080`; the image sets `0.0.0.0:8080` | +| `BUZZ_RELAY_GATEWAY_RELAY_URL` | yes | `wss://host[:port]`; `ws://` only on loopback | +| `BUZZ_RELAY_GATEWAY_LINK_URL` | yes | the `u` every link bearer must carry | +| `BUZZ_RELAY_GATEWAY_LINK_PUBKEY` | yes | the gateway-auth public key, 64 lowercase hex | +| `RUST_LOG` | no | `info` | + +`GET /healthz` answers `ok`, or 503 once shutdown has begun. SIGTERM or +Ctrl-C drains in-flight requests for up to 10 s, closes every relay socket, +then closes each link with 1012. + +## Testing + +``` +cargo test -p buzz-relay-gateway +cargo clippy -p buzz-relay-gateway --all-targets -- -D warnings +``` + +The tests run offline against an in-process mock relay (HTTP routes and a +NIP-42 socket) and a mock Worker. They are not yet in the `Justfile` unit-test +list, so CI does not run them; run them locally on every change. + +The image builds from the repo root. Pushing it is a deployment step, not a +build step: + +``` +docker build -f Dockerfile.relay-gateway --platform linux/amd64 -t buzz-relay-gateway:dev . +``` diff --git a/crates/buzz-relay-gateway/tests/logs.rs b/crates/buzz-relay-gateway/tests/logs.rs new file mode 100644 index 00000000000..56533745aa0 --- /dev/null +++ b/crates/buzz-relay-gateway/tests/logs.rs @@ -0,0 +1,70 @@ +//! Its own test binary: tracing caches callsite interest process-wide, so a +//! scoped subscriber misses events while other tests run on other threads. +mod common; + +use std::{ + io::Write, + sync::{Arc, Mutex}, +}; + +use common::{bearer_with, limits, Gateway, Relay, Worker}; +use nostr::{EventBuilder, Keys, RelayUrl}; +use serde_json::json; + +#[derive(Clone, Default)] +struct Capture(Arc>>); + +impl Write for Capture { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.0.lock().unwrap().extend_from_slice(buf); + Ok(buf.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +// current_thread (the tokio::test default) keeps every task under this thread's subscriber. +#[tokio::test] +async fn logs_never_carry_bearers_bodies_or_frames() { + const MARK: &str = "MARK7f3a"; + let capture = Capture::default(); + let writer = capture.clone(); + let subscriber = tracing_subscriber::fmt() + .json() + .with_max_level(tracing::Level::TRACE) + .with_writer(move || writer.clone()) + .finish(); + let _guard = tracing::subscriber::set_default(subscriber); + + let relay = Relay::start().await; + let gw = Gateway::start(&relay.ws_url(), limits()).await; + let mut w = Worker::connect(&gw, &bearer_with(&gw.keys, MARK)) + .await + .unwrap(); + let res = w + .http( + "h", + "POST", + "/query", + json!({"authorization": format!("Nostr {MARK}")}), + Some(MARK.as_bytes()), + ) + .await; + assert_eq!(res["status"], 200); + let challenge = w.open("s").await; + let url = RelayUrl::parse(&relay.ws_url()).unwrap(); + let event = EventBuilder::auth(challenge, url) + .sign_with_keys(&Keys::generate()) + .unwrap(); + w.say("s", json!(["AUTH", event, MARK])).await; + w.say("s", json!(["REQ", MARK])).await; + w.send(json!({"type":"ws_close","sid":"s","reason":MARK})) + .await; + w.frame().await; + + let logs = String::from_utf8(capture.0.lock().unwrap().clone()).unwrap(); + assert!(logs.contains("auth_reply_ms"), "the AUTH timing was logged"); + assert!(!logs.contains(MARK), "marker leaked into logs"); + assert!(!logs.contains("Nostr "), "a bearer leaked into logs"); +} From d56cc9ea7b7179eb5f2cd544ca31fb17e003e314 Mon Sep 17 00:00:00 2001 From: QuicksilverSlick Date: Tue, 27 Oct 2026 09:21:09 -0500 Subject: [PATCH 13/15] fix(relay-gateway): keep a stalled relay write under the socket's limits Review fixes for parts 5 to 7. They sit here because parts 5 and 6 are at the PR size limit. - A Worker frame's write to the relay ran inside the socket loop's select! arm, so a relay that stopped reading stalled the whole task: the queue overflow, the silence timer and the link's end went unseen, and later ws_send frames were dropped with no ws_closed. The write now races those three, so the socket still closes with backpressure, relay_silent or link_closed. - When the relay closed a socket, the task returned at once and dropped tungstenite's queued close reply. It now flushes it (1 s cap), so the relay gets its close echoed, as RFC 6455 5.5.1 asks. - tests/http.rs checks that a relay that never finishes the TLS handshake gets upstream_connect (the part 5 fix). - tests/logs.rs installs the global subscriber through log_filter("trace") as main.rs does, `log` bridge included, and checks that a dependency's `log` record arrives. Without the tungstenite pins it now fails. Signed-off-by: QuicksilverSlick Co-Authored-By: Claude Opus 5.5 --- crates/buzz-relay-gateway/README.md | 2 +- crates/buzz-relay-gateway/src/relay_ws.rs | 16 ++++++++++--- crates/buzz-relay-gateway/tests/http.rs | 23 +++++++++++++++++- crates/buzz-relay-gateway/tests/logs.rs | 16 +++++++------ crates/buzz-relay-gateway/tests/ws.rs | 29 +++++++++++++++++++++++ 5 files changed, 74 insertions(+), 12 deletions(-) diff --git a/crates/buzz-relay-gateway/README.md b/crates/buzz-relay-gateway/README.md index d44e351de14..69a839fd8ad 100644 --- a/crates/buzz-relay-gateway/README.md +++ b/crates/buzz-relay-gateway/README.md @@ -37,7 +37,7 @@ limits in `src/config.rs`, and the link close codes in `src/link.rs`. | `BUZZ_RELAY_GATEWAY_RELAY_URL` | yes | `wss://host[:port]`; `ws://` only on loopback | | `BUZZ_RELAY_GATEWAY_LINK_URL` | yes | the `u` every link bearer must carry | | `BUZZ_RELAY_GATEWAY_LINK_PUBKEY` | yes | the gateway-auth public key, 64 lowercase hex | -| `RUST_LOG` | no | `info` | +| `RUST_LOG` | no | `info`; `tungstenite` and `tokio_tungstenite` stay off at any level, since at trace they print whole frames | `GET /healthz` answers `ok`, or 503 once shutdown has begun. SIGTERM or Ctrl-C drains in-flight requests for up to 10 s, closes every relay socket, diff --git a/crates/buzz-relay-gateway/src/relay_ws.rs b/crates/buzz-relay-gateway/src/relay_ws.rs index 6be7e13c08b..641a9d091de 100644 --- a/crates/buzz-relay-gateway/src/relay_ws.rs +++ b/crates/buzz-relay-gateway/src/relay_ws.rs @@ -147,6 +147,8 @@ impl Socket { } Some(Ok(Message::Close(frame))) => { let (code, reason) = frame.map_or((None, String::new()), |f| (Some(f.code.into()), f.reason.as_str().to_owned())); + // tungstenite only queues the close reply; flushing sends it. + let _ = timeout(CLOSE_CAP, ws.flush()).await; return Closed { by: Side::Relay, code, reason, http_status: None }; } Some(Ok(_)) => continue, @@ -162,10 +164,18 @@ impl Socket { tracing::info!(link_id, sid, auth_reply_ms = ms(at), "worker answered AUTH"); challenge = None; } - if ws.send(Message::Text(text.into())).await.is_err() { - return Closed { by: Side::Relay, code: None, reason: "connection_lost".into(), http_status: None }; + // A relay that stops reading stalls this write, so the socket's other ends still apply. + tokio::select! { + sent = ws.send(Message::Text(text.into())) => { + if sent.is_err() { + return Closed { by: Side::Relay, code: None, reason: "connection_lost".into(), http_status: None }; + } + continue; + } + _ = self.overflow.cancelled() => (1000, "backpressure"), + _ = self.link_closed.cancelled() => self.end(), + _ = sleep_until(silent_at) => (1001, "relay_silent"), } - continue; } Some(Cmd::Close(code, reason)) => { let frame = CloseFrame { code: code.into(), reason: reason.clone().into() }; diff --git a/crates/buzz-relay-gateway/tests/http.rs b/crates/buzz-relay-gateway/tests/http.rs index 8bf16699d10..219a0486e63 100644 --- a/crates/buzz-relay-gateway/tests/http.rs +++ b/crates/buzz-relay-gateway/tests/http.rs @@ -4,7 +4,7 @@ use std::time::Duration; use base64::{engine::general_purpose::STANDARD, Engine as _}; use buzz_relay_gateway::Limits; -use common::{body_of, limits, setup, sha256_hex, Mode}; +use common::{body_of, limits, setup, sha256_hex, Gateway, Mode}; use nostr::{EventBuilder, Keys, Kind, Tag}; use serde_json::{json, Value}; @@ -203,6 +203,27 @@ async fn in_flight_cap_duplicates_and_timeouts() { assert_eq!(w.post("after", "/query", b"[]").await["status"], 200); } +#[tokio::test] +async fn a_handshake_that_never_ends_is_upstream_connect() { + // Takes the TCP connection and never answers, so the TLS handshake stalls. + let silent = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = silent.local_addr().unwrap(); + tokio::spawn(async move { + let mut held = Vec::new(); + while let Ok((tcp, _)) = silent.accept().await { + held.push(tcp); + } + }); + let quick = Limits { + connect_timeout: Duration::from_millis(300), + ..limits() + }; + let gw = Gateway::start(&format!("wss://{addr}"), quick).await; + let mut w = gw.worker().await; + let res = w.post("c", "/query", b"[]").await; + assert_eq!(res["code"], "upstream_connect", "{res}"); +} + #[tokio::test] async fn shutdown_drains_in_flight_requests() { let (relay, gw, mut w) = setup(limits()).await; diff --git a/crates/buzz-relay-gateway/tests/logs.rs b/crates/buzz-relay-gateway/tests/logs.rs index 56533745aa0..f3558172d63 100644 --- a/crates/buzz-relay-gateway/tests/logs.rs +++ b/crates/buzz-relay-gateway/tests/logs.rs @@ -1,5 +1,6 @@ -//! Its own test binary: tracing caches callsite interest process-wide, so a -//! scoped subscriber misses events while other tests run on other threads. +//! Its own test binary, because it installs the global subscriber the way +//! main.rs does: fmt().init() also bridges `log` records, which is how +//! tungstenite and reqwest log. mod common; use std::{ @@ -24,18 +25,17 @@ impl Write for Capture { } } -// current_thread (the tokio::test default) keeps every task under this thread's subscriber. #[tokio::test] async fn logs_never_carry_bearers_bodies_or_frames() { const MARK: &str = "MARK7f3a"; let capture = Capture::default(); let writer = capture.clone(); - let subscriber = tracing_subscriber::fmt() + // RUST_LOG=trace: the most anyone can turn on. + tracing_subscriber::fmt() .json() - .with_max_level(tracing::Level::TRACE) + .with_env_filter(buzz_relay_gateway::log_filter("trace")) .with_writer(move || writer.clone()) - .finish(); - let _guard = tracing::subscriber::set_default(subscriber); + .init(); let relay = Relay::start().await; let gw = Gateway::start(&relay.ws_url(), limits()).await; @@ -65,6 +65,8 @@ async fn logs_never_carry_bearers_bodies_or_frames() { let logs = String::from_utf8(capture.0.lock().unwrap().clone()).unwrap(); assert!(logs.contains("auth_reply_ms"), "the AUTH timing was logged"); + // reqwest logs through `log`, so this line shows the bridge is live. + assert!(logs.contains("starting new connection"), "no `log` records"); assert!(!logs.contains(MARK), "marker leaked into logs"); assert!(!logs.contains("Nostr "), "a bearer leaked into logs"); } diff --git a/crates/buzz-relay-gateway/tests/ws.rs b/crates/buzz-relay-gateway/tests/ws.rs index ea542a0e076..71f246e1c5a 100644 --- a/crates/buzz-relay-gateway/tests/ws.rs +++ b/crates/buzz-relay-gateway/tests/ws.rs @@ -46,6 +46,11 @@ async fn closes_reach_the_other_side() { w.frame().await, json!({"type":"ws_closed","sid":"r","by":"relay","code":1012,"reason":"relay restarting"}) ); + // RFC 6455 5.5.1: the relay gets its close echoed back. + wait_for("the close echo", || { + relay.socket_closes().contains(&Some(1012)) + }) + .await; w.open("w").await; w.send(json!({"type":"ws_close","sid":"w","code":4000,"reason":"done"})) .await; @@ -136,6 +141,30 @@ async fn relay_side_limits_close_one_socket() { assert_eq!(w.frame().await["reason"], "relay_silent"); } +#[tokio::test] +async fn a_relay_that_stops_reading_closes_that_socket() { + // A queue that never fills, so only the silence timer can end the socket. + let stalled = Limits { + relay_queue: 1024, + relay_silence: Duration::from_secs(5), + ..limits() + }; + let (_relay, _gw, mut w) = setup(stalled).await; + w.open("stuck").await; + // The mock relay stops reading and sending; the write stalls once the TCP buffers fill. + w.ws_send("stuck", r#"["SILENT"]"#).await; + let big = "x".repeat(256 << 10); + for _ in 0..100 { + w.ws_send("stuck", &big).await; + } + let closed = w.frame().await; + assert_eq!( + (closed["type"].as_str(), closed["reason"].as_str()), + (Some("ws_closed"), Some("relay_silent")), + "{closed}" + ); +} + #[tokio::test] async fn a_worker_that_stops_reading_loses_the_link() { let slow = Limits { From 6477e7a6e6a93c8ac72b6701a1bdc123b0335f51 Mon Sep 17 00:00:00 2001 From: QuicksilverSlick Date: Tue, 27 Oct 2026 09:36:27 -0500 Subject: [PATCH 14/15] ci(relay-gateway): run the gateway tests in just test-unit Workspace membership only buys clippy and check; the unit lane lists each crate it tests. nextest gives each test its own process, which tests/logs.rs needs for its global subscriber. Signed-off-by: QuicksilverSlick Co-Authored-By: Claude Opus 5.5 --- Justfile | 3 +++ crates/buzz-relay-gateway/README.md | 3 +-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Justfile b/Justfile index 312ec115b97..21c1fcea7ee 100644 --- a/Justfile +++ b/Justfile @@ -462,6 +462,9 @@ test-unit: # contract/race tests run in the dedicated CI job below. cargo nextest run -p buzz-push-gateway cargo nextest run -p buzz-push-gateway --features personal-dev-app-attest + # Relay gateway (Bishop's keyless relay forwarder): infra-free tests + # against a loopback mock relay and Worker. + cargo nextest run -p buzz-relay-gateway # Kubernetes backend provider: the decision layers (state machine, GC # planner, env precedence, naming, wire) are pure functions with a fake # substrate, so they belong in the unit job. Enumerated explicitly diff --git a/crates/buzz-relay-gateway/README.md b/crates/buzz-relay-gateway/README.md index 69a839fd8ad..f96f6004264 100644 --- a/crates/buzz-relay-gateway/README.md +++ b/crates/buzz-relay-gateway/README.md @@ -51,8 +51,7 @@ cargo clippy -p buzz-relay-gateway --all-targets -- -D warnings ``` The tests run offline against an in-process mock relay (HTTP routes and a -NIP-42 socket) and a mock Worker. They are not yet in the `Justfile` unit-test -list, so CI does not run them; run them locally on every change. +NIP-42 socket) and a mock Worker. CI runs them through `just test-unit`. The image builds from the repo root. Pushing it is a deployment step, not a build step: From d23f1a81fc37c9353a54c8269521432c2f3fd6d5 Mon Sep 17 00:00:00 2001 From: QuicksilverSlick Date: Fri, 30 Oct 2026 05:31:22 -0500 Subject: [PATCH 15/15] ci(relay-gateway): run the gateway tests from _ci-rust.yml, not the Justfile Editing the Justfile matches CI's desktop path filter, so this PR kept pulling in the desktop e2e suite and its flakes. The Unit Tests job now runs the same `cargo nextest run -p buzz-relay-gateway` as its own step, and the PR no longer touches the Justfile. Co-Authored-By: Claude Opus 5.5 Signed-off-by: QuicksilverSlick --- .github/workflows/_ci-rust.yml | 6 ++++++ Justfile | 3 --- crates/buzz-relay-gateway/README.md | 2 +- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/_ci-rust.yml b/.github/workflows/_ci-rust.yml index e25639d3ac7..a09ca691b71 100644 --- a/.github/workflows/_ci-rust.yml +++ b/.github/workflows/_ci-rust.yml @@ -66,6 +66,12 @@ jobs: tool: cargo-nextest@0.9.136 - name: Unit tests run: just test-unit + # Relay gateway (Bishop's keyless relay forwarder): infra-free tests + # against a loopback mock relay and Worker. Kept out of the Justfile so + # gateway PRs do not select the desktop suite. nextest gives each test + # its own process, which tests/logs.rs needs for its global subscriber. + - name: Unit tests (buzz-relay-gateway) + run: cargo nextest run -p buzz-relay-gateway server-cross-compile: name: Server Cross-Compile diff --git a/Justfile b/Justfile index 21c1fcea7ee..312ec115b97 100644 --- a/Justfile +++ b/Justfile @@ -462,9 +462,6 @@ test-unit: # contract/race tests run in the dedicated CI job below. cargo nextest run -p buzz-push-gateway cargo nextest run -p buzz-push-gateway --features personal-dev-app-attest - # Relay gateway (Bishop's keyless relay forwarder): infra-free tests - # against a loopback mock relay and Worker. - cargo nextest run -p buzz-relay-gateway # Kubernetes backend provider: the decision layers (state machine, GC # planner, env precedence, naming, wire) are pure functions with a fake # substrate, so they belong in the unit job. Enumerated explicitly diff --git a/crates/buzz-relay-gateway/README.md b/crates/buzz-relay-gateway/README.md index f96f6004264..7e03649a3db 100644 --- a/crates/buzz-relay-gateway/README.md +++ b/crates/buzz-relay-gateway/README.md @@ -51,7 +51,7 @@ cargo clippy -p buzz-relay-gateway --all-targets -- -D warnings ``` The tests run offline against an in-process mock relay (HTTP routes and a -NIP-42 socket) and a mock Worker. CI runs them through `just test-unit`. +NIP-42 socket) and a mock Worker. CI runs them in the Unit Tests job (`.github/workflows/_ci-rust.yml`). The image builds from the repo root. Pushing it is a deployment step, not a build step: