diff --git a/Cargo.lock b/Cargo.lock index 248d9970..96598a89 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -27,6 +27,16 @@ dependencies = [ "generic-array", ] +[[package]] +name = "aead" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1973cfbc1a2daf9cf550e74e1f088c28e7f7d8c1e1418fb6c9dc5184b7e84c99" +dependencies = [ + "crypto-common 0.2.2", + "inout 0.2.5", +] + [[package]] name = "aes" version = "0.8.4" @@ -34,7 +44,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" dependencies = [ "cfg-if", - "cipher", + "cipher 0.4.4", "cpufeatures 0.2.17", ] @@ -44,9 +54,9 @@ version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "831010a0f742e1209b3bcea8fab6a8e149051ba6099432c8cb2cc117dec3ead1" dependencies = [ - "aead", + "aead 0.5.2", "aes", - "cipher", + "cipher 0.4.4", "ctr", "ghash", "subtle", @@ -895,8 +905,7 @@ dependencies = [ [[package]] name = "ant-protocol" version = "5.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d95baa1de35f7183806372d59baa34b93c2de8f28c76c664f3225008ed17317" +source = "git+https://github.com/grumbach/ant-protocol?branch=feat%2Fpeer-swarm#cbfe49a4e2cb5f14ef04e642071cb4983cd49897" dependencies = [ "blake3", "bytes", @@ -1316,18 +1325,6 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" -[[package]] -name = "attohttpc" -version = "0.30.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16e2cdb6d5ed835199484bb92bb8b3edd526effe995c61732580439c1a67e2e9" -dependencies = [ - "base64 0.22.1", - "http", - "log", - "url", -] - [[package]] name = "auto_impl" version = "1.3.0" @@ -1682,7 +1679,7 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6" dependencies = [ - "cipher", + "cipher 0.4.4", ] [[package]] @@ -1703,8 +1700,8 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ae3c82e4355234767756212c570e29833699ab63e6ffd161887314cc5b43847" dependencies = [ - "aead", - "cipher", + "aead 0.5.2", + "cipher 0.4.4", "ctr", "subtle", ] @@ -1728,7 +1725,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3613f74bd2eac03dad61bd53dbe620703d4371614fe0bc3b9f04dd36fe4e818" dependencies = [ "cfg-if", - "cipher", + "cipher 0.4.4", "cpufeatures 0.2.17", ] @@ -1739,6 +1736,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" dependencies = [ "cfg-if", + "cipher 0.5.2", "cpufeatures 0.3.1", "rand_core 0.10.1", ] @@ -1749,13 +1747,25 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "10cd79432192d1c0f4e1a0fef9527696cc039165d729fb41b3f4f4f354c2dc35" dependencies = [ - "aead", + "aead 0.5.2", "chacha20 0.9.1", - "cipher", - "poly1305", + "cipher 0.4.4", + "poly1305 0.8.0", "zeroize", ] +[[package]] +name = "chacha20poly1305" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b89e1c441e926b9c82a8d023f6e1b7ae0adcfaa7d621814e4d60789bac751cb" +dependencies = [ + "aead 0.6.1", + "chacha20 0.10.2", + "cipher 0.5.2", + "poly1305 0.9.1", +] + [[package]] name = "chrono" version = "0.4.45" @@ -1777,10 +1787,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" dependencies = [ "crypto-common 0.1.7", - "inout", + "inout 0.1.4", "zeroize", ] +[[package]] +name = "cipher" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8cf2a2c93cd704877c0858356ed03480ff301ee950b43f1cbe4573b088bfa6c" +dependencies = [ + "block-buffer 0.12.1", + "crypto-common 0.2.2", + "inout 0.2.5", +] + [[package]] name = "clap" version = "4.6.6" @@ -1830,6 +1851,12 @@ dependencies = [ "cc", ] +[[package]] +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" + [[package]] name = "cobs" version = "0.3.0" @@ -2093,7 +2120,16 @@ version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835" dependencies = [ - "cipher", + "cipher 0.4.4", +] + +[[package]] +name = "ctutils" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe3e9d21d39e6f39ff46e03d5b621424f88ee23d8a1672c6bfc9ccc7f2da4f6a" +dependencies = [ + "cmov", ] [[package]] @@ -2339,6 +2375,15 @@ dependencies = [ "dirs-sys 0.5.0", ] +[[package]] +name = "dirs" +version = "7.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d57d423b3c82e89b9a24ca3091fee61f456a26edbd28d26c65906f4bc1dcd8f" +dependencies = [ + "dirs-sys 0.5.0", +] + [[package]] name = "dirs-sys" version = "0.4.1" @@ -2406,7 +2451,7 @@ dependencies = [ "byteorder", "cbc", "ccm", - "chacha20poly1305", + "chacha20poly1305 0.10.1", "der-parser 9.0.0", "hmac", "log", @@ -2981,9 +3026,9 @@ dependencies = [ [[package]] name = "granit-parser" -version = "1.3.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e20f99e46474f56bd905c56e817ebddcf377a611f94c53ac4649e4d3fa3c0cd0" +checksum = "da55feca5d1625466bae068f9fd4b87892caeae70c3fe671ddb0f50d41aebf37" dependencies = [ "arraydeque", "smallvec", @@ -3411,11 +3456,10 @@ dependencies = [ [[package]] name = "igd-next" -version = "0.17.1" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de7238d487a9aff61f81b5ab41c0a841532a115a398b5fa92a2fadd0885e2581" +checksum = "c71312255fdc5364bff7eec0adf68146bc4e72aba85075261af40d234c11be99" dependencies = [ - "attohttpc", "bytes", "futures", "http", @@ -3424,8 +3468,10 @@ dependencies = [ "hyper-util", "log", "rand 0.10.2", + "thiserror 2.0.20", "tokio", "url", + "xml-rs", "xmltree", ] @@ -3488,6 +3534,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "inout" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b5cf4372d022092674a83b8febf5865f7a573cc0c5baec5e49c879b6a20978f" +dependencies = [ + "hybrid-array", +] + [[package]] name = "interceptor" version = "0.15.0" @@ -4450,7 +4505,17 @@ checksum = "8159bd90725d2df49889a078b54f4f79e87f1f8a8444194cdca81d38f5393abf" dependencies = [ "cpufeatures 0.2.17", "opaque-debug", - "universal-hash", + "universal-hash 0.5.1", +] + +[[package]] +name = "poly1305" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e2d0073b297041425c7c3df6eb4792d598a15323fe63346852b092eca02904c" +dependencies = [ + "cpufeatures 0.3.1", + "universal-hash 0.6.1", ] [[package]] @@ -4462,7 +4527,7 @@ dependencies = [ "cfg-if", "cpufeatures 0.2.17", "opaque-debug", - "universal-hash", + "universal-hash 0.5.1", ] [[package]] @@ -5278,15 +5343,14 @@ dependencies = [ [[package]] name = "saorsa-core" version = "0.29.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49de4f92b6808afd54a206d7dcb6cf754f796ee876394d6838b79b39dd1a72dd" +source = "git+https://github.com/grumbach/saorsa-core?branch=feat%2Fpeer-swarm#990af9af0d5a0abe9c91a223c15b2ea7254043ee" dependencies = [ "anyhow", "async-trait", "blake3", "bytes", "dashmap", - "dirs", + "dirs 6.0.0", "futures", "hex", "libc", @@ -5314,12 +5378,12 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "211578bef9588f0f57162139f1c214bcc6207fb632e9368040d52e4e36471938" dependencies = [ - "aead", + "aead 0.5.2", "aes-gcm", "anyhow", "blake3", "bytes", - "chacha20poly1305", + "chacha20poly1305 0.10.1", "fips203", "fips204", "fips205", @@ -5347,20 +5411,19 @@ dependencies = [ [[package]] name = "saorsa-transport" version = "0.38.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd334b3d02ba2d0bcac8a30d07c3e79004fc504a193813aaa7bf89c11f0f44d0" +source = "git+https://github.com/grumbach/saorsa-transport?branch=feat%2Fpeer-swarm#a1664feb3fb3743a3b9336192f24b8c2e80e4de0" dependencies = [ "anyhow", "async-trait", "aws-lc-rs", - "base64 0.22.1", + "base64 0.23.1", "blake3", "bytes", - "chacha20poly1305", + "chacha20poly1305 0.11.0", "chrono", "clap", "dashmap", - "dirs", + "dirs 7.0.0", "enum_dispatch", "futures-util", "hex", @@ -5392,7 +5455,7 @@ dependencies = [ "serde_json", "slab", "socket2 0.6.5", - "stun 0.7.0", + "stun 0.17.2", "thiserror 2.0.20", "time", "tinyvec", @@ -5561,7 +5624,7 @@ dependencies = [ "blake3", "brotli", "bytes", - "chacha20poly1305", + "chacha20poly1305 0.10.1", "hex", "rand 0.8.8", "rand_chacha 0.3.1", @@ -5609,9 +5672,9 @@ dependencies = [ [[package]] name = "serde-saphyr" -version = "1.3.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8050abb251097357e24aff63ba2c52a6309ecb7d23a5474023df960a02694d8" +checksum = "a253edfdea34032da21af84e16fcc94f4c7a6ba237f9f3bc2dbf43cea8535c44" dependencies = [ "annotate-snippets", "encoding_rs_io", @@ -5948,28 +6011,28 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] name = "stun" -version = "0.7.0" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea256fb46a13f9204e9dee9982997b2c3097db175a9fddaa8350310d03c4d5a3" +checksum = "1a512c5d501e3e3b5a4bb3e8e31462d56d54a66b95a28b8596e14422bf21c32b" dependencies = [ "base64 0.22.1", "crc", "lazy_static", "md-5", - "rand 0.8.8", + "rand 0.9.5", "ring", "subtle", "thiserror 1.0.69", "tokio", "url", - "webrtc-util 0.10.0", + "webrtc-util 0.12.0", ] [[package]] name = "stun" -version = "0.9.0" +version = "0.17.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a512c5d501e3e3b5a4bb3e8e31462d56d54a66b95a28b8596e14422bf21c32b" +checksum = "2e3b8d5ad1dd4c8cd0595440f26521a71dd593cb5873ee8e06b91510b7d97269" dependencies = [ "base64 0.22.1", "crc", @@ -5981,7 +6044,7 @@ dependencies = [ "thiserror 1.0.69", "tokio", "url", - "webrtc-util 0.12.0", + "webrtc-util 0.17.2", ] [[package]] @@ -6627,6 +6690,16 @@ dependencies = [ "subtle", ] +[[package]] +name = "universal-hash" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4987bdc12753382e0bec4a65c50738ffaabc998b9cdd1f952fb5f39b0048a96" +dependencies = [ + "crypto-common 0.2.2", + "ctutils", +] + [[package]] name = "untrusted" version = "0.9.0" @@ -6965,7 +7038,7 @@ version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "66d8cdc33413f1d0192670a80ce93d17cb78d57fe3a2414be30d6f6dff121123" dependencies = [ - "aead", + "aead 0.5.2", "aes", "aes-gcm", "byteorder", @@ -6984,20 +7057,19 @@ dependencies = [ [[package]] name = "webrtc-util" -version = "0.10.0" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1438a8fd0d69c5775afb4a71470af92242dbd04059c61895163aa3c1ef933375" +checksum = "d1c0c7e0c8f280f2bbfae442701465777ac07adaf46ce0c5863cd58e13fe472a" dependencies = [ "async-trait", "bitflags 1.3.2", "bytes", "ipnet", "lazy_static", - "libc", "log", "nix 0.26.4", "portable-atomic", - "rand 0.8.8", + "rand 0.9.5", "thiserror 1.0.69", "tokio", "winapi", @@ -7005,9 +7077,9 @@ dependencies = [ [[package]] name = "webrtc-util" -version = "0.12.0" +version = "0.17.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1c0c7e0c8f280f2bbfae442701465777ac07adaf46ce0c5863cd58e13fe472a" +checksum = "1beae0b4f24969741c26ca282a257dc5823bd9cbe608f7e3ca3775102d323ad9" dependencies = [ "async-trait", "bitflags 1.3.2", @@ -7055,7 +7127,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index aa249a27..a8b297c9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -242,3 +242,10 @@ cognitive_complexity = "allow" # Allow non-const functions during initial development (may need runtime features later) missing_const_for_fn = "allow" +# The swarm's wire types (ant-protocol #50), the send on one named channel +# (saorsa-core #183) and the plain STUN answers (saorsa-transport #231) are on +# these branches until they merge. +[patch.crates-io] +ant-protocol = { git = "https://github.com/grumbach/ant-protocol", branch = "feat/peer-swarm" } +saorsa-core = { git = "https://github.com/grumbach/saorsa-core", branch = "feat/peer-swarm" } +saorsa-transport = { git = "https://github.com/grumbach/saorsa-transport", branch = "feat/peer-swarm" } diff --git a/docs/adr/ADR-0026-clients-share-records-nodes-introduce-them.md b/docs/adr/ADR-0026-clients-share-records-nodes-introduce-them.md new file mode 100644 index 00000000..ccbaf741 --- /dev/null +++ b/docs/adr/ADR-0026-clients-share-records-nodes-introduce-them.md @@ -0,0 +1,748 @@ +# ADR-0026: Clients reading the same file share its records; nodes introduce them and stay the last seed + +- **Status:** Proposed +- **Date:** 2026-10-27 +- **Decision owners:** +- **Reviewers:** +- **Supersedes:** none +- **Superseded by:** none +- **Related:** [ADR-0015](./ADR-0015-direct-browser-clients-over-webrtc-direct.md) + (browser sessions over WebRTC Direct, whose UDP socket also answers STUN + here), [ADR-0023](./ADR-0023-spread-hot-reads-and-refuse-full-browser-sessions.md) + (option 4, "browser peer-to-peer sharing", deferred to its own ADR: this + one), [RFC 8489](https://www.rfc-editor.org/rfc/rfc8489) (STUN), + [RFC 8445](https://www.rfc-editor.org/rfc/rfc8445) (ICE), + [RFC 8831](https://www.rfc-editor.org/rfc/rfc8831) (WebRTC data channels) +- **Implementation:** ant-protocol (swarm messages), saorsa-transport (plain + STUN Binding answers on the WebRTC Direct socket), saorsa-core (a send on + one named connection only), ant-node (the swarm rendezvous), ant-client + (the swarm in every client, native and browser) + +## Context + +Every reader of a file asks the same close group for the same records. ADR-0023 +spread hot reads across the group and made full browser sessions refuse +cleanly, but the group's capacity is still the ceiling: a node admits a bounded +number of browser sessions and bulk replies, so a popular file (a film many +people watch at once, a release everyone fetches) is served at the rate seven +nodes can serve it, however many readers there are. + +Readers of one file hold exactly the records the next reader needs. Records are +content addressed, every client checks each record against its address +(`ant-core/src/record.rs`), and public records are byte-identical for every +reader. So a record from another reader is as good as one from a node, once it +passes the check. BitTorrent peers share pieces the same way. What clients lack +is a way to find each other and connect: + +- Browsers cannot accept connections; two browsers connect only through WebRTC, + which needs a signalling channel to swap an offer and an answer, and a STUN + server to learn the public address the other side can reach. +- Native clients run in `NodeMode::Client`, are not DHT members, and are + usually behind NAT too. +- The network has no third-party service to lean on. A WebTorrent tracker or a + public STUN server would learn which addresses read which file, and would be + a single point of failure the network does not otherwise have. + +The nodes closest to a file are already the parties every reader talks to. They +can introduce readers to each other on the connections those readers already +have, and answer their STUN requests on the WebRTC Direct UDP socket they +already run. + +## Decision Drivers + +- Load on the close group should grow with the data, not with the number of + readers. +- No new binary and no third-party service: nodes and clients only. +- Every client takes part, browser and native alike, always: a swarm nobody + joins serves nobody. +- A peer can never make a client accept wrong bytes, and a slow or hostile + peer can only cost a short wait before the nodes are asked. +- A node's cost is small and bounded by hard limits, with no switch to turn it + off or tune it. +- A release that adds this must not break older clients or older nodes, in + either order of rollout. + +## Considered Options + +1. **A separate signalling service** next to the nodes (built as `ant-signal` + and reviewed, then dropped). It works, but it is one more binary to run and + secure, someone has to run it, and readers would depend on it. +2. **Third-party WebTorrent trackers and public STUN servers.** Free to use, + but they see who reads what, and the network would stop working the way it + should whenever they do. +3. **Peer records in the DHT** ("I hold file X, reach me at Y"). Clients are not + DHT members, browsers cannot be reached at an address, and two clients would + still need someone to carry their offer and answer. +4. **Nodes relay the data** (a TURN-like role). Every byte would still cross a + node, so nothing is offloaded. +5. **Nodes introduce readers and answer their STUN; readers share records over + WebRTC data channels; nodes stay the last seed** (chosen). + +## Decision + +We adopt option 5. + +### Rooms + +The readers of one file form a *room*. Its key is the address the file's +root DataMap would have as a public record: `compute_address(rmp_serde(root))`. +Every reader derives the same key whichever way it reached the file (by public +address, through a manifest, which may hold a shrunk or a root map, or from a +private map), once it has resolved a shrunk map to its root. For a file whose +public map is its root, the key is the file's public address. A reader +announces itself at the `SWARM_RENDEZVOUS_NODES` (2) nodes closest to the key +that host rooms, so two readers whose views of the close group differ +slightly still share one, and readers still meet while some of the closest +nodes predate rooms: it looks up the 8 nodes closest to the key and passes +over those that host none (see Older nodes). A node hosts rooms only once +browsers can reach it: until its browser listener has an endpoint, and so a +STUN server to name, it refuses every announce as `NotHosting`, and a client +passes it over at once, with no pause. Native readers so pass over the same +nodes browsers never see (a relay-only node, or one +built without the listener), and meet the browsers at the closest nodes +both can reach, rather than at nodes no browser comes to and that give them +no STUN. + +### Wire (ant-protocol, `src/swarm.rs`) + +Six `ChunkMessageBody` variants are appended after the pointer messages, at +discriminants 14 to 19; existing variants keep their discriminants. + +| Request | Reply | Purpose | +|---|---|---| +| `SwarmAnnounceRequest { room, node, membership, offers, wait_ms }` | `SwarmAnnounceResponse::{Joined, Refused, Error}` | Join or stay in a room, hand over WebRTC offers, collect signals | +| `SwarmAnswerRequest { room, node, to, offer_id, sdp }` | `SwarmAnswerResponse::{Delivered, Unmatched, Error}` | Answer an offer the node delivered | +| `SwarmLeaveRequest { room, node, membership }` | `SwarmLeaveResponse::{Left, Error}` | Leave | + +`node` is the peer id of the rendezvous node the request is for. +`membership` names one of the client's memberships of the room: a number that +grows each time the client joins the room anew. A node compares only the +numbers of one connection identity (a native client's peer id, made afresh +each run, or one browser connection). It keeps each connection's newest +membership of a room: an announce naming a newer one starts a member in place +of the old (whose signals and offers go with it), and a leave ends the +membership it names, or an older one in its place. Admitting a membership +marks every older one of that connection in that room as replaced. A +membership the client ended (left, or replaced by a newer one) is remembered for twice the +10-minute signed-time window and 2 s more, as long as a request sent before +it ended can still be taken (one per connection and room however often it +ends a newer one; at most 64 per connection, 256 per host and 16384 in all, +past which the connection's, host's or node's oldest goes, so the newest +ends are the ones kept; a forgotten one can come back only as a member that +is dropped once silent): an announce naming it, or an older one, is +refused +(`SwarmRefusal::Superseded`), even one that arrives after its own leave. So a +request of a membership the client left, however late it arrives, cannot end +or take over a newer one, nor bring the old one back. A membership dropped +because its connection closed or it went silent is not ended: the client +carries on in it over a new connection. A waiting announce whose member was +replaced or dropped meanwhile is answered `Superseded`, not `Joined`. + +`Joined` carries the member's id, the room's size, how many offers went out, +the signals waiting for the member (offers from others, answers to its own), +and the node's STUN URL (`stun:`). + +Limits both sides check: at most 8 offers per announce, 16 KiB per SDP, 32 +signals and 48 KiB of SDP per reply (inside the browser transport's 64 KiB +small-response bound, asserted at compile time), at most 4 STUN URLs of at +most 256 bytes, and a wait of at most 20 s. `MAX_SWARM_REQUEST_BYTES` bounds +an encoded request, so a node refuses a longer one from its length before +decoding it, and a client validates every reply before acting on it, +announce, answer and leave alike, an error's text included (at most 512 +bytes). + +**Candidates.** An offer or answer is another client's, and its receiver +sends ICE connectivity checks to every candidate in it before anything proves +who made it. Both sides therefore pass each SDP through the protocol's +`constrain_swarm_sdp`: UDP candidates only, naming an IP address that is not +unspecified, multicast or broadcast, or an mDNS `.local` name, at most 16. A +node relaying an SDP also drops every candidate whose address is public and +not the sender's own host, as the node reaches it on that request (an IPv4 +address, or an IPv6 /64). Local addresses and mDNS names stay only where the +node and the client reach each other at local addresses (a network of their +own, such as a test network or a LAN deployment): the node keeps them only +for a sender it reaches at a local address, and a client applies them only +from a node it reached at one, loopback ones only over loopback. On a network +reached at public addresses, readers meet by their public (server-reflexive) +addresses alone, so two readers behind one NAT need it to hairpin. A stranger +in a room therefore cannot point other readers' checks at a third party, nor +at the readers' own home networks, and a hostile node can point them at 16 +addresses at most per SDP. Receivers read SDP leniently (webrtc-rs trims an +attribute's value and skips a carriage return before a line, Chrome takes an +attribute name followed by a space, Firefox names attributes in any case), so +a line any of them may read as a candidate or a DTLS fingerprint stays only +in its one standard form (`a=candidate:` or `a=fingerprint:`, then a value +with no whitespace around it), and a line with a carriage return anywhere but +before its line feed is dropped: what a receiver reads is exactly what was +checked. The filtered SDP holds no more memory than its length, and the node +charges a stored SDP the memory it holds. Honest SDPs only lose candidates, +so a link's binding to both ends' fingerprints does not change. + +**Decoding.** The offers, inbox and STUN server lists are refused while +decoding once past their limits, so a request or reply of a legal length +cannot make its receiver build thousands of entries before `validate` runs; +and a message with bytes after its end is refused, so a small reply cannot be +padded out to the wire limit. Every enum in +the module is encoded by variant position; a test pins each tag, and variants +may only be appended. + +### The node's part (ant-node, `src/swarm.rs`) + +- **Identity.** The node names each member itself, from the authenticated + connection (a native peer id, or the number of a browser connection on this + node). A browser's session is its connection. A native client's request is + signed by its key but not bound to the connection it came on. So every + request names its rendezvous node, and a node refuses one naming another: + a request one node saw cannot be replayed at the room's other node. A node + also drops a native request whose signed time is more than 10 minutes from + its own clock, more than 5 s older than that client's newest, or whose id + it has seen, so one cannot be replayed at the same node either. A request + naming another node is answered with an error before it touches that + history, so a node that saw a client's request cannot send it on to make + the client's later requests elsewhere look stale. Only an + announce or a leave starts a client's history (an answer comes from a + member, which announced first; a leave may overtake the announce it ends); + a host keeps at most 256 histories, 16384 in all, + the oldest made forgotten first, and a history goes once nothing it holds + could pass the 10-minute check. +- **Introductions.** Each offer goes to a different other member, picked at + random. An answer is passed on only to the member whose offer the node + delivered to the answerer, once, before it expires. Unanswered offers, and + signals waiting for their member, expire after the protocol's + `SWARM_SIGNAL_TTL_MS` (70 s), which a client checks at compile time against + the slowest answer any client gives: 11 s to its next announce (a + browser's 3 s pause and one 8 s step to make that round's offers), 30 s + for an announce that may already be on its way (a native one may wait + 20 s, with 10 s for the request and reply), 8 s to make the answer and + 10 s for the answer's request. An offering client keeps its offer for its + own announce's time on each way, the poll gap and the node's 70 s: 141 s + natively, 101 s in a browser, whose announces do not wait. It drops + expired offers before and after every announce, and all of them before a + pause after a failure, so an unanswered offer holds its place in the + room's connection limits at most that long and one announce more. A link + a client answered has 79 s to open and be proven, since its answer still + has to reach the offerer; one it offered has 20 s once it has the answer. A member that stops + announcing is dropped after 60 s; a connection that closes leaves all its + rooms at once. +- **Long poll, native only.** A native client's announce with nothing waiting + may be held until a signal arrives, the wait ends, the member is dropped, or + the node stops; one announce per member waits, and no lock is held + meanwhile. Native swarm requests are routed apart from chunk requests, with + their own task budget, so a waiting announce never delays a GET. A browser's + announce is never held: it would keep one of the browser session's few + request slots, and its byte reservation, from the browser's reads. Browsers + ask again after 1 s while a signal is likely on its way (they just acted on + one, or made an offer within the last 10 s) and after 3 s otherwise: in a + room larger than its members' links can hold, most offers go unanswered, + and their makers wait at the slower pace. At a node's browser listener, + swarm requests run on a budget of their own (64 a second in all, 8 from one + source): every request takes a token from the budget the reads share as + it arrives, and a swarm request the swarm's budget has room for gives it + back, so announces never spend what reads need; one past the swarm's + budget keeps that token and is refused, without closing the channel the + reads use, and its maker backs off as from any refusal. A request that + arrives once the reads' budget is spent is taken on the swarm's instead, + since only the opened frame says whether it is a swarm request: one that + is goes on, and anything else gives the swarm its token back and is + refused as over the request rate, as before. Native swarm requests have a + budget of their own with the same figures (64 a second in all, 8 from one + host, an IPv4 address or an IPv6 /64, since a native client's identities + cost nothing to make), taken before the request is decoded or given a + task: one past it is dropped, and its client reads from nodes alone. The + node remembers at most 4096 hosts' budgets, forgetting first those that + have refilled, which leaves room for a new host however many came before. +- **Closing.** A connection that closes leaves all its rooms at once. No + request of it still being handled can act after the close (announce, answer + or leave), nor any request older than a minute. A member that goes takes + with it its signals still queued for others and the offers others made to + it, since none can be answered now. +- **Hard limits**, no configuration: 4096 rooms, 256 members per room, 16384 + members in all, 4 rooms per connection, 64 members per host (an IPv4 + address or an IPv6 /64, since a native client's identities cost nothing to + make), 32 unanswered offers per member, + 16 MiB of stored SDP (2 MiB for the offers and answers of one host's + members), 1024 waiting announces (more are answered at once), and + a fixed number of swarm tasks (more requests are dropped). +- **Every node.** A node hosts the rendezvous whether it stores chunks or + not, and devnets route native swarm requests the way nodes do. +- **Replies.** A reply to a native client whose connection closed meanwhile, + or from a node stopping, is dropped rather than sent, and a reply goes + only on a connection the client still has, so the node never dials a + client that left. A signed message is not bound to the connection it came + on, and saorsa-core maps a client's identity to every connection a message + signed by it arrived on: a node that saw a client's message could send it + on to another node, which would then take the sender's connection for the + client's too. So a native reply goes on the connection the request came + on and no other, through saorsa-core's new `send_message_on_channel` + (never another of the peer's connections, never a dial). Other protocols' + replies still go by peer id; moving them to the same call is a separate + change. Offers are handed on in order, so `offers_delivered` names exactly + the first ones, and the client drops the rest at once. +- **STUN.** The WebRTC Direct UDP socket also answers a plain STUN Binding + request (no USERNAME, no comprehension-required attribute, a valid + FINGERPRINT if present) with XOR-MAPPED-ADDRESS. The answer (32 bytes for a + 20-byte IPv4 request, 44 for IPv6) is larger than the request, and a UDP + source can be forged, so it is bounded three ways: only hosts (an IPv4 + address or an IPv6 /64, as the responder's own limits count them) that + announced in a room within the last 60 s are answered, so a victim's + address is answered only by the few nodes of its own rooms; each source (an + IPv4 address or an IPv6 /64) gets at most 10 answers a second (burst 20, + 4096 sources tracked); and the socket sends at most 2000 answers a second + in all, after a burst of at most 4000. A request counts against both + limits or neither: only requests that would be answered count, so one + source over its rate, or sending malformed requests, costs the others + nothing, and a request the socket's limit refuses costs its source + nothing. Every check runs on the raw bytes without allocating, the answer + is built only once both limits grant it, and a refused plain request is + dropped rather than passed to the connection handling that shares the + socket. A request with a comprehension-required + attribute is left alone rather than answered with `420 Unknown Attribute`: + no ICE agent sends one, and an error answer is one more reply a forged + source could draw. Policy lookups take a read lock, so the packet path + rarely waits on announces. +- **Capability.** A node advertises `swarm_rendezvous` in its browser HELLO. + +### The client's part (ant-client, `ant-core/src/swarm/`) + +Every client takes part, always, on both builds: a browser through its own +`RTCPeerConnection`, a native client through webrtc-rs (the stack +saorsa-transport's WebRTC Direct already uses). + +- **Joining.** Only a valid root map has a room (not a child map, one whose + records index, with at least the 3 records a file has), checked before + joining, so a bad map never takes a valid room's place; and only one with + at most 262,144 records (about 1 TiB of 4 MiB records), all of which a + room holds, so a room always shares its whole file. A larger file is read + from the nodes alone. Every read of a file joins its file's room as soon + as it has the root map (native `file_download*` and `data_download*`; + in a browser, opening a reader, a public or private download, and a + stream), and a browser reader joins again on a read if reading other + files has left the room since. Joining starts the room's own task and + returns at once: the other readers are found beside the reads from the + nodes, which never wait for that (see Reading), so a slow or failing + discovery leaves the reads from the nodes as they were. The small DataMap + records come from the nodes. A room joined by a read asks the nodes for + nothing (no lookup, no announce) until that read's first record of the file + is in, or for 30 s if none comes: its discovery so never competes with a + download's first record for the nodes, the client's connections or its + CPU. (With discovery started at the open, the first chunk came 0.3 to + 6.5 s later in 6 of 7 lab pairs whose nodes were the bottleneck.) A room + joined on request (`join_swarm`) starts at once. A client's read of a + file's record keeps the file's room in use for as long as it runs, however + long the nodes take. Joining past 4 rooms leaves the room unread longest, once no + read has used it for 30 s; with every room still read, the join waits (at + most 2 do) and is made once a room has gone unread, when the file is + joined again or a read of one of its records starts or ends, the file + meanwhile read from the nodes. A client + reading more files than that at once so cannot make every read a new + membership (a key, a lookup, links that never open), and a long download's + room is never left for a file opened beside it; at most 8 rooms left at a time tell their nodes so (each + leave bounded to 10 s in all), and one left past that leaves quietly, its + places there expiring after 60 s. A room left tells its nodes even if it + never saw its join confirmed (a reply can be lost). Two places are not + told: a room left quietly, and a node a rendezvous place stops using after + 3 failures or 10 minutes (the place may come back to it in the same + membership). Those memberships are not ended at the node; they go after + 60 s of silence, and a late announce of one can bring it back only until + it is silent that long again. A room joined again is a new membership: + its number is newer, so nothing the old one still sends can end it, and + links made for the old one are not kept by the new one. A room left or shut + down drops its record list at once. `Client::join_swarm` and the browser's + `joinSwarm` let an application join before reading, so the client meets + the others first. A page tells which file each link and count belongs to + from the channel's label, from the `Swarm room ` a browser read + reports when it resolves its file, and from per-room counts + (`swarm_room_stats`, `swarmRoomStats`). +- **One swarm per identity.** Rendezvous nodes know a native client's + memberships by its node identity, so clients built on one native node + share one swarm. A membership continued over a connection from another + address counts against that address from then on, if it has room. Each + browser client has its own sessions, and its own swarm. +- **Room-scoped records.** A room knows its file's records: those its root + map lists. A record is asked of peers, kept, announced and served only + within a room whose file holds it, so a peer never learns what else a + client read. Reads of any other record (existence checks, a `chunk get` by + address, DataMap records, a shrunk map's wrappers included) neither ask + peers nor keep the record. +- **Proven links.** The room's key is the root map's address; next to it a + client derives the room's *secret* from the root map's bytes, which a node + that knows only the key of a private file's room does not have. Each end + of a new link sends `Proof`: a keyed BLAKE3 MAC, under the secret, of its + role (offerer or answerer), the offer's id, and the DTLS fingerprints of + the offer and the answer as both ends saw them. Each WebRTC stack checks + the peer's DTLS certificate against those fingerprints, so a node that put + itself in the middle, with an offer and an answer of its own, cannot pass + one client's proof on to the other. The secret proves nothing about which + member a link reaches, since every member knows it, so each membership of + a room also has an ML-DSA-65 key of its own: `Hello` carries the public key + (1952 bytes), the member's name in the room is a hash of it, and `Proof` + also carries the member's signature (3309 bytes) over its role, the offer's + id and the same fingerprints. A member that names itself by another's key + cannot sign for it, and a signature made for one link proves nothing on + another, so no member can pose as another to settle away its links or take + its names. A link opens only after the peer's + `Hello` and then its `Proof`, which must check out within 20 s of the + answer on a link this client offered, or 79 s on one it answered (which + starts before the offerer has had the answer): until then + no `Have` is sent, nothing is served, no duplicate is settled, and any + other message ends the link. A public file's secret is open to anyone with + its address, as its records are. A client keeps one link per member, as + the node that introduced it names it, so a member cannot fill the room's + links by connecting again and again under new names: one answer at a time + per member and 8 in all, each held until the node has taken the answer, + taken before any negotiation; until its proof, one link each + way per member (two clients offering to each other at once make + one each way, and settle on one once open); once open, a member linked + again under another name keeps its first link. +- **STUN.** A client uses only the STUN URLs a rendezvous node names at an + IP address the client reached that node on, two per node: natively the + remote address of an authenticated connection to it, in a browser the + host of a WebRTC Direct session with it. The addresses a node advertises + are its own claim, so it cannot send a client's STUN requests, and with + them its address, to another host. With no such connection, none of that + node's STUN URLs are used. A native connection through a relay is named + by the relay's address, a host the client already sends to. +- **Meeting.** The two rendezvous places never use the same node: a place + whose lookup puts the other place's node at its own position takes the + first node of the lookup the other does not use. Offers go through every + rendezvous node, as many as that node + has members the client has no link or connection being made to there, + each node counted alone (so two readers whose views of the close group + share only one node still meet there); offers from others are answered + while the client wants links (8 per room, 16 in all, counting the + connections being made). A link made through one node to a peer that is + also a member of the other settles with its duplicate, and the link kept + then counts at both nodes, so the nodes do not keep doubling each other's + offers. A room looks its nodes up once for both places, and one lookup, + found or failed, serves them for 30 s; a place takes a node again when its membership ends + (after 3 failures in a row, or 10 minutes), so a node that goes is + replaced while the other carries on; its STUN URLs are dropped as the + membership ends. Two links between the same pair (met + through both nodes, or crossing offers) settle on the one both ends pick, + told apart by the name the client has in that room alone (from its key for + that membership), so peers meeting it in two rooms cannot tell it is the + same client by its name or key. Its addresses may still let them link + the two: each connection's WebRTC candidates carry whatever host and + public addresses it gathers (see below for native offers), which are often + the same across rooms, though not reliably so (shared NATs, changed + interfaces, browsers' mDNS names). The swarm gives no unlinkability at the + network level. The end that + closes a duplicate stops using it at once but keeps its connection 2 s + more, still counted against the link limits: the other end may not have this end's proof on that link yet, and + without it would see a link that failed rather than a duplicate, never + learn that both reach the same peer, and offer that member again every + few seconds (seen between a native and a browser client). Signalling is + non-trickle, with up to 3 s of ICE gathering, and a browser takes the page's + ICE gathering turn after any connection to a node waiting for it (see + Discovery is bounded). Every negotiation step (making an offer, answering + one, applying an answer) ends after 8 s, or as soon + as the room is left or the client shuts down, and a step that ends closes + the connection it was making; the swarm is never held across one, so a + native client dropped meanwhile still shuts its swarm down. A connection + being made through either node counts at both until it is a link, so a + reader starting out does not offer through both at once to a peer of + both. An announce is bounded as a whole, waiting for the transport + included, so a busy browser session cannot hold a round's offers past + their time. A native + announce that comes back empty well before the wait it asked for (the node + had no place for it to wait) is followed by a 2 s pause, not another at + once, and a native client starts announces at one node at least 1 s apart + whatever the replies hold. A room starts at most 8 answers to other + members' offers in 20 s, so a node that hands out offers of members it + makes up, well formed or not, costs a bounded number of negotiations + however fast it replies. A link has exactly one data + channel, labelled `ant-swarm-`: the tag is 16 hex digits derived + from the room's secret, so it names the room and says nothing else, and an + answerer refuses a channel labelled for another room. Another channel + opened by the peer, a text message, or a message longer than the wire + allows ends the link before anything is copied. +- **Sharing.** Over the data channel, peers say which records they hold + (`Have`), ask (`Get`), and answer with 16 KiB `Piece`s, `Missing` or `Busy`. + Every message is bounded before anything is allocated for it; every piece + but a record's last is a full frame; a channel holds at most 257 messages + unread (a bound of 256 plus the one sender's place) and, unsent, 256 KiB + plus the message that crossed it, about 272 KiB. +- **Reading.** A record the client's swarm holds already (checked when it + came) is used first. Otherwise (and except for two native downloads whose + data chunks come from nodes only: one with download diagnostics, whose + figures count the nodes' answers, and the peer-report download, which + still shares what it reads and may resolve a large file's map as any + other read does) a read asks peers only for a record a linked + peer in the record's room said it holds. Peers complement the nodes and + never hold a read up: a read that asks a peer gives it the read's hedge + delay (3 s in a browser, 1 s natively: the time a node gets before a + second is asked). Past that, the peer keeps the read while, at its average + rate so far, the whole record arrives within the read's patience, and for + the patience at most (measured on the clock from when the peer was asked, + so timers that fire late on a busy page do not stretch it): as long as reads that went to the nodes have lately + taken to have their records (a running average), at least twice the hedge + delay, and never past the 12 s a peer has to send a record. Then, or as + soon as the peer falls off that pace or fails, the read asks the nodes + beside it, and the first checked record wins: a node read that ends + without one (the nodes refuse, are busy or have none) leaves a peer still + sending to finish within its own deadline, and only then is the read + answered as the nodes answered; the node limiter still learns of the + nodes' miss. A slow or failing peer so + delays a read by the patience at most, which is about what a node read + started then would take, and by the hedge delay unless it was on pace to + beat one. (A node read started while the peer is still sending costs the + node's bytes even when the peer wins; with a fixed 3 s window, six + staggered readers in the lab used 1 % more node bytes than nodes alone.) A browser read still waiting + for its turn at the nodes, with no GET sent, takes its record from a + linked peer that offers it meanwhile (it looks every 250 ms) while it + keeps its place in that queue: if the peer fails or is slower, the node + read goes on as if no peer had been asked. A read asks up to 2 peers in + all, however often it asks (a + peer whose record fails its check counts as one; a link closed meanwhile as the + duplicate of one kept does not, and the read goes on to its twin, waiting + up to 2 s for the twin to say what it holds), with 2 s to send each next + piece, a deadline of 2 s plus the record's size at 512 KiB/s, and 12 s for + the read in all, from its first ask; a peer that stalls or runs past its record's time is + passed over for 30 s (the read's own time running out does not count + against it). A + browser download asks for its records in an order of its own, picked at + random, as BitTorrent picks pieces: a download keeps up to 8 records in + flight (4 at first outside Chromium), and readers of one file who start together and ask in the same + order would hold the same records at every moment, and want the same next + ones, so none would have a record another is still waiting for. A + download delivers its chunks in any order, so its order costs it nothing; + a stream or a range read asks in the order its reader reads. A client asks one link for one record at a time: in + Chromium, two records sent at once on one data channel, a few pieces of + each in turn, stalled it for seconds (a 2 MiB record took 3.6 to 5.3 s + that way, 66 to 656 ms alone). One more read may wait its turn at the + client, where the wait is seen, keeping the link busy back to back; reads + past that go to the nodes rather than queue behind a peer that may be + slower than they are (on a local network, queueing made the last of four + readers starting 1.5 s apart 11.5 s slower than with no peers at all); at most 8 records come from peers at once, and reads past that go + to the nodes, as do the reads of a hidden browser page (a tab in the + background), whose timers the browser slows to a second or more; a page + hidden mid-read leaves the peers for the nodes at once, on its + `visibilitychange` event. A record being received holds only the pieces that came + (grown as they arrive, doubling from 64 KiB and never past the record's + size), so a peer that names a large record and stops costs at most about + twice what it sent, and never less than 64 KiB, and records being + received hold at most 8 records' bytes. A record that fails its check ends + that link; pieces out of order end it too. +- **Existence checks never ask peers.** `chunk_exists` (used before uploads to + skip stored records) asks only nodes, and records from peers stay out of the + chunk cache that answers it: another client holding a record says nothing + about whether the network stores it. +- **The node limiter sees only nodes.** A record from a peer does not feed the + adaptive fetch limiter, which measures how the nodes are coping, including + the reads browser read-ahead holds for; after a fallback, only the time the + nodes took counts. +- **Serving.** Records fetched from nodes, or from peers and checked, go into a + store of 256 MiB (native) or 64 MiB (browser), oldest dropped first, and so + do records a read finds in the chunk cache, so records read before the + room was joined are offered too. Records a client already holds when it + joins a room are told to the room's links as the room learns its file's + records, so a link that opened first still hears of them. A client sends at + most 4 records at once, and starts no send past 1 while fetches of its own + are running, whatever they are for (a file's records or its map, a raw + read or an existence check; sends already running when one starts finish; the count + is taken in one step, so two links never both take the last place): a + record sent costs the sender's CPU and line, which its own download needs + first + (in the lab, readers each held to one CPU core and sending 4 at once while + downloading took up to 40 % longer than nodes alone; sending 1 at a time, + they were no slower); a client done reading sends 4, + 2 per link, one more than a peer asks for at once (a record the peer gave + up on counts until its next piece sees that, and the peer's next ask may + come first), and answers `Busy` past that; an ask cancelled and made again under the same id is a new transfer, + and the old one stops. A record, or any other message, + that takes more than 20 s to send (a peer that stopped reading) ends the + link. Messages take turns on a link in order, and a record lets the + others, and newly arrived asks, have the channel every 64 KiB, so records + sent at once interleave rather than one taking every turn (a browser send + that need not wait never yields by itself; the browser yields through a + posted message, not a timer, which background tabs slow). A browser waits on a full + channel for + its `bufferedamountlow` event rather than a timer: browsers slow timers in + tabs that are not in front to one a second, which would cap a tab left + seeding at a quarter of a megabyte a second. +- **Older nodes.** A browser sends swarm messages only to nodes that advertise + `swarm_rendezvous`: an older node closes the channel on a message it cannot + decode, which would fail the reads in flight on it. The browser client's + network adapter must say which nodes do (the client's own does); a client + on an adapter that cannot joins no room and reads from the nodes alone. Older nodes drop the + messages of a native client silently, which costs that client a timeout. + A rendezvous place passes over, for 10 minutes, a node a session says + hosts no rooms (at once, from its capabilities, and still once that + session has closed) and one that never let the client join in a + membership (after its 3 failures), and takes the next closest of the + nodes the room looked up (a room remembers as many such nodes as its + lookups can name in 10 minutes, 160); readers that see the same + nodes pass over the same ones, so they still meet. With none left, a place + tries again less and less often, doubling up to 5 minutes. Reads go to the + nodes as before throughout. (Without this, a room whose closest node + predated rooms never formed: in a lab network with 10 of 56 nodes on the + previous release, six readers starting together met no one.) +- **Discovery is bounded.** Per room: one lookup of its nodes at a time and + at most one per 30 s for both places, two places, each with one announce in + flight (a browser asks again after 1 s while a signal is likely on its way + and after 3 s otherwise, within the node's swarm budget; a native announce + is held at the node for up to its wait, and announces at a node start at + least 1 s apart), + at most 2 offers per announce, each a WebRTC connection with up to 3 s of + ICE gathering and every negotiation step bounded, and at most 8 links; + per room, at most 8 answers started in 20 s; per client: 4 rooms, 16 + links, 8 answers being made, 4 records sent (1 while any of its own fetches + runs) and 8 + received at once. In a browser that makes connections take turns to + gather ICE (every engine but Chromium), the swarm's own connections take + the page's turn only while it is free and no connection to a node waits + for it, and a room's lookup asks one node at a time, dials each only then, + and warms no connections to the nodes it learns of; in any browser a room's + lookup never starts the seeds' bootstrap (which opens several connections + at once): it starts from the routes the reads' own lookups found, waiting + up to 10 s for one, and with none (a tab left seeding whose routes went + with its network) from one seed, the seeds tried one at a time in the + discovery turn; discovery so holds at + most one turn at a time, and a read whose connection starts while it does + waits for that one turn, at most 1 s (the longest a connection holds the + turn). `swarm_stats().discovery` (`swarmStats` in a browser) + counts lookups, announces and their failures by cause, the signalling + bytes, offers made, delivered and received, answers sent and received, + channels opened, and links proven or failed. In lab runs of six browser + readers of a 128 MiB file, each staying about 3 minutes after its + download, each reader sent 28-47 KB of swarm requests (one lookup, about + one announce a second) and received 14-22 KB of offers, answers and STUN + names over its whole stay. + A browser's pool keeps 64 node sessions and closes the least recently used + idle one, which may be a rendezvous node's; the node then drops the + membership, with its offers. The client's next announce joins anew under a + new member id, which makes it drop its earlier offers at once rather than + hold their places for their lifetime. Only that announce connects again: + in the discovery turn, which it lets go before it waits on the node, like a + room's lookup; an answer or a leave on a session that has closed fails at + once, since the node has dropped the membership it belonged to. + +## Consequences + +### Positive + +- The close group serves a popular file about once per swarm rather than once + per reader. In the native end-to-end test the second reader of an 8 MiB file + took every record from the first, and the nodes received no GET for any of + them (`ant-core/tests/e2e_swarm.rs`). +- No new service and no third party: the readers' own nodes introduce them, on + connections they already have. +- A peer cannot corrupt a read, and a bad or slow peer costs one short wait. + +### Negative / Trade-offs + +- **Readers of a file learn each other's addresses.** That is how + BitTorrent works, and it is new here: until now only nodes saw a reader's + address. A browser hides its local addresses behind mDNS names; a native + client's offer lists its host addresses. The rendezvous nodes learn which + connections read which file; a reader's close-group nodes could already see + the records it fetches. +- **Private files too.** Two readers of the same private DataMap now meet; the + room key, a hash of the root map, does not reveal the map or the content. + Self-encryption is convergent, so the same content stored privately and + publicly has the same root map and records, and its private and public + readers meet in one room; its nodes could already see them fetch the same + records. +- **Clients upload.** A client sends up to 4 records at once to other readers + for as long as it is in their room, which costs bandwidth and, on phones, + data and battery. +- **Browsers poll.** A browser's announce every 1 to 3 s per room and + rendezvous node is a small request on its control lane; a held announce + would cost it a request slot instead. Native clients long-poll. A room of + a few hundred browsers can fill a node's swarm budget; its members are + then refused and back off, which slows their meeting but never their reads. +- **Nodes browsers cannot reach host no rooms**, so a network of nodes built + without the browser listener has no swarm, natively either. +- **Native clients link webrtc-rs**, which compiles in rustls's ring provider + alongside aws-lc-rs. rustls then cannot pick a process-wide provider by + itself, so the client installs aws-lc-rs, the one QUIC uses, before its + first peer connection, as saorsa-transport already does for nodes. +- **Two readers on different networks may not connect** when both NATs are + symmetric: there is no relay, by design. They read from the nodes. +- **Native readers behind relays get no STUN.** A client uses a node's STUN + servers only at an address it reached that node on. A native client whose + connection to a rendezvous node runs through a relay reached the relay, + not the node's WebRTC Direct address, so it uses none of that node's STUN + servers, and on a network reached at public addresses its host candidates + are dropped too. Two such native readers behind NATs then exchange offers + but cannot link, and read from the nodes. Browsers reach nodes directly + and are not affected. Open: whether a client may check a node's direct + address before trusting its STUN servers, or should prefer a directly + reached rendezvous node. +- **A request taken after its connection closed.** A node can take a native + request its transport queued before the connection closed, after the + close; such a membership, like any a client stops using without leaving, + stays at most 60 s of silence. Tying requests to the connection they came + on needs saorsa-core to tag packets with their connection; it does not. +- **Hidden tabs.** A hidden browser tab reads from the nodes, not peers, and + still serves. Its 20 s send bound runs on timers the browser slows in a + hidden tab (to about a minute at worst), so a peer that stops reading can + hold one of its 4 upload turns that long; stopping uploads when hidden + would end background seeding. + +### Neutral / Operational + +- Nodes count announces, refusals, offers delivered and undelivered, answers + delivered and unmatched (`SwarmService::counters`); clients report + `SwarmStats` (records and bytes from and to peers, rejected records, links, + rooms). +- Rollout order does not matter for correctness. Nodes first lets new clients + share at once; clients first costs native clients the backed-off timeouts + above until their rooms' nodes upgrade. + +## Validation + +- ant-protocol: the variants' discriminants are pinned, and a swarm announce + round-trips; the limits are tested at their bounds. +- saorsa-transport: plain Binding answers carry the source address (IPv4-mapped + sources as IPv4), check and return FINGERPRINT, leave connectivity checks and + other packets to the usual handling, drop refused plain requests, are + answered only for allowed sources within their rate, share one rate per IPv6 + /64, and stop at the socket's overall budget. +- saorsa-core: a send on a named connection is refused for a connection that + is not the peer's, and never falls back or dials. +- ant-node: unit tests for requests naming another node, introductions, answers nobody asked for, long-poll + wake-up and shutdown, caps and leaving, expiry, the STUN allowlist, request + classification from the header, the unanswered-offer cap, browsers never + waiting, one waiting announce per member, a signal already taken not cutting + a later wait short, requests from before a close joining nothing, + replayed or stale native requests, histories started only by announces and + leaves and capped per host, ended memberships refused however late their + announce, kept per connection and host, members capped per host, a departing member's signals + and the offers made to it going with it, and the native request budget (a + host's identities and IPv6 /64 sharing one, many hosts held to the node's, + the hosts remembered staying bounded, and a flood through the router held + to it while another host gets through). +- ant-client unit tests over an in-memory WebRTC stand-in: a record served to + a second and a third client, nothing asked for a record nobody announced, a + forged record and out-of-order pieces ending the link, a stalled peer costing + one stall and being sent `Cancel`, a dribbling peer running out of time, + `Missing` and `Busy` falling back at once, crossing links settling on one, + the per-link send limit, reads taking turns on one link, a record of another + room neither announced nor served, a full queue closing the channel, and + shutdown. +- ant-client end to end, native, on in-process nodes with real webrtc-rs: a + second reader served records by the first, the nodes asked for every record + it did not send, a reader whose peer left served by the nodes, and readers of + different files not meeting. (The race itself, a peer keeping a read or the + nodes taking it, is timed by unit tests on a paused clock.) +- ant-client end to end in real Chromium (`browser-tests/swarm.spec.js`): two + pages in separate contexts read one file; with nodes that host rooms, the + second takes data records from the first (CI runs this against the ant-node + the client is built with); with the pinned older node CI also runs, + it reads from the nodes and no swarm message is sent. +- A native client and Chromium pages in one room, on a local devnet: a page + took every data record from a native client (webrtc-rs), and a native client + every data record from a page, in 4 of 5 runs; in the other one record ran + out of time and came from the nodes. +- The browsers installed on the machine, started by the operating system + rather than a test driver: Brave took every data record from Chrome in 3 of + 3 runs. Firefox could not be checked locally: the local nodes listen only on + loopback, which Firefox does not use for WebRTC. +- Review triggers: a record from a peer ever reaching a caller unchecked, or + answering an existence check; a node limit that has to grow; the STUN answer + growing relative to the request. + +## Notes for AI-assisted work + +AI tools may help draft this ADR, but **must not mark it Accepted without human review**. Accepted ADRs are immutable: create a new superseding ADR rather than editing an Accepted ADR. diff --git a/docs/adr/README.md b/docs/adr/README.md index 43cc2028..1a76d1be 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -41,3 +41,4 @@ See [`TOOLING.md`](./TOOLING.md) for `adrs`, `adr-kit`, and AI harness setup. - [ADR-0019: Pointer audits serve the record round 1 bound](./ADR-0019-pointer-audits-serve-the-record-round-one-bound.md) - [ADR-0022: Remove the LMDB Chunk Store](./ADR-0022-remove-the-lmdb-chunk-store.md) - [ADR-0023: Spread hot-record reads across the close group, and refuse full browser sessions explicitly](./ADR-0023-spread-hot-reads-and-refuse-full-browser-sessions.md) +- [ADR-0026: Clients reading the same file share its records; nodes introduce them and stay the last seed](./ADR-0026-clients-share-records-nodes-introduce-them.md) diff --git a/scripts/third_party_notices/config.toml b/scripts/third_party_notices/config.toml index 617ba3e5..6512ce62 100644 --- a/scripts/third_party_notices/config.toml +++ b/scripts/third_party_notices/config.toml @@ -108,7 +108,7 @@ text_sha256 = "b05785f9f18e6716bab63424b11454513b9943a222595b70411009202fc592b5" note = "its COPYING, reproduced with its copyright lines, refers to LICENSE-MIT and LICENSE-APACHE files that neither the crate nor its repository contain; the MIT text is reproduced from the SPDX licence list." [missing.granit-parser] -version = "1.3.0" +version = "2.0.0" text_url = "https://raw.githubusercontent.com/spdx/license-list-data/v3.29.0/text/MIT.txt" text_sha256 = "b05785f9f18e6716bab63424b11454513b9943a222595b70411009202fc592b5" note = "its root LICENSE, reproduced with its notices for Chen Yuheng, Ethiraric and Bourumir Wyngs, refers to licence texts the crate ships under a hidden .licenses directory, which is not searched for licence files; the MIT text is reproduced from the SPDX licence list." diff --git a/scripts/third_party_notices/generate.py b/scripts/third_party_notices/generate.py index b3bf57ed..15f3ee32 100644 --- a/scripts/third_party_notices/generate.py +++ b/scripts/third_party_notices/generate.py @@ -38,6 +38,7 @@ import time import tomllib import urllib.error +import urllib.parse import urllib.request LICENCE_FILE = re.compile( @@ -152,7 +153,11 @@ def resolve_package(name, version, annotation, by_name_version): elif annotation.startswith("registry "): raise Gap(f"{name} {version} comes from {annotation}; only crates.io is supported") elif "://" in annotation: - matches = [p for p in candidates if (p["source"] or "").startswith("git+" + annotation)] + # cargo tree percent-encodes a git branch's slashes, cargo metadata + # does not: compare both decoded. + wanted = "git+" + urllib.parse.unquote(annotation) + matches = [p for p in candidates + if urllib.parse.unquote(p["source"] or "").startswith(wanted)] else: wanted = os.path.realpath(annotation) matches = [p for p in candidates if p["source"] is None diff --git a/src/devnet.rs b/src/devnet.rs index 18e459bb..69991c3b 100644 --- a/src/devnet.rs +++ b/src/devnet.rs @@ -27,6 +27,11 @@ use tokio::sync::RwLock; use tokio::task::JoinHandle; use tokio::time::Instant; use tokio_util::sync::CancellationToken; +use tokio_util::task::TaskTracker; + +use crate::node::NativeSwarmRouter; +#[cfg(feature = "webrtc-direct")] +use crate::swarm::SwarmService; #[cfg(all(feature = "webrtc-direct", feature = "test-utils"))] use crate::ant_protocol::{ChunkMessage, ChunkMessageBody, ChunkPutRequest, ChunkPutResponse}; @@ -1059,11 +1064,17 @@ impl Devnet { node.index )) })?; + let swarm = node + .ant_protocol + .as_ref() + .map_or_else(SwarmService::new, |protocol| Arc::clone(protocol.swarm())); + swarm.set_node_id(*p2p.peer_id().to_bytes()); let server = crate::web_rtc::spawn( &webrtc_direct_config, &node.data_dir, p2p, node.ant_protocol.clone(), + swarm, self.config .evm_network .as_ref() @@ -1091,8 +1102,18 @@ impl Devnet { let p2p_clone = Arc::clone(p2p); let protocol_clone = Arc::clone(protocol); let node_index = node.index; + // Native clients' swarm requests, as a node routes them. + let swarm = NativeSwarmRouter::start( + Arc::clone(protocol.swarm()), + Arc::clone(p2p), + TaskTracker::new(), + self.shutdown.clone(), + ); node.protocol_task = Some(tokio::spawn(async move { while let Ok(event) = events.recv().await { + if swarm.route(&event) { + continue; + } if let P2PEvent::Message { topic, source: Some(source), diff --git a/src/lib.rs b/src/lib.rs index dd07b8e9..df50e185 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -58,6 +58,7 @@ pub mod payment; pub mod pointer; pub mod replication; pub mod storage; +pub mod swarm; pub mod upgrade; #[cfg(feature = "webrtc-direct")] mod web_rtc; diff --git a/src/node.rs b/src/node.rs index ac1f3588..838abedf 100644 --- a/src/node.rs +++ b/src/node.rs @@ -1,6 +1,6 @@ //! Node implementation - thin wrapper around saorsa-core's `P2PNode`. -use crate::ant_protocol::CHUNK_PROTOCOL_ID; +use crate::ant_protocol::{ChunkMessage, CHUNK_PROTOCOL_ID}; use crate::config::{ default_nodes_dir, default_root_dir, NetworkMode, NodeConfig, NODE_IDENTITY_FILENAME, }; @@ -19,19 +19,22 @@ use crate::storage::legacy_artifacts; use crate::storage::traffic as storage_traffic; use crate::storage::MIB; use crate::storage::{AntProtocol, ChunkRequestContext, ChunkStore, ChunkStoreConfig}; +use crate::swarm::{self, SwarmService, SwarmSession}; use crate::upgrade::{ upgrade_cache_dir, AutoApplyUpgrader, BinaryCache, ReleaseCache, UpgradeMonitor, UpgradeResult, }; use rand::Rng; -use saorsa_core::identity::NodeIdentity; +use saorsa_core::identity::{NodeIdentity, PeerId}; use saorsa_core::{ IPDiversityConfig as CoreDiversityConfig, MultiAddr, NodeConfig as CoreNodeConfig, P2PEvent, P2PNode, }; +use saorsa_transport::shared::normalize_socket_addr; +use std::net::IpAddr; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicI32, Ordering}; use std::sync::Arc; -use std::time::Instant; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use tokio::sync::mpsc::UnboundedReceiver; use tokio::sync::Semaphore; use tokio::task::JoinHandle; @@ -57,6 +60,209 @@ fn user_agent() -> String { format!("node/{}", env!("CARGO_PKG_VERSION")) } +/// Routes native clients' swarm requests to the rendezvous, apart from chunk +/// requests, and sweeps it: the same for nodes and devnet nodes. +pub(crate) struct NativeSwarmRouter { + swarm: Arc, + /// Swarm requests have their own budget: an announce may wait for a + /// signal, and must never hold a permit chunk requests use. + slots: Arc, + p2p: Arc, + children: TaskTracker, + stopping: CancellationToken, +} + +impl NativeSwarmRouter { + /// Starts sweeping `swarm` every [`SWARM_SWEEP_INTERVAL`] until `stopping`. + pub(crate) fn start( + swarm: Arc, + p2p: Arc, + children: TaskTracker, + stopping: CancellationToken, + ) -> Self { + let sweeping = Arc::clone(&swarm); + let stop = stopping.clone(); + children.spawn(async move { + let mut every = tokio::time::interval(SWARM_SWEEP_INTERVAL); + loop { + tokio::select! { + () = stop.cancelled() => break, + _ = every.tick() => sweeping.sweep(Instant::now()), + } + } + }); + Self { + swarm, + slots: Arc::new(Semaphore::new(swarm::MAX_SWARM_TASKS)), + p2p, + children, + stopping, + } + } + + /// Handles `event` if it is the swarm's: a connection that closed, or a + /// swarm request, handed to its own task. Returns whether it was. + /// Saturated, a request is dropped: the client times out and fetches + /// from nodes alone. + pub(crate) fn route(&self, event: &P2PEvent) -> bool { + if let P2PEvent::PeerDisconnected(peer) = event { + self.swarm + .session_closed(SwarmSession::Native(*peer.to_bytes())); + return true; + } + let P2PEvent::Message { + topic, + source: Some(source), + transport_source, + timestamp, + data, + } = event + else { + return false; + }; + if topic != CHUNK_PROTOCOL_ID || !swarm::is_swarm_request_bytes(data) { + return false; + } + // No legal swarm request is this long; refuse it before decoding. + if data.len() > ant_protocol::MAX_SWARM_REQUEST_BYTES { + debug!( + "Swarm request from {source} dropped: {} bytes is too long", + data.len() + ); + return true; + } + // The reply goes back on the connection the request came on, named + // as the transport names it. + let Some(channel) = transport_source + .as_ref() + .and_then(MultiAddr::socket_addr) + .map(|address| normalize_socket_addr(address).to_string()) + else { + debug!("Swarm request from {source} dropped: no connection to answer on"); + return true; + }; + let source_ip = transport_source.as_ref().and_then(MultiAddr::ip); + if !self.swarm.admit_native(source_ip, Instant::now()) { + debug!("Swarm request from {source} dropped: over the swarm's request budget"); + return true; + } + let Ok(permit) = Arc::clone(&self.slots).try_acquire_owned() else { + debug!("Swarm request from {source} dropped: swarm slots full"); + return true; + }; + let swarm = Arc::clone(&self.swarm); + let p2p = Arc::clone(&self.p2p); + let stopping = self.stopping.clone(); + let request = SwarmRequest { + source: *source, + channel, + source_ip, + signed_at: *timestamp, + received: Instant::now(), + data: data.clone(), + }; + self.children.spawn(async move { + let _permit = permit; + Self::answer(&swarm, &p2p, request, &stopping).await; + }); + true + } + + /// Answers one swarm request and sends the reply. + async fn answer( + swarm: &SwarmService, + p2p: &Arc, + request: SwarmRequest, + stopping: &CancellationToken, + ) { + let SwarmRequest { + source, + channel, + source_ip, + signed_at, + received, + data, + } = request; + let source = &source; + let message = match ChunkMessage::decode(&data) { + Ok(message) => message, + Err(e) => { + debug!("Undecodable swarm request from {source}: {e}"); + return; + } + }; + // The encoded request is not kept while an announce waits. + drop(data); + let now_unix = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |since| since.as_secs()); + let peer = *source.to_bytes(); + let starts = swarm::starts_history(&message.body); + // One for another node gets its error reply without touching the + // client's history here. + if swarm.names_this_node(&message.body) + && !swarm.first_sight( + peer, + source_ip, + message.request_id, + signed_at, + now_unix, + starts, + ) + { + debug!("Swarm request from {source} dropped: stale or seen before"); + return; + } + let session = SwarmSession::Native(peer); + let Some(body) = swarm + .handle(session, source_ip, message.body, stopping, received) + .await + else { + return; + }; + // A client that left (or a node stopping) is not dialled to be told. + if stopping.is_cancelled() || swarm.reply_unwanted(session, received) { + return; + } + let reply = ChunkMessage { + request_id: message.request_id, + body, + }; + match reply.encode() { + // Only on the connection the request came on. A signed message + // is not bound to its connection, so a node that saw one of this + // client's could send it here and have the transport take its + // own connection for the client's too: the reply never goes on + // another of the client's connections, nor dials a client that + // left between the check above and the send. + Ok(bytes) => { + if let Err(e) = p2p + .send_message_on_channel(source, &channel, CHUNK_PROTOCOL_ID, bytes) + .await + { + debug!("Failed to send swarm reply to {source}: {e}"); + } + } + Err(e) => warn!("Failed to encode swarm reply: {e}"), + } + } +} + +/// A swarm request from a native client, as it reached the routing loop. +struct SwarmRequest { + source: PeerId, + /// The connection it came on. + channel: String, + source_ip: Option, + /// The client's signed send time, in Unix seconds. + signed_at: u64, + received: Instant, + data: Vec, +} + +/// How often silent swarm members and expired swarm signals are swept. +const SWARM_SWEEP_INTERVAL: Duration = Duration::from_secs(15); + /// Builder for constructing an Ant node. pub struct NodeBuilder { config: NodeConfig, @@ -214,6 +420,11 @@ impl NodeBuilder { _ => None, }; + // Every node hosts the swarm rendezvous, storing chunks or not. + let swarm = ant_protocol + .as_ref() + .map_or_else(SwarmService::new, |protocol| Arc::clone(protocol.swarm())); + swarm.set_node_id(*p2p_arc.peer_id().to_bytes()); let node = RunningNode { config: self.config, p2p_node: p2p_arc, @@ -222,6 +433,7 @@ impl NodeBuilder { events_rx: Some(events_rx), upgrade_monitor, ant_protocol, + swarm, replication_engine, protocol_task: None, protocol_children: TaskTracker::new(), @@ -559,6 +771,9 @@ pub struct RunningNode { upgrade_monitor: Option, /// ANT protocol handler for chunk storage. ant_protocol: Option>, + /// The swarm rendezvous: the storage handler's, or this node's own when + /// it stores nothing. + swarm: Arc, /// Replication engine (manages neighbor sync, verification, audits). replication_engine: Option, /// Protocol message routing background task. @@ -669,6 +884,7 @@ impl RunningNode { &self.config.root_dir, Arc::clone(&self.p2p_node), self.ant_protocol.clone(), + Arc::clone(&self.swarm), &evm_network, self.shutdown.clone(), None, @@ -1041,19 +1257,29 @@ impl RunningNode { /// Subscribes to P2P events and routes incoming chunk protocol messages /// to the `AntProtocol` handler, sending responses back to the sender. fn start_protocol_routing(&mut self) { - let protocol = match self.ant_protocol { - Some(ref p) => Arc::clone(p), - None => return, - }; + let protocol = self.ant_protocol.clone(); let mut events = self.p2p_node.subscribe_events(); let p2p = Arc::clone(&self.p2p_node); let semaphore = Arc::new(Semaphore::new(64)); let children = self.protocol_children.clone(); let stopping = self.shutdown.clone(); + let swarm = NativeSwarmRouter::start( + Arc::clone(&self.swarm), + Arc::clone(&p2p), + children.clone(), + stopping.clone(), + ); self.protocol_task = Some(tokio::spawn(async move { while let Ok(event) = events.recv().await { + if swarm.route(&event) { + continue; + } + // Chunk requests only where chunks are stored. + let Some(protocol) = protocol.as_ref() else { + continue; + }; if let P2PEvent::Message { topic, source: Some(source), @@ -1070,7 +1296,7 @@ impl RunningNode { if let Some((data_type, response_topic)) = handler_info { debug!("Received {data_type} protocol message from {source}"); let received_at = Instant::now(); - let protocol = Arc::clone(&protocol); + let protocol = Arc::clone(protocol); let p2p = Arc::clone(&p2p); let sem = semaphore.clone(); let stopping = stopping.clone(); @@ -1134,6 +1360,10 @@ fn jittered_interval(base: std::time::Duration) -> std::time::Duration { #[cfg(test)] #[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] mod tests { + use std::net::SocketAddr; + + use ant_protocol::swarm::SwarmAnnounceRequest; + use ant_protocol::ChunkMessageBody; use rand::Rng; use tempfile::TempDir; @@ -1219,6 +1449,77 @@ mod tests { node.shutdown.cancel(); } + /// A native host flooding swarm requests, under as many identities as it + /// likes, has its budget's worth routed to the rendezvous and no more, and + /// an honest host beside it still gets through. + #[tokio::test] + async fn a_native_hosts_swarm_flood_is_held_to_its_budget() { + let dir = TempDir::new().expect("temp dir"); + let root = dir.path().join("node"); + std::fs::create_dir_all(&root).expect("mkdir"); + let mut built = None; + let mut last_err = String::new(); + for _ in 0..BIND_ATTEMPTS { + let port = rand::thread_rng().gen_range(TEST_PORT_RANGE); + match NodeBuilder::new(local_node_config(&root, port)) + .build() + .await + { + Ok(node) => { + built = Some(node); + break; + } + Err(e) => { + last_err = e.to_string(); + tokio::time::sleep(std::time::Duration::from_millis(250)).await; + } + } + } + let Some(node) = built else { + panic!("could not build a node after {BIND_ATTEMPTS} attempts: {last_err}"); + }; + let swarm = SwarmService::new(); + let router = NativeSwarmRouter::start( + Arc::clone(&swarm), + Arc::clone(&node.p2p_node), + TaskTracker::new(), + node.shutdown.clone(), + ); + let request = ChunkMessage { + request_id: 1, + body: ChunkMessageBody::SwarmAnnounceRequest(SwarmAnnounceRequest { + room: [1; 32], + node: [2; 32], + membership: 1, + offers: Vec::new(), + wait_ms: 0, + }), + } + .encode() + .expect("encode"); + let from = |ip: &str, identity: u8| P2PEvent::Message { + topic: CHUNK_PROTOCOL_ID.to_string(), + source: Some(PeerId::from_bytes([identity; 32])), + transport_source: Some(MultiAddr::quic(SocketAddr::new( + ip.parse().expect("ip"), + 5000, + ))), + timestamp: 0, + data: request.clone(), + }; + let over_budget = || swarm.counters().native_over_budget.load(Ordering::Relaxed); + let flood: u32 = 100; + for identity in 0..flood { + let identity = u8::try_from(identity).expect("fits"); + assert!(router.route(&from("198.51.100.4", identity))); + } + let dropped = flood - swarm::NATIVE_SWARM_REQUESTS_PER_SECOND_PER_HOST; + assert_eq!(over_budget(), u64::from(dropped)); + assert!(router.route(&from("198.51.100.5", 0))); + assert_eq!(over_budget(), u64::from(dropped)); + node.shutdown.cancel(); + } + /// A node with chunks in a store this build cannot read STARTS, however it is /// configured, and does not lose them. /// diff --git a/src/storage/handler.rs b/src/storage/handler.rs index 62e2db75..74c4e822 100644 --- a/src/storage/handler.rs +++ b/src/storage/handler.rs @@ -45,6 +45,7 @@ use crate::replication::config::K_BUCKET_SIZE; use crate::replication::fresh::FreshWriteEvent; use crate::storage::traffic::{self, ChunkRequestKind, ChunkResponseKey}; use crate::storage::ChunkStore; +use crate::swarm::{SwarmService, SwarmSession}; use ant_protocol::chunk::{PointerGetResponse, PointerPutResponse}; use bytes::Bytes; use parking_lot::RwLock; @@ -349,6 +350,8 @@ pub struct AntProtocol { /// message to a node that does not keep pointers gets a clean refusal /// rather than a silent drop. pointers: Option, + /// Introduces clients downloading the same file to each other. + swarm: Arc, } impl AntProtocol { @@ -387,6 +390,7 @@ impl AntProtocol { fresh_write_tx: None, p2p_node: RwLock::new(None), pointers: None, + swarm: SwarmService::new(), } } @@ -406,6 +410,39 @@ impl AntProtocol { self.pointers.as_ref() } + /// The swarm rendezvous. Every node runs one: it is how clients + /// downloading the same file find each other. + #[must_use] + pub fn swarm(&self) -> &Arc { + &self.swarm + } + + /// Answers an encoded swarm request from `session`, which arrived from + /// `source_ip`, and returns the encoded reply; `None` if `data` is not a + /// swarm request. An announce may wait for a signal until `shutdown`. + /// Callers route swarm requests here apart from chunk requests, which + /// must not wait behind them (see [`crate::swarm::is_swarm_request_bytes`]). + pub async fn handle_swarm_request( + &self, + session: SwarmSession, + source_ip: Option, + data: &[u8], + shutdown: &tokio_util::sync::CancellationToken, + ) -> Option { + let message = ChunkMessage::decode(data).ok()?; + let body = self + .swarm + .handle(session, source_ip, message.body, shutdown, Instant::now()) + .await?; + ChunkMessage { + request_id: message.request_id, + body, + } + .encode() + .ok() + .map(Bytes::from) + } + /// Attach the node's P2P handle for payment live-DHT checks. /// /// Wires the handle into the payment verifier so payment-proof closeness @@ -413,6 +450,8 @@ impl AntProtocol { /// replaces the verifier handle. pub fn attach_p2p_node(&self, node: Arc) { *self.p2p_node.write() = Some(Arc::clone(&node)); + // Swarm requests must name this node. + self.swarm.set_node_id(*node.peer_id().to_bytes()); if let Some(pointers) = &self.pointers { // Pointers take the same self-closeness gate as chunks, judged at // the pointer address because that is what the network routes on. diff --git a/src/swarm.rs b/src/swarm.rs new file mode 100644 index 00000000..a3ca9a9f --- /dev/null +++ b/src/swarm.rs @@ -0,0 +1,3143 @@ +//! Swarm rendezvous: the node introduces clients that are downloading the +//! same file, so they can share its records with each other. +//! +//! A client announces itself in a *room* (a file) at the nodes closest to +//! the room's key and hands them WebRTC offers. The node passes each offer to +//! another member and carries the answer back; the two clients then talk +//! directly and records never pass through the node. The node stays the last +//! seed: whatever no client has is fetched from it as before. +//! +//! The rules, all enforced here: +//! - members are named by the node ([`MemberId`]), one per connection and +//! room, so a client cannot speak for another; +//! - an answer is passed on only for an offer this node delivered to the +//! answering member, once, before the offer expires; +//! - only `{offer_id, sdp}` is relayed, never anything else a client sent; +//! - rooms, members, rooms per connection, remembered offers, waiting +//! announces and stored SDP bytes are all capped. The caps are fixed: +//! swarm work is small, and a node that is short of room simply refuses, +//! which leaves its clients fetching from nodes alone. +//! +//! Signals for a member wait in a small inbox until its next announce, which +//! may wait for them (a long poll). A waiting announce holds no lock and +//! ends at the deadline, on the first signal, or when the node shuts down. + +use std::collections::{HashMap, VecDeque}; +use std::net::{IpAddr, Ipv6Addr}; +use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; +use std::sync::Arc; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +use ant_protocol::chunk::{ChunkMessageBody, ProtocolError}; +use ant_protocol::swarm::{ + constrain_swarm_sdp, is_global_ip, MemberId, Membership, NodeId, OfferId, RoomKey, + SwarmAnnounceRequest, SwarmAnnounceResponse, SwarmAnswerRequest, SwarmAnswerResponse, + SwarmLeaveRequest, SwarmLeaveResponse, SwarmRefusal, SwarmSignal, MAX_SWARM_ICE_SERVERS, + MAX_SWARM_ICE_SERVER_BYTES, MAX_SWARM_INBOX, MAX_SWARM_REPLY_SDP_BYTES, SWARM_SIGNAL_TTL_MS, +}; +use parking_lot::{Mutex, RwLock}; +use rand::seq::SliceRandom; +use tokio::sync::Notify; +use tokio_util::sync::CancellationToken; + +use crate::logging::debug; + +/// Rooms one node hosts. +pub const MAX_ROOMS: usize = 4096; +/// Members in one room. +pub const MAX_MEMBERS_PER_ROOM: usize = 256; +/// Members across every room. +pub const MAX_MEMBERS: usize = 16_384; +/// Rooms one connection may be in. +pub const MAX_ROOMS_PER_SESSION: usize = 4; +/// Members from one host (an IPv4 address or an IPv6 /64) across every +/// room: a native client's identities cost nothing to make, so a host could +/// otherwise take every place with new ones. +pub const MAX_MEMBERS_PER_HOST: usize = 64; +/// Signals waiting for one member. +pub const MAX_INBOX_PER_MEMBER: usize = MAX_SWARM_INBOX; +/// Unanswered offers remembered for one member. Past it, its new offers are +/// not passed on until older ones are answered or expire. +pub const MAX_PENDING_PER_MEMBER: usize = 32; +/// SDP bytes held in every inbox together. +pub const MAX_STORED_SDP_BYTES: usize = 16 * 1024 * 1024; +/// SDP bytes held for the offers and answers of one host's members, so one +/// host cannot take the whole budget and stop every other introduction. +pub const MAX_STORED_SDP_BYTES_PER_HOST: usize = MAX_STORED_SDP_BYTES / 8; +/// Announces waiting for a signal at once; past it, announces reply at once. +pub const MAX_WAITING: usize = 1024; +/// A member that has not announced for this long leaves its room. +pub const MEMBER_TTL: Duration = Duration::from_secs(60); +/// How long a delivered offer waits for its answer, and a signal for its +/// member's next announce: the protocol's, which clients check their own +/// answering time against. +pub const SIGNAL_TTL: Duration = Duration::from_millis(SWARM_SIGNAL_TTL_MS); +/// Client addresses remembered as recent announcers, which the node's STUN +/// responder answers. +pub const MAX_STUN_SOURCES: usize = 16_384; +/// Memberships that ended remembered, in all, per connection and per host. +const MAX_ENDED_MEMBERSHIPS: usize = MAX_MEMBERS; +const MAX_ENDED_PER_SESSION: usize = 64; +const MAX_ENDED_PER_HOST: usize = 4 * MAX_ENDED_PER_SESSION; +/// How long an ended membership is remembered: as long as a request sent +/// before it ended can still be taken. A native request's signed time may be +/// up to [`MAX_REQUEST_SKEW`] ahead of this node's clock and is taken until +/// it is that far behind, so twice the skew, and a little for rounding. +const ENDED_MEMBERSHIP_TTL: Duration = Duration::from_secs(2 * MAX_REQUEST_SKEW.as_secs() + 2); +/// Closed connections remembered, and for how long: longer than any request +/// takes to reach its handler. +const MAX_CLOSED_SESSIONS: usize = MAX_MEMBERS; +const CLOSED_SESSION_TTL: Duration = Duration::from_secs(60); +/// How far a native client's signed timestamp may be from this node's clock. +/// A wide window: clients' clocks drift, and a request outside it only costs +/// the client its swarm, not its reads. +pub const MAX_REQUEST_SKEW: Duration = Duration::from_mins(10); +/// How much older than a native client's newest request one of its requests +/// may be, for requests overtaking each other on the way. +const REQUEST_REORDER: u64 = 5; +/// Request ids remembered per native client within [`REQUEST_REORDER`]. +const MAX_RECENT_REQUESTS: usize = 256; +/// Native clients whose requests are remembered. +const MAX_REQUEST_HISTORIES: usize = MAX_MEMBERS; +/// Native clients of one host whose requests are remembered: identities cost +/// nothing to make, so a host could otherwise push every other client's +/// history out. +const MAX_HISTORIES_PER_HOST: usize = 4 * MAX_MEMBERS_PER_HOST; + +/// A native client's recent requests, to refuse one sent again. Its own +/// requests are the only ones that can change it. +#[derive(Default)] +struct RequestHistory { + /// The newest signed timestamp seen. + latest: u64, + /// Ids signed within [`REQUEST_REORDER`] of `latest`, with their times. + recent: VecDeque<(u64, u64)>, + /// When it was made, among all histories. + number: u64, + /// The client's host, if known. + host: Option, +} + +impl RequestHistory { + /// Records a request; `false` if it was here, or is much older than the + /// newest. + fn note(&mut self, request_id: u64, signed_at: u64) -> bool { + if signed_at.saturating_add(REQUEST_REORDER) < self.latest + || self.recent.iter().any(|(_, id)| *id == request_id) + { + return false; + } + self.latest = self.latest.max(signed_at); + self.recent.push_back((signed_at, request_id)); + let oldest = self.latest.saturating_sub(REQUEST_REORDER); + self.recent.retain(|(at, _)| *at >= oldest); + while self.recent.len() > MAX_RECENT_REQUESTS { + self.recent.pop_front(); + } + true + } +} + +/// Native clients' request histories: at most [`MAX_HISTORIES_PER_HOST`] per +/// host and [`MAX_REQUEST_HISTORIES`] in all, the oldest made forgotten first, +/// and each forgotten once nothing it holds could pass the age check. +#[derive(Default)] +struct Histories { + by_peer: HashMap<[u8; 32], RequestHistory>, + /// The histories in the order they were made, numbered, so an entry for + /// a history forgotten since never removes a newer one of its client. + order: VecDeque<([u8; 32], u64)>, + made: u64, + by_host: HashMap, +} + +impl Histories { + fn forget(&mut self, peer: &[u8; 32]) { + let Some(history) = self.by_peer.remove(peer) else { + return; + }; + if let Some(host) = history.host { + if let Some(count) = self.by_host.get_mut(&host) { + *count = count.saturating_sub(1); + if *count == 0 { + self.by_host.remove(&host); + } + } + } + } + + /// Starts `peer`'s history, unless its host has all it may have. With + /// every place taken, the oldest history goes. + fn start(&mut self, peer: [u8; 32], host: Option) -> bool { + let host_full = host.is_some_and(|host| { + self.by_host.get(&host).copied().unwrap_or(0) >= MAX_HISTORIES_PER_HOST + }); + if host_full { + return false; + } + while self.by_peer.len() >= MAX_REQUEST_HISTORIES { + let Some((oldest, number)) = self.order.pop_front() else { + break; + }; + if self.by_peer.get(&oldest).map(|history| history.number) == Some(number) { + self.forget(&oldest); + } + } + self.made = self.made.wrapping_add(1); + self.by_peer.insert( + peer, + RequestHistory { + number: self.made, + host, + ..RequestHistory::default() + }, + ); + self.order.push_back((peer, self.made)); + if let Some(host) = host { + *self.by_host.entry(host).or_default() += 1; + } + true + } + + /// Forgets the histories whose newest request was signed before + /// `oldest`: nothing they hold could pass the age check now. + fn forget_older_than(&mut self, oldest: u64) { + let idle: Vec<[u8; 32]> = self + .by_peer + .iter() + .filter(|(_, history)| history.latest < oldest) + .map(|(peer, _)| *peer) + .collect(); + for peer in &idle { + self.forget(peer); + } + let by_peer = &self.by_peer; + self.order.retain(|(peer, number)| { + by_peer.get(peer).map(|history| history.number) == Some(*number) + }); + } +} + +/// The connection a request came on. Members are tied to it: a native +/// client's authenticated peer id, or a browser connection. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum SwarmSession { + /// A native client, by its authenticated peer id. + Native([u8; 32]), + /// A browser, by its WebRTC Direct connection number on this node. + Browser(u64), +} + +/// Counters for metrics and logs. +#[derive(Debug, Default)] +pub struct SwarmCounters { + /// Announces received. + pub announces: AtomicU64, + /// Answers received. + pub answers: AtomicU64, + /// Offers handed to a member. + pub offers_delivered: AtomicU64, + /// Offers not handed on: no room in an inbox, a pending cap, or the + /// stored-SDP budget. + pub offers_undelivered: AtomicU64, + /// Answers handed back to the member that made the offer. + pub answers_delivered: AtomicU64, + /// Answers with no matching offer, or that could not be kept. + pub answers_unmatched: AtomicU64, + /// Announces refused by a cap. + pub refused: AtomicU64, + /// Members dropped because they stopped announcing. + pub members_expired: AtomicU64, + /// Native requests dropped past the node's or their host's budget. + pub native_over_budget: AtomicU64, +} + +/// Counts at one moment. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct SwarmStats { + /// Rooms with at least one member. + pub rooms: usize, + /// Members across all rooms. + pub members: usize, + /// Offers delivered and waiting for an answer. + pub pending_offers: usize, + /// SDP bytes held in inboxes. + pub stored_sdp_bytes: usize, + /// Announces waiting for a signal. + pub waiting: usize, +} + +/// One close of a native client's connection. +#[derive(Debug, Clone, Copy)] +struct Closed { + at: Instant, + number: u64, +} + +/// A membership the client ended, see [`State::ended`]. +#[derive(Clone, Copy)] +struct Ended { + membership: Membership, + /// When it was noted, among all ended memberships. + number: u64, + host: Option, +} + +struct Pending { + offer_id: OfferId, + to: MemberId, + expires: Instant, +} + +struct Queued { + signal: SwarmSignal, + bytes: usize, + /// The host of the member that sent it, whose share of the budget it + /// takes. + host: Option, + expires: Instant, +} + +struct Member { + session: SwarmSession, + /// The client's membership this member is: a newer one takes its place, + /// and only a leave naming it ends it. + membership: Membership, + /// The host it announced from, if known: see [`MAX_MEMBERS_PER_HOST`]. + host: Option, + last_seen: Instant, + inbox: VecDeque, + pending: VecDeque, + /// Woken, for every announce waiting then, when a signal arrives or the + /// member is dropped. + wake: Arc, + /// Whether an announce of this member is waiting: one at a time. + waiting: bool, +} + +#[derive(Default)] +struct Room { + members: HashMap, + by_session: HashMap, +} + +#[derive(Default)] +struct State { + rooms: HashMap, + sessions: HashMap>, + /// When native clients' connections last closed, so a request of theirs + /// still being handled cannot act after the close (a browser connection's + /// own token tells), and the closes in the order they happened, to forget + /// the old ones in constant time. + closed: HashMap, + closed_order: VecDeque<(SwarmSession, Closed)>, + /// Numbers closes, so an old queue entry never removes a newer close. + closes: u64, + /// The newest membership of each connection and room the client ended + /// (left, or replaced by a newer one), numbered, and in the order they + /// ended: an announce naming it or an older one is refused. A membership + /// dropped because its connection closed or it went silent is not ended: + /// the client may still be in it, and carry on over a new connection. + ended: HashMap<(SwarmSession, RoomKey), Ended>, + /// When each was noted, oldest first; an entry whose number is no longer + /// its membership's is stale and skipped. + ended_order: VecDeque<((SwarmSession, RoomKey), u64, Instant)>, + ends: u64, + /// Each connection's and each host's ended memberships, oldest first. + ended_by_session: HashMap>, + ended_by_host: HashMap>, + members: usize, + /// Members from each host, see [`MAX_MEMBERS_PER_HOST`]. + by_host: HashMap, + stored_sdp_bytes: usize, + /// Stored SDP bytes by the host that sent them. + sdp_by_host: HashMap, + pending_offers: usize, +} + +/// The node's swarm rendezvous. +#[derive(Default)] +pub struct SwarmService { + state: Mutex, + waiting: AtomicUsize, + counters: SwarmCounters, + /// This node's STUN servers, as handed to members. + ice_servers: RwLock>, + /// Hosts (IPv4 addresses, IPv6 /64s) that announced recently: the only + /// ones the node's STUN responder answers. A client reaches this list + /// only over a connection it proved (QUIC, or DTLS and the PQ handshake), + /// so a spoofed source cannot turn the responder on a third party. + announcers: RwLock>, + /// Native clients' recent requests: a signed request sent again is + /// refused. + histories: Mutex, + /// This node's peer id: requests naming another node are refused, so a + /// signed request this node saw cannot be replayed at another, nor one + /// another node saw here. Until it is known, every request is refused. + node_id: RwLock>, + /// See [`NATIVE_SWARM_REQUESTS_PER_SECOND`]. + native_budget: Mutex, +} + +impl std::fmt::Debug for SwarmService { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SwarmService") + .field("stats", &self.stats()) + .finish() + } +} + +/// Swarm requests handled at once on the native path. Above +/// [`MAX_WAITING`] so announces that cannot wait still get through. +pub const MAX_SWARM_TASKS: usize = MAX_WAITING + 256; +/// Native clients' swarm requests a node takes in a second, in all. +/// +/// With [`NATIVE_SWARM_REQUESTS_PER_SECOND_PER_HOST`], the native path's own +/// budget, as the browser listener's swarm budget is for browsers. An identity +/// costs a native client nothing, so its requests are counted by host (an +/// IPv4 address or an IPv6 /64). Past either, a request is dropped before any +/// work; its client times out and reads from nodes alone. +pub const NATIVE_SWARM_REQUESTS_PER_SECOND: u32 = 64; +/// See [`NATIVE_SWARM_REQUESTS_PER_SECOND`]. +pub const NATIVE_SWARM_REQUESTS_PER_SECOND_PER_HOST: u32 = 8; +/// Hosts whose native swarm budget is remembered at once. A host whose +/// budget has refilled is as good as new, so those go first when it is full; +/// past it, a request from a host not yet counted is dropped. +pub const MAX_BUDGETED_HOSTS: usize = 4096; + +/// A token bucket of `rate` requests a second, holding at most one second's. +#[derive(Debug)] +struct RateBucket { + rate: u128, + /// Tokens, in billionths. + units: u128, + last: Instant, +} + +const TOKEN: u128 = 1_000_000_000; + +impl RateBucket { + fn new(rate: u32, now: Instant) -> Self { + let rate = u128::from(rate); + Self { + rate, + units: rate.saturating_mul(TOKEN), + last: now, + } + } + + fn refill(&mut self, now: Instant) { + let elapsed = now.saturating_duration_since(self.last); + self.last = self.last.max(now); + let capacity = self.rate.saturating_mul(TOKEN); + self.units = self + .units + .saturating_add(elapsed.as_nanos().saturating_mul(self.rate)) + .min(capacity); + } + + fn take(&mut self, now: Instant) -> bool { + self.refill(now); + if self.units < TOKEN { + return false; + } + self.units -= TOKEN; + true + } + + fn refund(&mut self) { + self.units = self + .units + .saturating_add(TOKEN) + .min(self.rate.saturating_mul(TOKEN)); + } + + fn is_full(&mut self, now: Instant) -> bool { + self.refill(now); + self.units >= self.rate.saturating_mul(TOKEN) + } +} + +/// The native path's swarm request budget: the node's and each host's. +#[derive(Debug)] +struct NativeBudget { + all: RateBucket, + hosts: HashMap, +} + +impl NativeBudget { + /// Takes a place from the node's budget and from `ip`'s host's, both or + /// neither. The node's is taken first, so once it is spent a flood costs + /// one check a request. + fn admit(&mut self, ip: IpAddr, now: Instant) -> bool { + if !self.all.take(now) { + return false; + } + let host = host_of(ip); + if !self.hosts.contains_key(&host) && self.hosts.len() >= MAX_BUDGETED_HOSTS { + self.hosts.retain(|_, bucket| !bucket.is_full(now)); + } + let taken = (self.hosts.contains_key(&host) || self.hosts.len() < MAX_BUDGETED_HOSTS) + && self + .hosts + .entry(host) + .or_insert_with(|| RateBucket::new(NATIVE_SWARM_REQUESTS_PER_SECOND_PER_HOST, now)) + .take(now); + if !taken { + self.all.refund(); + } + taken + } +} + +impl Default for NativeBudget { + fn default() -> Self { + Self { + all: RateBucket::new(NATIVE_SWARM_REQUESTS_PER_SECOND, Instant::now()), + hosts: HashMap::new(), + } + } +} + +// The wire discriminants of the swarm requests in `ChunkMessageBody` +// (ant-protocol pins them in a test). +const ANNOUNCE_REQUEST: u64 = 14; +const ANSWER_REQUEST: u64 = 16; +const LEAVE_REQUEST: u64 = 18; + +/// Whether an encoded `ChunkMessage` is a swarm request, read from its +/// header alone so the message is not decoded twice: a postcard varint +/// request id, then the body's varint discriminant. +#[must_use] +pub fn is_swarm_request_bytes(data: &[u8]) -> bool { + let mut bytes = data.iter().copied(); + let mut read_varint = || -> Option { + let mut value = 0u64; + for shift in (0..70).step_by(7) { + let byte = bytes.next()?; + value |= u64::from(byte & 0x7f).checked_shl(shift)?; + if byte & 0x80 == 0 { + return Some(value); + } + } + None + }; + let request_id = read_varint(); + let discriminant = read_varint(); + request_id.is_some() + && matches!( + discriminant, + Some(ANNOUNCE_REQUEST | ANSWER_REQUEST | LEAVE_REQUEST) + ) +} + +/// An SDP from a member, as this node passes it on: its candidates may name +/// the member's own host, as this node reaches it, and no other public +/// address, at most `MAX_SWARM_CANDIDATES` of them. Local addresses and mDNS +/// names stay only for a member this node reaches at a local address itself +/// (a network of its own, such as a test network): elsewhere they would let a +/// stranger point readers at their own home networks. The member an SDP goes +/// to sends connectivity checks to each candidate before anything proves who +/// made it. +fn relayable_sdp(sdp: &str, host: Option) -> String { + let local = host.is_some_and(|host| !is_global_ip(host)); + constrain_swarm_sdp(sdp, |ip| match ip { + Some(ip) if is_global_ip(ip) => Some(host_of(ip)) == host, + _ => local, + }) +} + +/// The host `ip` counts as: an IPv4 address, or an IPv6 /64, which one +/// client is usually given whole. +fn host_of(ip: IpAddr) -> IpAddr { + match ip.to_canonical() { + IpAddr::V6(v6) => { + let mut octets = v6.octets(); + if let Some(interface) = octets.get_mut(8..) { + interface.fill(0); + } + IpAddr::V6(Ipv6Addr::from(octets)) + } + v4 @ IpAddr::V4(_) => v4, + } +} + +fn other_node() -> ProtocolError { + ProtocolError::Internal("this swarm request is for another node".into()) +} + +/// Whether a native client's request `body` may start its request history +/// (see [`SwarmService::first_sight`]): an announce, or a leave, which may +/// overtake the announce it ends. +#[must_use] +pub fn starts_history(body: &ChunkMessageBody) -> bool { + matches!( + body, + ChunkMessageBody::SwarmAnnounceRequest(_) | ChunkMessageBody::SwarmLeaveRequest(_) + ) +} + +/// Whether a message body is one this service answers. +#[must_use] +pub fn is_swarm_request(body: &ChunkMessageBody) -> bool { + matches!( + body, + ChunkMessageBody::SwarmAnnounceRequest(_) + | ChunkMessageBody::SwarmAnswerRequest(_) + | ChunkMessageBody::SwarmLeaveRequest(_) + ) +} + +impl State { + /// Remembers a native client's close, forgetting closes past + /// [`CLOSED_SESSION_TTL`] and, past [`MAX_CLOSED_SESSIONS`] in the queue, + /// the oldest; the newest is always kept. + fn note_close(&mut self, session: SwarmSession, now: Instant) { + self.closes = self.closes.wrapping_add(1); + let close = Closed { + at: now, + number: self.closes, + }; + self.closed.insert(session, close); + self.closed_order.push_back((session, close)); + self.forget_closes(now); + } + + fn forget_closes(&mut self, now: Instant) { + while let Some((session, close)) = self.closed_order.front().copied() { + let expired = now.saturating_duration_since(close.at) >= CLOSED_SESSION_TTL; + if !expired && self.closed_order.len() <= MAX_CLOSED_SESSIONS { + break; + } + self.closed_order.pop_front(); + // A later close of the same session keeps its own entry. + if self.closed.get(&session).map(|kept| kept.number) == Some(close.number) { + self.closed.remove(&session); + } + } + } + + /// Whether `session`'s connection closed since `received`. + fn closed_since(&self, session: SwarmSession, received: Instant) -> bool { + self.closed + .get(&session) + .is_some_and(|closed| closed.at >= received) + } + + /// Whether a request of `session` received at `received` must not act: + /// its connection closed since, it is older than any close is + /// remembered, or (a browser's) its connection is closing. + fn stale( + &self, + session: SwarmSession, + received: Instant, + closing: &CancellationToken, + now: Instant, + ) -> bool { + closing.is_cancelled() + || now.saturating_duration_since(received) >= CLOSED_SESSION_TTL + || self.closed_since(session, received) + } + + /// Remembers that `membership` of `session` (from `host`) in `room_key` + /// ended, unless a newer one of theirs already did. A connection's place + /// in a room counts once however often it ends a newer membership there; + /// past its own share, or its host's, the oldest of its own (or its + /// host's) goes, so the newest ends are the ones kept. + fn note_ended( + &mut self, + session: SwarmSession, + host: Option, + room_key: RoomKey, + membership: Membership, + now: Instant, + ) { + let key = (session, room_key); + let number = self.ends.wrapping_add(1); + // A newer end of the same place counts as the newest: it goes to the + // back of every queue, under the host it came from now. + if let Some(kept) = self.ended.get(&key) { + if kept.membership >= membership { + return; + } + self.remove_ended(&key); + } + { + let oldest_of_session = self + .ended_by_session + .get(&session) + .filter(|ended| ended.len() >= MAX_ENDED_PER_SESSION) + .and_then(|ended| ended.front().copied()); + if let Some(room) = oldest_of_session { + self.remove_ended(&(session, room)); + } + let oldest_of_host = host + .and_then(|host| self.ended_by_host.get(&host)) + .filter(|ended| ended.len() >= MAX_ENDED_PER_HOST) + .and_then(|ended| ended.front().copied()); + if let Some(oldest) = oldest_of_host { + self.remove_ended(&oldest); + } + while self.ended.len() >= MAX_ENDED_MEMBERSHIPS && self.forget_oldest_ended() {} + self.ended.insert( + key, + Ended { + membership, + number, + host, + }, + ); + self.ended_by_session + .entry(session) + .or_default() + .push_back(room_key); + if let Some(host) = host { + self.ended_by_host.entry(host).or_default().push_back(key); + } + } + self.ends = number; + self.ended_order.push_back((key, number, now)); + self.forget_ended(now); + } + + fn remove_ended(&mut self, key: &(SwarmSession, RoomKey)) { + let Some(ended) = self.ended.remove(key) else { + return; + }; + let (session, room) = *key; + if let Some(rooms) = self.ended_by_session.get_mut(&session) { + rooms.retain(|kept| *kept != room); + if rooms.is_empty() { + self.ended_by_session.remove(&session); + } + } + if let Some(host) = ended.host { + if let Some(keys) = self.ended_by_host.get_mut(&host) { + keys.retain(|kept| kept != key); + if keys.is_empty() { + self.ended_by_host.remove(&host); + } + } + } + } + + /// Whether the queue entry `(key, number)` is its membership's newest. + fn ended_live(&self, key: &(SwarmSession, RoomKey), number: u64) -> bool { + self.ended.get(key).map(|ended| ended.number) == Some(number) + } + + /// Forgets the oldest ended membership; `false` if there is none. + fn forget_oldest_ended(&mut self) -> bool { + while let Some((key, number, _)) = self.ended_order.pop_front() { + if self.ended_live(&key, number) { + self.remove_ended(&key); + return true; + } + } + false + } + + /// Forgets ended memberships past [`ENDED_MEMBERSHIP_TTL`], and drops + /// stale queue entries, so the queue stays near the memberships' number. + fn forget_ended(&mut self, now: Instant) { + while let Some((key, number, at)) = self.ended_order.front().copied() { + let live = self.ended_live(&key, number); + if live && now.saturating_duration_since(at) < ENDED_MEMBERSHIP_TTL { + break; + } + self.ended_order.pop_front(); + if live { + self.remove_ended(&key); + } + } + if self.ended_order.len() > 2 * self.ended.len() + 1024 { + let ended = &self.ended; + self.ended_order + .retain(|(key, number, _)| ended.get(key).map(|kept| kept.number) == Some(*number)); + } + } + + /// Whether `membership` of `session` in `room_key`, or a newer one of + /// theirs, ended. + fn ended_covers( + &self, + session: SwarmSession, + room_key: &RoomKey, + membership: Membership, + ) -> bool { + self.ended + .get(&(session, *room_key)) + .is_some_and(|ended| ended.membership >= membership) + } + + /// Takes `bytes` of the stored-SDP budget, and of `host`'s share of it, + /// if both fit. + fn reserve(&mut self, bytes: usize, host: Option) -> bool { + let Some(total) = self + .stored_sdp_bytes + .checked_add(bytes) + .filter(|total| *total <= MAX_STORED_SDP_BYTES) + else { + return false; + }; + if let Some(host) = host { + let held = self.sdp_by_host.get(&host).copied().unwrap_or(0); + let Some(share) = held + .checked_add(bytes) + .filter(|share| *share <= MAX_STORED_SDP_BYTES_PER_HOST) + else { + return false; + }; + self.sdp_by_host.insert(host, share); + } + self.stored_sdp_bytes = total; + true + } + + fn release(&mut self, bytes: usize, host: Option) { + self.stored_sdp_bytes = self.stored_sdp_bytes.saturating_sub(bytes); + if let Some(host) = host { + if let Some(share) = self.sdp_by_host.get_mut(&host) { + *share = share.saturating_sub(bytes); + if *share == 0 { + self.sdp_by_host.remove(&host); + } + } + } + } + + fn release_all(&mut self, freed: Vec<(usize, Option)>) { + for (bytes, host) in freed { + self.release(bytes, host); + } + } + + /// Forgets a member and everything it held, and every signal of its + /// still queued for others and every offer made to it: none can be + /// answered now, so none keeps an inbox place or the budget. + fn drop_member(&mut self, room_key: &RoomKey, member_id: &MemberId) { + let Some(room) = self.rooms.get_mut(room_key) else { + return; + }; + let Some(member) = room.members.remove(member_id) else { + return; + }; + room.by_session.remove(&member.session); + let mut freed: Vec<(usize, Option)> = member + .inbox + .iter() + .map(|queued| (queued.bytes, queued.host)) + .collect(); + let mut pending = member.pending.len(); + for other in room.members.values_mut() { + other.inbox.retain(|queued| { + let (SwarmSignal::Offer { from, .. } | SwarmSignal::Answer { from, .. }) = + &queued.signal; + let keep = from != member_id; + if !keep { + freed.push((queued.bytes, queued.host)); + } + keep + }); + let before = other.pending.len(); + other.pending.retain(|offer| offer.to != *member_id); + pending += before - other.pending.len(); + } + let empty = room.members.is_empty(); + if empty { + self.rooms.remove(room_key); + } + if let Some(rooms) = self.sessions.get_mut(&member.session) { + rooms.retain(|room| room != room_key); + if rooms.is_empty() { + self.sessions.remove(&member.session); + } + } + self.release_all(freed); + self.pending_offers = self.pending_offers.saturating_sub(pending); + self.members = self.members.saturating_sub(1); + if let Some(host) = member.host { + if let Some(count) = self.by_host.get_mut(&host) { + *count = count.saturating_sub(1); + if *count == 0 { + self.by_host.remove(&host); + } + } + } + // Waiting announces for this member end now rather than at their deadline. + member.wake.notify_waiters(); + } +} + +impl SwarmService { + /// A service with no rooms. + #[must_use] + pub fn new() -> Arc { + Arc::new(Self::default()) + } + + /// Counters for metrics. + pub fn counters(&self) -> &SwarmCounters { + &self.counters + } + + /// Whether a native swarm request from `ip` may be handled now: it takes + /// a place from the node's budget and from its host's, both or neither + /// (see [`NATIVE_SWARM_REQUESTS_PER_SECOND`]). A request with no source + /// address is refused. + pub fn admit_native(&self, ip: Option, now: Instant) -> bool { + let admitted = ip.is_some_and(|ip| self.native_budget.lock().admit(ip, now)); + if !admitted { + self.counters + .native_over_budget + .fetch_add(1, Ordering::Relaxed); + } + admitted + } + + /// Whether a reply to a request of `session` received at `received` + /// should be dropped: the connection closed since, so sending it would + /// only make the node dial a client that left. + pub fn reply_unwanted(&self, session: SwarmSession, received: Instant) -> bool { + self.state.lock().closed_since(session, received) + } + + /// Sets this node's peer id, which every request it answers must name. + pub fn set_node_id(&self, node_id: NodeId) { + *self.node_id.write() = Some(node_id); + } + + /// Sets the STUN servers handed to members (this node's own, once its + /// public address is known). + pub fn set_ice_servers(&self, servers: Vec) { + *self.ice_servers.write() = servers + .into_iter() + .filter(|server| server.len() <= MAX_SWARM_ICE_SERVER_BYTES) + .take(MAX_SWARM_ICE_SERVERS) + .collect(); + } + + /// Whether the STUN responder should answer `ip`: it announced in a room + /// within [`MEMBER_TTL`]. + pub fn stun_allows(&self, ip: IpAddr, now: Instant) -> bool { + self.announcers + .read() + .get(&host_of(ip)) + .is_some_and(|seen| now.saturating_duration_since(*seen) < MEMBER_TTL) + } + + /// Takes a place among the waiting announces, if there is one. + fn admit_waiter(&self) -> bool { + let mut waiting = self.waiting.load(Ordering::Acquire); + loop { + if waiting >= MAX_WAITING { + return false; + } + match self.waiting.compare_exchange_weak( + waiting, + waiting + 1, + Ordering::AcqRel, + Ordering::Acquire, + ) { + Ok(_) => return true, + Err(actual) => waiting = actual, + } + } + } + + fn note_announcer(&self, ip: IpAddr, now: Instant) { + // By host (an IPv4 address, or an IPv6 /64, as the STUN responder's + // own limits count), so one host cannot fill the list by announcing + // from address after address of its own. + let ip = host_of(ip); + // Most announces refresh a known address: a read lock does, so the + // STUN responder's lookups on the packet path rarely wait. + if let Some(seen) = self.announcers.read().get(&ip) { + if now.saturating_duration_since(*seen) < MEMBER_TTL / 2 { + return; + } + } + let mut announcers = self.announcers.write(); + if !announcers.contains_key(&ip) && announcers.len() >= MAX_STUN_SOURCES { + announcers.retain(|_, seen| now.saturating_duration_since(*seen) < MEMBER_TTL); + if announcers.len() >= MAX_STUN_SOURCES { + return; + } + } + announcers.insert(ip, now); + } + + /// Whether a native client's signed swarm request, from `source_ip`, is + /// new: signed within [`MAX_REQUEST_SKEW`] of `now_unix`, not more than a + /// few seconds older than the client's newest, and not seen before. A + /// signed request is authentic but not bound to the connection it came + /// on, so a node that saw it could send it again in the client's name. + /// Only an announce or a leave (`starts`) makes a client's history: an + /// answer comes from a member, which announced first, while a leave may + /// overtake the announce it ends. + pub fn first_sight( + &self, + peer: [u8; 32], + source_ip: Option, + request_id: u64, + signed_at: u64, + now_unix: u64, + starts: bool, + ) -> bool { + let skew = MAX_REQUEST_SKEW.as_secs(); + if signed_at.saturating_add(skew) < now_unix || signed_at > now_unix.saturating_add(skew) { + return false; + } + let mut histories = self.histories.lock(); + if !histories.by_peer.contains_key(&peer) + && (!starts || !histories.start(peer, source_ip.map(host_of))) + { + return false; + } + histories + .by_peer + .get_mut(&peer) + .is_some_and(|history| history.note(request_id, signed_at)) + } + + /// Counts for metrics. + pub fn stats(&self) -> SwarmStats { + let state = self.state.lock(); + SwarmStats { + rooms: state.rooms.len(), + members: state.members, + pending_offers: state.pending_offers, + stored_sdp_bytes: state.stored_sdp_bytes, + waiting: self.waiting.load(Ordering::Relaxed), + } + } + + /// Whether swarm request `body` names this node. A native client's request + /// for another node is answered with an error and leaves its history + /// alone: a node that saw a client's request could otherwise send it on + /// here to make the client's later requests here look stale. + pub fn names_this_node(&self, body: &ChunkMessageBody) -> bool { + let node = match body { + ChunkMessageBody::SwarmAnnounceRequest(request) => &request.node, + ChunkMessageBody::SwarmAnswerRequest(request) => &request.node, + ChunkMessageBody::SwarmLeaveRequest(request) => &request.node, + _ => return false, + }; + self.node_id.read().as_ref() == Some(node) + } + + /// Answers one swarm request from `session`, which arrived from + /// `source_ip` at `received`, or returns `None` if `body` is not a swarm + /// request. A native client's announce may wait for a signal; it ends at + /// its deadline, on the first signal, when its member is dropped, or when + /// `shutdown` is cancelled. A request received before its connection + /// closed (or, for a browser, whose `shutdown` is cancelled) joins nothing. + pub async fn handle( + &self, + session: SwarmSession, + source_ip: Option, + body: ChunkMessageBody, + shutdown: &CancellationToken, + received: Instant, + ) -> Option { + let for_this_node = |node: &NodeId| self.node_id.read().as_ref() == Some(node); + match body { + ChunkMessageBody::SwarmAnnounceRequest(request) => { + if !for_this_node(&request.node) { + return Some(ChunkMessageBody::SwarmAnnounceResponse( + SwarmAnnounceResponse::Error(other_node()), + )); + } + Some(ChunkMessageBody::SwarmAnnounceResponse( + self.announce(session, source_ip, request, shutdown, received) + .await, + )) + } + ChunkMessageBody::SwarmAnswerRequest(request) => { + if !for_this_node(&request.node) { + return Some(ChunkMessageBody::SwarmAnswerResponse( + SwarmAnswerResponse::Error(other_node()), + )); + } + Some(ChunkMessageBody::SwarmAnswerResponse(self.answer( + session, + source_ip.map(host_of), + &request, + received, + shutdown, + ))) + } + ChunkMessageBody::SwarmLeaveRequest(request) => { + if !for_this_node(&request.node) { + return Some(ChunkMessageBody::SwarmLeaveResponse( + SwarmLeaveResponse::Error(other_node()), + )); + } + Some(ChunkMessageBody::SwarmLeaveResponse( + self.leave(session, source_ip, &request, received, shutdown), + )) + } + _ => None, + } + } + + async fn announce( + &self, + session: SwarmSession, + source_ip: Option, + request: SwarmAnnounceRequest, + shutdown: &CancellationToken, + received: Instant, + ) -> SwarmAnnounceResponse { + self.counters.announces.fetch_add(1, Ordering::Relaxed); + if let Err(error) = request.validate() { + return SwarmAnnounceResponse::Error(error); + } + // A node browsers cannot reach (no WebRTC Direct endpoint, so no STUN + // server to name either) is no meeting place: native readers that met + // here would never meet the browsers, which look only for nodes they + // can reach, and would get no STUN from it. Refusing sends every + // client on to the closest nodes that have both, at once. + if self.ice_servers.read().is_empty() { + self.counters.refused.fetch_add(1, Ordering::Relaxed); + return SwarmAnnounceResponse::Refused(SwarmRefusal::NotHosting); + } + let mut request = request; + let host = source_ip.map(host_of); + for offer in &mut request.offers { + offer.sdp = relayable_sdp(&offer.sdp, host); + } + let now = Instant::now(); + let room = request.room; + // A browser's waiting request would hold one of its session's few + // request slots, and its byte reservation, at this node, delaying its + // reads: a browser never waits, and asks again instead. + let wait = match session { + SwarmSession::Browser(_) => Duration::ZERO, + SwarmSession::Native(_) => Duration::from_millis(u64::from(request.wait_ms)), + }; + let joined = match self.join_and_offer(session, source_ip, request, now, received, shutdown) + { + Ok(joined) => joined, + Err(refusal) => { + self.counters.refused.fetch_add(1, Ordering::Relaxed); + return SwarmAnnounceResponse::Refused(refusal); + } + }; + if let Some(ip) = source_ip { + self.note_announcer(ip, now); + } + + // Nothing waiting yet: hold the reply until a signal comes, the + // deadline passes, the member is dropped, or the node stops. One + // announce per member waits; no lock is held meanwhile. + if !wait.is_zero() { + if let Some(_turn) = self.wait_turn(&room, &joined.member) { + let deadline = tokio::time::Instant::now() + wait; + loop { + // Registered before the inbox is looked at, so a signal + // that arrives in between still wakes it. + let notified = joined.wake.notified(); + tokio::pin!(notified); + notified.as_mut().enable(); + if self.inbox_waits(&room, &joined.member) != Some(true) { + break; + } + tokio::select! { + () = &mut notified => {} + () = tokio::time::sleep_until(deadline) => break, + () = shutdown.cancelled() => break, + } + } + } + } + + let mut state = self.state.lock(); + // A member replaced or dropped while the announce waited is not + // reported as joined. + let current = state + .rooms + .get(&room) + .and_then(|kept| kept.by_session.get(&session)) + == Some(&joined.member); + let drained = + current.then(|| Self::drain(&mut state, &room, &joined.member, Instant::now())); + drop(state); + let Some((members, inbox)) = drained else { + return SwarmAnnounceResponse::Refused(SwarmRefusal::Superseded); + }; + SwarmAnnounceResponse::Joined { + member: joined.member, + members, + offers_delivered: joined.offers_delivered, + inbox, + ice_servers: self.ice_servers.read().clone(), + } + } + + /// Whether the member is still there with nothing for it yet (`None` if + /// it was dropped). + fn inbox_waits(&self, room_key: &RoomKey, member_id: &MemberId) -> Option { + self.state + .lock() + .rooms + .get(room_key) + .and_then(|room| room.members.get(member_id)) + .map(|member| member.inbox.is_empty()) + } + + /// A place among the waiting announces for this member, unless one of + /// its announces waits already or every place is taken. + fn wait_turn<'a>(&'a self, room_key: &RoomKey, member_id: &MemberId) -> Option> { + if !Self::claim_wait(&mut self.state.lock(), room_key, member_id, || { + self.admit_waiter() + }) { + return None; + } + Some(WaitTurn { + service: self, + room: *room_key, + member: *member_id, + }) + } + + /// Marks the member as waiting, if none of its announces is and `admit` + /// gives a place. + fn claim_wait( + state: &mut State, + room_key: &RoomKey, + member_id: &MemberId, + admit: impl FnOnce() -> bool, + ) -> bool { + let Some(member) = state + .rooms + .get_mut(room_key) + .and_then(|room| room.members.get_mut(member_id)) + else { + return false; + }; + if member.waiting || !admit() { + return false; + } + member.waiting = true; + true + } + + /// Joins (or refreshes) `session` in the room, and hands each offer to a + /// different member picked at random. + fn join_and_offer( + &self, + session: SwarmSession, + source_ip: Option, + request: SwarmAnnounceRequest, + now: Instant, + received: Instant, + shutdown: &CancellationToken, + ) -> Result { + let room_key = request.room; + let mut state = self.state.lock(); + // Under the lock that closing takes too: either the close comes after + // and removes this membership, or it came before and is seen here. + if state.stale(session, received, shutdown, now) { + return Err(SwarmRefusal::Busy); + } + let member_id = Self::admit_member( + &mut state, + session, + source_ip, + room_key, + request.membership, + now, + )?; + let wake = Self::refresh_member(&mut state, &room_key, &member_id, now) + .ok_or(SwarmRefusal::Busy)?; + let offers_delivered = self.deliver_offers(&mut state, &room_key, member_id, request, now); + drop(state); + Ok(Joined { + member: member_id, + offers_delivered, + wake, + }) + } + + /// The member `session` already is in the room for `membership`, or a + /// new one if the caps allow. A newer membership takes the place of the + /// member there; an older one is refused, as the client has left it. + fn admit_member( + state: &mut State, + session: SwarmSession, + source_ip: Option, + room_key: RoomKey, + membership: Membership, + now: Instant, + ) -> Result { + let current = state.rooms.get(&room_key).and_then(|room| { + let id = room.by_session.get(&session).copied()?; + room.members.get(&id).map(|member| (id, member.membership)) + }); + if let Some((id, kept)) = current { + match membership.cmp(&kept) { + std::cmp::Ordering::Equal => { + // The same membership over another connection (a native + // client's address changed) counts against its new host. + Self::rehost_member(state, &room_key, &id, source_ip.map(host_of))?; + return Ok(id); + } + std::cmp::Ordering::Less => return Err(SwarmRefusal::Superseded), + std::cmp::Ordering::Greater => { + state.note_ended(session, source_ip.map(host_of), room_key, kept, now); + state.drop_member(&room_key, &id); + } + } + } + // A membership that ended (or an older one) does not come back, however + // late its announce arrives. + if state.ended_covers(session, &room_key, membership) { + return Err(SwarmRefusal::Superseded); + } + if !state.rooms.contains_key(&room_key) && state.rooms.len() >= MAX_ROOMS { + return Err(SwarmRefusal::TooManyRooms); + } + let room_full = state + .rooms + .get(&room_key) + .is_some_and(|room| room.members.len() >= MAX_MEMBERS_PER_ROOM); + if room_full || state.members >= MAX_MEMBERS { + return Err(SwarmRefusal::RoomFull); + } + let host = source_ip.map(host_of); + let host_full = host.is_some_and(|host| { + state.by_host.get(&host).copied().unwrap_or(0) >= MAX_MEMBERS_PER_HOST + }); + if host_full || state.sessions.get(&session).map_or(0, Vec::len) >= MAX_ROOMS_PER_SESSION { + return Err(SwarmRefusal::TooManyRoomsForPeer); + } + // Every older membership of the client's here is replaced by this + // one: none comes back, even after this one goes silent. + if let Some(older) = membership.checked_sub(1) { + state.note_ended(session, host, room_key, older, now); + } + let member_id: MemberId = rand::random(); + let room = state.rooms.entry(room_key).or_default(); + room.by_session.insert(session, member_id); + room.members.insert( + member_id, + Member { + session, + membership, + host, + last_seen: now, + inbox: VecDeque::new(), + pending: VecDeque::new(), + wake: Arc::new(Notify::new()), + waiting: false, + }, + ); + state.sessions.entry(session).or_default().push(room_key); + state.members += 1; + if let Some(host) = host { + *state.by_host.entry(host).or_default() += 1; + } + Ok(member_id) + } + + /// Counts member `id` against `host` from now on, if it moved there and + /// that host has room for one more member; its signals already held stay + /// counted where they were taken. + fn rehost_member( + state: &mut State, + room_key: &RoomKey, + id: &MemberId, + host: Option, + ) -> Result<(), SwarmRefusal> { + let Some(old) = state + .rooms + .get(room_key) + .and_then(|room| room.members.get(id)) + .map(|member| member.host) + else { + return Ok(()); + }; + if old == host { + return Ok(()); + } + if host.is_some_and(|host| { + state.by_host.get(&host).copied().unwrap_or(0) >= MAX_MEMBERS_PER_HOST + }) { + return Err(SwarmRefusal::TooManyRoomsForPeer); + } + if let Some(old) = old { + if let Some(count) = state.by_host.get_mut(&old) { + *count = count.saturating_sub(1); + if *count == 0 { + state.by_host.remove(&old); + } + } + } + if let Some(host) = host { + *state.by_host.entry(host).or_default() += 1; + } + if let Some(member) = state + .rooms + .get_mut(room_key) + .and_then(|room| room.members.get_mut(id)) + { + member.host = host; + } + Ok(()) + } + + /// Marks the member as seen and forgets its offers nobody answered in + /// time. Returns its wake-up handle. + fn refresh_member( + state: &mut State, + room_key: &RoomKey, + member_id: &MemberId, + now: Instant, + ) -> Option> { + let member = state + .rooms + .get_mut(room_key) + .and_then(|room| room.members.get_mut(member_id))?; + member.last_seen = now; + let before = member.pending.len(); + member.pending.retain(|pending| pending.expires > now); + let expired = before - member.pending.len(); + let refreshed = Arc::clone(&member.wake); + state.pending_offers = state.pending_offers.saturating_sub(expired); + Some(refreshed) + } + + /// Hands each offer, in order, to a different other member picked at + /// random, and returns how many went: always the first ones. A member + /// whose inbox is full is passed over; the sender's cap on unanswered + /// offers, or the stored-SDP budget, stops the rest. + fn deliver_offers( + &self, + state: &mut State, + room_key: &RoomKey, + sender_id: MemberId, + request: SwarmAnnounceRequest, + now: Instant, + ) -> u32 { + if request.offers.is_empty() { + return 0; + } + let mut candidates: Vec = state + .rooms + .get(room_key) + .map(|room| { + room.members + .keys() + .filter(|other| **other != sender_id) + .copied() + .collect() + }) + .unwrap_or_default(); + candidates.shuffle(&mut rand::thread_rng()); + let deadline = now.checked_add(SIGNAL_TTL).unwrap_or(now); + let total = request.offers.len(); + let mut offers = request.offers.into_iter().peekable(); + let mut delivered = 0u32; + // Offers go in order, so the client knows which went: the first + // `delivered`. A recipient with a full inbox is passed over; the + // sender's pending cap or the SDP budget stops the rest. + for target in candidates { + // What the stored SDP holds, which its length alone may not say. + let Some(bytes) = offers.peek().map(|offer| offer.sdp.capacity()) else { + break; + }; + let Some(room) = state.rooms.get(room_key) else { + break; + }; + let sender_ok = room + .members + .get(&sender_id) + .is_some_and(|member| member.pending.len() < MAX_PENDING_PER_MEMBER); + if !sender_ok { + break; + } + let target_ok = room + .members + .get(&target) + .is_some_and(|member| member.inbox.len() < MAX_INBOX_PER_MEMBER); + if !target_ok { + continue; + } + let host = room.members.get(&sender_id).and_then(|member| member.host); + if !state.reserve(bytes, host) { + break; + } + let (Some(offer), Some(room)) = (offers.next(), state.rooms.get_mut(room_key)) else { + state.release(bytes, host); + break; + }; + let Some(recipient) = room.members.get_mut(&target) else { + state.release(bytes, host); + break; + }; + recipient.inbox.push_back(Queued { + signal: SwarmSignal::Offer { + from: sender_id, + offer_id: offer.offer_id, + sdp: offer.sdp, + }, + bytes, + host, + expires: deadline, + }); + recipient.wake.notify_waiters(); + if let Some(sender) = room.members.get_mut(&sender_id) { + sender.pending.push_back(Pending { + offer_id: offer.offer_id, + to: target, + expires: deadline, + }); + } + state.pending_offers += 1; + delivered += 1; + } + let undelivered = total.saturating_sub(usize::try_from(delivered).unwrap_or(usize::MAX)); + self.counters.offers_undelivered.fetch_add( + u64::try_from(undelivered).unwrap_or(u64::MAX), + Ordering::Relaxed, + ); + self.counters + .offers_delivered + .fetch_add(u64::from(delivered), Ordering::Relaxed); + delivered + } + + /// Takes the member's signals that fit in one reply, oldest first. + fn drain( + state: &mut State, + room_key: &RoomKey, + member_id: &MemberId, + now: Instant, + ) -> (u32, Vec) { + let mut signals = Vec::new(); + let mut freed = Vec::new(); + let mut reply_bytes = 0usize; + let members = state.rooms.get(room_key).map_or(0, |room| { + u32::try_from(room.members.len()).unwrap_or(u32::MAX) + }); + if let Some(member) = state + .rooms + .get_mut(room_key) + .and_then(|room| room.members.get_mut(member_id)) + { + while let Some(front) = member.inbox.front() { + if front.expires <= now { + freed.push((front.bytes, front.host)); + member.inbox.pop_front(); + continue; + } + let fits = signals.len() < MAX_SWARM_INBOX + && reply_bytes + front.bytes <= MAX_SWARM_REPLY_SDP_BYTES; + // The first signal always goes, so one large SDP cannot wedge the inbox. + if !fits && !signals.is_empty() { + break; + } + let Some(queued) = member.inbox.pop_front() else { + break; + }; + reply_bytes += queued.bytes; + freed.push((queued.bytes, queued.host)); + signals.push(queued.signal); + } + } + state.release_all(freed); + (members, signals) + } + + /// `host` is the one this request came from: the answer's candidates are + /// kept to it, and its bytes counted against it. + fn answer( + &self, + session: SwarmSession, + host: Option, + request: &SwarmAnswerRequest, + received: Instant, + closing: &CancellationToken, + ) -> SwarmAnswerResponse { + self.counters.answers.fetch_add(1, Ordering::Relaxed); + if let Err(error) = request.validate() { + return SwarmAnswerResponse::Error(error); + } + let now = Instant::now(); + let delivered = { + let mut state = self.state.lock(); + !state.stale(session, received, closing, now) + && Self::relay_answer(&mut state, session, host, request, now) + }; + let counter = if delivered { + &self.counters.answers_delivered + } else { + &self.counters.answers_unmatched + }; + counter.fetch_add(1, Ordering::Relaxed); + if delivered { + SwarmAnswerResponse::Delivered + } else { + SwarmAnswerResponse::Unmatched + } + } + + /// Queues the answer for the member that made the offer, if that member + /// delivered the offer to this one and it has not expired or been + /// answered. The offer is consumed whether or not the answer can be kept. + fn relay_answer( + state: &mut State, + session: SwarmSession, + host: Option, + request: &SwarmAnswerRequest, + now: Instant, + ) -> bool { + let Some(answerer) = state + .rooms + .get(&request.room) + .and_then(|room| room.by_session.get(&session).copied()) + else { + return false; + }; + let Some(offerer) = state + .rooms + .get_mut(&request.room) + .and_then(|room| room.members.get_mut(&request.to)) + else { + return false; + }; + let Some(position) = offerer.pending.iter().position(|pending| { + pending.offer_id == request.offer_id && pending.to == answerer && pending.expires > now + }) else { + return false; + }; + offerer.pending.remove(position); + let has_room = offerer.inbox.len() < MAX_INBOX_PER_MEMBER; + state.pending_offers = state.pending_offers.saturating_sub(1); + let sdp = relayable_sdp(&request.sdp, host); + let bytes = sdp.capacity(); + if !has_room || !state.reserve(bytes, host) { + return false; + } + let Some(offerer) = state + .rooms + .get_mut(&request.room) + .and_then(|room| room.members.get_mut(&request.to)) + else { + state.release(bytes, host); + return false; + }; + offerer.inbox.push_back(Queued { + signal: SwarmSignal::Answer { + from: answerer, + offer_id: request.offer_id, + sdp, + }, + bytes, + host, + expires: now.checked_add(SIGNAL_TTL).unwrap_or(now), + }); + offerer.wake.notify_waiters(); + true + } + + fn leave( + &self, + session: SwarmSession, + source_ip: Option, + request: &SwarmLeaveRequest, + received: Instant, + closing: &CancellationToken, + ) -> SwarmLeaveResponse { + let mut state = self.state.lock(); + // A leave from before a close would remove a later connection's + // membership; the close already removed its own. + if state.stale(session, received, closing, Instant::now()) { + return SwarmLeaveResponse::Left; + } + // The membership the leave names ends, and so does an older one in + // its place; a newer one stays. It is remembered as ended even if its + // announce has not arrived yet, so that announce cannot join. + let now = Instant::now(); + let member = state.rooms.get(&request.room).and_then(|room| { + let id = room.by_session.get(&session).copied()?; + room.members + .get(&id) + .is_some_and(|member| member.membership <= request.membership) + .then_some(id) + }); + if let Some(member) = member { + state.drop_member(&request.room, &member); + } + state.note_ended( + session, + source_ip.map(host_of), + request.room, + request.membership, + now, + ); + SwarmLeaveResponse::Left + } + + /// Removes every membership of a connection that closed, and remembers + /// the close a while, so a request of the connection still being handled + /// cannot join again. + pub fn session_closed(&self, session: SwarmSession) { + let mut state = self.state.lock(); + if let SwarmSession::Native(_) = session { + state.note_close(session, Instant::now()); + } + let rooms = state.sessions.get(&session).cloned().unwrap_or_default(); + for room_key in rooms { + let member = state + .rooms + .get(&room_key) + .and_then(|room| room.by_session.get(&session).copied()); + if let Some(member) = member { + state.drop_member(&room_key, &member); + } + } + } + + /// Drops members that stopped announcing, and signals and offers that + /// expired. + pub fn sweep(&self, now: Instant) { + let mut guard = self.state.lock(); + let state = &mut *guard; + let mut silent = Vec::new(); + let mut freed = Vec::new(); + let mut expired_offers = 0usize; + for (room_key, room) in &mut state.rooms { + for (member_id, member) in &mut room.members { + if now.saturating_duration_since(member.last_seen) >= MEMBER_TTL { + silent.push((*room_key, *member_id)); + continue; + } + member.inbox.retain(|queued| { + let keep = queued.expires > now; + if !keep { + freed.push((queued.bytes, queued.host)); + } + keep + }); + let before = member.pending.len(); + member.pending.retain(|pending| pending.expires > now); + expired_offers += before - member.pending.len(); + } + } + state.release_all(freed); + state.pending_offers = state.pending_offers.saturating_sub(expired_offers); + for (room_key, member_id) in &silent { + state.drop_member(room_key, member_id); + } + state.forget_closes(now); + state.forget_ended(now); + drop(guard); + let now_unix = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |since| since.as_secs()); + self.histories + .lock() + .forget_older_than(now_unix.saturating_sub(MAX_REQUEST_SKEW.as_secs())); + self.announcers + .write() + .retain(|_, seen| now.saturating_duration_since(*seen) < MEMBER_TTL); + if !silent.is_empty() { + self.counters.members_expired.fetch_add( + u64::try_from(silent.len()).unwrap_or(u64::MAX), + Ordering::Relaxed, + ); + debug!(expired = silent.len(), "swarm members expired"); + } + } +} + +struct Joined { + member: MemberId, + offers_delivered: u32, + wake: Arc, +} + +/// A waiting announce's place, given back however the wait ends. +struct WaitTurn<'a> { + service: &'a SwarmService, + room: RoomKey, + member: MemberId, +} + +impl Drop for WaitTurn<'_> { + fn drop(&mut self) { + self.service.waiting.fetch_sub(1, Ordering::AcqRel); + if let Some(member) = self + .service + .state + .lock() + .rooms + .get_mut(&self.room) + .and_then(|room| room.members.get_mut(&self.member)) + { + member.waiting = false; + } + } +} + +#[cfg(test)] +#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] +mod tests { + use super::*; + use ant_protocol::swarm::{SwarmOffer, MAX_SWARM_SDP_BYTES}; + + /// The node the tests' requests are for. + const NODE: NodeId = [0xee; 32]; + + /// A node's STUN server, as one with a browser endpoint names it. + const STUN: &str = "stun:203.0.113.9:10000"; + + fn service() -> Arc { + let service = SwarmService::new(); + service.set_node_id(NODE); + service.set_ice_servers(vec![STUN.into()]); + service + } + + fn announce(room: u8, offers: &[u8], wait_ms: u32) -> ChunkMessageBody { + ChunkMessageBody::SwarmAnnounceRequest(SwarmAnnounceRequest { + room: [room; 32], + node: NODE, + membership: 1, + offers: offers + .iter() + .map(|id| SwarmOffer { + offer_id: [*id; 16], + sdp: format!("v=0 offer {id}"), + }) + .collect(), + wait_ms, + }) + } + + struct Reply { + member: MemberId, + members: u32, + delivered: u32, + inbox: Vec, + } + + async fn call(service: &SwarmService, session: SwarmSession, body: ChunkMessageBody) -> Reply { + let token = CancellationToken::new(); + match service + .handle(session, None, body, &token, Instant::now()) + .await + { + Some(ChunkMessageBody::SwarmAnnounceResponse(SwarmAnnounceResponse::Joined { + member, + members, + offers_delivered, + inbox, + .. + })) => Reply { + member, + members, + delivered: offers_delivered, + inbox, + }, + other => panic!("unexpected reply {other:?}"), + } + } + + fn answer(room: u8, to: MemberId, offer: u8) -> ChunkMessageBody { + ChunkMessageBody::SwarmAnswerRequest(SwarmAnswerRequest { + room: [room; 32], + node: NODE, + to, + offer_id: [offer; 16], + sdp: "v=0 answer".into(), + }) + } + + async fn answer_reply( + service: &SwarmService, + session: SwarmSession, + body: ChunkMessageBody, + ) -> SwarmAnswerResponse { + match service + .handle( + session, + None, + body, + &CancellationToken::new(), + Instant::now(), + ) + .await + { + Some(ChunkMessageBody::SwarmAnswerResponse(response)) => response, + other => panic!("unexpected reply {other:?}"), + } + } + + const A: SwarmSession = SwarmSession::Browser(1); + const B: SwarmSession = SwarmSession::Native([2; 32]); + const C: SwarmSession = SwarmSession::Browser(3); + + #[tokio::test] + async fn an_offer_reaches_another_member_and_its_answer_comes_back() { + let service = service(); + let b = call(&service, B, announce(1, &[], 0)).await; + assert_eq!(b.members, 1); + let a = call(&service, A, announce(1, &[30, 31], 0)).await; + assert_eq!((a.members, a.delivered), (2, 1)); + assert_ne!(a.member, b.member); + + let inbox = call(&service, B, announce(1, &[], 0)).await.inbox; + let [SwarmSignal::Offer { from, offer_id, .. }] = inbox.as_slice() else { + panic!("expected one offer: {inbox:?}"); + }; + assert_eq!(*from, a.member); + let offer = offer_id.first().copied().unwrap_or(0); + assert_eq!( + answer_reply(&service, B, answer(1, a.member, offer)).await, + SwarmAnswerResponse::Delivered + ); + let inbox = call(&service, A, announce(1, &[], 0)).await.inbox; + assert!(matches!( + inbox.as_slice(), + [SwarmSignal::Answer { from, .. }] if *from == b.member + )); + // Answered once only. + assert_eq!( + answer_reply(&service, B, answer(1, a.member, offer)).await, + SwarmAnswerResponse::Unmatched + ); + assert_eq!(service.stats().pending_offers, 0); + assert_eq!(service.stats().stored_sdp_bytes, 0); + } + + /// An SDP naming `own` (its sender's public address), a stranger's public + /// address and a local one. + fn pointing(own: &str) -> String { + format!( + "v=0\r\n\ + a=candidate:1 1 udp 1 {own} 5000 typ srflx raddr 0.0.0.0 rport 0\r\n\ + a=candidate:2 1 udp 1 9.9.9.99 5001 typ host\r\n\ + a=candidate:3 1 udp 1 192.168.1.4 5002 typ host\r\n" + ) + } + + /// Sends `body` from `session` at `ip`; the inbox and member id of an + /// announce's reply, if it was one. + async fn from( + service: &SwarmService, + session: SwarmSession, + ip: &str, + body: ChunkMessageBody, + ) -> Option<(MemberId, Vec)> { + let ip: IpAddr = ip.parse().expect("ip"); + match service + .handle( + session, + Some(ip), + body, + &CancellationToken::new(), + Instant::now(), + ) + .await + { + Some(ChunkMessageBody::SwarmAnnounceResponse(SwarmAnnounceResponse::Joined { + member, + inbox, + .. + })) => Some((member, inbox)), + _ => None, + } + } + + #[tokio::test] + async fn a_relayed_sdp_names_no_public_address_but_its_senders() { + let service = service(); + from(&service, B, "1.1.1.5", announce(1, &[], 0)).await; + let mut offer = announce(1, &[30], 0); + if let ChunkMessageBody::SwarmAnnounceRequest(request) = &mut offer { + for sent in &mut request.offers { + sent.sdp = pointing("1.1.1.4"); + } + } + let (a, _) = from(&service, A, "1.1.1.4", offer).await.expect("joined"); + let (_, inbox) = from(&service, B, "1.1.1.5", announce(1, &[], 0)) + .await + .expect("joined"); + let [SwarmSignal::Offer { sdp, .. }] = inbox.as_slice() else { + panic!("expected one offer: {inbox:?}"); + }; + assert!(sdp.contains("1.1.1.4 5000"), "{sdp}"); + // A member reached at a public address keeps no local candidate. + assert!(!sdp.contains("192.168.1.4"), "{sdp}"); + assert!(!sdp.contains("9.9.9.99"), "{sdp}"); + + // The answer, likewise, names its own sender's public address only. + let mut reply = answer(1, a, 30); + if let ChunkMessageBody::SwarmAnswerRequest(request) = &mut reply { + request.sdp = pointing("1.1.1.5"); + } + from(&service, B, "1.1.1.5", reply).await; + let (_, inbox) = from(&service, A, "1.1.1.4", announce(1, &[], 0)) + .await + .expect("joined"); + let [SwarmSignal::Answer { sdp, .. }] = inbox.as_slice() else { + panic!("expected one answer: {inbox:?}"); + }; + assert!(sdp.contains("1.1.1.5 5000"), "{sdp}"); + assert!(!sdp.contains("9.9.9.99"), "{sdp}"); + // What is held is what goes, and all of it is let go. + assert_eq!(service.stats().stored_sdp_bytes, 0); + } + + #[tokio::test] + async fn a_member_that_moves_counts_and_answers_from_its_new_host() { + let service = service(); + let (old, new): (IpAddr, IpAddr) = ( + "1.1.1.5".parse().expect("ip"), + "1.1.1.6".parse().expect("ip"), + ); + from(&service, B, "1.1.1.5", announce(1, &[], 0)).await; + // The same membership, over a connection from another address. + from(&service, B, "1.1.1.6", announce(1, &[], 0)).await; + { + let state = service.state.lock(); + assert_eq!(state.by_host.get(&old), None); + assert_eq!(state.by_host.get(&new), Some(&1)); + } + let (a, _) = from(&service, A, "1.1.1.4", announce(1, &[30], 0)) + .await + .expect("joined"); + from(&service, B, "1.1.1.6", announce(1, &[], 0)).await; + let mut reply = answer(1, a, 30); + if let ChunkMessageBody::SwarmAnswerRequest(request) = &mut reply { + request.sdp = pointing("1.1.1.6"); + } + from(&service, B, "1.1.1.6", reply).await; + let (_, inbox) = from(&service, A, "1.1.1.4", announce(1, &[], 0)) + .await + .expect("joined"); + let [SwarmSignal::Answer { sdp, .. }] = inbox.as_slice() else { + panic!("expected one answer: {inbox:?}"); + }; + // The answer names the address it came from, which is the member's own. + assert!(sdp.contains("1.1.1.6 5000"), "{sdp}"); + } + + #[tokio::test] + async fn an_sdp_of_dropped_candidates_is_charged_only_what_it_keeps() { + let service = service(); + from(&service, B, "1.1.1.5", announce(1, &[], 0)).await; + let mut offer = announce(1, &[31], 0); + if let ChunkMessageBody::SwarmAnnounceRequest(request) = &mut offer { + for sent in &mut request.offers { + // A stranger's address, spelled as a receiver still reads + // it, filling the SDP: none of it is passed on. + sent.sdp = String::from("v=0\r\n"); + while sent.sdp.len() < MAX_SWARM_SDP_BYTES - 64 { + sent.sdp + .push_str("a= candidate:2 1 udp 1 9.9.9.99 5001 typ host\r\n"); + } + } + } + from(&service, A, "1.1.1.4", offer).await; + assert_eq!(service.stats().pending_offers, 1); + let held = service.stats().stored_sdp_bytes; + assert!(held < 64, "{held} bytes charged for a 5-byte SDP"); + let (_, inbox) = from(&service, B, "1.1.1.5", announce(1, &[], 0)) + .await + .expect("joined"); + let [SwarmSignal::Offer { sdp, .. }] = inbox.as_slice() else { + panic!("expected one offer: {inbox:?}"); + }; + assert_eq!(sdp, "v=0\r\n"); + } + + #[test] + fn a_member_on_this_nodes_own_network_keeps_its_local_candidates() { + let sdp = format!( + "{}a=candidate:4 1 udp 1 1f2e3d4c.local 5003 typ host\r\n", + pointing("10.78.3.2") + ); + let kept = relayable_sdp(&sdp, Some(host_of("10.78.3.2".parse().expect("ip")))); + assert!(kept.contains("10.78.3.2 5000"), "{kept}"); + assert!(kept.contains("192.168.1.4 5002"), "{kept}"); + assert!(kept.contains("1f2e3d4c.local"), "{kept}"); + assert!(!kept.contains("9.9.9.99"), "{kept}"); + let public = relayable_sdp(&sdp, Some(host_of("1.1.1.4".parse().expect("ip")))); + assert!( + !public.contains(".local") && !public.contains("192.168.1.4"), + "{public}" + ); + // A member whose address this node cannot tell keeps nothing local. + assert!(!relayable_sdp(&sdp, None).contains("192.168.1.4")); + } + + #[tokio::test] + async fn answers_nobody_asked_for_go_nowhere() { + let service = service(); + let a = call(&service, A, announce(1, &[], 0)).await; + let _b = call(&service, B, announce(1, &[], 0)).await; + let _c = call(&service, C, announce(2, &[], 0)).await; + // No offer 9 was delivered to B; C is not in room 1. + assert_eq!( + answer_reply(&service, B, answer(1, a.member, 9)).await, + SwarmAnswerResponse::Unmatched + ); + assert_eq!( + answer_reply(&service, C, answer(1, a.member, 9)).await, + SwarmAnswerResponse::Unmatched + ); + assert_eq!(call(&service, A, announce(1, &[], 0)).await.inbox, []); + } + + #[test] + fn a_hosts_native_requests_share_one_budget() { + let service = service(); + let now = Instant::now(); + let host: IpAddr = "198.51.100.4".parse().expect("ip"); + for _ in 0..NATIVE_SWARM_REQUESTS_PER_SECOND_PER_HOST { + assert!(service.admit_native(Some(host), now)); + } + // Whatever identities it uses, the host has spent its second. + assert!(!service.admit_native(Some(host), now)); + assert_eq!( + service + .counters() + .native_over_budget + .load(Ordering::Relaxed), + 1 + ); + // Another host is not held back, and the refused request took + // nothing of the node's budget. + let other: IpAddr = "198.51.100.5".parse().expect("ip"); + assert!(service.admit_native(Some(other), now)); + let all = service.native_budget.lock().all.units / TOKEN; + assert_eq!( + all, + u128::from( + NATIVE_SWARM_REQUESTS_PER_SECOND - NATIVE_SWARM_REQUESTS_PER_SECOND_PER_HOST - 1 + ) + ); + // A second later the host has its budget back. + assert!(service.admit_native(Some(host), now + Duration::from_secs(1))); + // A request with no source address is never admitted. + assert!(!service.admit_native(None, now)); + } + + #[test] + fn an_ipv6_hosts_addresses_share_its_budget() { + let service = service(); + let now = Instant::now(); + for n in 0..NATIVE_SWARM_REQUESTS_PER_SECOND_PER_HOST { + let ip: IpAddr = format!("2001:db8:1:2::{n:x}").parse().expect("ip"); + assert!(service.admit_native(Some(ip), now)); + } + let same_64: IpAddr = "2001:db8:1:2:ffff::1".parse().expect("ip"); + assert!(!service.admit_native(Some(same_64), now)); + let next_64: IpAddr = "2001:db8:1:3::1".parse().expect("ip"); + assert!(service.admit_native(Some(next_64), now)); + } + + #[test] + fn many_hosts_together_get_the_nodes_budget_and_no_more() { + let service = service(); + let now = Instant::now(); + let host = |n: u32| IpAddr::from(std::net::Ipv4Addr::from(0x0a00_0000 + n)); + for n in 0..NATIVE_SWARM_REQUESTS_PER_SECOND { + assert!(service.admit_native(Some(host(n)), now)); + } + // A host that has sent nothing is refused too: the node's second is spent. + assert!(!service.admit_native(Some(host(10_000)), now)); + // It refills at its rate. + let later = now + Duration::from_secs(1) / NATIVE_SWARM_REQUESTS_PER_SECOND; + assert!(service.admit_native(Some(host(10_000)), later)); + assert!(!service.admit_native(Some(host(10_001)), later)); + } + + #[test] + fn the_hosts_remembered_stay_bounded_and_a_new_host_still_gets_in() { + let service = service(); + let start = Instant::now(); + // More hosts than are remembered, each once, slower than the node's + // budget refills. + let spacing = Duration::from_millis(20); + let total = u32::try_from(MAX_BUDGETED_HOSTS).expect("fits") + 100; + let mut at = start; + for n in 0..total { + at = start + spacing * n; + let ip = IpAddr::from(std::net::Ipv4Addr::from(0x0a00_0000 + n)); + assert!(service.admit_native(Some(ip), at), "host {n}"); + assert!(service.native_budget.lock().hosts.len() <= MAX_BUDGETED_HOSTS); + } + // A host it forgot (its budget had refilled) starts afresh. + let first = IpAddr::from(std::net::Ipv4Addr::from(0x0a00_0000)); + assert!(service.admit_native(Some(first), at)); + } + + #[tokio::test] + async fn only_the_member_an_offer_went_to_can_answer_it() { + let service = service(); + let _b = call(&service, B, announce(1, &[], 0)).await; + // A's offer 50 goes to B, the only other member then. + let a = call(&service, A, announce(1, &[50], 0)).await; + assert_eq!(a.delivered, 1); + // C joins the same room and answers it with its real id. + let _c = call(&service, C, announce(1, &[], 0)).await; + assert_eq!( + answer_reply(&service, C, answer(1, a.member, 50)).await, + SwarmAnswerResponse::Unmatched + ); + // That took nothing: the offer still waits for B, and A has no answer. + assert_eq!(service.stats().pending_offers, 1); + assert_eq!(call(&service, A, announce(1, &[], 0)).await.inbox, []); + assert_eq!( + answer_reply(&service, B, answer(1, a.member, 50)).await, + SwarmAnswerResponse::Delivered + ); + assert_eq!(service.stats().pending_offers, 0); + assert_eq!(call(&service, A, announce(1, &[], 0)).await.inbox.len(), 1); + } + + #[tokio::test] + async fn a_waiting_announce_wakes_on_the_first_signal() { + let service = service(); + let _b = call(&service, B, announce(1, &[], 0)).await; + let waiting = { + let service = Arc::clone(&service); + tokio::spawn(async move { call(&service, B, announce(1, &[], 10_000)).await }) + }; + tokio::time::sleep(Duration::from_millis(50)).await; + assert_eq!(service.stats().waiting, 1); + let started = Instant::now(); + let _a = call(&service, A, announce(1, &[40], 0)).await; + let reply = waiting.await.expect("task"); + assert_eq!(reply.inbox.len(), 1); + assert!(started.elapsed() < Duration::from_secs(2)); + assert_eq!(service.stats().waiting, 0); + } + + #[tokio::test] + async fn a_waiting_announce_ends_when_the_node_stops() { + let service = service(); + let token = CancellationToken::new(); + let waiting = { + let service = Arc::clone(&service); + let token = token.clone(); + tokio::spawn(async move { + service + .handle(B, None, announce(1, &[], 20_000), &token, Instant::now()) + .await + }) + }; + tokio::time::sleep(Duration::from_millis(50)).await; + token.cancel(); + let reply = tokio::time::timeout(Duration::from_secs(2), waiting).await; + assert!(matches!( + reply, + Ok(Ok(Some(ChunkMessageBody::SwarmAnnounceResponse( + SwarmAnnounceResponse::Joined { .. } + )))) + )); + } + + #[tokio::test] + async fn caps_refuse_newcomers_and_leaving_frees_the_place() { + let service = service(); + for n in 0..MAX_ROOMS_PER_SESSION { + let room = u8::try_from(n).unwrap_or(0); + let _ = call(&service, A, announce(room, &[], 0)).await; + } + let refused = service + .handle( + A, + None, + announce(200, &[], 0), + &CancellationToken::new(), + Instant::now(), + ) + .await; + assert!(matches!( + refused, + Some(ChunkMessageBody::SwarmAnnounceResponse( + SwarmAnnounceResponse::Refused(SwarmRefusal::TooManyRoomsForPeer) + )) + )); + service.session_closed(A); + assert_eq!(service.stats(), SwarmStats::default()); + let _ = call(&service, A, announce(200, &[], 0)).await; + let _ = service + .handle( + A, + None, + ChunkMessageBody::SwarmLeaveRequest(SwarmLeaveRequest { + room: [200; 32], + node: NODE, + membership: 1, + }), + &CancellationToken::new(), + Instant::now(), + ) + .await; + assert_eq!(service.stats().members, 0); + } + + /// Marks every member of `room` as having announced at `at`, without the + /// announce itself, which would take the member's inbox. + fn seen_at(service: &SwarmService, room: u8, at: Instant) { + if let Some(room) = service.state.lock().rooms.get_mut(&[room; 32]) { + for member in room.members.values_mut() { + member.last_seen = at; + } + } + } + + #[tokio::test] + async fn unanswered_offers_expire_while_their_members_stay() { + let service = service(); + let start = Instant::now(); + let _b = call(&service, B, announce(1, &[], 0)).await; + let _a = call(&service, A, announce(1, &[50], 0)).await; + assert_eq!(service.stats().pending_offers, 1); + assert!(service.stats().stored_sdp_bytes > 0); + // Both members keep announcing, so only the signal's own deadline can + // end the offer: it outlives MEMBER_TTL, which would otherwise take + // the members and their offers with them. + let before = start + SIGNAL_TTL.saturating_sub(Duration::from_secs(1)); + seen_at(&service, 1, before); + service.sweep(before); + assert_eq!(service.stats().members, 2); + assert_eq!(service.stats().pending_offers, 1); + assert!(service.stats().stored_sdp_bytes > 0); + let after = start + SIGNAL_TTL + Duration::from_secs(1); + seen_at(&service, 1, after); + service.sweep(after); + assert_eq!(service.stats().members, 2); + assert_eq!(service.stats().pending_offers, 0); + assert_eq!(service.stats().stored_sdp_bytes, 0); + } + + #[tokio::test] + async fn silent_members_expire_with_their_offers() { + let service = service(); + let start = Instant::now(); + let _b = call(&service, B, announce(1, &[], 0)).await; + let _a = call(&service, A, announce(1, &[50], 0)).await; + service.sweep(start + MEMBER_TTL.saturating_sub(Duration::from_secs(1))); + assert_eq!(service.stats().members, 2); + // Silent for MEMBER_TTL, well before the offer's own deadline. + service.sweep(start + MEMBER_TTL + Duration::from_secs(1)); + assert_eq!(service.stats(), SwarmStats::default()); + } + + #[tokio::test] + async fn a_node_browsers_cannot_reach_hosts_no_rooms() { + let service = SwarmService::new(); + service.set_node_id(NODE); + let token = CancellationToken::new(); + let refused = service + .handle(A, None, announce(1, &[], 0), &token, Instant::now()) + .await; + assert!(matches!( + refused, + Some(ChunkMessageBody::SwarmAnnounceResponse( + SwarmAnnounceResponse::Refused(SwarmRefusal::NotHosting) + )) + )); + assert_eq!(service.stats().members, 0); + // Once it has a browser endpoint, and so a STUN server to name, it does. + service.set_ice_servers(vec![STUN.into()]); + let joined = service + .handle(A, None, announce(1, &[], 0), &token, Instant::now()) + .await; + assert!(matches!( + joined, + Some(ChunkMessageBody::SwarmAnnounceResponse( + SwarmAnnounceResponse::Joined { .. } + )) + )); + } + + #[tokio::test] + async fn stun_answers_only_recent_announcers_and_replies_name_it() { + let service = service(); + let ip: IpAddr = "198.51.100.4".parse().expect("ip"); + let other: IpAddr = "198.51.100.5".parse().expect("ip"); + let now = Instant::now(); + assert!(!service.stun_allows(ip, now)); + let reply = service + .handle( + A, + Some(ip), + announce(1, &[], 0), + &CancellationToken::new(), + Instant::now(), + ) + .await; + assert!(matches!( + reply, + Some(ChunkMessageBody::SwarmAnnounceResponse(SwarmAnnounceResponse::Joined { ice_servers, .. })) + if ice_servers == vec![STUN.to_string()] + )); + assert!(service.stun_allows(ip, Instant::now())); + assert!(!service.stun_allows(other, Instant::now())); + // An IPv4-mapped form of the same address counts as the same. + let mapped: IpAddr = "::ffff:198.51.100.4".parse().expect("ip"); + assert!(service.stun_allows(mapped, Instant::now())); + service.sweep(Instant::now() + MEMBER_TTL + Duration::from_secs(1)); + assert!(!service.stun_allows(ip, Instant::now() + MEMBER_TTL + Duration::from_secs(1))); + } + + #[test] + fn swarm_requests_are_told_apart_from_their_header() { + use ant_protocol::chunk::{ChunkGetRequest, ChunkMessage}; + let encode = |body: ChunkMessageBody, request_id: u64| { + ChunkMessage { request_id, body }.encode().expect("encode") + }; + for request_id in [0, 1, 300, u64::MAX] { + assert!(is_swarm_request_bytes(&encode( + announce(1, &[1], 0), + request_id + ))); + assert!(is_swarm_request_bytes(&encode( + answer(1, [2; 16], 3), + request_id + ))); + assert!(is_swarm_request_bytes(&encode( + ChunkMessageBody::SwarmLeaveRequest(SwarmLeaveRequest { + room: [1; 32], + node: NODE, + membership: 1, + }), + request_id + ))); + assert!(!is_swarm_request_bytes(&encode( + ChunkMessageBody::GetRequest(ChunkGetRequest::new([1; 32])), + request_id + ))); + } + assert!(!is_swarm_request_bytes(&[])); + assert!(!is_swarm_request_bytes(&[0xff; 12])); + } + + #[tokio::test] + async fn a_browser_never_waits() { + let service = service(); + let started = Instant::now(); + let _ = call(&service, A, announce(1, &[], 10_000)).await; + assert!(started.elapsed() < Duration::from_secs(1)); + } + + #[tokio::test] + async fn a_member_waits_with_one_announce_at_a_time() { + let service = service(); + let _ = call(&service, B, announce(1, &[], 0)).await; + let waiting = { + let service = Arc::clone(&service); + tokio::spawn(async move { call(&service, B, announce(1, &[], 10_000)).await }) + }; + tokio::time::sleep(Duration::from_millis(50)).await; + let started = Instant::now(); + let _ = call(&service, B, announce(1, &[], 10_000)).await; + assert!(started.elapsed() < Duration::from_secs(1)); + assert_eq!(service.stats().waiting, 1); + // Closing the connection ends the wait, and frees the place. + service.session_closed(B); + let ended = tokio::time::timeout(Duration::from_secs(2), waiting).await; + assert!(ended.is_ok()); + assert_eq!(service.stats().waiting, 0); + } + + #[tokio::test] + async fn a_signal_already_taken_does_not_cut_a_later_wait_short() { + let service = service(); + let _ = call(&service, B, announce(1, &[], 0)).await; + // An offer arrives while B is not waiting, and B takes it at once. + let _ = call(&service, A, announce(1, &[60], 0)).await; + assert_eq!(call(&service, B, announce(1, &[], 0)).await.inbox.len(), 1); + let started = Instant::now(); + let reply = call(&service, B, announce(1, &[], 300)).await; + assert_eq!(reply.inbox, []); + assert!(started.elapsed() >= Duration::from_millis(250)); + } + + #[tokio::test] + async fn a_request_from_before_its_connection_closed_joins_nothing() { + let service = service(); + let before = Instant::now(); + tokio::time::sleep(Duration::from_millis(5)).await; + service.session_closed(B); + let late = service + .handle( + B, + None, + announce(1, &[], 0), + &CancellationToken::new(), + before, + ) + .await; + assert!(matches!( + late, + Some(ChunkMessageBody::SwarmAnnounceResponse( + SwarmAnnounceResponse::Refused(SwarmRefusal::Busy) + )) + )); + assert_eq!(service.stats().members, 0); + // A browser whose connection is closing joins nothing either. + let closing = CancellationToken::new(); + closing.cancel(); + let refused = service + .handle(A, None, announce(1, &[], 0), &closing, Instant::now()) + .await; + assert!(matches!( + refused, + Some(ChunkMessageBody::SwarmAnnounceResponse( + SwarmAnnounceResponse::Refused(SwarmRefusal::Busy) + )) + )); + // A request after the close is a new connection's. + let _ = call(&service, B, announce(1, &[], 0)).await; + assert_eq!(service.stats().members, 1); + } + + #[tokio::test] + async fn a_request_for_another_node_is_refused() { + let service = service(); + let mut elsewhere = SwarmAnnounceRequest { + room: [1; 32], + node: [1; 32], + membership: 1, + offers: Vec::new(), + wait_ms: 0, + }; + let token = CancellationToken::new(); + let reply = service + .handle( + B, + None, + ChunkMessageBody::SwarmAnnounceRequest(elsewhere.clone()), + &token, + Instant::now(), + ) + .await; + assert!(matches!( + reply, + Some(ChunkMessageBody::SwarmAnnounceResponse( + SwarmAnnounceResponse::Error(_) + )) + )); + assert_eq!(service.stats().members, 0); + // A node that does not know its own id yet answers nothing. + let unknown = SwarmService::new(); + elsewhere.node = NODE; + let reply = unknown + .handle( + B, + None, + ChunkMessageBody::SwarmAnnounceRequest(elsewhere), + &token, + Instant::now(), + ) + .await; + assert!(matches!( + reply, + Some(ChunkMessageBody::SwarmAnnounceResponse( + SwarmAnnounceResponse::Error(_) + )) + )); + // A leave for another node is told so too, and ends nothing here. + let _ = call(&service, A, announce(1, &[], 0)).await; + let leave = service + .handle( + A, + None, + ChunkMessageBody::SwarmLeaveRequest(SwarmLeaveRequest { + room: [1; 32], + node: [1; 32], + membership: 1, + }), + &token, + Instant::now(), + ) + .await; + assert!(matches!( + leave, + Some(ChunkMessageBody::SwarmLeaveResponse( + SwarmLeaveResponse::Error(_) + )) + )); + assert_eq!(service.stats().members, 1); + } + + #[tokio::test] + async fn a_leave_from_before_a_close_spares_the_next_connection() { + let service = service(); + let leave_received = Instant::now(); + tokio::time::sleep(Duration::from_millis(5)).await; + service.session_closed(B); + // The same client, connected again, joins. + let _ = call(&service, B, announce(1, &[], 0)).await; + let leave = ChunkMessageBody::SwarmLeaveRequest(SwarmLeaveRequest { + room: [1; 32], + node: NODE, + membership: 1, + }); + let _ = service + .handle(B, None, leave, &CancellationToken::new(), leave_received) + .await; + assert_eq!(service.stats().members, 1); + } + + #[tokio::test] + async fn offers_go_in_order_and_the_rest_are_counted() { + let service = service(); + let _b = call(&service, B, announce(1, &[], 0)).await; + // Two offers, one other member: the first goes, the second does not. + let a = call(&service, A, announce(1, &[70, 71], 0)).await; + assert_eq!(a.delivered, 1); + let inbox = call(&service, B, announce(1, &[], 0)).await.inbox; + assert!(matches!( + inbox.as_slice(), + [SwarmSignal::Offer { offer_id, .. }] if *offer_id == [70; 16] + )); + assert_eq!( + service + .counters() + .offers_undelivered + .load(Ordering::Relaxed), + 1 + ); + } + + #[test] + fn closes_of_one_session_keep_the_queue_bounded() { + let mut state = State::default(); + let start = Instant::now(); + let session = SwarmSession::Native([7; 32]); + for _ in 0..(3 * MAX_CLOSED_SESSIONS) { + state.note_close(session, start); + } + assert!(state.closed_order.len() <= MAX_CLOSED_SESSIONS); + assert!(state.closed_since(session, start)); + } + + #[test] + fn closes_are_forgotten_in_order_and_the_newest_is_kept() { + let mut state = State::default(); + let start = Instant::now(); + for n in 0..(MAX_CLOSED_SESSIONS + 10) { + let mut peer = [0u8; 32]; + peer[..8].copy_from_slice(&(n as u64).to_be_bytes()); + state.note_close(SwarmSession::Native(peer), start); + } + assert_eq!(state.closed.len(), MAX_CLOSED_SESSIONS); + // The newest close is always remembered. + let newest = SwarmSession::Native([0xff; 32]); + state.note_close(newest, start); + assert!(state.closed.contains_key(&newest)); + state.forget_closes(start + CLOSED_SESSION_TTL); + assert!(state.closed.is_empty() && state.closed_order.is_empty()); + } + + #[test] + fn a_signed_request_counts_once_and_only_while_fresh() { + let service = service(); + let now = 1_000_000; + let skew = MAX_REQUEST_SKEW.as_secs(); + let seen = + |peer: u8, id: u64, at: u64| service.first_sight([peer; 32], None, id, at, now, true); + assert!(seen(1, 7, now)); + // The same request again, as a node that saw it might send it. + assert!(!seen(1, 7, now)); + // Another client's request with the same id is its own. + assert!(seen(2, 7, now)); + // Too old, or from too far ahead. + assert!(!seen(1, 8, now - skew - 1)); + assert!(!seen(1, 9, now + skew + 1)); + // Overtaken by a newer request by a few seconds: still fine. + assert!(seen(1, 10, now + 4)); + assert!(seen(1, 11, now)); + // Much older than the newest: an old request sent again. + assert!(!seen(1, 12, now - REQUEST_REORDER - 1)); + } + + #[test] + fn only_an_announce_or_a_leave_starts_a_client_s_history() { + assert!(starts_history(&announce(1, &[], 0))); + assert!(starts_history(&ChunkMessageBody::SwarmLeaveRequest( + SwarmLeaveRequest { + room: [1; 32], + node: NODE, + membership: 1, + } + ))); + assert!(!starts_history(&answer(1, [2; 16], 3))); + let service = service(); + let now = 1_000_000; + assert!(!service.first_sight([3; 32], None, 1, now, now, false)); + assert!(service.histories.lock().by_peer.is_empty()); + assert!(service.first_sight([3; 32], None, 2, now, now, true)); + assert!(service.first_sight([3; 32], None, 3, now, now, false)); + } + + #[test] + fn one_host_keeps_only_so_many_histories_and_idle_ones_go() { + let service = service(); + let now = 1_000_000; + let host: IpAddr = "198.51.100.4".parse().expect("ip"); + let peer = |n: u32| { + let mut peer = [0u8; 32]; + peer[..4].copy_from_slice(&n.to_be_bytes()); + peer + }; + for n in 0..u32::try_from(MAX_HISTORIES_PER_HOST).unwrap_or(u32::MAX) { + assert!(service.first_sight(peer(n), Some(host), 1, now, now, true)); + } + assert!(!service.first_sight(peer(100_000), Some(host), 1, now, now, true)); + // Another host is not held back. + let other: IpAddr = "198.51.100.5".parse().expect("ip"); + assert!(service.first_sight(peer(100_001), Some(other), 1, now, now, true)); + // Once nothing they hold could pass the age check, they go. + let later = now + MAX_REQUEST_SKEW.as_secs() + 1; + service + .histories + .lock() + .forget_older_than(later - MAX_REQUEST_SKEW.as_secs()); + assert!(service.histories.lock().by_peer.is_empty()); + assert!(service.histories.lock().order.is_empty()); + assert!(service.first_sight(peer(100_000), Some(host), 1, later, later, true)); + } + + #[test] + fn with_every_history_taken_the_oldest_goes() { + let mut histories = Histories::default(); + let peer = |n: u32| { + let mut peer = [0u8; 32]; + peer[..4].copy_from_slice(&n.to_be_bytes()); + peer + }; + for n in 0..=u32::try_from(MAX_REQUEST_HISTORIES).unwrap_or(u32::MAX) { + assert!(histories.start(peer(n), None)); + } + assert_eq!(histories.by_peer.len(), MAX_REQUEST_HISTORIES); + assert!(!histories.by_peer.contains_key(&peer(0))); + assert!(histories.by_peer.contains_key(&peer(1))); + } + + #[tokio::test] + async fn a_member_can_hold_only_so_many_unanswered_offers() { + let service = service(); + let _b = call(&service, B, announce(1, &[], 0)).await; + let mut delivered = 0; + let mut received = 0; + for offer in 0..40u8 { + delivered += call(&service, A, announce(1, &[offer], 0)).await.delivered; + // B collects every offer and answers none, so its inbox always has + // room: only A's own cap on unanswered offers can stop them. + received += call(&service, B, announce(1, &[], 0)).await.inbox.len(); + } + assert_eq!(delivered, 32); + assert_eq!(received, 32); + assert_eq!(service.stats().pending_offers, 32); + // A's next offer is still refused while B's inbox is empty. + assert_eq!(call(&service, A, announce(1, &[40], 0)).await.delivered, 0); + assert_eq!(call(&service, B, announce(1, &[], 0)).await.inbox, []); + } + + #[tokio::test] + async fn a_member_that_goes_takes_the_offers_made_to_it() { + let service = service(); + let _b = call(&service, B, announce(1, &[], 0)).await; + let a = call(&service, A, announce(1, &[80], 0)).await; + assert_eq!(a.delivered, 1); + assert_eq!(service.stats().pending_offers, 1); + // B can no longer answer A's offer: A's place for it is free again. + service.session_closed(B); + assert_eq!(service.stats().pending_offers, 0); + assert_eq!(service.stats().stored_sdp_bytes, 0); + } + + #[tokio::test] + async fn a_newer_membership_takes_the_place_and_an_old_leave_spares_it() { + let service = service(); + let with = |membership: u64, body: ChunkMessageBody| match body { + ChunkMessageBody::SwarmAnnounceRequest(mut request) => { + request.membership = membership; + ChunkMessageBody::SwarmAnnounceRequest(request) + } + other => other, + }; + let leave = |membership: u64| { + ChunkMessageBody::SwarmLeaveRequest(SwarmLeaveRequest { + room: [1; 32], + node: NODE, + membership, + }) + }; + let _b = call(&service, B, announce(1, &[], 0)).await; + let old = call(&service, A, with(5, announce(1, &[90], 0))).await; + assert_eq!(old.delivered, 1); + // The client rejoins: a new member, and the old one's offer is gone. + let new = call(&service, A, with(6, announce(1, &[], 0))).await; + assert_ne!(new.member, old.member); + assert_eq!(service.stats().members, 2); + assert_eq!(service.stats().pending_offers, 0); + // The old membership's announce is refused, and its leave ends nothing. + let stale = service + .handle( + A, + None, + with(5, announce(1, &[], 0)), + &CancellationToken::new(), + Instant::now(), + ) + .await; + assert!(matches!( + stale, + Some(ChunkMessageBody::SwarmAnnounceResponse( + SwarmAnnounceResponse::Refused(SwarmRefusal::Superseded) + )) + )); + let _ = service + .handle(A, None, leave(5), &CancellationToken::new(), Instant::now()) + .await; + assert_eq!(service.stats().members, 2); + // Its own leave ends it. + let _ = service + .handle(A, None, leave(6), &CancellationToken::new(), Instant::now()) + .await; + assert_eq!(service.stats().members, 1); + } + + #[tokio::test] + async fn an_ended_membership_does_not_come_back() { + let service = service(); + let token = CancellationToken::new(); + let membership = |membership: u64| { + let ChunkMessageBody::SwarmAnnounceRequest(mut request) = announce(1, &[], 0) else { + unreachable!("an announce"); + }; + request.membership = membership; + ChunkMessageBody::SwarmAnnounceRequest(request) + }; + let leave = |membership: u64| { + ChunkMessageBody::SwarmLeaveRequest(SwarmLeaveRequest { + room: [1; 32], + node: NODE, + membership, + }) + }; + let superseded = |reply: &Option| { + matches!( + reply, + Some(ChunkMessageBody::SwarmAnnounceResponse( + SwarmAnnounceResponse::Refused(SwarmRefusal::Superseded) + )) + ) + }; + // A leave that overtook its own announce: the announce joins nothing. + let _ = service + .handle(A, None, leave(6), &token, Instant::now()) + .await; + let late = service + .handle(A, None, membership(6), &token, Instant::now()) + .await; + assert!(superseded(&late)); + // Nor does an older membership's, after the newer one left. + let late = service + .handle(A, None, membership(5), &token, Instant::now()) + .await; + assert!(superseded(&late)); + assert_eq!(service.stats().members, 0); + // A newer membership joins. + let _ = call(&service, A, membership(7)).await; + assert_eq!(service.stats().members, 1); + // A connection that closes ends no membership: the client carries on + // in the same one over its next connection. + service.session_closed(A); + assert_eq!(service.stats().members, 0); + let _ = call(&service, A, membership(7)).await; + assert_eq!(service.stats().members, 1); + } + + #[tokio::test] + async fn an_older_membership_does_not_come_back_after_a_newer_one_goes_silent() { + let service = service(); + let with = |membership: u64| { + let ChunkMessageBody::SwarmAnnounceRequest(mut request) = announce(1, &[], 0) else { + unreachable!("an announce"); + }; + request.membership = membership; + ChunkMessageBody::SwarmAnnounceRequest(request) + }; + // Membership 7 arrives first, then goes silent. + let _ = call(&service, A, with(7)).await; + service.sweep(Instant::now() + MEMBER_TTL + Duration::from_secs(1)); + assert_eq!(service.stats().members, 0); + // The older 6, arriving late, does not come back; 7 itself may. + let late = service + .handle(A, None, with(6), &CancellationToken::new(), Instant::now()) + .await; + assert!(matches!( + late, + Some(ChunkMessageBody::SwarmAnnounceResponse( + SwarmAnnounceResponse::Refused(SwarmRefusal::Superseded) + )) + )); + let _ = call(&service, A, with(7)).await; + assert_eq!(service.stats().members, 1); + } + + #[test] + fn a_request_for_another_node_is_told_apart_before_its_history() { + let service = service(); + assert!(service.names_this_node(&announce(1, &[], 0))); + let ChunkMessageBody::SwarmAnnounceRequest(mut elsewhere) = announce(1, &[], 0) else { + unreachable!("an announce"); + }; + elsewhere.node = [1; 32]; + assert!(!service.names_this_node(&ChunkMessageBody::SwarmAnnounceRequest(elsewhere))); + assert!(service.names_this_node(&answer(1, [2; 16], 3))); + } + + #[tokio::test] + async fn one_host_holds_only_its_share_of_the_stored_sdp() { + let service = service(); + let token = CancellationToken::new(); + let sdp = MAX_SWARM_SDP_BYTES; + let offers = |room: u8| { + ChunkMessageBody::SwarmAnnounceRequest(SwarmAnnounceRequest { + room: [room; 32], + node: NODE, + membership: 1, + offers: (0..8u8) + .map(|id| SwarmOffer { + offer_id: [id; 16], + sdp: "v".repeat(sdp), + }) + .collect(), + wait_ms: 0, + }) + }; + // 24 rooms of 8 receivers each, from hosts of their own. + for room in 0..24u8 { + let receivers: IpAddr = format!("203.0.113.{room}").parse().expect("ip"); + for n in 0..8u64 { + let session = SwarmSession::Browser(1000 + u64::from(room) * 8 + n); + let _ = service + .handle( + session, + Some(receivers), + announce(room, &[], 0), + &token, + Instant::now(), + ) + .await; + } + } + // One host offers 8 into each room: more than its share. + let host: IpAddr = "198.51.100.4".parse().expect("ip"); + let mut delivered = 0u32; + for room in 0..24u8 { + if let Some(ChunkMessageBody::SwarmAnnounceResponse(SwarmAnnounceResponse::Joined { + offers_delivered, + .. + })) = service + .handle( + SwarmSession::Browser(u64::from(room)), + Some(host), + offers(room), + &token, + Instant::now(), + ) + .await + { + delivered += offers_delivered; + } + } + assert_eq!( + usize::try_from(delivered).unwrap_or(usize::MAX), + MAX_STORED_SDP_BYTES_PER_HOST / sdp + ); + // The rest of the budget is left for every other host. + assert_eq!( + service.stats().stored_sdp_bytes, + MAX_STORED_SDP_BYTES_PER_HOST + ); + } + + #[tokio::test] + async fn the_stun_allowlist_counts_an_ipv6_network_once() { + let service = service(); + let token = CancellationToken::new(); + for n in 0..32u16 { + let ip = IpAddr::V6(Ipv6Addr::new(0x2001, 0xdb8, 0, 9, 0, 0, 0, n)); + let _ = service + .handle( + SwarmSession::Browser(u64::from(n)), + Some(ip), + announce(1, &[], 0), + &token, + Instant::now(), + ) + .await; + } + assert_eq!(service.announcers.read().len(), 1); + let neighbour = IpAddr::V6(Ipv6Addr::new(0x2001, 0xdb8, 0, 9, 0, 0, 1, 1)); + assert!(service.stun_allows(neighbour, Instant::now())); + } + + #[test] + fn ended_memberships_outlast_every_request_still_taken() { + assert!(ENDED_MEMBERSHIP_TTL > 2 * MAX_REQUEST_SKEW); + } + + #[test] + fn one_connection_cannot_push_out_the_others_ended_memberships() { + let mut state = State::default(); + let now = Instant::now(); + let host: IpAddr = "198.51.100.4".parse().expect("ip"); + let honest = SwarmSession::Browser(1); + state.note_ended(honest, Some(host_of(host)), [1; 32], 5, now); + // A connection ending ever newer memberships of one room holds one + // place, and the queue does not grow past its bound. + let flooder = SwarmSession::Browser(2); + let other: IpAddr = "203.0.113.9".parse().expect("ip"); + for membership in 1..=(3 * MAX_ENDED_MEMBERSHIPS as u64) { + state.note_ended(flooder, Some(host_of(other)), [2; 32], membership, now); + } + assert_eq!(state.ended.len(), 2); + assert!(state.ended_order.len() <= 2 * state.ended.len() + 1024); + // Nor many rooms past its own share. + for room in 0..=u8::MAX { + state.note_ended(flooder, Some(host_of(other)), [room; 32], 1, now); + } + assert!( + state + .ended_by_session + .get(&flooder) + .map_or(0, VecDeque::len) + <= MAX_ENDED_PER_SESSION + ); + // The newest of them are the ones kept. + assert!(state.ended_covers(flooder, &[u8::MAX; 32], 1)); + assert!(state.ended_covers(honest, &[1; 32], 5)); + // A place ended again goes to the back: the oldest other goes first. + let mut rotating = State::default(); + let session = SwarmSession::Browser(9); + for room in 0..MAX_ENDED_PER_SESSION { + let room = u8::try_from(room).unwrap_or(u8::MAX); + rotating.note_ended(session, None, [room; 32], 1, now); + } + rotating.note_ended(session, None, [0; 32], 2, now); + rotating.note_ended(session, None, [200; 32], 1, now); + assert!(rotating.ended_covers(session, &[0; 32], 2)); + assert!(!rotating.ended_covers(session, &[1; 32], 1)); + assert!(rotating.ended_covers(session, &[200; 32], 1)); + // All go once nothing they cover could be taken. + state.forget_ended(now + ENDED_MEMBERSHIP_TTL); + assert!(state.ended.is_empty() && state.ended_order.is_empty()); + assert!(state.ended_by_session.is_empty() && state.ended_by_host.is_empty()); + } + + #[tokio::test] + async fn a_waiting_announce_of_a_replaced_membership_is_not_told_it_joined() { + let service = service(); + let with = |membership: u64, wait_ms: u32| { + let ChunkMessageBody::SwarmAnnounceRequest(mut request) = announce(1, &[], wait_ms) + else { + unreachable!("an announce"); + }; + request.membership = membership; + ChunkMessageBody::SwarmAnnounceRequest(request) + }; + let waiting = { + let service = Arc::clone(&service); + tokio::spawn(async move { + service + .handle( + B, + None, + with(5, 10_000), + &CancellationToken::new(), + Instant::now(), + ) + .await + }) + }; + tokio::time::sleep(Duration::from_millis(50)).await; + assert_eq!(service.stats().waiting, 1); + let _ = call(&service, B, with(6, 0)).await; + let reply = tokio::time::timeout(Duration::from_secs(2), waiting).await; + assert!(matches!( + reply, + Ok(Ok(Some(ChunkMessageBody::SwarmAnnounceResponse( + SwarmAnnounceResponse::Refused(SwarmRefusal::Superseded) + )))) + )); + } + + #[tokio::test] + async fn one_host_holds_only_so_many_places() { + let service = service(); + let join = |n: u16, ip: IpAddr| { + let service = Arc::clone(&service); + async move { + let mut peer = [0u8; 32]; + peer[..2].copy_from_slice(&n.to_be_bytes()); + service + .handle( + SwarmSession::Native(peer), + Some(ip), + announce(1, &[], 0), + &CancellationToken::new(), + Instant::now(), + ) + .await + } + }; + let joined = |reply: &Option| { + matches!( + reply, + Some(ChunkMessageBody::SwarmAnnounceResponse( + SwarmAnnounceResponse::Joined { .. } + )) + ) + }; + // Identities of one IPv6 /64, each on its own address in it. + let host = |n: u16| IpAddr::V6(Ipv6Addr::new(0x2001, 0xdb8, 0, 7, 0, 0, 0, n)); + for n in 0..u16::try_from(MAX_MEMBERS_PER_HOST).unwrap_or(u16::MAX) { + assert!(joined(&join(n, host(n)).await)); + } + let refused = join(1000, host(1000)).await; + assert!(matches!( + refused, + Some(ChunkMessageBody::SwarmAnnounceResponse( + SwarmAnnounceResponse::Refused(SwarmRefusal::TooManyRoomsForPeer) + )) + )); + // Another host is not held back; one that goes frees its place. + assert!(joined( + &join(2000, "198.51.100.4".parse().expect("ip")).await + )); + let mut first = [0u8; 32]; + first[..2].copy_from_slice(&0u16.to_be_bytes()); + service.session_closed(SwarmSession::Native(first)); + assert!(joined(&join(1000, host(1000)).await)); + } + + #[tokio::test] + async fn a_member_that_goes_leaves_no_offer_in_another_inbox() { + let service = service(); + let _b = call(&service, B, announce(1, &[], 0)).await; + for offer in 0..4u8 { + let _ = call(&service, A, announce(1, &[offer], 0)).await; + service.session_closed(A); + } + assert_eq!(service.stats().pending_offers, 0); + assert_eq!(service.stats().stored_sdp_bytes, 0); + assert_eq!(call(&service, B, announce(1, &[], 0)).await.inbox, []); + } +} diff --git a/src/web_rtc.rs b/src/web_rtc.rs index ddbb1b91..a4ea5971 100644 --- a/src/web_rtc.rs +++ b/src/web_rtc.rs @@ -19,6 +19,7 @@ use crate::error::{Error, Result}; use crate::logging::{debug, info, warn}; use crate::payment::{serialize_single_node_proof, PaymentProof}; use crate::storage::AntProtocol; +use crate::swarm::{is_swarm_request, is_swarm_request_bytes, SwarmService, SwarmSession}; use certificate::load_or_generate_certificate; use errors::{error_response, public_error, sanitize_response}; use evmlib::common::{Amount, TxHash}; @@ -60,6 +61,11 @@ const MAX_FIND_NODE_RESULTS: usize = 20; const MAX_CHANNEL_REQUESTS: usize = 4; const RPC_MULTIPLEX_CAPABILITY: &str = "rpc-multiplex-4"; +/// Advertised when `chunk_protocol` admits the swarm messages: a browser +/// sends them only to a node that names this, since a node that predates +/// them closes the channel on an unknown chunk protocol request. +pub const SWARM_CAPABILITY: &str = "swarm_rendezvous"; + /// Advertised when `chunk_protocol` admits pointer reads and paid pointer /// writes (ADR-0016), so a browser client can tell a node that would refuse /// them from one that failed. @@ -78,6 +84,16 @@ const FIRST_DATA_CHANNEL_TIMEOUT_ERROR: &str = "DataChannel opening timed out"; const CHANNEL_CAPACITY_ERROR: &str = "global DataChannel capacity exhausted"; const REQUEST_CAPACITY_ERROR: &str = "global request capacity exhausted"; const REQUEST_RATE_ERROR: &str = "request rate limit exceeded"; +const SWARM_RATE_ERROR: &str = "swarm request rate limit exceeded"; +/// Swarm requests a listener takes in a second, in all and from one source, +/// on budgets of their own: a browser announces about once a second per room +/// it shares a node with, and those announces must never spend the request +/// budget its own and every other browser's reads need. A swarm request the +/// swarm's budget has room for gives the read budget its token back; one past +/// it keeps that charge and is refused, without closing the channel the +/// reads share. +const SWARM_REQUESTS_PER_SECOND: usize = 64; +const SWARM_REQUESTS_PER_SECOND_PER_SOURCE: usize = 8; const GLOBAL_BYTE_CAPACITY_ERROR: &str = "global in-flight byte capacity exhausted"; const SOURCE_BYTE_CAPACITY_ERROR: &str = "source in-flight byte capacity exhausted"; @@ -140,6 +156,13 @@ impl RequestRateBucket { self.token_units -= 1_000_000_000; true } + + /// Gives back a token taken by [`Self::allow`], up to the bucket's + /// capacity. + fn refund(&mut self) { + let capacity = self.rate_per_second.saturating_mul(1_000_000_000); + self.token_units = self.token_units.saturating_add(1_000_000_000).min(capacity); + } } /// Atomic byte budget and an RAII reservation within it. @@ -271,6 +294,8 @@ impl TrackedBytes { struct SourceQuota { bulk_responses: Arc, request_rate: Mutex, + /// See [`SWARM_REQUESTS_PER_SECOND_PER_SOURCE`]. + swarm_rate: Mutex, bytes: Arc, } @@ -288,6 +313,8 @@ struct SourceAdmissionState { /// Admission and accounting shared by every association on one listener. struct ListenerResources { running: AtomicBool, + /// Numbers browser connections for the swarm rendezvous. + next_connection: AtomicU64, connection_rejections: AtomicU64, channel_rejections: AtomicU64, request_rejections: AtomicU64, @@ -303,6 +330,8 @@ struct ListenerResources { bulk_responses: Arc, request_tasks: Mutex, global_request_rate: Mutex, + /// See [`SWARM_REQUESTS_PER_SECOND`]. + global_swarm_rate: Mutex, global_bytes: Arc, source_state: Mutex, max_connections_per_ip: usize, @@ -317,6 +346,7 @@ impl ListenerResources { let byte_rejections = Arc::new(AtomicU64::new(0)); Arc::new(Self { running: AtomicBool::new(false), + next_connection: AtomicU64::new(0), connection_rejections: AtomicU64::new(0), channel_rejections: AtomicU64::new(0), request_rejections: AtomicU64::new(0), @@ -334,6 +364,7 @@ impl ListenerResources { ))), request_tasks: Mutex::new(TaskTracker::new()), global_request_rate: Mutex::new(RequestRateBucket::new(config.max_requests_per_second)), + global_swarm_rate: Mutex::new(RequestRateBucket::new(SWARM_REQUESTS_PER_SECOND)), global_bytes: Arc::new(ByteBudget::with_rejections( config.max_in_flight_bytes, byte_rejections, @@ -396,6 +427,9 @@ impl ListenerResources { request_rate: Mutex::new(RequestRateBucket::new( self.max_requests_per_second_per_ip, )), + swarm_rate: Mutex::new(RequestRateBucket::new( + SWARM_REQUESTS_PER_SECOND_PER_SOURCE, + )), bytes: Arc::new(ByteBudget::with_rejections( self.max_in_flight_bytes_per_ip, Arc::clone(&self.byte_rejections), @@ -415,6 +449,9 @@ impl ListenerResources { request_rate: Mutex::new(RequestRateBucket::new( self.max_requests_per_second_per_connection, )), + swarm: SwarmSession::Browser(self.next_connection.fetch_add(1, Ordering::Relaxed)), + remote_ip: remote_addr.ip(), + closed: CancellationToken::new(), }); Ok(ConnectionAdmission { listener: Arc::clone(self), @@ -457,10 +494,28 @@ impl ListenerResources { } } +/// The budget a request frame was admitted on. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum FrameCharge { + /// The request budget the reads share; a swarm request moves its charge + /// onto the swarm's own once it is known for one. + Requests, + /// The swarm's own, with the reads' spent: a frame then goes on only if + /// it is a swarm request, and anything else is refused as over the + /// request rate, its swarm charge given back. + Swarm, +} + struct ConnectionResources { listener: Arc, source: Arc, request_rate: Mutex, + /// This connection, as the swarm rendezvous names it. + swarm: SwarmSession, + /// The browser's address, for the STUN allowlist. + remote_ip: IpAddr, + /// Cancelled when the connection ends, which ends its waiting announces. + closed: CancellationToken, } impl ConnectionResources { @@ -486,6 +541,92 @@ impl ConnectionResources { Ok(permit) } + /// Admits a request frame as [`Self::try_admit_request`] does, or, when + /// the request budget the reads share has no token left, on the swarm's + /// own budget: only once the frame is opened is it known whether it is a + /// swarm request, which that budget is for (see [`FrameCharge`]). + fn try_admit_frame(&self) -> ServerResult<(OwnedSemaphorePermit, FrameCharge)> { + let permit = Arc::clone(&self.listener.request_limit) + .try_acquire_owned() + .map_err(|_| { + self.listener + .request_rejections + .fetch_add(1, Ordering::Relaxed); + REQUEST_CAPACITY_ERROR.to_string() + })?; + let now = Instant::now(); + let charge = if self.take_request_tokens(now) { + FrameCharge::Requests + } else if self.take_swarm_tokens(now) { + FrameCharge::Swarm + } else { + drop(permit); + self.listener + .rate_rejections + .fetch_add(1, Ordering::Relaxed); + return Err(REQUEST_RATE_ERROR.to_string()); + }; + Ok((permit, charge)) + } + + /// Takes a token from each request budget the reads share (the source's, + /// the connection's and the listener's), or from none: a budget with no + /// token left gives the ones before it theirs back, so a frame taken on + /// the swarm's budget instead has spent nothing of the reads'. + fn take_request_tokens(&self, now: Instant) -> bool { + if !self.source.request_rate.lock().allow(now) { + return false; + } + if !self.request_rate.lock().allow(now) { + self.source.request_rate.lock().refund(); + return false; + } + if !self.listener.global_request_rate.lock().allow(now) { + self.request_rate.lock().refund(); + self.source.request_rate.lock().refund(); + return false; + } + true + } + + /// As [`Self::take_request_tokens`], for the swarm's own budgets (the + /// source's and the listener's). + fn take_swarm_tokens(&self, now: Instant) -> bool { + if !self.source.swarm_rate.lock().allow(now) { + return false; + } + if !self.listener.global_swarm_rate.lock().allow(now) { + self.source.swarm_rate.lock().refund(); + return false; + } + true + } + + /// Gives the swarm's budget back a frame's charge: it was no swarm + /// request. + fn refund_swarm(&self) { + self.source.swarm_rate.lock().refund(); + self.listener.global_swarm_rate.lock().refund(); + } + + /// Moves an admitted swarm request's charge from the request budget the + /// reads share onto the swarm's own: true if the swarm's budget had room + /// (the read budget gets its tokens back), false if not (the request + /// keeps its charge, and is refused). + fn charge_to_swarm(&self) -> bool { + let now = Instant::now(); + if !self.take_swarm_tokens(now) { + self.listener + .rate_rejections + .fetch_add(1, Ordering::Relaxed); + return false; + } + self.source.request_rate.lock().refund(); + self.request_rate.lock().refund(); + self.listener.global_request_rate.lock().refund(); + true + } + fn try_reserve_bytes(&self, amount: usize) -> ServerResult { let source = self .source @@ -626,11 +767,13 @@ pub struct WebRtcDirectServer { } /// Start the feature-gated browser listener and return its endpoint and task. +#[allow(clippy::too_many_arguments)] pub async fn spawn( config: &WebRtcDirectConfig, root_dir: &Path, p2p: Arc, ant_protocol: Option>, + swarm: Arc, evm_network: &evmlib::Network, shutdown: CancellationToken, endpoint_catalog: Option>, @@ -652,6 +795,12 @@ pub async fn spawn( ) .await .map_err(|error| Error::Startup(format!("failed to bind WebRTC Direct listener: {error}")))?; + // This socket is also the STUN server for swarm members' WebRTC + // connections to each other, answering clients that announced recently. + let stun = Arc::clone(&swarm); + listener.set_plain_binding_policy(Some(Arc::new(move |ip| { + stun.stun_allows(ip, Instant::now()) + }))); let local_addr = listener.local_addr(); let identity = Arc::clone(p2p.transport().node_identity()); let state = Arc::new(ServerState { @@ -659,6 +808,7 @@ pub async fn spawn( identity, p2p, ant_protocol, + swarm, payment, endpoint: RwLock::new(None), endpoint_catalog, @@ -754,6 +904,8 @@ async fn refresh_browser_endpoint( if let Some(catalog) = state.endpoint_catalog.as_ref() { catalog.insert(peer_id, endpoint.clone()); } + // The same UDP socket answers swarm members' STUN requests. + state.swarm.set_ice_servers(vec![format!("stun:{address}")]); *state.endpoint.write() = Some(endpoint); state .p2p @@ -1067,6 +1219,10 @@ async fn handle_connection( // listener's request tracker and keep their storage state and admission // until processing finishes, independently of this connection. channel_shutdown.cancel(); + // The browser has gone: its waiting announces end now, and it leaves every + // swarm room it was in. + resources.closed.cancel(); + state.swarm.session_closed(resources.swarm); if let Err(error) = connection.close().await { debug!(remote = %remote_addr, %error, "Failed to close WebRTC Direct connection"); } @@ -1322,6 +1478,7 @@ async fn establish_pq_session( struct AdmittedRequest { request: Request, content: Vec, + charge: FrameCharge, _request_permit: OwnedSemaphorePermit, _in_flight_bytes: InFlightByteReservation, } @@ -1362,6 +1519,7 @@ async fn start_request( let AdmittedRequest { request, content, + charge, _request_permit: request_permit, _in_flight_bytes: in_flight_bytes, } = admitted; @@ -1381,7 +1539,8 @@ async fn start_request( } else { None }; - let (response, content) = process_request(request, content, &state, &resources).await?; + let (response, content) = + process_request(request, content, charge, &state, &resources).await?; Ok(CompletedRequest { _bulk_permits: bulk_permits, response, @@ -1415,7 +1574,7 @@ async fn read_webrtc_request( } // Admission happens as soon as a client starts a frame. Idle persistent // channels consume neither request-rate tokens nor request worker slots. - let request_permit = resources.try_admit_request()?; + let (request_permit, charge) = resources.try_admit_frame()?; let max_plaintext_bytes = MAX_BROWSER_FRAME_BYTES; let mut encrypted = read_pq_payload_after_first( first_message, @@ -1451,9 +1610,21 @@ async fn read_webrtc_request( let content = frame[content_offset..].to_vec(); drop(frame); reservation.resize(accounted_request_bytes)?; + // Admitted on the swarm's budget, only a swarm request goes on. + if charge == FrameCharge::Swarm + && !(matches!(request.body, RequestBody::ChunkProtocol) && is_swarm_request_bytes(&content)) + { + resources.refund_swarm(); + resources + .listener + .rate_rejections + .fetch_add(1, Ordering::Relaxed); + return Err(REQUEST_RATE_ERROR.to_string()); + } Ok(AdmittedRequest { request, content, + charge, _request_permit: request_permit, _in_flight_bytes: reservation, }) @@ -1647,9 +1818,38 @@ async fn write_framed_pq_payload(channel: &WebRtcDataChannel, payload: &[u8]) -> Ok(()) } +/// A swarm request's refusal before it is decoded: no legal one is longer +/// than `MAX_SWARM_REQUEST_BYTES`, and swarm requests run on a budget of their +/// own, never the reads' (see [`SWARM_REQUESTS_PER_SECOND`]). `None` lets it +/// go on, its charge moved off the read budget (or there from the start, for +/// a frame `charge` says was admitted on the swarm's). +fn swarm_request_refusal( + request_id: u64, + content_len: usize, + charge: FrameCharge, + resources: &ConnectionResources, +) -> Option { + if content_len > ant_protocol::MAX_SWARM_REQUEST_BYTES { + return Some(Response::error( + request_id, + "invalid_request", + "swarm request too long", + )); + } + if charge == FrameCharge::Requests && !resources.charge_to_swarm() { + return Some(Response::error( + request_id, + "rate_limited", + SWARM_RATE_ERROR, + )); + } + None +} + async fn process_request( request: Request, content: Vec, + charge: FrameCharge, state: &ServerState, resources: &ConnectionResources, ) -> ServerResult<(Response, Option)> { @@ -1669,16 +1869,29 @@ async fn process_request( } match request.body { RequestBody::ChunkProtocol => { - let Some(protocol) = state.ant_protocol.as_ref() else { - return Ok(( + // Every node hosts the swarm rendezvous, storing chunks or not. + let swarm_request = is_swarm_request_bytes(&content); + let storage_disabled = || { + Ok(( Response::error( request.request_id, "storage_disabled", "chunk storage is disabled".to_string(), ), None, - )); + )) }; + let protocol = state.ant_protocol.as_ref(); + if protocol.is_none() && !swarm_request { + return storage_disabled(); + } + if swarm_request { + if let Some(refused) = + swarm_request_refusal(request.request_id, content.len(), charge, resources) + { + return Ok((refused, None)); + } + } // The admitted frame remains charged for the whole request. Charge // decoding before it copies the input, then retain that allowance // for the PUT handler's disk buffer after dropping the wire buffer. @@ -1688,7 +1901,13 @@ async fn process_request( drop(content); let response_limit = binary_response_limit(&message.body)?; reservation.resize(reservation.source.amount.max(2 * response_limit))?; - let response = handle_ant_message(protocol, message).await?; + let response = match protocol { + _ if is_swarm_request(&message.body) => { + handle_swarm_message(&state.swarm, message, resources).await? + } + Some(protocol) => handle_ant_message(protocol, message).await?, + None => return storage_disabled(), + }; let bytes = encode_binary_response(&response, response_limit)?; drop(response); reservation.resize(bytes.len())?; @@ -1756,11 +1975,44 @@ fn binary_response_limit(body: &ChunkMessageBody) -> ServerResult { | ChunkMessageBody::MerkleCandidateQuoteRequest(_) | ChunkMessageBody::MerkleCandidateQuoteRequestV2(_) | ChunkMessageBody::PointerGetRequest(_) - | ChunkMessageBody::PointerPutRequest(_) => Ok(SMALL_BINARY_RESPONSE_BYTES), + | ChunkMessageBody::PointerPutRequest(_) + | ChunkMessageBody::SwarmAnnounceRequest(_) + | ChunkMessageBody::SwarmAnswerRequest(_) + | ChunkMessageBody::SwarmLeaveRequest(_) => Ok(SMALL_BINARY_RESPONSE_BYTES), _ => Err("unsupported chunk protocol request".to_string()), } } +// A swarm reply carries at most MAX_SWARM_REPLY_SDP_BYTES of SDP, plus ids and +// framing for up to MAX_SWARM_INBOX signals: well inside the small bound. +const _: () = assert!( + ant_protocol::swarm::MAX_SWARM_REPLY_SDP_BYTES + 4096 < SMALL_BINARY_RESPONSE_BYTES, + "a swarm reply must fit the small response bound" +); + +/// Answers a swarm request on this browser connection. A waiting announce +/// ends when the connection does. +async fn handle_swarm_message( + swarm: &SwarmService, + message: ChunkMessage, + resources: &ConnectionResources, +) -> ServerResult { + let body = swarm + .handle( + resources.swarm, + Some(resources.remote_ip), + message.body, + &resources.closed, + Instant::now(), + ) + .await + .ok_or_else(|| "unsupported chunk protocol request".to_string())?; + Ok(ChunkMessage { + request_id: message.request_id, + body, + }) +} + fn encode_binary_response(message: &ChunkMessage, limit: usize) -> ServerResult> { // Size without allocating, then encode into exactly the charged space. encode_exact(message, limit).map_err(|error| match error { @@ -1813,6 +2065,8 @@ fn hello_capabilities(pointers: bool) -> Vec { if pointers { capabilities.push(POINTER_PROTOCOL_CAPABILITY.into()); } + // Every node hosts the swarm rendezvous. + capabilities.push(SWARM_CAPABILITY.into()); capabilities } @@ -2269,6 +2523,8 @@ struct ServerState { identity: Arc, p2p: Arc, ant_protocol: Option>, + /// The swarm rendezvous, hosted whether chunks are stored or not. + swarm: Arc, payment: BrowserPaymentNetwork, endpoint: RwLock>, endpoint_catalog: Option>, @@ -2592,6 +2848,176 @@ mod tests { assert!(honest.context.try_admit_request().is_ok()); } + #[test] + fn swarm_requests_within_their_budget_give_the_reads_their_tokens_back() { + let config = WebRtcDirectConfig { + max_requests_per_second: 100, + max_requests_per_second_per_ip: 1, + max_requests_per_second_per_connection: 1, + ..WebRtcDirectConfig::default() + }; + let resources = ListenerResources::new(&config); + let address = "198.51.100.1:1000" + .parse() + .unwrap_or_else(|error| panic!("browser address: {error}")); + let browser = resources + .try_admit_connection(address) + .unwrap_or_else(|error| panic!("browser connection: {error}")); + // An announce takes the source's only token as it arrives, and gives + // it back once it is known for a swarm request. + let announce = browser + .context + .try_admit_request() + .unwrap_or_else(|error| panic!("announce admitted: {error}")); + assert!(browser.context.charge_to_swarm()); + drop(announce); + // So the browser's next read still has it. + assert!(browser.context.try_admit_request().is_ok()); + } + + #[test] + fn a_frame_past_the_read_budget_is_admitted_on_the_swarms_own() { + let config = WebRtcDirectConfig { + max_requests_per_second: 100, + max_requests_per_second_per_ip: 1, + max_requests_per_second_per_connection: 1, + ..WebRtcDirectConfig::default() + }; + let resources = ListenerResources::new(&config); + let address = "198.51.100.1:1000" + .parse() + .unwrap_or_else(|error| panic!("browser address: {error}")); + let browser = resources + .try_admit_connection(address) + .unwrap_or_else(|error| panic!("browser connection: {error}")); + // A read takes the source's only token. + let (read, charge) = browser + .context + .try_admit_frame() + .unwrap_or_else(|error| panic!("read admitted: {error}")); + assert_eq!(charge, FrameCharge::Requests); + drop(read); + // An announce right after it is still admitted, on the swarm's + // budget, rather than refused with the channel the reads share. + let (announce, charge) = browser + .context + .try_admit_frame() + .unwrap_or_else(|error| panic!("announce admitted: {error}")); + assert_eq!(charge, FrameCharge::Swarm); + drop(announce); + // A frame that turns out to be no swarm request gives the swarm its + // charge back: the swarm's budget lets as many announces through as + // before, and the reads gain nothing from it. + for _ in 0..SWARM_REQUESTS_PER_SECOND_PER_SOURCE - 1 { + let (frame, charge) = browser + .context + .try_admit_frame() + .unwrap_or_else(|error| panic!("frame admitted: {error}")); + assert_eq!(charge, FrameCharge::Swarm); + browser.context.refund_swarm(); + drop(frame); + } + assert!(browser.context.try_admit_frame().is_ok()); + } + + #[test] + fn a_frame_taken_on_the_swarms_budget_spends_none_of_the_reads() { + // The connection's budget runs out before the source's. + let config = WebRtcDirectConfig { + max_requests_per_second: 100, + max_requests_per_second_per_ip: 2, + max_requests_per_second_per_connection: 1, + ..WebRtcDirectConfig::default() + }; + let resources = ListenerResources::new(&config); + let connect = |address: &str| { + let address = address + .parse() + .unwrap_or_else(|error| panic!("address: {error}")); + resources + .try_admit_connection(address) + .unwrap_or_else(|error| panic!("connection: {error}")) + }; + let browser = connect("198.51.100.1:1000"); + let (_read, charge) = browser + .context + .try_admit_frame() + .unwrap_or_else(|error| panic!("read admitted: {error}")); + assert_eq!(charge, FrameCharge::Requests); + let (_announce, charge) = browser + .context + .try_admit_frame() + .unwrap_or_else(|error| panic!("announce admitted: {error}")); + assert_eq!(charge, FrameCharge::Swarm); + // The source's second token is still there for the reads of another + // of its connections. + let other = connect("198.51.100.1:1001"); + let (_read, charge) = other + .context + .try_admit_frame() + .unwrap_or_else(|error| panic!("other read admitted: {error}")); + assert_eq!(charge, FrameCharge::Requests); + + // The listener's budget runs out first: the source's and the + // connection's tokens come back. + let config = WebRtcDirectConfig { + max_requests_per_second: 1, + max_requests_per_second_per_ip: 10, + max_requests_per_second_per_connection: 10, + ..WebRtcDirectConfig::default() + }; + let resources = ListenerResources::new(&config); + let address = "198.51.100.2:1000" + .parse() + .unwrap_or_else(|error| panic!("address: {error}")); + let browser = resources + .try_admit_connection(address) + .unwrap_or_else(|error| panic!("connection: {error}")); + let (_read, charge) = browser + .context + .try_admit_frame() + .unwrap_or_else(|error| panic!("read admitted: {error}")); + assert_eq!(charge, FrameCharge::Requests); + let (_announce, charge) = browser + .context + .try_admit_frame() + .unwrap_or_else(|error| panic!("announce admitted: {error}")); + assert_eq!(charge, FrameCharge::Swarm); + let now = Instant::now(); + let left = |bucket: &Mutex| { + (0..20).take_while(|_| bucket.lock().allow(now)).count() + }; + assert_eq!(left(&browser.context.request_rate), 9); + assert_eq!(left(&browser.context.source.request_rate), 9); + } + + #[test] + fn swarm_requests_past_their_budget_are_refused_and_keep_their_charge() { + let config = WebRtcDirectConfig { + max_requests_per_second: 1000, + max_requests_per_second_per_ip: 1000, + max_requests_per_second_per_connection: 1000, + ..WebRtcDirectConfig::default() + }; + let resources = ListenerResources::new(&config); + let connect = |address: &str| { + let address = address + .parse() + .unwrap_or_else(|error| panic!("address: {error}")); + resources + .try_admit_connection(address) + .unwrap_or_else(|error| panic!("connection: {error}")) + }; + let busy = connect("198.51.100.1:1000"); + for _ in 0..SWARM_REQUESTS_PER_SECOND_PER_SOURCE { + assert!(busy.context.charge_to_swarm()); + } + assert!(!busy.context.charge_to_swarm()); + // Another source has a budget of its own. + let other = connect("203.0.113.2:2000"); + assert!(other.context.charge_to_swarm()); + } + #[test] fn reconnecting_does_not_reset_the_source_request_bucket() { let config = WebRtcDirectConfig {