feat(hosting): 4 TiB site cap (stacked on #222)

apps/web/src/app/getting-started/page.tsx

@@ -93,7 +93,7 @@ $ c0mpute hosting ls          # sites a gateway knows, with live copy counts`}</
           <code>--name</code> to update it. Any gateway serves it at{" "}
           <code>/s/&lt;site_id&gt;/</code>, for example{" "}
           <code>https://gateway.c0mpute.com/s/&lt;site_id&gt;/</code>, with
-          byte ranges so video seeks. Limits: 64 GiB per free site, 256 GiB
+          byte ranges so video seeks. Limits: 64 GiB per free site, 4 TiB
           with <code>--disks auto</code>. Sites over 1 GiB are striped over the
           network in 64 MiB groups; run the same command again to resume an
           interrupted publish.

docs/hosting.md

@@ -44,7 +44,7 @@ is used for misses.
 
 | | |
 |---|---|
-| Site, paid (`--disks`, a v2 record) | 256 GiB |
+| Site, paid (`--disks`, a v2 record) | 4 TiB |
 | Site, free (v1 record) | 64 GiB |
 | Files per site | 10,000,000 |
 | One file | 128 GiB |
@@ -63,15 +63,16 @@ The budget advertised to d1sks (`max_gib`) is the same number, so `--disks auto`
 picks disks with room for their share of the site.
 
 Errors say which limit was hit and what to do (`--disks auto` for a free site over
-64 GiB, split the folder above 256 GiB). A hosting node logs the limit that made it
+64 GiB, split the folder above 4 TiB). A hosting node logs the limit that made it
 refuse a copy (`over this node's hosting budget`, `disk nearly full`, `per-owner limit`).
 
 ## Large sites: striped placement
 
 A site up to 1 GiB is replicated whole, exactly as before. Above that it is
 **striped**: its chunks are bucketed by hash into groups, one group per 64 MiB of the
-site (the count comes from the size in the signed record), and every group is placed on
-its own:
+site up to 8,192 groups (512 GiB); a bigger site keeps 8,192 groups and makes each one
+bigger (512 MiB at the 4 TiB cap), so a group bitmap on the gossip stays at most 1 KiB.
+The count comes from the size in the signed record, and every group is placed on its own:
 
 * The hosting nodes that rank highest for `(site, group)` (rendezvous hashing, the same
   answer on every node) take that group until it has three copies. No node needs the
@@ -85,14 +86,15 @@ its own:
   A node holds each group as soon as it is complete, so a restart resumes group by group.
 * Paid sites: preferred hosts take every group when there are at most three of them (a
   paid host holds the whole site), else the three that rank highest per group; each is
-  paid for its share (`size x 3 / preferred`).
+  paid for its share (`size x 3 / preferred`). A multi-TiB site therefore needs disks
+  with that much room each; `--copies 5` cuts each host's share to 60% of the site.
 * `c0mpute hosting ls` and `/hosting/v1/sites` show, for a striped site, `copies` = the
   fewest copies of any group, plus `groups`, `short_groups` and `missing_groups`.
 
 Manifests: up to 5,000 files and 1 GiB the manifest is one chunk, byte for byte what
 older nodes read. Above either, it is a root listing parts (by first path) of at most
 4,000 files and 65,536 chunk hashes each, so every part is one chunk fetch even for a
-256 GiB site; a gateway fetches only the part a request needs. Every holder of any group
+4 TiB site; a gateway fetches only the part a request needs. Every holder of any group
 keeps the root and all parts.
 
 Groups are defined over chunk hashes only (the group of a chunk is its hash modulo the

node/crates/c0mpute-cli/src/main.rs

@@ -821,8 +821,8 @@ enum HostingCmd {
     /// The site is signed with this machine's identity, so only this machine
     /// can update it; publishing the same --name again ships a new version.
     /// Stays up until three hosting nodes hold copies, then exits.
-    /// Sites over 1 GiB (up to 64 GiB free, 256 GiB with --disks) are striped
-    /// over the network in 64 MiB groups; files are served from the folder
+    /// Sites over 1 GiB (up to 64 GiB free, 4 TiB with --disks) are striped
+    /// over the network in 64 MiB groups (bigger past 512 GiB); files are served from the folder
     /// itself while the network copies them, so keep it running, and run the
     /// same command again to resume if it stops.
     #[command(verbatim_doc_comment)]

node/crates/c0mpute-core/src/hosting/mod.rs

@@ -2686,7 +2686,7 @@ mod tests {
     #[test]
     fn a_holding_message_of_striped_sites_fits_gossipsub() {
         // Entries are packed up to HOLDING_MESSAGE_BYTES; the largest bitmap
-        // (every group of a 256 GiB site) still leaves room.
+        // (every group of a 4 TiB site, MAX_GROUPS) still leaves room.
         let all: BTreeSet<u32> = (0..stripe::group_count(site::MAX_SITE_BYTES) as u32).collect();
         let g = stripe::encode_groups(&all);
         let per = HOLDING_MESSAGE_BYTES / (120 + g.len()) + 1;
@@ -3102,9 +3102,10 @@ pub async fn publish_dir_with(
         scanned.manifest.files.len(),
         gib(record.size),
         if striped {
-            ", striped in 64 MiB groups"
+            let n = stripe::group_count(record.size) as u64;
+            format!(", striped in {n} groups of ~{} MiB", record.size.div_ceil(n) >> 20)
         } else {
-            ""
+            String::new()
         },
         record.site_id
     );

node/crates/c0mpute-core/src/hosting/site.rs

@@ -18,8 +18,10 @@ use serde::{Deserialize, Serialize};
 /// as a CBOR integer array (about 2x on the wire and slow to decode), so
 /// small chunks keep each fetch quick and well under the 10 MiB response cap.
 pub const FILE_CHUNK_BYTES: usize = 1024 * 1024;
-/// Largest site a node will publish or replicate (with paid preferred hosts).
-pub const MAX_SITE_BYTES: u64 = 256 * GIB;
+/// Largest site a node will publish or replicate (with paid preferred hosts):
+/// 4 TiB. Striping caps a site at [`super::stripe::MAX_GROUPS`] groups, so
+/// past 512 GiB the groups grow (512 MiB each at 4 TiB) instead.
+pub const MAX_SITE_BYTES: u64 = 4 * 1024 * GIB;
 /// Largest free site (a v1 record, no preferred hosts). Volunteers hold free
 /// copies, so a free site may use at most a quarter of what a paid one may.
 pub const MAX_FREE_SITE_BYTES: u64 = 64 * GIB;
@@ -638,7 +640,7 @@ impl SiteManifest {
     /// Encode the manifest: one chunk up to [`INLINE_FILES`] files and
     /// [`STRIPE_ABOVE`] bytes (byte for byte what older nodes read), else a
     /// root plus parts of at most [`PART_FILES`] files and [`PART_CHUNKS`]
-    /// chunks, so even a 256 GiB site's chunk lists are fetched a part at a
+    /// chunks, so even a 4 TiB site's chunk lists are fetched a part at a
     /// time.
     pub fn stored(&self) -> StoredManifest {
         let m = self;
@@ -1371,7 +1373,8 @@ mod tests {
         let e = check_site_size(70 * GIB, false).unwrap_err().to_string();
         assert!(e.contains("--disks auto"), "{e}");
         assert!(check_site_size(70 * GIB, true).is_ok());
-        assert!(check_site_size(300 * GIB, true).unwrap_err().to_string().contains("Split"));
+        assert!(check_site_size(300 * GIB, true).is_ok(), "300 GiB is under the 4 TiB cap");
+        assert!(check_site_size(MAX_SITE_BYTES + GIB, true).unwrap_err().to_string().contains("Split"));
     }
 
     #[test]

node/crates/c0mpute-core/src/hosting/stripe.rs

@@ -1,8 +1,9 @@
 //! Striped placement for large sites.
 //!
 //! A site at or below [`STRIPE_ABOVE`] is replicated whole, as it always was.
-//! Above it, the site's chunks are bucketed by hash into one group per 64 MiB
-//! ([`group_count`] of the signed size) and every group is placed on its own: the hosting nodes that rank
+//! Above it, the site's chunks are bucketed by hash into one group per 64 MiB,
+//! at most [`MAX_GROUPS`] ([`group_count`] of the signed size; a site over
+//! 512 GiB gets bigger groups, 512 MiB each at the 4 TiB cap) and every group is placed on its own: the hosting nodes that rank
 //! highest for `(site, group)` (rendezvous hashing, the same answer on every
 //! node) take that group. No node needs the whole site, a 50 GiB folder spreads
 //! over the whole network, and healing, copy counting, garbage collection,
@@ -22,9 +23,9 @@ use super::site::{FILE_CHUNK_BYTES, FileEntry, STRIPE_ABOVE, SiteManifest};
 
 /// Chunks per group: 64 MiB of a site.
 pub const GROUP_CHUNKS: usize = 64;
-/// Most groups any site can have (256 GiB of 1 MiB chunks, plus one partial
-/// chunk per file, in 64-chunk groups, with room to spare). Bounds the size
-/// of a group bitmap read off the gossip.
+/// Most groups any site can have: 512 GiB of 64 MiB groups. Larger sites (up
+/// to the 4 TiB cap) keep this many groups and make each one bigger, so a
+/// group bitmap read off the gossip stays at most 1 KiB.
 pub const MAX_GROUPS: usize = 8192;
 
 static THRESHOLD: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(STRIPE_ABOVE);
@@ -51,7 +52,8 @@ pub fn is_striped(size: u64) -> bool {
 /// Bytes of a site per group, on average.
 pub const GROUP_BYTES: u64 = (GROUP_CHUNKS * FILE_CHUNK_BYTES) as u64;
 
-/// How many groups a striped site of `size` bytes has: one per 64 MiB. The
+/// How many groups a striped site of `size` bytes has: one per 64 MiB, at
+/// most [`MAX_GROUPS`]. The
 /// size is in the signed record, so every node agrees without the manifest.
 pub fn group_count(size: u64) -> usize {
     (size.div_ceil(GROUP_BYTES).max(1) as usize).min(MAX_GROUPS)
@@ -354,7 +356,11 @@ mod tests {
         // Buckets are roughly even (64 chunks on average).
         assert!((0..a.len() as u32).all(|g| (20..=110).contains(&a.group(g).len())));
         assert_eq!(group_count(1), 1);
-        assert_eq!(group_count(super::super::site::MAX_SITE_BYTES), 4096);
+        assert_eq!(group_count(256 * 1024 * 1024 * 1024), 4096);
+        assert_eq!(group_count(512 * 1024 * 1024 * 1024), MAX_GROUPS);
+        // The 4 TiB cap: the group count stops at MAX_GROUPS, groups grow to 512 MiB.
+        assert_eq!(group_count(super::super::site::MAX_SITE_BYTES), MAX_GROUPS);
+        assert_eq!(super::super::site::MAX_SITE_BYTES / MAX_GROUPS as u64, 512 * 1024 * 1024);
     }
 
     #[test]