diff --git a/apps/web/src/app/getting-started/page.tsx b/apps/web/src/app/getting-started/page.tsx
index 9f743f8b..2456a525 100644
--- a/apps/web/src/app/getting-started/page.tsx
+++ b/apps/web/src/app/getting-started/page.tsx
@@ -93,7 +93,7 @@ $ c0mpute hosting ls # sites a gateway knows, with live copy counts`}--name to update it. Any gateway serves it at{" "}
/s/<site_id>/, for example{" "}
https://gateway.c0mpute.com/s/<site_id>/, with
- byte ranges so video seeks. Limits: 64 GiB per free site, 256 GiB
+ byte ranges so video seeks. Limits: 64 GiB per free site, 4 TiB
with --disks auto. Sites over 1 GiB are striped over the
network in 64 MiB groups; run the same command again to resume an
interrupted publish.
diff --git a/docs/hosting.md b/docs/hosting.md
index afa75c90..2e3e4c6e 100644
--- a/docs/hosting.md
+++ b/docs/hosting.md
@@ -44,7 +44,7 @@ is used for misses.
| | |
|---|---|
-| Site, paid (`--disks`, a v2 record) | 256 GiB |
+| Site, paid (`--disks`, a v2 record) | 4 TiB |
| Site, free (v1 record) | 64 GiB |
| Files per site | 10,000,000 |
| One file | 128 GiB |
@@ -63,15 +63,16 @@ The budget advertised to d1sks (`max_gib`) is the same number, so `--disks auto`
picks disks with room for their share of the site.
Errors say which limit was hit and what to do (`--disks auto` for a free site over
-64 GiB, split the folder above 256 GiB). A hosting node logs the limit that made it
+64 GiB, split the folder above 4 TiB). A hosting node logs the limit that made it
refuse a copy (`over this node's hosting budget`, `disk nearly full`, `per-owner limit`).
## Large sites: striped placement
A site up to 1 GiB is replicated whole, exactly as before. Above that it is
**striped**: its chunks are bucketed by hash into groups, one group per 64 MiB of the
-site (the count comes from the size in the signed record), and every group is placed on
-its own:
+site up to 8,192 groups (512 GiB); a bigger site keeps 8,192 groups and makes each one
+bigger (512 MiB at the 4 TiB cap), so a group bitmap on the gossip stays at most 1 KiB.
+The count comes from the size in the signed record, and every group is placed on its own:
* The hosting nodes that rank highest for `(site, group)` (rendezvous hashing, the same
answer on every node) take that group until it has three copies. No node needs the
@@ -85,14 +86,15 @@ its own:
A node holds each group as soon as it is complete, so a restart resumes group by group.
* Paid sites: preferred hosts take every group when there are at most three of them (a
paid host holds the whole site), else the three that rank highest per group; each is
- paid for its share (`size x 3 / preferred`).
+ paid for its share (`size x 3 / preferred`). A multi-TiB site therefore needs disks
+ with that much room each; `--copies 5` cuts each host's share to 60% of the site.
* `c0mpute hosting ls` and `/hosting/v1/sites` show, for a striped site, `copies` = the
fewest copies of any group, plus `groups`, `short_groups` and `missing_groups`.
Manifests: up to 5,000 files and 1 GiB the manifest is one chunk, byte for byte what
older nodes read. Above either, it is a root listing parts (by first path) of at most
4,000 files and 65,536 chunk hashes each, so every part is one chunk fetch even for a
-256 GiB site; a gateway fetches only the part a request needs. Every holder of any group
+4 TiB site; a gateway fetches only the part a request needs. Every holder of any group
keeps the root and all parts.
Groups are defined over chunk hashes only (the group of a chunk is its hash modulo the
diff --git a/node/crates/c0mpute-cli/src/main.rs b/node/crates/c0mpute-cli/src/main.rs
index daf53325..907816cc 100644
--- a/node/crates/c0mpute-cli/src/main.rs
+++ b/node/crates/c0mpute-cli/src/main.rs
@@ -821,8 +821,8 @@ enum HostingCmd {
/// The site is signed with this machine's identity, so only this machine
/// can update it; publishing the same --name again ships a new version.
/// Stays up until three hosting nodes hold copies, then exits.
- /// Sites over 1 GiB (up to 64 GiB free, 256 GiB with --disks) are striped
- /// over the network in 64 MiB groups; files are served from the folder
+ /// Sites over 1 GiB (up to 64 GiB free, 4 TiB with --disks) are striped
+ /// over the network in 64 MiB groups (bigger past 512 GiB); files are served from the folder
/// itself while the network copies them, so keep it running, and run the
/// same command again to resume if it stops.
#[command(verbatim_doc_comment)]
diff --git a/node/crates/c0mpute-core/src/hosting/mod.rs b/node/crates/c0mpute-core/src/hosting/mod.rs
index 46cfa6f9..2e46c1f9 100644
--- a/node/crates/c0mpute-core/src/hosting/mod.rs
+++ b/node/crates/c0mpute-core/src/hosting/mod.rs
@@ -2686,7 +2686,7 @@ mod tests {
#[test]
fn a_holding_message_of_striped_sites_fits_gossipsub() {
// Entries are packed up to HOLDING_MESSAGE_BYTES; the largest bitmap
- // (every group of a 256 GiB site) still leaves room.
+ // (every group of a 4 TiB site, MAX_GROUPS) still leaves room.
let all: BTreeSet = (0..stripe::group_count(site::MAX_SITE_BYTES) as u32).collect();
let g = stripe::encode_groups(&all);
let per = HOLDING_MESSAGE_BYTES / (120 + g.len()) + 1;
@@ -3102,9 +3102,10 @@ pub async fn publish_dir_with(
scanned.manifest.files.len(),
gib(record.size),
if striped {
- ", striped in 64 MiB groups"
+ let n = stripe::group_count(record.size) as u64;
+ format!(", striped in {n} groups of ~{} MiB", record.size.div_ceil(n) >> 20)
} else {
- ""
+ String::new()
},
record.site_id
);
diff --git a/node/crates/c0mpute-core/src/hosting/site.rs b/node/crates/c0mpute-core/src/hosting/site.rs
index 433198dd..1b62c2e5 100644
--- a/node/crates/c0mpute-core/src/hosting/site.rs
+++ b/node/crates/c0mpute-core/src/hosting/site.rs
@@ -18,8 +18,10 @@ use serde::{Deserialize, Serialize};
/// as a CBOR integer array (about 2x on the wire and slow to decode), so
/// small chunks keep each fetch quick and well under the 10 MiB response cap.
pub const FILE_CHUNK_BYTES: usize = 1024 * 1024;
-/// Largest site a node will publish or replicate (with paid preferred hosts).
-pub const MAX_SITE_BYTES: u64 = 256 * GIB;
+/// Largest site a node will publish or replicate (with paid preferred hosts):
+/// 4 TiB. Striping caps a site at [`super::stripe::MAX_GROUPS`] groups, so
+/// past 512 GiB the groups grow (512 MiB each at 4 TiB) instead.
+pub const MAX_SITE_BYTES: u64 = 4 * 1024 * GIB;
/// Largest free site (a v1 record, no preferred hosts). Volunteers hold free
/// copies, so a free site may use at most a quarter of what a paid one may.
pub const MAX_FREE_SITE_BYTES: u64 = 64 * GIB;
@@ -638,7 +640,7 @@ impl SiteManifest {
/// Encode the manifest: one chunk up to [`INLINE_FILES`] files and
/// [`STRIPE_ABOVE`] bytes (byte for byte what older nodes read), else a
/// root plus parts of at most [`PART_FILES`] files and [`PART_CHUNKS`]
- /// chunks, so even a 256 GiB site's chunk lists are fetched a part at a
+ /// chunks, so even a 4 TiB site's chunk lists are fetched a part at a
/// time.
pub fn stored(&self) -> StoredManifest {
let m = self;
@@ -1371,7 +1373,8 @@ mod tests {
let e = check_site_size(70 * GIB, false).unwrap_err().to_string();
assert!(e.contains("--disks auto"), "{e}");
assert!(check_site_size(70 * GIB, true).is_ok());
- assert!(check_site_size(300 * GIB, true).unwrap_err().to_string().contains("Split"));
+ assert!(check_site_size(300 * GIB, true).is_ok(), "300 GiB is under the 4 TiB cap");
+ assert!(check_site_size(MAX_SITE_BYTES + GIB, true).unwrap_err().to_string().contains("Split"));
}
#[test]
diff --git a/node/crates/c0mpute-core/src/hosting/stripe.rs b/node/crates/c0mpute-core/src/hosting/stripe.rs
index a17552c6..85d5644e 100644
--- a/node/crates/c0mpute-core/src/hosting/stripe.rs
+++ b/node/crates/c0mpute-core/src/hosting/stripe.rs
@@ -1,8 +1,9 @@
//! Striped placement for large sites.
//!
//! A site at or below [`STRIPE_ABOVE`] is replicated whole, as it always was.
-//! Above it, the site's chunks are bucketed by hash into one group per 64 MiB
-//! ([`group_count`] of the signed size) and every group is placed on its own: the hosting nodes that rank
+//! Above it, the site's chunks are bucketed by hash into one group per 64 MiB,
+//! at most [`MAX_GROUPS`] ([`group_count`] of the signed size; a site over
+//! 512 GiB gets bigger groups, 512 MiB each at the 4 TiB cap) and every group is placed on its own: the hosting nodes that rank
//! highest for `(site, group)` (rendezvous hashing, the same answer on every
//! node) take that group. No node needs the whole site, a 50 GiB folder spreads
//! over the whole network, and healing, copy counting, garbage collection,
@@ -22,9 +23,9 @@ use super::site::{FILE_CHUNK_BYTES, FileEntry, STRIPE_ABOVE, SiteManifest};
/// Chunks per group: 64 MiB of a site.
pub const GROUP_CHUNKS: usize = 64;
-/// Most groups any site can have (256 GiB of 1 MiB chunks, plus one partial
-/// chunk per file, in 64-chunk groups, with room to spare). Bounds the size
-/// of a group bitmap read off the gossip.
+/// Most groups any site can have: 512 GiB of 64 MiB groups. Larger sites (up
+/// to the 4 TiB cap) keep this many groups and make each one bigger, so a
+/// group bitmap read off the gossip stays at most 1 KiB.
pub const MAX_GROUPS: usize = 8192;
static THRESHOLD: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(STRIPE_ABOVE);
@@ -51,7 +52,8 @@ pub fn is_striped(size: u64) -> bool {
/// Bytes of a site per group, on average.
pub const GROUP_BYTES: u64 = (GROUP_CHUNKS * FILE_CHUNK_BYTES) as u64;
-/// How many groups a striped site of `size` bytes has: one per 64 MiB. The
+/// How many groups a striped site of `size` bytes has: one per 64 MiB, at
+/// most [`MAX_GROUPS`]. The
/// size is in the signed record, so every node agrees without the manifest.
pub fn group_count(size: u64) -> usize {
(size.div_ceil(GROUP_BYTES).max(1) as usize).min(MAX_GROUPS)
@@ -354,7 +356,11 @@ mod tests {
// Buckets are roughly even (64 chunks on average).
assert!((0..a.len() as u32).all(|g| (20..=110).contains(&a.group(g).len())));
assert_eq!(group_count(1), 1);
- assert_eq!(group_count(super::super::site::MAX_SITE_BYTES), 4096);
+ assert_eq!(group_count(256 * 1024 * 1024 * 1024), 4096);
+ assert_eq!(group_count(512 * 1024 * 1024 * 1024), MAX_GROUPS);
+ // The 4 TiB cap: the group count stops at MAX_GROUPS, groups grow to 512 MiB.
+ assert_eq!(group_count(super::super::site::MAX_SITE_BYTES), MAX_GROUPS);
+ assert_eq!(super::super::site::MAX_SITE_BYTES / MAX_GROUPS as u64, 512 * 1024 * 1024);
}
#[test]