From 9b3b84ca36af5286a40765adcda2c7cc6b9ea803 Mon Sep 17 00:00:00 2001 From: Michael Taylor Date: Mon, 31 Aug 2026 06:39:01 -0700 Subject: [PATCH 1/6] chore: open cache/serve batch lane (#450, #284, #265) Stub commit so the lane's branch and draft PR exist before implementation. Co-Authored-By: Claude From 278c4c0c68eafb7295bccd68acb55a76992bf77c Mon Sep 17 00:00:00 2001 From: Michael Taylor Date: Mon, 31 Aug 2026 06:57:54 -0700 Subject: [PATCH 2/6] fix(landing): a page the node serves at /s/ can no longer land capsules bondable A store's own page could make the node land attacker-chosen capsules Held, and so bondable with the operator's $DIG staked on a stranger's content. `/s/*path` and `POST /` are the same router on the same port, so a store page is same-origin with the RPC endpoint, and STORE_CSP grants store pages `script-src 'unsafe-inline'` with `connect-src 'self'` -- where 'self' IS that endpoint. The browser then truthfully reports `Sec-Fetch-Site: same-origin`, which mapped to FirstParty, folded to Local, and returned Announce. Because Announce also removes an existing marker, the same call could un-suppress a capsule that had been correctly relayed. The two-axis model could not express the fix: the operator's own read and attacker content served at /s/ are both (Local, FirstParty). Same-origin stopped being a trust signal the moment the node began serving untrusted content on its control origin, so this adds a THIRD provenance rather than a tighter reading of two. RequestProvenance::StoreServed covers every page-driven Sec-Fetch-Site value and folds to Peer. `none` (a user-initiated top-level navigation, unforgeable by page script) stays FirstParty, so opening a store in a browser still lands its capsule and the reshare flywheel survives. An absent header still means a non-browser CLI/SDK client and still lands. An unknown value now fails CLOSED. Deliberately not Referer-derived: a page controls its own referrer-policy and can strip the path or the header, so a Referer rule is bypassable by exactly the party it constrains. Sec-Fetch-* is browser-set and forbidden to script. Refs #450 Co-Authored-By: Claude --- crates/dig-node-core/src/download.rs | 116 +++++++++++++++--- .../src/seams/dig_rpc/dispatch.rs | 23 ++++ .../dig-node-service/tests/content_serve.rs | 26 +++- 3 files changed, 144 insertions(+), 21 deletions(-) diff --git a/crates/dig-node-core/src/download.rs b/crates/dig-node-core/src/download.rs index 6c5c4e84..357b4aef 100644 --- a/crates/dig-node-core/src/download.rs +++ b/crates/dig-node-core/src/download.rs @@ -375,9 +375,19 @@ pub enum ReadOrigin { /// provenance closes that CSRF door WITHOUT ever throttling the read: the bytes always serve. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum RequestProvenance { - /// A first-party request: the user's own navigation, a same-origin/same-site subresource, a - /// direct address-bar hit, OR any non-browser client (CLI/SDK send no `Sec-Fetch-*` header). + /// A first-party request: the user's own top-level navigation (`Sec-Fetch-Site: none` — the + /// address bar or a bookmark, which no page script can forge), OR any non-browser client + /// (CLI/SDK send no `Sec-Fetch-*` header at all). FirstParty, + /// A request driven by a page THIS NODE is serving on its own control origin — i.e. by store + /// content published by a stranger (`Sec-Fetch-Site: same-origin`/`same-site`, or any value the + /// browser reports that is neither `none` nor `cross-site`). + /// + /// `/s/*` is the ONLY HTML surface this router serves (`server.rs` — `/`, `/health`, + /// `/version`, `/openrpc.json`, `/.well-known/*`, `/ws*`, `/verify/*` are all non-HTML), so a + /// page-driven request arriving on this origin was, by construction, authored by store content. + /// The read still serves; landing does not. + StoreServed, /// A cross-site subresource: the browser explicitly reported `Sec-Fetch-Site: cross-site`, /// meaning some OTHER origin's page drove this request. The read still serves; landing does not. CrossSite, @@ -386,16 +396,41 @@ pub enum RequestProvenance { /// Classify a request's provenance from its `Sec-Fetch-Site` header value (already extracted from /// the header map; `None` when the header is absent). /// -/// ONLY an explicit, case-insensitive `cross-site` denies landing. Everything else — `same-origin`, -/// `same-site`, `none`, an unknown value, AND (critically) an ABSENT header — is [`FirstParty`], so -/// non-browser clients that never send `Sec-Fetch-*` (the CLI, the SDK) are never mistaken for a -/// cross-site attacker. Absence must NEVER map to `CrossSite`. +/// The mapping, and why each arm is where it is: +/// +/// | `Sec-Fetch-Site` | provenance | why | +/// |---|---|---| +/// | absent | [`FirstParty`] | a non-browser client (CLI/SDK). Absence must NEVER deny a CLI read's landing. | +/// | `none` | [`FirstParty`] | a user-initiated top-level navigation — address bar or bookmark. A page-driven fetch can NEVER produce `none`, and `Sec-*` is a forbidden header name, so script cannot forge it. | +/// | `cross-site` | [`CrossSite`] | another origin's page drove it. | +/// | anything else (`same-origin`, `same-site`, an unknown future value) | [`StoreServed`] | page-driven on an origin whose only HTML is `/s/*`. | +/// +/// **Why `same-origin` is not first-party here (dig-node#450).** `/s/*path` and `POST /` are the +/// same router on the same port, and `STORE_CSP` grants store pages `script-src 'unsafe-inline'` +/// plus `connect-src 'self'` — where `'self'` IS the RPC endpoint. So a stranger's store page can +/// script a request that the browser truthfully labels `same-origin`, and under the old mapping +/// that landed the attacker's chosen capsule `Held` — bondable, with the operator's $DIG staked on +/// it. Same-origin stopped being a trust signal the moment the node began serving untrusted content +/// on its control origin, so the fix is a THIRD provenance rather than a tighter reading of two. +/// +/// This deliberately does NOT consult `Referer`: a page controls its own referrer-policy and can +/// strip the path (or the whole header), so a `Referer`-derived rule is bypassable by the exact +/// party it constrains. `Sec-Fetch-Site` is browser-set and unforgeable by page script. +/// +/// **Cost, stated plainly:** a subresource of an operator-initiated store view no longer lands on +/// its own. The flywheel survives because the top-level navigation that opened the store IS `none` +/// and lands the capsule, and every subresource is then served from that already-landed capsule. /// /// [`FirstParty`]: RequestProvenance::FirstParty +/// [`StoreServed`]: RequestProvenance::StoreServed +/// [`CrossSite`]: RequestProvenance::CrossSite pub fn from_sec_fetch_site(hdr: Option<&str>) -> RequestProvenance { match hdr.map(|v| v.trim().to_ascii_lowercase()).as_deref() { + None | Some("none") => RequestProvenance::FirstParty, Some("cross-site") => RequestProvenance::CrossSite, - _ => RequestProvenance::FirstParty, + // Fails CLOSED: an unknown/future value is page-driven until proven otherwise. It still + // serves; it merely does not land. + Some(_) => RequestProvenance::StoreServed, } } @@ -412,7 +447,10 @@ pub fn from_sec_fetch_site(hdr: Option<&str>) -> RequestProvenance { pub(crate) fn landing_origin(origin: ReadOrigin, provenance: RequestProvenance) -> ReadOrigin { match provenance { RequestProvenance::FirstParty => origin, - RequestProvenance::CrossSite => ReadOrigin::Peer, + // A page THIS NODE serves at `/s/` is a stranger's content running on the control origin + // (dig-node#450). It folds exactly as a cross-site page does: the bytes serve, nothing + // durable lands, so the operator never bonds a capsule an attacker chose. + RequestProvenance::StoreServed | RequestProvenance::CrossSite => ReadOrigin::Peer, } } @@ -5267,16 +5305,33 @@ pub(crate) mod tests { } #[test] - fn sec_fetch_site_first_party_values_are_first_party() { - for value in ["same-origin", "same-site", "none"] { + fn sec_fetch_site_page_driven_values_are_store_served() { + // dig-node#450. `same-origin` and `same-site` mean A PAGE drove this request, and the only + // HTML this router serves is `/s/*` — a stranger's store. Classifying either as FirstParty + // let that page choose which capsule the operator bonds $DIG against. + for value in ["same-origin", "same-site"] { assert_eq!( from_sec_fetch_site(Some(value)), - RequestProvenance::FirstParty, - "{value} is a first-party fetch and must land normally" + RequestProvenance::StoreServed, + "{value} is page-driven on a node whose only HTML surface is store content" ); } } + #[test] + fn sec_fetch_site_none_is_first_party_so_the_operator_still_lands_a_store_they_opened() { + // The counterpart to the test above, and the reason the fix does not cost the flywheel: + // `none` is a USER-initiated top-level navigation (address bar / bookmark). A page-driven + // fetch can never produce it, and `Sec-*` is a forbidden header name so script cannot forge + // it. If this arm ever folded to StoreServed, opening a store in a browser would stop + // landing it at all. + assert_eq!( + from_sec_fetch_site(Some("none")), + RequestProvenance::FirstParty, + "a user-initiated top-level navigation must still land" + ); + } + #[test] fn sec_fetch_site_absent_is_first_party() { // LOAD-BEARING: non-browser clients (CLI/SDK) send no Sec-Fetch-* header. Absence must map @@ -5295,16 +5350,43 @@ pub(crate) mod tests { RequestProvenance::CrossSite, "the header match must be trimmed + case-insensitive" ); + assert_eq!( + from_sec_fetch_site(Some(" Same-Origin ")), + RequestProvenance::StoreServed, + "the page-driven arm must be trimmed + case-insensitive too, or a browser that cases \ + the value differently would land an attacker's capsule" + ); } #[test] - fn sec_fetch_site_unknown_value_is_first_party() { - // Only an explicit "cross-site" denies landing; an unrecognized value fails OPEN (serves + - // lands) so a future/odd Sec-Fetch-Site value never silently breaks landing. + fn sec_fetch_site_unknown_value_fails_closed_to_store_served() { + // Inverted by dig-node#450: an unrecognised value is PAGE-DRIVEN until proven otherwise. + // The request still serves; it merely does not land. Failing open here would make the whole + // gate optional to any browser that ships a new value. assert_eq!( from_sec_fetch_site(Some("wat")), - RequestProvenance::FirstParty, - "an unknown Sec-Fetch-Site value must default to first-party" + RequestProvenance::StoreServed, + "an unknown Sec-Fetch-Site value must fail closed" + ); + } + + // -- landing_origin: the fold every landing leg shares ------------------------------------- + + #[test] + fn a_store_served_request_never_lands_however_local_its_transport_is() { + // The decision under test is the FOLD, not the marker a leg later writes. A loopback + // transport is the strongest case for landing and is exactly the one dig-node#450 exploits, + // so `Local` is the input that distinguishes this fix from doing nothing. + assert_eq!( + landing_origin(ReadOrigin::Local, RequestProvenance::StoreServed), + ReadOrigin::Peer, + "a page the node serves at /s/ must not land on the operator's behalf" + ); + // The truthful control, on the same axis: same transport, only the provenance differs. + assert_eq!( + landing_origin(ReadOrigin::Local, RequestProvenance::FirstParty), + ReadOrigin::Local, + "the operator's own read must still land, or the fix has disabled the flywheel" ); } } diff --git a/crates/dig-node-core/src/seams/dig_rpc/dispatch.rs b/crates/dig-node-core/src/seams/dig_rpc/dispatch.rs index 4803fc6e..7698f0c8 100644 --- a/crates/dig-node-core/src/seams/dig_rpc/dispatch.rs +++ b/crates/dig-node-core/src/seams/dig_rpc/dispatch.rs @@ -1164,4 +1164,27 @@ mod holder_claim_tests { "folding both axes is only ever restrictive" ); } + /// **Proves (dig-node#450, the same-origin variant):** a request driven by a page THIS NODE + /// serves at `/s/` backfills unbonded, even over a loopback socket the browser truthfully + /// labels `same-origin`. + /// + /// This is the variant the two-axis fold could not express. `/s/*path` and `POST /` are the + /// same router on the same port, and `STORE_CSP` grants store pages `script-src 'unsafe-inline'` + /// with `connect-src 'self'` — where `'self'` IS the RPC endpoint. So a stranger's store page + /// scripts a request the browser reports as `same-origin`, which folded to `Local` → `Announce` + /// → the operator's $DIG staked on the attacker's chosen capsule. `Announce` also REMOVES an + /// existing marker, so the same call could un-suppress a capsule already correctly relayed. + /// + /// **Catches:** any collapse of `StoreServed` back into `FirstParty`. Every other test in this + /// module passes with that collapse present — including the two `#436` ones — which is exactly + /// why this one is written against `StoreServed` over a `Local` transport rather than against a + /// landed marker further down. + #[test] + fn a_store_served_read_over_a_local_socket_backfills_suppressed() { + assert_eq!( + holder_claim_for_landing(ReadOrigin::Local, RequestProvenance::StoreServed), + HolderClaim::Suppress, + "a stranger's store page must not choose what this operator bonds $DIG against" + ); + } } diff --git a/crates/dig-node-service/tests/content_serve.rs b/crates/dig-node-service/tests/content_serve.rs index 4db453fd..f5580524 100644 --- a/crates/dig-node-service/tests/content_serve.rs +++ b/crates/dig-node-service/tests/content_serve.rs @@ -902,17 +902,35 @@ async fn store_serve_labels_provenance_from_sec_fetch_site_without_blocking_the_ cross.recorded_provenances() ); - // Same-site: a first-party subresource — must land normally. - let (same, same_status) = drive_s_get("127.0.0.1:51241", &path, None, Some("same-site")).await; + // Same-origin: dig-node#450. `/s/*` and `POST /` share a router, a port and therefore an + // ORIGIN, and STORE_CSP lets a store page script a call the browser labels `same-origin`. It + // must reach the seam as StoreServed so landing folds to `Peer` — otherwise a stranger's page + // chooses which capsule this operator bonds $DIG against. + let (same, same_status) = + drive_s_get("127.0.0.1:51241", &path, None, Some("same-origin")).await; assert!( same.recorded_provenances() .iter() - .all(|p| *p == RequestProvenance::FirstParty) + .all(|p| *p == RequestProvenance::StoreServed) && !same.recorded_provenances().is_empty(), - "a same-site read must reach the seam as FirstParty, got {:?}", + "a same-origin read must reach the seam as StoreServed, got {:?}", same.recorded_provenances() ); + // The control that keeps the flywheel honest: a USER-initiated top-level navigation + // (`Sec-Fetch-Site: none`, unforgeable by page script) is still FirstParty and still lands. If + // this arm ever folded too, opening a store in a browser would stop landing it at all — a + // regression a StoreServed-only assertion could not see. + let (nav, _) = drive_s_get("127.0.0.1:51243", &path, None, Some("none")).await; + assert!( + nav.recorded_provenances() + .iter() + .all(|p| *p == RequestProvenance::FirstParty) + && !nav.recorded_provenances().is_empty(), + "a top-level navigation must reach the seam as FirstParty, got {:?}", + nav.recorded_provenances() + ); + // Header ABSENT (a CLI/SDK client sends no Sec-Fetch-*): must be FirstParty, never CrossSite. let (absent, absent_status) = drive_s_get("127.0.0.1:51242", &path, None, None).await; assert!( From 4d5d3cfa35530a93232c145647f9f9f63df87e31 Mon Sep 17 00:00:00 2001 From: Michael Taylor Date: Mon, 31 Aug 2026 07:13:41 -0700 Subject: [PATCH 3/6] fix(cache): one budget bounds the whole cache, and unrecognised bytes count against it Two defects with one root: what the configured cap is a bound ON. #284 -- `cache_cap_bytes()` was read independently by both eviction paths over two different subtrees, so neither knew the other existed and a node configured for N bytes held close to 2N. `cache_budget()` now splits the one cap into a responses share (an eighth) and a modules share, so the two sweeps spend halves of one budget. A reserved share rather than 'modules take what responses leave' because the latter starves the response cache: its small regenerable windows always lose the race to ~135 MiB capsules. #265 -- the scan's hex64 filter governed BOTH the total measured AND the candidate set, so anything under /modules the node could not identify was invisible to the bound while consuming the disk the bound protects, and could grow without limit. Semantics chosen and now written down in the code: COUNT, never DELETE. Unrecognised bytes are charged against the modules share, so recognised capsules are evicted to compensate; the sweep never removes a file it cannot identify from a directory it does not exclusively own. used_bytes and cap_bytes now describe the same thing. Refs #284, #265 Co-Authored-By: Claude --- crates/dig-node-core/src/lib.rs | 266 +++++++++++++++++++++++++++++++- 1 file changed, 262 insertions(+), 4 deletions(-) diff --git a/crates/dig-node-core/src/lib.rs b/crates/dig-node-core/src/lib.rs index b0a7f1e3..0e63faa0 100644 --- a/crates/dig-node-core/src/lib.rs +++ b/crates/dig-node-core/src/lib.rs @@ -732,6 +732,50 @@ pub fn cache_cap_bytes() -> u64 { .and_then(|s| s.parse().ok()) .unwrap_or(DEFAULT_CACHE_CAP) } +/// The share of [`cache_cap_bytes`] reserved for the per-resource response cache +/// (`/responses`), expressed as a divisor: `cap / RESPONSE_CACHE_SHARE_DIVISOR`. +/// +/// **Why a split rather than one number applied twice (dig-node#284).** The cap used to be read +/// independently by BOTH eviction paths over BOTH subtrees, so a node configured for N bytes held +/// close to 2N: neither sweep knew the other existed. A user setting a cap did not get it. +/// +/// **Why a reserved share rather than "modules take whatever responses leave".** Sizing each sweep +/// against the other's live usage converges to the right total but starves whichever subtree loses +/// the race — and the response cache, being made of small regenerable windows, always loses to +/// ~135 MiB capsules. A cache that evicts every response the instant it is written is a bound that +/// starves the work it protects. An eighth is small enough that capsules keep the bulk of the disk +/// and large enough that the response cache is never pointless. +const RESPONSE_CACHE_SHARE_DIVISOR: u64 = 8; + +/// How the single [`cache_cap_bytes`] budget is divided between the two evicting subtrees, so their +/// SUM is bounded by the cap the operator configured and `used_bytes` and `cap_bytes` describe the +/// same thing (dig-node#284). +/// +/// Unrecognised bytes are charged to the modules half by [`Node::evict_modules_locked`] rather than +/// being represented here, because only the sweep that walks `/modules` can measure them. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct CacheBudget { + /// The whole operator-configured cap. The sum of the two shares below never exceeds it. + pub total: u64, + /// Bytes `/responses` may hold. + pub responses: u64, + /// Bytes `/modules` may hold, BEFORE unrecognised bytes are charged against it. + pub modules: u64, +} + +/// Split the configured cap into its two shares. See [`CacheBudget`]. +pub fn cache_budget() -> CacheBudget { + let total = cache_cap_bytes(); + let responses = total / RESPONSE_CACHE_SHARE_DIVISOR; + CacheBudget { + total, + responses, + // Saturating rather than plain subtraction so the invariant holds even if the divisor is + // ever changed to something that could exceed the total. + modules: total.saturating_sub(responses), + } +} + /// Persist the cache size cap (bytes) to config.json (the DIG settings page). /// Read-modify-write under the cross-process lock so a concurrent writer (e.g. @@ -2157,9 +2201,11 @@ impl Node { } } } - // Read the cap dynamically so changes from the DIG settings page apply - // without restarting the browser. `self.cache_cap` is the startup default. - let cap = cache_cap_bytes(); + // Read the budget dynamically so changes from the DIG settings page apply without + // restarting the browser (`self.cache_cap` is the startup default), and take only the + // RESPONSES SHARE of it — the modules sweep spends the rest. Reading the whole cap here, + // as this did, let the two subtrees each grow to the full cap (dig-node#284). + let cap = cache_budget().responses; for victim in plan_eviction(&entries, cap) { // Size of the victim, looked up from the scan, so the reclaimed-bytes // counter is accurate even though the file is about to be unlinked. @@ -2268,6 +2314,44 @@ impl Node { self.refresh_dht_inventory().await; } + /// Total bytes under `/modules` that [`Node::scan_cached_modules`] does NOT return as an + /// eviction candidate — anything whose store directory is not lowercase 64-hex, plus any file + /// inside a recognised store directory that is not a capsule (a `.tier` sidecar, a `.tmp-*` + /// write-atomic scratch file, a stray subdirectory). + /// + /// **Why this exists (dig-node#265).** The scan's hex64 filter governed BOTH the total the sweep + /// measured AND the candidate set it could evict, so unrecognised bytes were invisible to the + /// bound while still consuming the disk the bound protects. They could grow without limit. + /// + /// **The semantics chosen, stated here because nowhere else records it:** COUNT, never DELETE. + /// These bytes are charged against the modules budget, so recognised capsules are evicted to + /// compensate; the sweep never removes a file it cannot identify. + fn unrecognised_module_bytes(&self) -> u64 { + fn walk(p: &Path, total: &mut u64) { + let Ok(rd) = std::fs::read_dir(p) else { return }; + for e in rd.flatten() { + let path = e.path(); + if path.is_dir() { + walk(&path, total); + } else if let Ok(md) = e.metadata() { + *total += md.len(); + } + } + } + + let modules = self.cache_dir.join("modules"); + let mut everything = 0u64; + walk(&modules, &mut everything); + // Subtract exactly what the scan WOULD evict, so the two halves partition the subtree by + // construction rather than by two filters that must be kept in agreement by hand. + let recognised: u64 = self + .scan_cached_modules() + .iter() + .map(|m| m.size_bytes) + .sum(); + everything.saturating_sub(recognised) + } + /// Every capsule file under `/modules`, as the eviction decision needs to see it. /// /// Also refreshes each store's persisted `.tier` sidecar from the live composition, so tier @@ -2347,10 +2431,25 @@ impl Node { /// advertising content it had deleted (#267). fn evict_modules_locked(&self) -> Vec { let _xproc = acquire_cache_lock(); - let cap = cache_cap_bytes(); + // The MODULES SHARE of the one budget, not the whole cap (dig-node#284) — the response + // cache spends the rest, so the two subtrees together stay under what the operator set. + let budget = cache_budget().modules; let cached = self.scan_cached_modules(); let total: u64 = cached.iter().map(|m| m.size_bytes).sum(); + // dig-node#265: bytes under `/modules` that this sweep cannot identify as a capsule + // are COUNTED but never DELETED. Counted, because they consume exactly the disk the bound + // exists to protect, and a bound that cannot see half the directory is not a bound. + // Never deleted, because the sweep does not exclusively own this directory and removing + // files it does not understand is the kind of thing that is fine until the once it is not. + // + // The consequence is deliberate and worth stating plainly: unrecognised bytes shrink the + // budget available to recognised capsules, so the sweep evicts MORE capsules to stay under + // the cap. That is the honest direction — the operator asked for a disk bound, not for a + // bound on the subset of the disk this version happens to recognise. + let unrecognised = self.unrecognised_module_bytes(); + let cap = budget.saturating_sub(unrecognised); + // The DECISION — whose capsules to sacrifice, and in what order — belongs to `dig-sex`. This // node supplies only the facts (`tier_algorithms`) and performs only the I/O. Note what is // deliberately NOT supplied: the file mtime. It is bumped by `touch` on the SERVE path, so an @@ -6471,6 +6570,165 @@ mod tests { std::env::remove_var("DIG_NODE_CACHE"); } + /// Total bytes actually on disk under `dir`, recursively. The tests below assert against THIS + /// rather than against a function having been called: dig-node#284 is a defect about how many + /// bytes survive a sweep, so only the bytes can witness it. + fn bytes_on_disk(dir: &Path) -> u64 { + let mut total = 0u64; + let Ok(rd) = std::fs::read_dir(dir) else { + return 0; + }; + for e in rd.flatten() { + let p = e.path(); + if p.is_dir() { + total += bytes_on_disk(&p); + } else if let Ok(md) = e.metadata() { + total += md.len(); + } + } + total + } + + /// **Proves (dig-node#284):** the configured cap bounds the WHOLE cache, not each subtree + /// separately. Both subtrees are filled well past the cap, both sweeps run, and the bytes left + /// on disk are measured against the number the operator configured. + /// + /// **Catches:** either sweep reading `cache_cap_bytes()` instead of its share of + /// [`cache_budget`]. With that defect the cache settles near 2x the cap, which is exactly what + /// this asserts against — and no assertion about a call, a policy, or a victim list can see it, + /// because both sweeps behave *correctly* in isolation. That is the whole shape of the bug. + #[test] + fn one_budget_bounds_both_cache_subtrees_together() { + let _g = ENV_GUARD.lock().unwrap_or_else(|p| p.into_inner()); + let (node, _td) = test_node(None); + let cfg = tempfile::tempdir().unwrap(); + std::env::set_var("DIG_NODE_CACHE", cfg.path()); + let _ = std::fs::remove_file(config_path()); + + const CAP: u64 = 40_960; + set_cache_cap_bytes(CAP).unwrap(); + + // Fill `modules` with sacrificial (tier-0) capsules far past the whole cap, so the modules + // sweep has real work and cannot pass by holding nothing. + let root = "cd".repeat(32); + for i in 0..12u8 { + let store = format!("{:02x}", i).repeat(32); + crate::tier0_live::mark_tier0_land(&store); + let p = module_path(&node.cache_dir, &store, &root); + std::fs::create_dir_all(p.parent().unwrap()).unwrap(); + std::fs::write(&p, vec![0u8; 8_192]).unwrap(); + } + + // Fill `responses` past the cap too. + let responses = node.cache_dir.join("responses"); + std::fs::create_dir_all(&responses).unwrap(); + for i in 0..12u8 { + std::fs::write(responses.join(format!("window-{i}")), vec![0u8; 8_192]).unwrap(); + } + + let before = bytes_on_disk(&node.cache_dir); + assert!( + before > CAP * 2, + "the fixture must start ABOVE twice the cap or it cannot distinguish one budget from \ + two, got {before} against a cap of {CAP}" + ); + + pin_test_rt().block_on(node.evict_modules_if_needed()); + node.evict_if_needed(&responses); + + let after = bytes_on_disk(&node.cache_dir); + assert!( + after <= CAP, + "the whole cache must fit the ONE configured cap after both sweeps: {after} bytes on \ + disk against a cap of {CAP}. Roughly 2x the cap means each subtree enforced the cap \ + independently (dig-node#284)." + ); + // The truthful control: the bound must not be satisfied by deleting everything. A cache + // that evicts itself to nothing is a different defect wearing this test's green. + assert!( + after > 0, + "the sweeps must leave the cache populated, not empty — an empty cache trivially fits \ + any cap and would prove nothing" + ); + + std::env::remove_var("DIG_NODE_CACHE"); + } + + /// **Proves (dig-node#265):** bytes under `/modules` that the scan cannot identify are + /// COUNTED against the budget and are NEVER DELETED — the two halves of the semantics chosen in + /// [`Node::unrecognised_module_bytes`], asserted separately because a fix could get either one + /// right alone. + /// + /// **Catches:** the hex64 filter governing both the measured total and the candidate set, which + /// made unrecognised bytes invisible to the bound. Under that defect the recognised capsule fits + /// the budget on its own and survives, so the eviction assertion below fails — the unrecognised + /// directory is the ONLY reason the sweep is under pressure at all. + #[test] + fn unrecognised_bytes_under_modules_are_counted_but_never_deleted() { + let _g = ENV_GUARD.lock().unwrap_or_else(|p| p.into_inner()); + let (node, _td) = test_node(None); + let cfg = tempfile::tempdir().unwrap(); + std::env::set_var("DIG_NODE_CACHE", cfg.path()); + let _ = std::fs::remove_file(config_path()); + + // Sized FROM the split, not guessed: the modules share is 7/8 of the cap. One 4 KiB capsule + // fits that share comfortably on its own; adding 8 KiB of unrecognised bytes puts the + // subtree over it. So the two worlds — counted and not-counted — give opposite verdicts. + // Sized FROM the split rather than guessed, and the sizing IS the fixture. The modules + // share is 7/8 of the cap = 10_752. A lone 4 KiB capsule fits it, so under the OLD + // behaviour (unrecognised bytes invisible) nothing is evicted. Charging the 8 KiB stray + // leaves 2_560, which the same capsule does NOT fit. The two worlds give opposite verdicts + // on the identical fixture, which is what makes the eviction assertion evidence about the + // COUNTING decision rather than about eviction working at all. + const CAP: u64 = 12_288; + const CAPSULE: u64 = 4_096; + const STRAY: u64 = 8_192; + set_cache_cap_bytes(CAP).unwrap(); + let modules_budget = cache_budget().modules; + assert!( + modules_budget > CAPSULE, + "the capsule must fit the modules share ALONE, or it would be evicted whether or not the stray bytes were counted" + ); + assert!( + modules_budget.saturating_sub(STRAY) < CAPSULE, + "charging the stray bytes must push the capsule OVER the share, or counting them changes nothing observable" + ); + + let store = "9a".repeat(32); + let root = "cd".repeat(32); + crate::tier0_live::mark_tier0_land(&store); // sacrificial, so pressure CAN evict it + let capsule = module_path(&node.cache_dir, &store, &root); + std::fs::create_dir_all(capsule.parent().unwrap()).unwrap(); + std::fs::write(&capsule, vec![0u8; CAPSULE as usize]).unwrap(); + + // A directory whose name is not a lowercase 64-hex store id — invisible to the scan. + let stray = node.cache_dir.join("modules").join("not-a-store-id"); + std::fs::create_dir_all(&stray).unwrap(); + let stray_file = stray.join("blob.bin"); + std::fs::write(&stray_file, vec![0u8; STRAY as usize]).unwrap(); + + assert_eq!( + node.unrecognised_module_bytes(), + STRAY, + "the unrecognised bytes must be measured, or nothing downstream can charge for them" + ); + + pin_test_rt().block_on(node.evict_modules_if_needed()); + + assert!( + !capsule.exists(), + "the recognised capsule must be evicted to make room for bytes the sweep counted but \ + cannot remove — under the old filter it fits the budget alone and survives" + ); + assert!( + stray_file.exists(), + "the sweep must NEVER delete a file it cannot identify from a directory it does not \ + exclusively own" + ); + + std::env::remove_var("DIG_NODE_CACHE"); + } + /// Pin a two-capsule cache under a tiny cap so the sweep MUST sacrifice exactly the tier-0 store, /// returning `(node, cache tempdir, config tempdir, victim path, survivor path)`. /// From 8f10563fafbea959df01679dec5ea4932df21579 Mon Sep 17 00:00:00 2001 From: Michael Taylor Date: Mon, 31 Aug 2026 07:22:44 -0700 Subject: [PATCH 4/6] docs(spec): record the third landing provenance and the split cache budget SPEC 21.8 gains the three-outcome Sec-Fetch-Site mapping (StoreServed, the unknown arm failing closed, and the explicit ban on deriving provenance from Referer/Origin), and states why 'none' must stay FirstParty. SPEC 7.10 states that cache_cap_bytes is ONE budget over the whole tree, split into a reserved responses share and a modules share, and that unrecognised bytes under modules are counted but never deleted. Refs #450, #284, #265 Co-Authored-By: Claude --- Cargo.toml | 2 +- SPEC.md | 42 +++++++++++++++++++++++++++++++++++------- 2 files changed, 36 insertions(+), 8 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 08e7541a..1005fc3d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,7 +32,7 @@ edition = "2021" # the ROOT manifest (`[workspace.package].version`), so it MUST be set here for a # release to fire (§3.6). The library crates (dig-node-core/dig-runtime/dig-wallet) # keep their own independent versions — only the released binary tracks the workspace version. -version = "0.189.0" +version = "0.190.0" # Release hardening, matching digstore: keep integer-overflow checks ON in release. # The node parses untrusted serialized input and does offset/length arithmetic over diff --git a/SPEC.md b/SPEC.md index f0d62d3a..3f643a60 100644 --- a/SPEC.md +++ b/SPEC.md @@ -1818,6 +1818,17 @@ eviction. These additive fields/methods complete that surface; all are `served: controller renders the eviction queue directly without re-deriving it. `last_used_unix_ms` is the file mtime, bumped to now on every local serve. +- **The cap bounds the WHOLE cache, and is SPLIT across the evicting subtrees.** `cache_cap_bytes` is + ONE budget over the entire cache tree, never a per-subtree limit. It is divided into a reserved + **responses share** (`cap / 8`, for `/responses`) and a **modules share** (the remainder, for + `/modules`); each eviction sweep spends only its own share, so the SUM on disk is bounded by + the configured cap and `used_bytes` and `cap_bytes` describe the same thing. Reserving a share for + responses (rather than letting modules take whatever responses leave) is required so the small + regenerable response windows are not starved by whole capsules. Bytes under `/modules` that + the scan cannot identify as a capsule are COUNTED against the modules share and MUST NOT be deleted: + the bound is a bound on disk consumed, and the sweep does not exclusively own that directory, so + recognised capsules are evicted to compensate for bytes it cannot remove. + - **`cache.setCapBytes { cap_bytes }` — the RESERVED cap.** Sets the reserved disk space for cached content, **floored at 64 MiB** (a `cap_bytes` below the floor is raised to it), and returns the applied `{ cap_bytes }`. `cache.getConfig` returns the live `{ cap_bytes, used_bytes, cache_dir, @@ -7151,13 +7162,30 @@ Landing therefore gates on BOTH axes: serve surface AND the `POST /` JSON-RPC read methods (`dig.getContent`, `dig.fetchRange`), whose miss-path landing legs (the implicit warm/backfill/reshare fired when the resource is not held locally) would otherwise let a SAME-ORIGIN capsule page `POST dig.getContent` and drive landing — - the loopback address labels it `Local`, so §21.7 alone permits it. ONLY an explicit, case-insensitive - `cross-site` value is `CrossSite`; `same-origin`, `same-site`, `none`, an unknown value, AND an ABSENT - header are all `FirstParty`. Absence MUST map to `FirstParty` — non-browser clients (the CLI, the SDK) - send no `Sec-Fetch-*` header, and treating absence as cross-site would silently stop every CLI/SDK - read from landing. -2. **A read lands only when it is BOTH `Local` (§21.7) AND `FirstParty`.** A `CrossSite` request collapses - its landing origin to `Peer`: the bytes are served identically, but no warm, reshare, promotion, or + the loopback address labels it `Local`, so §21.7 alone permits it. The mapping has THREE outcomes, + because same-origin is not a trust signal on a node that serves untrusted content on its control + origin (`/s/*` and `POST /` are the same router on the same port, and the store CSP grants store + pages `script-src 'unsafe-inline'` with `connect-src 'self'`): + + | `Sec-Fetch-Site` | provenance | lands? | + |---|---|---| + | ABSENT | `FirstParty` | yes | + | `none` | `FirstParty` | yes | + | `same-origin`, `same-site`, any unknown value | `StoreServed` | NO | + | `cross-site` (case-insensitive, trimmed) | `CrossSite` | NO | + + Absence MUST map to `FirstParty` — non-browser clients (the CLI, the SDK) send no `Sec-Fetch-*` + header, and treating absence as cross-site would silently stop every CLI/SDK read from landing. + `none` MUST map to `FirstParty`: it denotes a USER-initiated top-level navigation (address bar or + bookmark), a page-driven fetch can never produce it, and `Sec-*` is a forbidden header name so page + script cannot forge it. This is what keeps the reshare flywheel intact — opening a store in a + browser still lands its capsule, and every subresource is then served from that landed capsule. + Every other browser-reported value is PAGE-DRIVEN and MUST map to `StoreServed`, including values + this specification does not enumerate: the unknown arm fails CLOSED. Provenance MUST NOT be derived + from `Referer` or `Origin` — a page controls its own referrer-policy and can strip the path or the + whole header, so a `Referer`-derived rule is bypassable by exactly the party it constrains. +2. **A read lands only when it is BOTH `Local` (§21.7) AND `FirstParty`.** A `CrossSite` or + `StoreServed` request collapses its landing origin to `Peer`: the bytes are served identically, but no warm, reshare, promotion, or announce fires. The READ MUST NEVER be blocked, throttled, or altered by provenance — only the side effect is suppressed. 3. **The collapse is applied ONCE per landing site via the shared `landing_origin(origin, provenance)` From 308cd8e2d18485a568d9d443a533f6b81faa8012 Mon Sep 17 00:00:00 2001 From: Michael Taylor Date: Mon, 31 Aug 2026 07:47:15 -0700 Subject: [PATCH 5/6] style: cargo fmt after the cache-budget insertion Co-Authored-By: Claude --- Cargo.lock | 2 +- crates/dig-node-core/src/lib.rs | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6f84132a..d80efae9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3031,7 +3031,7 @@ dependencies = [ [[package]] name = "dig-node-service" -version = "0.189.0" +version = "0.190.0" dependencies = [ "async-trait", "axum", diff --git a/crates/dig-node-core/src/lib.rs b/crates/dig-node-core/src/lib.rs index 0e63faa0..83266364 100644 --- a/crates/dig-node-core/src/lib.rs +++ b/crates/dig-node-core/src/lib.rs @@ -776,7 +776,6 @@ pub fn cache_budget() -> CacheBudget { } } - /// Persist the cache size cap (bytes) to config.json (the DIG settings page). /// Read-modify-write under the cross-process lock so a concurrent writer (e.g. /// dig-companion setting `wc_project_id`) can't lose this update or vice-versa. From 47577b5b478bdbe960d21740fbcc671ece96fb97 Mon Sep 17 00:00:00 2001 From: Michael Taylor Date: Mon, 31 Aug 2026 10:38:05 -0700 Subject: [PATCH 6/6] test(serve): stop asserting the same-origin landing that #450 removes `provenance_is_read_on_the_post_path` pinned `same-origin -> FirstParty` with the comment "a same-origin POST still lands". That is the dig-node#450 defect stated as a requirement: `/s/*path` and `POST /` share one router and port, and `STORE_CSP` grants `connect-src 'self'` -- where `'self'` IS the RPC endpoint -- so a store's own page reached `dig.getContent` as same-origin, landed its capsule `Held`, and staked this operator's $DIG on a stranger's content. The test asserted the behaviour the fix exists to remove, which is why it went RED on the fix rather than on the defect. Third instance of this shape found today. Now asserts the real contract, with the two controls that stop an over-correction: `none` (a user-initiated navigation, which script can never produce) stays FirstParty so the operator's own reads still land, and an unrecognised value fails CLOSED where it previously failed open. Refs #450 Co-Authored-By: Claude --- crates/dig-node-service/src/server.rs | 36 +++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/crates/dig-node-service/src/server.rs b/crates/dig-node-service/src/server.rs index c8ce20bc..19bbb0f8 100644 --- a/crates/dig-node-service/src/server.rs +++ b/crates/dig-node-service/src/server.rs @@ -3276,19 +3276,51 @@ mod tests { "a cross-site POST must classify as CrossSite so its landing legs are denied" ); - // No Sec-Fetch-Site header (a non-browser client, or a same-origin request) ⇒ first-party. + // No Sec-Fetch-Site header ⇒ a NON-BROWSER client (CLI, SDK, FFI). A browser always sends + // the header, so its absence is the one case that is genuinely not page-driven. assert_eq!( provenance_for(&HeaderMap::new()), RequestProvenance::FirstParty, "an absent header must be first-party — a CLI/SDK read must never be mistaken for cross-site" ); + // `same-origin` is PAGE-DRIVEN, and on this node the only HTML surface is `/s/*` — so the + // page that sent it was authored by store content, not by the operator. + // + // This assertion previously read `FirstParty`, with the comment "a same-origin POST still + // lands". That was the dig-node#450 defect stated as a requirement: `/s/*path` and `POST /` + // share one router and port, and `STORE_CSP` grants `connect-src 'self'` — where `'self'` + // IS the RPC endpoint — so a store's own page reached `dig.getContent` as `same-origin`, + // landed its capsule `Held`, and staked this operator's $DIG on a stranger's content. + // + // The test asserted the behaviour the fix exists to remove, which is why it went red on the + // fix rather than on the defect. let mut same = HeaderMap::new(); same.insert("sec-fetch-site", "same-origin".parse().unwrap()); assert_eq!( provenance_for(&same), + RequestProvenance::StoreServed, + "a same-origin POST is page-driven, and every page this node serves is store content" + ); + + // `none` is a user-initiated navigation, which script can never produce — so it stays + // first-party and the reshare flywheel survives: the operator navigating to a store still + // lands it. + let mut nav = HeaderMap::new(); + nav.insert("sec-fetch-site", "none".parse().unwrap()); + assert_eq!( + provenance_for(&nav), RequestProvenance::FirstParty, - "a same-origin POST still lands — only an explicit cross-site value denies landing" + "a user-initiated navigation is the operator acting, not store content acting" + ); + + // An unrecognised value fails CLOSED. It failed open before. + let mut odd = HeaderMap::new(); + odd.insert("sec-fetch-site", "future-value".parse().unwrap()); + assert_eq!( + provenance_for(&odd), + RequestProvenance::StoreServed, + "an unknown Sec-Fetch-Site must not be treated as the operator" ); }