From 532fa4e656d562ed10bc089c845055c83df8a5d7 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 13 Nov 2026 18:38:08 -0800 Subject: [PATCH] A write finds the cached read ranges it can touch from an index ordered by end Every write to a page ran `invalidate_chunk_cache`, which walked every read range cached for that page to find the ones the write intersects. The ranges a page has cached grow with the reads of it -- an entry stays in the index until it is evicted, and after an append it is stale but still there -- so the walk grew with the page's read history, on every write. The per-page index is now ordered by the END of each cached range. The ranges a write can touch are the ones that end past its start, so the walk begins there instead of at the first range there is. An append starts at the end of the page, every cached range ends at or before it, and the walk finds nothing without visiting anything. The ranges returned are exactly those the intersection test accepts; a randomized comparison against that definition, written out separately, aims its writes at the places the boundary is. Tests: a write against random sets of cached ranges leaves cached exactly the ranges it does not intersect (starting where one ends, ending where one starts, past the page, empty, one byte inside); and 40 appends leave every cached range of the page in the index. Co-Authored-By: Claude Sonnet 5.5 --- src/lib.rs | 2 +- src/object_store_regression_tests.rs | 125 ++++++++++++++++++ .../cache_reclaim_snapshot/read_cache.inc.rs | 6 +- src/services/chunk/chunk_cache_control.inc.rs | 47 ++++--- src/services/chunk/chunk_footprint.rs | 4 +- .../admission_cache_batch/cache_control.rs | 12 ++ src/services/chunk/service_state.inc.rs | 2 +- 7 files changed, 171 insertions(+), 27 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index b9bead43..d1eef42e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -12,7 +12,7 @@ pub mod placement_outcome; use std::cell::{Cell, RefCell}; use std::cmp::Ordering; -use std::collections::{BTreeMap, BTreeSet, BinaryHeap, HashMap, HashSet, VecDeque}; +use std::collections::{BTreeMap, BTreeSet, BinaryHeap, HashMap, VecDeque}; use std::fmt; use std::time::{SystemTime, UNIX_EPOCH}; diff --git a/src/object_store_regression_tests.rs b/src/object_store_regression_tests.rs index 3e9f5808..0d71692e 100644 --- a/src/object_store_regression_tests.rs +++ b/src/object_store_regression_tests.rs @@ -55660,3 +55660,128 @@ fn malformed_manifest_publications_leave_all_metadata_unchanged() { assert!(PlacementOutcome::publish_manifest(&service, replace.manifest.unwrap(), 20).is_err()); assert_eq!(service.export_snapshot(), saved); } + +/// The intersection a cached read range has with a write, stated independently of the code under +/// test: both ranges are non-empty and each starts before the other ends. +fn oracle_ranges_intersect(a_offset: u64, a_length: u64, b_offset: u64, b_length: u64) -> bool { + a_length > 0 && b_length > 0 && a_offset < b_offset + b_length && b_offset < a_offset + a_length +} + +#[test] +fn a_write_invalidates_exactly_the_cached_ranges_it_intersects() { + // THE PER-PAGE INDEX OF CACHED RANGES IS ORDERED BY END, so that a write finds what it can + // touch by starting at the first range that ends past its start. That is an index change, + // and an index change is where an off-by-one decides which cached bytes a write leaves + // standing. So the answer is compared, over many random pages, against the definition of + // intersection written out independently, and the writes are aimed at the places the + // boundary is: a write that starts exactly where a cached range ends, one that ends exactly + // where one starts, one that touches nothing at the tail of the page, and an empty one. + let mut seed: u64 = 0x9E37_79B9_7F4A_7C15; + let mut next = move |bound: u64| { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + seed % bound + }; + const PAGE_BYTES: u64 = 20_000; + let mut shapes = [0usize; 6]; + for _trial in 0..200u32 { + let mut chunk = MatrixReplicaService::new(); + chunk.set_read_cache_max_bytes(8 << 20); + let page_id = PageId { + chunk_id: ChunkId::new(0, 996).unwrap(), + page_index: 0, + }; + let data: Vec = (0..PAGE_BYTES as u32).map(|i| (i * 7) as u8).collect(); + chunk.write_chunk(page_id.clone(), 0, &data).unwrap(); + + let mut ranges: Vec<(u64, u64)> = Vec::new(); + for _ in 0..(3 + next(10)) { + let offset = next(PAGE_BYTES - 1); + let length = 1 + next((PAGE_BYTES - offset).min(2_500)); + if !ranges.contains(&(offset, length)) { + chunk.read_chunk(&page_id, offset, length).unwrap(); + ranges.push((offset, length)); + } + } + // They are all cached: this is what makes "still cached" mean something below. + for (offset, length) in &ranges { + let (_, hit) = chunk + .read_chunk_with_cache_status(&page_id, *offset, *length, ReadCacheIntent::Fill) + .unwrap(); + assert!(hit, "range ({offset}, {length}) was not cached before the write"); + } + + let (cached_offset, cached_length) = ranges[next(ranges.len() as u64) as usize]; + let shape = next(6) as usize; + shapes[shape] += 1; + let (write_offset, write_length) = match shape { + // starts exactly where a cached range ends + 0 => (cached_offset + cached_length, 1 + next(50)), + // ends exactly where a cached range starts + 1 if cached_offset > 0 => { + let length = 1 + next(cached_offset.min(30)); + (cached_offset - length, length) + } + // at or past the end of the page: an append + 2 => (PAGE_BYTES + next(100), 1 + next(100)), + // empty + 3 => (next(PAGE_BYTES), 0), + // one byte inside a cached range + 4 => (cached_offset + next(cached_length), 1), + _ => (next(PAGE_BYTES), 1 + next(4_000)), + }; + chunk.invalidate_chunk_cache(&page_id, write_offset, write_length); + + for (offset, length) in &ranges { + let (_, hit) = chunk + .read_chunk_with_cache_status(&page_id, *offset, *length, ReadCacheIntent::Fill) + .unwrap(); + let should_survive = + !oracle_ranges_intersect(write_offset, write_length, *offset, *length); + assert_eq!( + hit, should_survive, + "write ({write_offset}, {write_length}) against cached ({offset}, {length}): \ + cached afterwards = {hit}, expected {should_survive}" + ); + } + } + // The aim at boundaries has to have happened, or the comparison above proves little. + assert!( + shapes.iter().all(|count| *count > 5), + "a write shape was never exercised: {shapes:?}" + ); +} + +#[test] +fn an_append_leaves_every_cached_range_of_the_page_alone() { + // The case the index is ordered for. Every range cached for a page ends at or before the + // page's length, an append starts there, so none can intersect and none may be removed. + let mut chunk = MatrixReplicaService::new(); + chunk.set_read_cache_max_bytes(8 << 20); + let page_id = PageId { + chunk_id: ChunkId::new(0, 997).unwrap(), + page_index: 0, + }; + let mut length = 0u64; + let mut cached: Vec<(u64, u64)> = Vec::new(); + for round in 0..40u64 { + let bytes: Vec = (0..300).map(|i| (round * 3 + i) as u8).collect(); + chunk.write_chunk(page_id.clone(), length, &bytes).unwrap(); + let stats = chunk.invalidate_chunk_cache(&page_id, length, bytes.len() as u64); + assert_eq!(stats.invalidated_entries, 0, "round {round}: an append invalidated a range"); + chunk.read_chunk(&page_id, length, 300).unwrap(); + cached.push((length, 300)); + length += 300; + } + // None of them was dropped by the appends. They are STALE -- a hit also needs the page's + // sequence and checksum to be the ones the entry was filled under, and every append changed + // both -- so asking for them back would miss; what the appends must not have done is lose + // them from the index, which is what invalidating the whole span finds out. + let stats = chunk.invalidate_chunk_cache(&page_id, 0, length); + assert_eq!( + stats.invalidated_entries as usize, + cached.len(), + "the index lost cached ranges that no write intersected" + ); +} diff --git a/src/services/chunk/cache_reclaim_snapshot/read_cache.inc.rs b/src/services/chunk/cache_reclaim_snapshot/read_cache.inc.rs index 9ad0ec07..990fbd2b 100644 --- a/src/services/chunk/cache_reclaim_snapshot/read_cache.inc.rs +++ b/src/services/chunk/cache_reclaim_snapshot/read_cache.inc.rs @@ -80,7 +80,7 @@ impl MatrixReplicaService { .borrow_mut() .entry(key.0.clone()) .or_default() - .insert((key.1, key.2)); + .insert(read_cache_span(key.1, key.2)); let mut cache = self.read_cache.borrow_mut(); if let Some(previous) = cache.insert( key, @@ -268,7 +268,7 @@ impl MatrixReplicaService { if let Some(entry) = &removed { let mut by_chunk = self.read_cache_by_chunk.borrow_mut(); if let Some(keys) = by_chunk.get_mut(&key.0) { - keys.remove(&(key.1, key.2)); + keys.remove(&read_cache_span(key.1, key.2)); if keys.is_empty() { by_chunk.remove(&key.0); } @@ -308,7 +308,7 @@ impl MatrixReplicaService { if let Some(removed) = cache.remove(&key) { let mut by_chunk = self.read_cache_by_chunk.borrow_mut(); if let Some(keys) = by_chunk.get_mut(&key.0) { - keys.remove(&(key.1, key.2)); + keys.remove(&read_cache_span(key.1, key.2)); if keys.is_empty() { by_chunk.remove(&key.0); } diff --git a/src/services/chunk/chunk_cache_control.inc.rs b/src/services/chunk/chunk_cache_control.inc.rs index e27ce5ea..49e7442f 100644 --- a/src/services/chunk/chunk_cache_control.inc.rs +++ b/src/services/chunk/chunk_cache_control.inc.rs @@ -115,6 +115,29 @@ impl MatrixReplicaService { self.invalidate_chunk_cache_bounded(page_id, offset, length, 0) } + /// The cached ranges of `page_id` that intersect `[offset, offset + length)`, as + /// `(offset, length)`. Exactly the ranges `ranges_intersect` accepts, found by walking the + /// per-page index from the first range that ends past `offset` rather than from its start. + fn cached_ranges_a_write_can_touch( + &self, + page_id: &PageId, + offset: u64, + length: u64, + ) -> Vec { + let Some(write_end) = offset.checked_add(length).filter(|_| length > 0) else { + return Vec::new(); + }; + let by_chunk = self.read_cache_by_chunk.borrow(); + let Some(spans) = by_chunk.get(page_id) else { + return Vec::new(); + }; + spans + .range((offset.saturating_add(1), 0)..) + .filter(|(end, cached_offset)| *cached_offset < write_end && *end > *cached_offset) + .map(|(end, cached_offset)| (*cached_offset, end - cached_offset)) + .collect() + } + pub fn invalidate_chunk_cache_bounded( &self, page_id: &PageId, @@ -130,17 +153,7 @@ impl MatrixReplicaService { self.read_cache_invalidations .set(self.read_cache_invalidations.get().saturating_add(1)); if max_bytes == 0 { - let keys: Vec = self - .read_cache_by_chunk - .borrow() - .get(page_id) - .into_iter() - .flat_map(|keys| keys.iter()) - .filter(|(cached_offset, cached_length)| { - ranges_intersect(offset, length, *cached_offset, *cached_length) - }) - .copied() - .collect(); + let keys = self.cached_ranges_a_write_can_touch(page_id, offset, length); for (cached_offset, cached_length) in keys { let key = (page_id.clone(), cached_offset, cached_length); if let Some(entry) = self.remove_read_cache_entry(&key) { @@ -161,19 +174,13 @@ impl MatrixReplicaService { } let mut keys: Vec<(ReplicaReadCacheRangeKey, u64)> = { let cache = self.read_cache.borrow(); - self.read_cache_by_chunk - .borrow() - .get(page_id) + self.cached_ranges_a_write_can_touch(page_id, offset, length) .into_iter() - .flat_map(|page_keys| page_keys.iter()) .filter_map(|(cached_offset, cached_length)| { - if !ranges_intersect(offset, length, *cached_offset, *cached_length) { - return None; - } - let key = (page_id.clone(), *cached_offset, *cached_length); + let key = (page_id.clone(), cached_offset, cached_length); cache .get(&key) - .map(|entry| ((*cached_offset, *cached_length), entry.result.data.len() as u64)) + .map(|entry| ((cached_offset, cached_length), entry.result.data.len() as u64)) }) .collect() }; diff --git a/src/services/chunk/chunk_footprint.rs b/src/services/chunk/chunk_footprint.rs index 2043dd98..c4a98983 100644 --- a/src/services/chunk/chunk_footprint.rs +++ b/src/services/chunk/chunk_footprint.rs @@ -48,13 +48,13 @@ impl MatrixReplicaService { read_cache_overhead = read_cache_overhead.saturating_add( (by_chunk.len() as u64).saturating_mul( (std::mem::size_of::() - + std::mem::size_of::>()) as u64, + + std::mem::size_of::>()) as u64, ), ); for ranges in by_chunk.values() { read_cache_overhead = read_cache_overhead.saturating_add( (ranges.len() as u64) - .saturating_mul(std::mem::size_of::() as u64), + .saturating_mul(std::mem::size_of::() as u64), ); } let lru = self.read_cache_lru.borrow(); diff --git a/src/services/chunk/protocol_receipts/admission_cache_batch/cache_control.rs b/src/services/chunk/protocol_receipts/admission_cache_batch/cache_control.rs index 2d1e4b9a..e4f3b680 100644 --- a/src/services/chunk/protocol_receipts/admission_cache_batch/cache_control.rs +++ b/src/services/chunk/protocol_receipts/admission_cache_batch/cache_control.rs @@ -3,6 +3,18 @@ pub(crate) type ReplicaReadCacheKey = (PageId, u64, u64); pub(crate) type ReplicaReadCacheRangeKey = (u64, u64); +/// A cached range as the per-page index holds it: `(end, offset)`. +/// +/// The index is ordered by END, so "the cached ranges a write can touch" starts at the first +/// range that ends past the write's start instead of at the first range there is. A write at +/// or past the end of every range cached for its page -- an append -- then finds none without +/// looking at any, where it used to look at all of them. +pub(crate) type ReplicaReadCacheSpan = (u64, u64); + +pub(crate) fn read_cache_span(offset: u64, length: u64) -> ReplicaReadCacheSpan { + (offset.saturating_add(length), offset) +} + #[derive(Clone, Debug, Eq, PartialEq)] pub(crate) struct ReplicaReadCacheEntry { pub(crate) result: ReplicaReadResult, diff --git a/src/services/chunk/service_state.inc.rs b/src/services/chunk/service_state.inc.rs index c7761206..7562f44a 100644 --- a/src/services/chunk/service_state.inc.rs +++ b/src/services/chunk/service_state.inc.rs @@ -82,7 +82,7 @@ pub struct MatrixReplicaService { /// payload has been fully validated once, so later reads verify only their own /// pages instead of re-hashing the whole chunk. Never persisted. page_validation: RefCell)>>, - read_cache_by_chunk: RefCell>>, + read_cache_by_chunk: RefCell>>, read_cache_lru: RefCell>, read_cache_bytes: Cell, max_read_cache_bytes: Cell,