diff --git a/src/lib.rs b/src/lib.rs index b9bead43..d1eef42e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -12,7 +12,7 @@ pub mod placement_outcome; use std::cell::{Cell, RefCell}; use std::cmp::Ordering; -use std::collections::{BTreeMap, BTreeSet, BinaryHeap, HashMap, HashSet, VecDeque}; +use std::collections::{BTreeMap, BTreeSet, BinaryHeap, HashMap, VecDeque}; use std::fmt; use std::time::{SystemTime, UNIX_EPOCH}; diff --git a/src/object_store_regression_tests.rs b/src/object_store_regression_tests.rs index 3e9f5808..0d71692e 100644 --- a/src/object_store_regression_tests.rs +++ b/src/object_store_regression_tests.rs @@ -55660,3 +55660,128 @@ fn malformed_manifest_publications_leave_all_metadata_unchanged() { assert!(PlacementOutcome::publish_manifest(&service, replace.manifest.unwrap(), 20).is_err()); assert_eq!(service.export_snapshot(), saved); } + +/// The intersection a cached read range has with a write, stated independently of the code under +/// test: both ranges are non-empty and each starts before the other ends. +fn oracle_ranges_intersect(a_offset: u64, a_length: u64, b_offset: u64, b_length: u64) -> bool { + a_length > 0 && b_length > 0 && a_offset < b_offset + b_length && b_offset < a_offset + a_length +} + +#[test] +fn a_write_invalidates_exactly_the_cached_ranges_it_intersects() { + // THE PER-PAGE INDEX OF CACHED RANGES IS ORDERED BY END, so that a write finds what it can + // touch by starting at the first range that ends past its start. That is an index change, + // and an index change is where an off-by-one decides which cached bytes a write leaves + // standing. So the answer is compared, over many random pages, against the definition of + // intersection written out independently, and the writes are aimed at the places the + // boundary is: a write that starts exactly where a cached range ends, one that ends exactly + // where one starts, one that touches nothing at the tail of the page, and an empty one. + let mut seed: u64 = 0x9E37_79B9_7F4A_7C15; + let mut next = move |bound: u64| { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + seed % bound + }; + const PAGE_BYTES: u64 = 20_000; + let mut shapes = [0usize; 6]; + for _trial in 0..200u32 { + let mut chunk = MatrixReplicaService::new(); + chunk.set_read_cache_max_bytes(8 << 20); + let page_id = PageId { + chunk_id: ChunkId::new(0, 996).unwrap(), + page_index: 0, + }; + let data: Vec = (0..PAGE_BYTES as u32).map(|i| (i * 7) as u8).collect(); + chunk.write_chunk(page_id.clone(), 0, &data).unwrap(); + + let mut ranges: Vec<(u64, u64)> = Vec::new(); + for _ in 0..(3 + next(10)) { + let offset = next(PAGE_BYTES - 1); + let length = 1 + next((PAGE_BYTES - offset).min(2_500)); + if !ranges.contains(&(offset, length)) { + chunk.read_chunk(&page_id, offset, length).unwrap(); + ranges.push((offset, length)); + } + } + // They are all cached: this is what makes "still cached" mean something below. + for (offset, length) in &ranges { + let (_, hit) = chunk + .read_chunk_with_cache_status(&page_id, *offset, *length, ReadCacheIntent::Fill) + .unwrap(); + assert!(hit, "range ({offset}, {length}) was not cached before the write"); + } + + let (cached_offset, cached_length) = ranges[next(ranges.len() as u64) as usize]; + let shape = next(6) as usize; + shapes[shape] += 1; + let (write_offset, write_length) = match shape { + // starts exactly where a cached range ends + 0 => (cached_offset + cached_length, 1 + next(50)), + // ends exactly where a cached range starts + 1 if cached_offset > 0 => { + let length = 1 + next(cached_offset.min(30)); + (cached_offset - length, length) + } + // at or past the end of the page: an append + 2 => (PAGE_BYTES + next(100), 1 + next(100)), + // empty + 3 => (next(PAGE_BYTES), 0), + // one byte inside a cached range + 4 => (cached_offset + next(cached_length), 1), + _ => (next(PAGE_BYTES), 1 + next(4_000)), + }; + chunk.invalidate_chunk_cache(&page_id, write_offset, write_length); + + for (offset, length) in &ranges { + let (_, hit) = chunk + .read_chunk_with_cache_status(&page_id, *offset, *length, ReadCacheIntent::Fill) + .unwrap(); + let should_survive = + !oracle_ranges_intersect(write_offset, write_length, *offset, *length); + assert_eq!( + hit, should_survive, + "write ({write_offset}, {write_length}) against cached ({offset}, {length}): \ + cached afterwards = {hit}, expected {should_survive}" + ); + } + } + // The aim at boundaries has to have happened, or the comparison above proves little. + assert!( + shapes.iter().all(|count| *count > 5), + "a write shape was never exercised: {shapes:?}" + ); +} + +#[test] +fn an_append_leaves_every_cached_range_of_the_page_alone() { + // The case the index is ordered for. Every range cached for a page ends at or before the + // page's length, an append starts there, so none can intersect and none may be removed. + let mut chunk = MatrixReplicaService::new(); + chunk.set_read_cache_max_bytes(8 << 20); + let page_id = PageId { + chunk_id: ChunkId::new(0, 997).unwrap(), + page_index: 0, + }; + let mut length = 0u64; + let mut cached: Vec<(u64, u64)> = Vec::new(); + for round in 0..40u64 { + let bytes: Vec = (0..300).map(|i| (round * 3 + i) as u8).collect(); + chunk.write_chunk(page_id.clone(), length, &bytes).unwrap(); + let stats = chunk.invalidate_chunk_cache(&page_id, length, bytes.len() as u64); + assert_eq!(stats.invalidated_entries, 0, "round {round}: an append invalidated a range"); + chunk.read_chunk(&page_id, length, 300).unwrap(); + cached.push((length, 300)); + length += 300; + } + // None of them was dropped by the appends. They are STALE -- a hit also needs the page's + // sequence and checksum to be the ones the entry was filled under, and every append changed + // both -- so asking for them back would miss; what the appends must not have done is lose + // them from the index, which is what invalidating the whole span finds out. + let stats = chunk.invalidate_chunk_cache(&page_id, 0, length); + assert_eq!( + stats.invalidated_entries as usize, + cached.len(), + "the index lost cached ranges that no write intersected" + ); +} diff --git a/src/services/chunk/cache_reclaim_snapshot/read_cache.inc.rs b/src/services/chunk/cache_reclaim_snapshot/read_cache.inc.rs index 9ad0ec07..990fbd2b 100644 --- a/src/services/chunk/cache_reclaim_snapshot/read_cache.inc.rs +++ b/src/services/chunk/cache_reclaim_snapshot/read_cache.inc.rs @@ -80,7 +80,7 @@ impl MatrixReplicaService { .borrow_mut() .entry(key.0.clone()) .or_default() - .insert((key.1, key.2)); + .insert(read_cache_span(key.1, key.2)); let mut cache = self.read_cache.borrow_mut(); if let Some(previous) = cache.insert( key, @@ -268,7 +268,7 @@ impl MatrixReplicaService { if let Some(entry) = &removed { let mut by_chunk = self.read_cache_by_chunk.borrow_mut(); if let Some(keys) = by_chunk.get_mut(&key.0) { - keys.remove(&(key.1, key.2)); + keys.remove(&read_cache_span(key.1, key.2)); if keys.is_empty() { by_chunk.remove(&key.0); } @@ -308,7 +308,7 @@ impl MatrixReplicaService { if let Some(removed) = cache.remove(&key) { let mut by_chunk = self.read_cache_by_chunk.borrow_mut(); if let Some(keys) = by_chunk.get_mut(&key.0) { - keys.remove(&(key.1, key.2)); + keys.remove(&read_cache_span(key.1, key.2)); if keys.is_empty() { by_chunk.remove(&key.0); } diff --git a/src/services/chunk/chunk_cache_control.inc.rs b/src/services/chunk/chunk_cache_control.inc.rs index e27ce5ea..49e7442f 100644 --- a/src/services/chunk/chunk_cache_control.inc.rs +++ b/src/services/chunk/chunk_cache_control.inc.rs @@ -115,6 +115,29 @@ impl MatrixReplicaService { self.invalidate_chunk_cache_bounded(page_id, offset, length, 0) } + /// The cached ranges of `page_id` that intersect `[offset, offset + length)`, as + /// `(offset, length)`. Exactly the ranges `ranges_intersect` accepts, found by walking the + /// per-page index from the first range that ends past `offset` rather than from its start. + fn cached_ranges_a_write_can_touch( + &self, + page_id: &PageId, + offset: u64, + length: u64, + ) -> Vec { + let Some(write_end) = offset.checked_add(length).filter(|_| length > 0) else { + return Vec::new(); + }; + let by_chunk = self.read_cache_by_chunk.borrow(); + let Some(spans) = by_chunk.get(page_id) else { + return Vec::new(); + }; + spans + .range((offset.saturating_add(1), 0)..) + .filter(|(end, cached_offset)| *cached_offset < write_end && *end > *cached_offset) + .map(|(end, cached_offset)| (*cached_offset, end - cached_offset)) + .collect() + } + pub fn invalidate_chunk_cache_bounded( &self, page_id: &PageId, @@ -130,17 +153,7 @@ impl MatrixReplicaService { self.read_cache_invalidations .set(self.read_cache_invalidations.get().saturating_add(1)); if max_bytes == 0 { - let keys: Vec = self - .read_cache_by_chunk - .borrow() - .get(page_id) - .into_iter() - .flat_map(|keys| keys.iter()) - .filter(|(cached_offset, cached_length)| { - ranges_intersect(offset, length, *cached_offset, *cached_length) - }) - .copied() - .collect(); + let keys = self.cached_ranges_a_write_can_touch(page_id, offset, length); for (cached_offset, cached_length) in keys { let key = (page_id.clone(), cached_offset, cached_length); if let Some(entry) = self.remove_read_cache_entry(&key) { @@ -161,19 +174,13 @@ impl MatrixReplicaService { } let mut keys: Vec<(ReplicaReadCacheRangeKey, u64)> = { let cache = self.read_cache.borrow(); - self.read_cache_by_chunk - .borrow() - .get(page_id) + self.cached_ranges_a_write_can_touch(page_id, offset, length) .into_iter() - .flat_map(|page_keys| page_keys.iter()) .filter_map(|(cached_offset, cached_length)| { - if !ranges_intersect(offset, length, *cached_offset, *cached_length) { - return None; - } - let key = (page_id.clone(), *cached_offset, *cached_length); + let key = (page_id.clone(), cached_offset, cached_length); cache .get(&key) - .map(|entry| ((*cached_offset, *cached_length), entry.result.data.len() as u64)) + .map(|entry| ((cached_offset, cached_length), entry.result.data.len() as u64)) }) .collect() }; diff --git a/src/services/chunk/chunk_footprint.rs b/src/services/chunk/chunk_footprint.rs index 2043dd98..c4a98983 100644 --- a/src/services/chunk/chunk_footprint.rs +++ b/src/services/chunk/chunk_footprint.rs @@ -48,13 +48,13 @@ impl MatrixReplicaService { read_cache_overhead = read_cache_overhead.saturating_add( (by_chunk.len() as u64).saturating_mul( (std::mem::size_of::() - + std::mem::size_of::>()) as u64, + + std::mem::size_of::>()) as u64, ), ); for ranges in by_chunk.values() { read_cache_overhead = read_cache_overhead.saturating_add( (ranges.len() as u64) - .saturating_mul(std::mem::size_of::() as u64), + .saturating_mul(std::mem::size_of::() as u64), ); } let lru = self.read_cache_lru.borrow(); diff --git a/src/services/chunk/protocol_receipts/admission_cache_batch/cache_control.rs b/src/services/chunk/protocol_receipts/admission_cache_batch/cache_control.rs index 2d1e4b9a..e4f3b680 100644 --- a/src/services/chunk/protocol_receipts/admission_cache_batch/cache_control.rs +++ b/src/services/chunk/protocol_receipts/admission_cache_batch/cache_control.rs @@ -3,6 +3,18 @@ pub(crate) type ReplicaReadCacheKey = (PageId, u64, u64); pub(crate) type ReplicaReadCacheRangeKey = (u64, u64); +/// A cached range as the per-page index holds it: `(end, offset)`. +/// +/// The index is ordered by END, so "the cached ranges a write can touch" starts at the first +/// range that ends past the write's start instead of at the first range there is. A write at +/// or past the end of every range cached for its page -- an append -- then finds none without +/// looking at any, where it used to look at all of them. +pub(crate) type ReplicaReadCacheSpan = (u64, u64); + +pub(crate) fn read_cache_span(offset: u64, length: u64) -> ReplicaReadCacheSpan { + (offset.saturating_add(length), offset) +} + #[derive(Clone, Debug, Eq, PartialEq)] pub(crate) struct ReplicaReadCacheEntry { pub(crate) result: ReplicaReadResult, diff --git a/src/services/chunk/service_state.inc.rs b/src/services/chunk/service_state.inc.rs index c7761206..7562f44a 100644 --- a/src/services/chunk/service_state.inc.rs +++ b/src/services/chunk/service_state.inc.rs @@ -82,7 +82,7 @@ pub struct MatrixReplicaService { /// payload has been fully validated once, so later reads verify only their own /// pages instead of re-hashing the whole chunk. Never persisted. page_validation: RefCell)>>, - read_cache_by_chunk: RefCell>>, + read_cache_by_chunk: RefCell>>, read_cache_lru: RefCell>, read_cache_bytes: Cell, max_read_cache_bytes: Cell,