Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ pub mod placement_outcome;

use std::cell::{Cell, RefCell};
use std::cmp::Ordering;
use std::collections::{BTreeMap, BTreeSet, BinaryHeap, HashMap, HashSet, VecDeque};
use std::collections::{BTreeMap, BTreeSet, BinaryHeap, HashMap, VecDeque};
use std::fmt;
use std::time::{SystemTime, UNIX_EPOCH};

Expand Down
125 changes: 125 additions & 0 deletions src/object_store_regression_tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -55660,3 +55660,128 @@ fn malformed_manifest_publications_leave_all_metadata_unchanged() {
assert!(PlacementOutcome::publish_manifest(&service, replace.manifest.unwrap(), 20).is_err());
assert_eq!(service.export_snapshot(), saved);
}

/// The intersection a cached read range has with a write, stated independently of the code under
/// test: both ranges are non-empty and each starts before the other ends.
fn oracle_ranges_intersect(a_offset: u64, a_length: u64, b_offset: u64, b_length: u64) -> bool {
a_length > 0 && b_length > 0 && a_offset < b_offset + b_length && b_offset < a_offset + a_length
}

#[test]
fn a_write_invalidates_exactly_the_cached_ranges_it_intersects() {
// THE PER-PAGE INDEX OF CACHED RANGES IS ORDERED BY END, so that a write finds what it can
// touch by starting at the first range that ends past its start. That is an index change,
// and an index change is where an off-by-one decides which cached bytes a write leaves
// standing. So the answer is compared, over many random pages, against the definition of
// intersection written out independently, and the writes are aimed at the places the
// boundary is: a write that starts exactly where a cached range ends, one that ends exactly
// where one starts, one that touches nothing at the tail of the page, and an empty one.
let mut seed: u64 = 0x9E37_79B9_7F4A_7C15;
let mut next = move |bound: u64| {
seed ^= seed << 13;
seed ^= seed >> 7;
seed ^= seed << 17;
seed % bound
};
const PAGE_BYTES: u64 = 20_000;
let mut shapes = [0usize; 6];
for _trial in 0..200u32 {
let mut chunk = MatrixReplicaService::new();
chunk.set_read_cache_max_bytes(8 << 20);
let page_id = PageId {
chunk_id: ChunkId::new(0, 996).unwrap(),
page_index: 0,
};
let data: Vec<u8> = (0..PAGE_BYTES as u32).map(|i| (i * 7) as u8).collect();
chunk.write_chunk(page_id.clone(), 0, &data).unwrap();

let mut ranges: Vec<(u64, u64)> = Vec::new();
for _ in 0..(3 + next(10)) {
let offset = next(PAGE_BYTES - 1);
let length = 1 + next((PAGE_BYTES - offset).min(2_500));
if !ranges.contains(&(offset, length)) {
chunk.read_chunk(&page_id, offset, length).unwrap();
ranges.push((offset, length));
}
}
// They are all cached: this is what makes "still cached" mean something below.
for (offset, length) in &ranges {
let (_, hit) = chunk
.read_chunk_with_cache_status(&page_id, *offset, *length, ReadCacheIntent::Fill)
.unwrap();
assert!(hit, "range ({offset}, {length}) was not cached before the write");
}

let (cached_offset, cached_length) = ranges[next(ranges.len() as u64) as usize];
let shape = next(6) as usize;
shapes[shape] += 1;
let (write_offset, write_length) = match shape {
// starts exactly where a cached range ends
0 => (cached_offset + cached_length, 1 + next(50)),
// ends exactly where a cached range starts
1 if cached_offset > 0 => {
let length = 1 + next(cached_offset.min(30));
(cached_offset - length, length)
}
// at or past the end of the page: an append
2 => (PAGE_BYTES + next(100), 1 + next(100)),
// empty
3 => (next(PAGE_BYTES), 0),
// one byte inside a cached range
4 => (cached_offset + next(cached_length), 1),
_ => (next(PAGE_BYTES), 1 + next(4_000)),
};
chunk.invalidate_chunk_cache(&page_id, write_offset, write_length);

for (offset, length) in &ranges {
let (_, hit) = chunk
.read_chunk_with_cache_status(&page_id, *offset, *length, ReadCacheIntent::Fill)
.unwrap();
let should_survive =
!oracle_ranges_intersect(write_offset, write_length, *offset, *length);
assert_eq!(
hit, should_survive,
"write ({write_offset}, {write_length}) against cached ({offset}, {length}): \
cached afterwards = {hit}, expected {should_survive}"
);
}
}
// The aim at boundaries has to have happened, or the comparison above proves little.
assert!(
shapes.iter().all(|count| *count > 5),
"a write shape was never exercised: {shapes:?}"
);
}

#[test]
fn an_append_leaves_every_cached_range_of_the_page_alone() {
// The case the index is ordered for. Every range cached for a page ends at or before the
// page's length, an append starts there, so none can intersect and none may be removed.
let mut chunk = MatrixReplicaService::new();
chunk.set_read_cache_max_bytes(8 << 20);
let page_id = PageId {
chunk_id: ChunkId::new(0, 997).unwrap(),
page_index: 0,
};
let mut length = 0u64;
let mut cached: Vec<(u64, u64)> = Vec::new();
for round in 0..40u64 {
let bytes: Vec<u8> = (0..300).map(|i| (round * 3 + i) as u8).collect();
chunk.write_chunk(page_id.clone(), length, &bytes).unwrap();
let stats = chunk.invalidate_chunk_cache(&page_id, length, bytes.len() as u64);
assert_eq!(stats.invalidated_entries, 0, "round {round}: an append invalidated a range");
chunk.read_chunk(&page_id, length, 300).unwrap();
cached.push((length, 300));
length += 300;
}
// None of them was dropped by the appends. They are STALE -- a hit also needs the page's
// sequence and checksum to be the ones the entry was filled under, and every append changed
// both -- so asking for them back would miss; what the appends must not have done is lose
// them from the index, which is what invalidating the whole span finds out.
let stats = chunk.invalidate_chunk_cache(&page_id, 0, length);
assert_eq!(
stats.invalidated_entries as usize,
cached.len(),
"the index lost cached ranges that no write intersected"
);
}
6 changes: 3 additions & 3 deletions src/services/chunk/cache_reclaim_snapshot/read_cache.inc.rs
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ impl MatrixReplicaService {
.borrow_mut()
.entry(key.0.clone())
.or_default()
.insert((key.1, key.2));
.insert(read_cache_span(key.1, key.2));
let mut cache = self.read_cache.borrow_mut();
if let Some(previous) = cache.insert(
key,
Expand Down Expand Up @@ -268,7 +268,7 @@ impl MatrixReplicaService {
if let Some(entry) = &removed {
let mut by_chunk = self.read_cache_by_chunk.borrow_mut();
if let Some(keys) = by_chunk.get_mut(&key.0) {
keys.remove(&(key.1, key.2));
keys.remove(&read_cache_span(key.1, key.2));
if keys.is_empty() {
by_chunk.remove(&key.0);
}
Expand Down Expand Up @@ -308,7 +308,7 @@ impl MatrixReplicaService {
if let Some(removed) = cache.remove(&key) {
let mut by_chunk = self.read_cache_by_chunk.borrow_mut();
if let Some(keys) = by_chunk.get_mut(&key.0) {
keys.remove(&(key.1, key.2));
keys.remove(&read_cache_span(key.1, key.2));
if keys.is_empty() {
by_chunk.remove(&key.0);
}
Expand Down
47 changes: 27 additions & 20 deletions src/services/chunk/chunk_cache_control.inc.rs
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,29 @@ impl MatrixReplicaService {
self.invalidate_chunk_cache_bounded(page_id, offset, length, 0)
}

/// The cached ranges of `page_id` that intersect `[offset, offset + length)`, as
/// `(offset, length)`. Exactly the ranges `ranges_intersect` accepts, found by walking the
/// per-page index from the first range that ends past `offset` rather than from its start.
fn cached_ranges_a_write_can_touch(
&self,
page_id: &PageId,
offset: u64,
length: u64,
) -> Vec<ReplicaReadCacheRangeKey> {
let Some(write_end) = offset.checked_add(length).filter(|_| length > 0) else {
return Vec::new();
};
let by_chunk = self.read_cache_by_chunk.borrow();
let Some(spans) = by_chunk.get(page_id) else {
return Vec::new();
};
spans
.range((offset.saturating_add(1), 0)..)
.filter(|(end, cached_offset)| *cached_offset < write_end && *end > *cached_offset)
.map(|(end, cached_offset)| (*cached_offset, end - cached_offset))
.collect()
}

pub fn invalidate_chunk_cache_bounded(
&self,
page_id: &PageId,
Expand All @@ -130,17 +153,7 @@ impl MatrixReplicaService {
self.read_cache_invalidations
.set(self.read_cache_invalidations.get().saturating_add(1));
if max_bytes == 0 {
let keys: Vec<ReplicaReadCacheRangeKey> = self
.read_cache_by_chunk
.borrow()
.get(page_id)
.into_iter()
.flat_map(|keys| keys.iter())
.filter(|(cached_offset, cached_length)| {
ranges_intersect(offset, length, *cached_offset, *cached_length)
})
.copied()
.collect();
let keys = self.cached_ranges_a_write_can_touch(page_id, offset, length);
for (cached_offset, cached_length) in keys {
let key = (page_id.clone(), cached_offset, cached_length);
if let Some(entry) = self.remove_read_cache_entry(&key) {
Expand All @@ -161,19 +174,13 @@ impl MatrixReplicaService {
}
let mut keys: Vec<(ReplicaReadCacheRangeKey, u64)> = {
let cache = self.read_cache.borrow();
self.read_cache_by_chunk
.borrow()
.get(page_id)
self.cached_ranges_a_write_can_touch(page_id, offset, length)
.into_iter()
.flat_map(|page_keys| page_keys.iter())
.filter_map(|(cached_offset, cached_length)| {
if !ranges_intersect(offset, length, *cached_offset, *cached_length) {
return None;
}
let key = (page_id.clone(), *cached_offset, *cached_length);
let key = (page_id.clone(), cached_offset, cached_length);
cache
.get(&key)
.map(|entry| ((*cached_offset, *cached_length), entry.result.data.len() as u64))
.map(|entry| ((cached_offset, cached_length), entry.result.data.len() as u64))
})
.collect()
};
Expand Down
4 changes: 2 additions & 2 deletions src/services/chunk/chunk_footprint.rs
Original file line number Diff line number Diff line change
Expand Up @@ -48,13 +48,13 @@ impl MatrixReplicaService {
read_cache_overhead = read_cache_overhead.saturating_add(
(by_chunk.len() as u64).saturating_mul(
(std::mem::size_of::<PageId>()
+ std::mem::size_of::<HashSet<ReplicaReadCacheRangeKey>>()) as u64,
+ std::mem::size_of::<std::collections::BTreeSet<ReplicaReadCacheSpan>>()) as u64,
),
);
for ranges in by_chunk.values() {
read_cache_overhead = read_cache_overhead.saturating_add(
(ranges.len() as u64)
.saturating_mul(std::mem::size_of::<ReplicaReadCacheRangeKey>() as u64),
.saturating_mul(std::mem::size_of::<ReplicaReadCacheSpan>() as u64),
);
}
let lru = self.read_cache_lru.borrow();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,18 @@
pub(crate) type ReplicaReadCacheKey = (PageId, u64, u64);
pub(crate) type ReplicaReadCacheRangeKey = (u64, u64);

/// A cached range as the per-page index holds it: `(end, offset)`.
///
/// The index is ordered by END, so "the cached ranges a write can touch" starts at the first
/// range that ends past the write's start instead of at the first range there is. A write at
/// or past the end of every range cached for its page -- an append -- then finds none without
/// looking at any, where it used to look at all of them.
pub(crate) type ReplicaReadCacheSpan = (u64, u64);

pub(crate) fn read_cache_span(offset: u64, length: u64) -> ReplicaReadCacheSpan {
(offset.saturating_add(length), offset)
}

#[derive(Clone, Debug, Eq, PartialEq)]
pub(crate) struct ReplicaReadCacheEntry {
pub(crate) result: ReplicaReadResult,
Expand Down
2 changes: 1 addition & 1 deletion src/services/chunk/service_state.inc.rs
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ pub struct MatrixReplicaService {
/// payload has been fully validated once, so later reads verify only their own
/// pages instead of re-hashing the whole chunk. Never persisted.
page_validation: RefCell<HashMap<PageId, (u64, String, Vec<u64>)>>,
read_cache_by_chunk: RefCell<HashMap<PageId, HashSet<ReplicaReadCacheRangeKey>>>,
read_cache_by_chunk: RefCell<HashMap<PageId, std::collections::BTreeSet<ReplicaReadCacheSpan>>>,
read_cache_lru: RefCell<VecDeque<(u64, ReplicaReadCacheKey)>>,
read_cache_bytes: Cell<u64>,
max_read_cache_bytes: Cell<u64>,
Expand Down
Loading