Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
140 changes: 140 additions & 0 deletions src/object_store_regression_tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3159,6 +3159,146 @@ fn a_page_filled_by_small_appends_reallocates_rarely_and_holds_bounded_slack() {
);
}

/// What a cold build of the validation memo of `page` would be: a hash per validation page.
fn cold_page_validation_hashes(data: &[u8]) -> Vec<u64> {
data.chunks(CHUNK_VALIDATION_PAGE_BYTES)
.map(crate::fnv1a64)
.collect()
}

#[test]
fn an_append_carries_the_page_validation_memo_and_it_equals_a_cold_build() {
// A READ AFTER EVERY APPEND USED TO HASH THE WHOLE PAGE EVERY TIME. The memo is keyed by
// (write_sequence, checksum), an append changes both, and the write also dropped the memo, so
// each read started from nothing: the work per read grew with the page. The memo is carried
// across an append now, and what it carries has to be what a cold build would have made --
// including the page that held the old end, whose hash is stale once the page grows.
//
// Widths straddle the validation page (4,096) on purpose: an append that fills the partial
// page, one that exactly reaches a boundary, and one that spans several pages are the shapes
// a carry gets wrong by one.
let widths: [usize; 14] = [
1, 100, 3995, 1, 4096, 5000, 3, 8189, 4096, 4095, 1, 12288, 7, 4093,
];
let mut chunk = MatrixReplicaService::new();
let page_id = PageId {
chunk_id: ChunkId::new(0, 993).unwrap(),
page_index: 0,
};
let mut expected: Vec<u8> = Vec::new();
for (round, width) in widths.iter().enumerate() {
let bytes: Vec<u8> = (0..*width).map(|i| (round * 37 + i * 11) as u8).collect();
chunk
.write_chunk(page_id.clone(), expected.len() as u64, &bytes)
.unwrap();
expected.extend_from_slice(&bytes);

// The first round has no memo to carry; every later round does, because a read ran
// after the round before. Checked BEFORE any read, so it is the write that left it.
let memo = chunk.page_validation_for_test(&page_id);
if round > 0 {
let (sequence, checksum, hashes) =
memo.unwrap_or_else(|| panic!("round {round}: the append dropped the memo"));
let page = chunk.pages_local().get(&page_id).unwrap();
assert_eq!(sequence, page.write_sequence, "round {round}: stale sequence");
assert_eq!(checksum, page.checksum(), "round {round}: stale checksum");
assert_eq!(
hashes,
cold_page_validation_hashes(&expected),
"round {round}: the carried memo differs from a cold build"
);
}

// A read covering the whole page, which also builds the memo on round 0.
let read = chunk.read_chunk(&page_id, 0, expected.len() as u64).unwrap();
assert_eq!(read.data, expected, "round {round}: the bytes read back differ");
}
}

#[test]
fn a_write_that_is_not_a_pure_append_does_not_carry_the_page_validation_memo() {
let mut chunk = MatrixReplicaService::new();
let page_id = PageId {
chunk_id: ChunkId::new(0, 994).unwrap(),
page_index: 0,
};
let mut expected: Vec<u8> = (0..10_000).map(|i| (i * 7) as u8).collect();
chunk.write_chunk(page_id.clone(), 0, &expected).unwrap();
chunk.read_chunk(&page_id, 0, 10_000).unwrap();
assert!(chunk.page_validation_for_test(&page_id).is_some(), "the read builds a memo");

// OVERWRITE INSIDE THE PAGE: the bytes before the old end changed, so nothing is carried.
chunk.write_chunk(page_id.clone(), 5_000, b"overwritten").unwrap();
expected[5_000..5_011].copy_from_slice(b"overwritten");
assert!(
chunk.page_validation_for_test(&page_id).is_none(),
"an overwrite carried a memo that no longer describes the page"
);
let read = chunk.read_chunk(&page_id, 0, 10_000).unwrap();
assert_eq!(read.data, expected, "the overwritten bytes must read back");
let (_, _, hashes) = chunk.page_validation_for_test(&page_id).unwrap();
assert_eq!(hashes, cold_page_validation_hashes(&expected));

// A WRITE PAST THE END WITH A HOLE: the skipped bytes are zero padding, an append, carried.
chunk.write_chunk(page_id.clone(), 12_000, b"tail").unwrap();
expected.resize(12_000, 0);
expected.extend_from_slice(b"tail");
let (_, _, hashes) = chunk
.page_validation_for_test(&page_id)
.expect("a write past the end is an append and carries the memo");
assert_eq!(hashes, cold_page_validation_hashes(&expected));
assert_eq!(
chunk.read_chunk(&page_id, 0, expected.len() as u64).unwrap().data,
expected
);
}

#[test]
fn a_carried_page_validation_memo_still_catches_a_damaged_prefix() {
// The memo is what a read checks the bytes against, so a carry that kept hashes of bytes that
// were not the bytes would stop catching damage. Damage lands in a page that was full before
// the last append, so its hash is one the carry kept rather than one it recomputed.
let mut chunk = MatrixReplicaService::new();
let page_id = PageId {
chunk_id: ChunkId::new(0, 995).unwrap(),
page_index: 0,
};
let mut expected: Vec<u8> = Vec::new();
for round in 0..8usize {
let bytes: Vec<u8> = (0..2_500).map(|i| (round * 13 + i * 5) as u8).collect();
chunk
.write_chunk(page_id.clone(), expected.len() as u64, &bytes)
.unwrap();
expected.extend_from_slice(&bytes);
chunk.read_chunk(&page_id, 0, expected.len() as u64).unwrap();
}
// One more append, and no read: the memo in place is the carried one.
let last: Vec<u8> = (0..2_500).map(|i| (i * 3 + 1) as u8).collect();
chunk
.write_chunk(page_id.clone(), expected.len() as u64, &last)
.unwrap();
expected.extend_from_slice(&last);
assert!(chunk.page_validation_for_test(&page_id).is_some());

// Damage a byte in the SECOND validation page (offset 4,096..8,192), full before the append.
let damaged_at = 5_000usize;
{
let page = std::sync::Arc::make_mut(chunk.pages_local_mut().get_mut(&page_id).unwrap());
std::sync::Arc::make_mut(&mut page.data)[damaged_at] ^= 0xff;
}
let read = chunk.read_chunk(&page_id, 4_096, 4_096).unwrap();
assert_eq!(
read.data.as_slice(),
&expected[4_096..8_192],
"damaged bytes were served: the carried memo no longer catches damage in a kept page"
);
// A read that does not touch the damaged page is unaffected.
assert_eq!(
chunk.read_chunk(&page_id, 0, 4_096).unwrap().data.as_slice(),
&expected[0..4_096]
);
}

#[test]
fn the_metadata_change_log_is_a_deque_because_it_drops_one_record_per_append() {
// THE LOG IS ALWAYS AT ITS BOUND once it has filled, and every append then drops its oldest
Expand Down
76 changes: 70 additions & 6 deletions src/services/chunk/cache_reclaim_snapshot/read_cache.inc.rs
Original file line number Diff line number Diff line change
Expand Up @@ -176,12 +176,7 @@ impl MatrixReplicaService {
return false;
}
let mut hashes = Vec::with_capacity(chunk.data.len() / page + 1);
let mut from = 0usize;
while from < chunk.data.len() {
let to = (from + page).min(chunk.data.len());
hashes.push(fnv1a64(&chunk.data[from..to]));
from = to;
}
Self::extend_page_hashes(&chunk.data, 0, &mut hashes);
self.page_validation.borrow_mut().insert(
page_id.clone(),
// `checksum()` returns an owned String; the clone was a second allocation.
Expand All @@ -190,6 +185,75 @@ impl MatrixReplicaService {
true
}

/// Hashes of the validation pages of `data` from the page that starts at `from` to the end.
/// `from` is a multiple of the page size, so every hash pushed is of a whole page except
/// possibly the last, which is the partial page at the end of the payload.
fn extend_page_hashes(data: &[u8], from: usize, hashes: &mut Vec<u64>) {
let page = CHUNK_VALIDATION_PAGE_BYTES;
debug_assert_eq!(from % page, 0);
let mut from = from;
while from < data.len() {
let to = (from + page).min(data.len());
hashes.push(fnv1a64(&data[from..to]));
from = to;
}
}

/// Takes the page's validation memo out, so a write that can prove it only appended may put
/// a corrected one back with `carry_page_validation_over_append`.
fn take_page_validation(&self, page_id: &PageId) -> Option<(u64, String, Vec<u64>)> {
self.page_validation.borrow_mut().remove(page_id)
}

/// A WRITE THAT ONLY APPENDED KEEPS THE MEMO IT TOOK, CORRECTED FOR WHAT IT ADDED.
///
/// The memo is keyed by (write_sequence, checksum), and an append changes both, so before this
/// every append made the next read start from nothing: `chunk_pages_valid` hashed the whole
/// payload and then every page of it again. A page filled by appends and read after each one
/// therefore cost a read time proportional to the page, which is the shape a log that is
/// followed while it is written has.
///
/// An append leaves every byte before the old end untouched, so the hash of each page that was
/// FULL at the old end is still the hash of that page. Those are kept. The page that held the
/// old end was partial and its hash is stale, and anything past it is new, so those are hashed
/// from the current bytes. The memo comes out describing the page exactly as a cold build of the
/// new payload would -- a test builds one and compares them -- so the integrity a read checks
/// is the same: a damaged byte fails its page.
///
/// It carries only a memo that described the page as it was just before this write:
/// `prior_sequence` and `prior_checksum` are that page's, captured before the write changed
/// them. A memo that describes anything else is dropped and the next read builds a fresh one.
fn carry_page_validation_over_append(
&self,
page_id: &PageId,
taken: (u64, String, Vec<u64>),
prior_sequence: u64,
prior_checksum: &str,
bytes_before_write: usize,
chunk: &PageState,
) {
let page = CHUNK_VALIDATION_PAGE_BYTES;
let (sequence, checksum, mut hashes) = taken;
if sequence != prior_sequence || checksum != prior_checksum {
return;
}
// The memo must have been built over exactly the bytes that were here.
if hashes.len() != bytes_before_write.div_ceil(page) || chunk.data.len() < bytes_before_write {
return;
}
let kept_pages = bytes_before_write / page;
hashes.truncate(kept_pages);
Self::extend_page_hashes(&chunk.data, kept_pages * page, &mut hashes);
self.page_validation
.borrow_mut()
.insert(page_id.clone(), (chunk.write_sequence, chunk.checksum(), hashes));
}

#[cfg(test)]
pub(crate) fn page_validation_for_test(&self, page_id: &PageId) -> Option<(u64, String, Vec<u64>)> {
self.page_validation.borrow().get(page_id).cloned()
}

fn can_insert_read_cache_payload(&self, bytes: u64) -> bool {
let max_bytes = self.max_read_cache_bytes.get();
let max_cacheable = self.max_cacheable_read_bytes.get();
Expand Down
28 changes: 28 additions & 0 deletions src/services/chunk/chunk_writes.inc.rs
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,10 @@ impl MatrixReplicaService {
// An append leaves the undo a length, so `make_mut` below copies nothing.
self.keep_page_undo_cheap_for_write(&page_id, start_index, end_index);
let accounting_before = self.page_accounting(&page_id);
// TAKEN BEFORE THE WRITE, put back after it if the write only appended: see
// `carry_page_validation_over_append`. Nothing between here and the end can fail, so a
// memo taken here is never lost to an early return.
let validation_before_write = self.take_page_validation(&cache_chunk_id);
// `make_mut` copies this one chunk's state if a snapshot still names
// it, so the copy is per chunk WRITTEN rather than per chunk held.
let chunk = std::sync::Arc::make_mut(
Expand All @@ -67,6 +71,11 @@ impl MatrixReplicaService {
// of the bytes that were already here instead of re-reading them.
let bytes_before_write = chunk.data.len();
let checksum_before_write = chunk.checksum_fnv1a64;
let sequence_before_write = chunk.write_sequence;
// Only built when there is a memo to compare it with: `checksum()` allocates.
let checksum_text_before_write = validation_before_write
.as_ref()
.map(|_| chunk.checksum());
if end_index > chunk.data.len() {
let payload = std::sync::Arc::make_mut(&mut chunk.data);
reserve_page_growth(payload, end_index);
Expand Down Expand Up @@ -108,7 +117,26 @@ impl MatrixReplicaService {
} else {
clear_chunk_ec_sidecars(chunk);
}
// The pure-append test is the one the checksum above uses: the write begins at or past
// the old end, so no byte before the old end changed.
let appended_only = start_index >= bytes_before_write;
self.invalidate_chunk_cache(&cache_chunk_id, offset, data.len() as u64);
if appended_only {
if let (Some(taken), Some(checksum_text)) =
(validation_before_write, checksum_text_before_write)
{
if let Some(page) = self.pages.get(&cache_chunk_id) {
self.carry_page_validation_over_append(
&cache_chunk_id,
taken,
sequence_before_write,
&checksum_text,
bytes_before_write,
page,
);
}
}
}
self.reaccount_page(&cache_chunk_id, accounting_before);
self.physical_write_count = self.physical_write_count.saturating_add(1);
self.physical_write_bytes = self.physical_write_bytes.saturating_add(data.len() as u64);
Expand Down
Loading