diff --git a/src/object_store_regression_tests.rs b/src/object_store_regression_tests.rs index d8d9e7ee..3e9f5808 100644 --- a/src/object_store_regression_tests.rs +++ b/src/object_store_regression_tests.rs @@ -3159,6 +3159,146 @@ fn a_page_filled_by_small_appends_reallocates_rarely_and_holds_bounded_slack() { ); } +/// What a cold build of the validation memo of `page` would be: a hash per validation page. +fn cold_page_validation_hashes(data: &[u8]) -> Vec { + data.chunks(CHUNK_VALIDATION_PAGE_BYTES) + .map(crate::fnv1a64) + .collect() +} + +#[test] +fn an_append_carries_the_page_validation_memo_and_it_equals_a_cold_build() { + // A READ AFTER EVERY APPEND USED TO HASH THE WHOLE PAGE EVERY TIME. The memo is keyed by + // (write_sequence, checksum), an append changes both, and the write also dropped the memo, so + // each read started from nothing: the work per read grew with the page. The memo is carried + // across an append now, and what it carries has to be what a cold build would have made -- + // including the page that held the old end, whose hash is stale once the page grows. + // + // Widths straddle the validation page (4,096) on purpose: an append that fills the partial + // page, one that exactly reaches a boundary, and one that spans several pages are the shapes + // a carry gets wrong by one. + let widths: [usize; 14] = [ + 1, 100, 3995, 1, 4096, 5000, 3, 8189, 4096, 4095, 1, 12288, 7, 4093, + ]; + let mut chunk = MatrixReplicaService::new(); + let page_id = PageId { + chunk_id: ChunkId::new(0, 993).unwrap(), + page_index: 0, + }; + let mut expected: Vec = Vec::new(); + for (round, width) in widths.iter().enumerate() { + let bytes: Vec = (0..*width).map(|i| (round * 37 + i * 11) as u8).collect(); + chunk + .write_chunk(page_id.clone(), expected.len() as u64, &bytes) + .unwrap(); + expected.extend_from_slice(&bytes); + + // The first round has no memo to carry; every later round does, because a read ran + // after the round before. Checked BEFORE any read, so it is the write that left it. + let memo = chunk.page_validation_for_test(&page_id); + if round > 0 { + let (sequence, checksum, hashes) = + memo.unwrap_or_else(|| panic!("round {round}: the append dropped the memo")); + let page = chunk.pages_local().get(&page_id).unwrap(); + assert_eq!(sequence, page.write_sequence, "round {round}: stale sequence"); + assert_eq!(checksum, page.checksum(), "round {round}: stale checksum"); + assert_eq!( + hashes, + cold_page_validation_hashes(&expected), + "round {round}: the carried memo differs from a cold build" + ); + } + + // A read covering the whole page, which also builds the memo on round 0. + let read = chunk.read_chunk(&page_id, 0, expected.len() as u64).unwrap(); + assert_eq!(read.data, expected, "round {round}: the bytes read back differ"); + } +} + +#[test] +fn a_write_that_is_not_a_pure_append_does_not_carry_the_page_validation_memo() { + let mut chunk = MatrixReplicaService::new(); + let page_id = PageId { + chunk_id: ChunkId::new(0, 994).unwrap(), + page_index: 0, + }; + let mut expected: Vec = (0..10_000).map(|i| (i * 7) as u8).collect(); + chunk.write_chunk(page_id.clone(), 0, &expected).unwrap(); + chunk.read_chunk(&page_id, 0, 10_000).unwrap(); + assert!(chunk.page_validation_for_test(&page_id).is_some(), "the read builds a memo"); + + // OVERWRITE INSIDE THE PAGE: the bytes before the old end changed, so nothing is carried. + chunk.write_chunk(page_id.clone(), 5_000, b"overwritten").unwrap(); + expected[5_000..5_011].copy_from_slice(b"overwritten"); + assert!( + chunk.page_validation_for_test(&page_id).is_none(), + "an overwrite carried a memo that no longer describes the page" + ); + let read = chunk.read_chunk(&page_id, 0, 10_000).unwrap(); + assert_eq!(read.data, expected, "the overwritten bytes must read back"); + let (_, _, hashes) = chunk.page_validation_for_test(&page_id).unwrap(); + assert_eq!(hashes, cold_page_validation_hashes(&expected)); + + // A WRITE PAST THE END WITH A HOLE: the skipped bytes are zero padding, an append, carried. + chunk.write_chunk(page_id.clone(), 12_000, b"tail").unwrap(); + expected.resize(12_000, 0); + expected.extend_from_slice(b"tail"); + let (_, _, hashes) = chunk + .page_validation_for_test(&page_id) + .expect("a write past the end is an append and carries the memo"); + assert_eq!(hashes, cold_page_validation_hashes(&expected)); + assert_eq!( + chunk.read_chunk(&page_id, 0, expected.len() as u64).unwrap().data, + expected + ); +} + +#[test] +fn a_carried_page_validation_memo_still_catches_a_damaged_prefix() { + // The memo is what a read checks the bytes against, so a carry that kept hashes of bytes that + // were not the bytes would stop catching damage. Damage lands in a page that was full before + // the last append, so its hash is one the carry kept rather than one it recomputed. + let mut chunk = MatrixReplicaService::new(); + let page_id = PageId { + chunk_id: ChunkId::new(0, 995).unwrap(), + page_index: 0, + }; + let mut expected: Vec = Vec::new(); + for round in 0..8usize { + let bytes: Vec = (0..2_500).map(|i| (round * 13 + i * 5) as u8).collect(); + chunk + .write_chunk(page_id.clone(), expected.len() as u64, &bytes) + .unwrap(); + expected.extend_from_slice(&bytes); + chunk.read_chunk(&page_id, 0, expected.len() as u64).unwrap(); + } + // One more append, and no read: the memo in place is the carried one. + let last: Vec = (0..2_500).map(|i| (i * 3 + 1) as u8).collect(); + chunk + .write_chunk(page_id.clone(), expected.len() as u64, &last) + .unwrap(); + expected.extend_from_slice(&last); + assert!(chunk.page_validation_for_test(&page_id).is_some()); + + // Damage a byte in the SECOND validation page (offset 4,096..8,192), full before the append. + let damaged_at = 5_000usize; + { + let page = std::sync::Arc::make_mut(chunk.pages_local_mut().get_mut(&page_id).unwrap()); + std::sync::Arc::make_mut(&mut page.data)[damaged_at] ^= 0xff; + } + let read = chunk.read_chunk(&page_id, 4_096, 4_096).unwrap(); + assert_eq!( + read.data.as_slice(), + &expected[4_096..8_192], + "damaged bytes were served: the carried memo no longer catches damage in a kept page" + ); + // A read that does not touch the damaged page is unaffected. + assert_eq!( + chunk.read_chunk(&page_id, 0, 4_096).unwrap().data.as_slice(), + &expected[0..4_096] + ); +} + #[test] fn the_metadata_change_log_is_a_deque_because_it_drops_one_record_per_append() { // THE LOG IS ALWAYS AT ITS BOUND once it has filled, and every append then drops its oldest diff --git a/src/services/chunk/cache_reclaim_snapshot/read_cache.inc.rs b/src/services/chunk/cache_reclaim_snapshot/read_cache.inc.rs index 4648345a..9ad0ec07 100644 --- a/src/services/chunk/cache_reclaim_snapshot/read_cache.inc.rs +++ b/src/services/chunk/cache_reclaim_snapshot/read_cache.inc.rs @@ -176,12 +176,7 @@ impl MatrixReplicaService { return false; } let mut hashes = Vec::with_capacity(chunk.data.len() / page + 1); - let mut from = 0usize; - while from < chunk.data.len() { - let to = (from + page).min(chunk.data.len()); - hashes.push(fnv1a64(&chunk.data[from..to])); - from = to; - } + Self::extend_page_hashes(&chunk.data, 0, &mut hashes); self.page_validation.borrow_mut().insert( page_id.clone(), // `checksum()` returns an owned String; the clone was a second allocation. @@ -190,6 +185,75 @@ impl MatrixReplicaService { true } + /// Hashes of the validation pages of `data` from the page that starts at `from` to the end. + /// `from` is a multiple of the page size, so every hash pushed is of a whole page except + /// possibly the last, which is the partial page at the end of the payload. + fn extend_page_hashes(data: &[u8], from: usize, hashes: &mut Vec) { + let page = CHUNK_VALIDATION_PAGE_BYTES; + debug_assert_eq!(from % page, 0); + let mut from = from; + while from < data.len() { + let to = (from + page).min(data.len()); + hashes.push(fnv1a64(&data[from..to])); + from = to; + } + } + + /// Takes the page's validation memo out, so a write that can prove it only appended may put + /// a corrected one back with `carry_page_validation_over_append`. + fn take_page_validation(&self, page_id: &PageId) -> Option<(u64, String, Vec)> { + self.page_validation.borrow_mut().remove(page_id) + } + + /// A WRITE THAT ONLY APPENDED KEEPS THE MEMO IT TOOK, CORRECTED FOR WHAT IT ADDED. + /// + /// The memo is keyed by (write_sequence, checksum), and an append changes both, so before this + /// every append made the next read start from nothing: `chunk_pages_valid` hashed the whole + /// payload and then every page of it again. A page filled by appends and read after each one + /// therefore cost a read time proportional to the page, which is the shape a log that is + /// followed while it is written has. + /// + /// An append leaves every byte before the old end untouched, so the hash of each page that was + /// FULL at the old end is still the hash of that page. Those are kept. The page that held the + /// old end was partial and its hash is stale, and anything past it is new, so those are hashed + /// from the current bytes. The memo comes out describing the page exactly as a cold build of the + /// new payload would -- a test builds one and compares them -- so the integrity a read checks + /// is the same: a damaged byte fails its page. + /// + /// It carries only a memo that described the page as it was just before this write: + /// `prior_sequence` and `prior_checksum` are that page's, captured before the write changed + /// them. A memo that describes anything else is dropped and the next read builds a fresh one. + fn carry_page_validation_over_append( + &self, + page_id: &PageId, + taken: (u64, String, Vec), + prior_sequence: u64, + prior_checksum: &str, + bytes_before_write: usize, + chunk: &PageState, + ) { + let page = CHUNK_VALIDATION_PAGE_BYTES; + let (sequence, checksum, mut hashes) = taken; + if sequence != prior_sequence || checksum != prior_checksum { + return; + } + // The memo must have been built over exactly the bytes that were here. + if hashes.len() != bytes_before_write.div_ceil(page) || chunk.data.len() < bytes_before_write { + return; + } + let kept_pages = bytes_before_write / page; + hashes.truncate(kept_pages); + Self::extend_page_hashes(&chunk.data, kept_pages * page, &mut hashes); + self.page_validation + .borrow_mut() + .insert(page_id.clone(), (chunk.write_sequence, chunk.checksum(), hashes)); + } + + #[cfg(test)] + pub(crate) fn page_validation_for_test(&self, page_id: &PageId) -> Option<(u64, String, Vec)> { + self.page_validation.borrow().get(page_id).cloned() + } + fn can_insert_read_cache_payload(&self, bytes: u64) -> bool { let max_bytes = self.max_read_cache_bytes.get(); let max_cacheable = self.max_cacheable_read_bytes.get(); diff --git a/src/services/chunk/chunk_writes.inc.rs b/src/services/chunk/chunk_writes.inc.rs index 9011a23f..cef177ca 100644 --- a/src/services/chunk/chunk_writes.inc.rs +++ b/src/services/chunk/chunk_writes.inc.rs @@ -56,6 +56,10 @@ impl MatrixReplicaService { // An append leaves the undo a length, so `make_mut` below copies nothing. self.keep_page_undo_cheap_for_write(&page_id, start_index, end_index); let accounting_before = self.page_accounting(&page_id); + // TAKEN BEFORE THE WRITE, put back after it if the write only appended: see + // `carry_page_validation_over_append`. Nothing between here and the end can fail, so a + // memo taken here is never lost to an early return. + let validation_before_write = self.take_page_validation(&cache_chunk_id); // `make_mut` copies this one chunk's state if a snapshot still names // it, so the copy is per chunk WRITTEN rather than per chunk held. let chunk = std::sync::Arc::make_mut( @@ -67,6 +71,11 @@ impl MatrixReplicaService { // of the bytes that were already here instead of re-reading them. let bytes_before_write = chunk.data.len(); let checksum_before_write = chunk.checksum_fnv1a64; + let sequence_before_write = chunk.write_sequence; + // Only built when there is a memo to compare it with: `checksum()` allocates. + let checksum_text_before_write = validation_before_write + .as_ref() + .map(|_| chunk.checksum()); if end_index > chunk.data.len() { let payload = std::sync::Arc::make_mut(&mut chunk.data); reserve_page_growth(payload, end_index); @@ -108,7 +117,26 @@ impl MatrixReplicaService { } else { clear_chunk_ec_sidecars(chunk); } + // The pure-append test is the one the checksum above uses: the write begins at or past + // the old end, so no byte before the old end changed. + let appended_only = start_index >= bytes_before_write; self.invalidate_chunk_cache(&cache_chunk_id, offset, data.len() as u64); + if appended_only { + if let (Some(taken), Some(checksum_text)) = + (validation_before_write, checksum_text_before_write) + { + if let Some(page) = self.pages.get(&cache_chunk_id) { + self.carry_page_validation_over_append( + &cache_chunk_id, + taken, + sequence_before_write, + &checksum_text, + bytes_before_write, + page, + ); + } + } + } self.reaccount_page(&cache_chunk_id, accounting_before); self.physical_write_count = self.physical_write_count.saturating_add(1); self.physical_write_bytes = self.physical_write_bytes.saturating_add(data.len() as u64);