From 6354184ff0b91c0e5fce7f3f0a8a3fced059280c Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 13 Nov 2026 18:04:25 -0800 Subject: [PATCH] An append keeps the page's validation memo, so a read after it stops hashing the page A read of a page checks its bytes against a memo of one hash per 4 KiB page. The memo is keyed by (write_sequence, checksum), an append changes both, and the write also dropped the memo outright, so the first read after every append started from nothing: it hashed the whole payload and then every page of it again. For a page filled by appends and read after each one -- a log that is followed while it is written -- the cost of a read grew with the page. An append leaves every byte before the old end untouched, so the hash of each page that was full at the old end is still that page's hash. The write now takes the memo out before it, and puts back one that keeps those hashes and recomputes only the page that held the old end and anything past it. A write that is not a pure append still drops the memo, as before, and a memo that described anything other than the page just before the write is not carried. Measured on the door under callgrind, driven by the shared-store harness: the per-write cost of the read path grew 2.5x between 300 and 1,200 writes, and its own cost inside the replica read was 81% of the whole growth. With the cold validation skipped (a measuring plant, never shippable) a 1,200-write run executed 40.5% fewer instructions. Tests: the carried memo equals a cold build after every append of widths that straddle the 4 KiB boundary; an overwrite drops it and a write past the end with a hole carries it; and a byte damaged in a page whose hash was carried is still not served. Co-Authored-By: Claude Sonnet 5.5 --- src/object_store_regression_tests.rs | 140 ++++++++++++++++++ .../cache_reclaim_snapshot/read_cache.inc.rs | 76 +++++++++- src/services/chunk/chunk_writes.inc.rs | 28 ++++ 3 files changed, 238 insertions(+), 6 deletions(-) diff --git a/src/object_store_regression_tests.rs b/src/object_store_regression_tests.rs index d8d9e7ee..3e9f5808 100644 --- a/src/object_store_regression_tests.rs +++ b/src/object_store_regression_tests.rs @@ -3159,6 +3159,146 @@ fn a_page_filled_by_small_appends_reallocates_rarely_and_holds_bounded_slack() { ); } +/// What a cold build of the validation memo of `page` would be: a hash per validation page. +fn cold_page_validation_hashes(data: &[u8]) -> Vec { + data.chunks(CHUNK_VALIDATION_PAGE_BYTES) + .map(crate::fnv1a64) + .collect() +} + +#[test] +fn an_append_carries_the_page_validation_memo_and_it_equals_a_cold_build() { + // A READ AFTER EVERY APPEND USED TO HASH THE WHOLE PAGE EVERY TIME. The memo is keyed by + // (write_sequence, checksum), an append changes both, and the write also dropped the memo, so + // each read started from nothing: the work per read grew with the page. The memo is carried + // across an append now, and what it carries has to be what a cold build would have made -- + // including the page that held the old end, whose hash is stale once the page grows. + // + // Widths straddle the validation page (4,096) on purpose: an append that fills the partial + // page, one that exactly reaches a boundary, and one that spans several pages are the shapes + // a carry gets wrong by one. + let widths: [usize; 14] = [ + 1, 100, 3995, 1, 4096, 5000, 3, 8189, 4096, 4095, 1, 12288, 7, 4093, + ]; + let mut chunk = MatrixReplicaService::new(); + let page_id = PageId { + chunk_id: ChunkId::new(0, 993).unwrap(), + page_index: 0, + }; + let mut expected: Vec = Vec::new(); + for (round, width) in widths.iter().enumerate() { + let bytes: Vec = (0..*width).map(|i| (round * 37 + i * 11) as u8).collect(); + chunk + .write_chunk(page_id.clone(), expected.len() as u64, &bytes) + .unwrap(); + expected.extend_from_slice(&bytes); + + // The first round has no memo to carry; every later round does, because a read ran + // after the round before. Checked BEFORE any read, so it is the write that left it. + let memo = chunk.page_validation_for_test(&page_id); + if round > 0 { + let (sequence, checksum, hashes) = + memo.unwrap_or_else(|| panic!("round {round}: the append dropped the memo")); + let page = chunk.pages_local().get(&page_id).unwrap(); + assert_eq!(sequence, page.write_sequence, "round {round}: stale sequence"); + assert_eq!(checksum, page.checksum(), "round {round}: stale checksum"); + assert_eq!( + hashes, + cold_page_validation_hashes(&expected), + "round {round}: the carried memo differs from a cold build" + ); + } + + // A read covering the whole page, which also builds the memo on round 0. + let read = chunk.read_chunk(&page_id, 0, expected.len() as u64).unwrap(); + assert_eq!(read.data, expected, "round {round}: the bytes read back differ"); + } +} + +#[test] +fn a_write_that_is_not_a_pure_append_does_not_carry_the_page_validation_memo() { + let mut chunk = MatrixReplicaService::new(); + let page_id = PageId { + chunk_id: ChunkId::new(0, 994).unwrap(), + page_index: 0, + }; + let mut expected: Vec = (0..10_000).map(|i| (i * 7) as u8).collect(); + chunk.write_chunk(page_id.clone(), 0, &expected).unwrap(); + chunk.read_chunk(&page_id, 0, 10_000).unwrap(); + assert!(chunk.page_validation_for_test(&page_id).is_some(), "the read builds a memo"); + + // OVERWRITE INSIDE THE PAGE: the bytes before the old end changed, so nothing is carried. + chunk.write_chunk(page_id.clone(), 5_000, b"overwritten").unwrap(); + expected[5_000..5_011].copy_from_slice(b"overwritten"); + assert!( + chunk.page_validation_for_test(&page_id).is_none(), + "an overwrite carried a memo that no longer describes the page" + ); + let read = chunk.read_chunk(&page_id, 0, 10_000).unwrap(); + assert_eq!(read.data, expected, "the overwritten bytes must read back"); + let (_, _, hashes) = chunk.page_validation_for_test(&page_id).unwrap(); + assert_eq!(hashes, cold_page_validation_hashes(&expected)); + + // A WRITE PAST THE END WITH A HOLE: the skipped bytes are zero padding, an append, carried. + chunk.write_chunk(page_id.clone(), 12_000, b"tail").unwrap(); + expected.resize(12_000, 0); + expected.extend_from_slice(b"tail"); + let (_, _, hashes) = chunk + .page_validation_for_test(&page_id) + .expect("a write past the end is an append and carries the memo"); + assert_eq!(hashes, cold_page_validation_hashes(&expected)); + assert_eq!( + chunk.read_chunk(&page_id, 0, expected.len() as u64).unwrap().data, + expected + ); +} + +#[test] +fn a_carried_page_validation_memo_still_catches_a_damaged_prefix() { + // The memo is what a read checks the bytes against, so a carry that kept hashes of bytes that + // were not the bytes would stop catching damage. Damage lands in a page that was full before + // the last append, so its hash is one the carry kept rather than one it recomputed. + let mut chunk = MatrixReplicaService::new(); + let page_id = PageId { + chunk_id: ChunkId::new(0, 995).unwrap(), + page_index: 0, + }; + let mut expected: Vec = Vec::new(); + for round in 0..8usize { + let bytes: Vec = (0..2_500).map(|i| (round * 13 + i * 5) as u8).collect(); + chunk + .write_chunk(page_id.clone(), expected.len() as u64, &bytes) + .unwrap(); + expected.extend_from_slice(&bytes); + chunk.read_chunk(&page_id, 0, expected.len() as u64).unwrap(); + } + // One more append, and no read: the memo in place is the carried one. + let last: Vec = (0..2_500).map(|i| (i * 3 + 1) as u8).collect(); + chunk + .write_chunk(page_id.clone(), expected.len() as u64, &last) + .unwrap(); + expected.extend_from_slice(&last); + assert!(chunk.page_validation_for_test(&page_id).is_some()); + + // Damage a byte in the SECOND validation page (offset 4,096..8,192), full before the append. + let damaged_at = 5_000usize; + { + let page = std::sync::Arc::make_mut(chunk.pages_local_mut().get_mut(&page_id).unwrap()); + std::sync::Arc::make_mut(&mut page.data)[damaged_at] ^= 0xff; + } + let read = chunk.read_chunk(&page_id, 4_096, 4_096).unwrap(); + assert_eq!( + read.data.as_slice(), + &expected[4_096..8_192], + "damaged bytes were served: the carried memo no longer catches damage in a kept page" + ); + // A read that does not touch the damaged page is unaffected. + assert_eq!( + chunk.read_chunk(&page_id, 0, 4_096).unwrap().data.as_slice(), + &expected[0..4_096] + ); +} + #[test] fn the_metadata_change_log_is_a_deque_because_it_drops_one_record_per_append() { // THE LOG IS ALWAYS AT ITS BOUND once it has filled, and every append then drops its oldest diff --git a/src/services/chunk/cache_reclaim_snapshot/read_cache.inc.rs b/src/services/chunk/cache_reclaim_snapshot/read_cache.inc.rs index 4648345a..9ad0ec07 100644 --- a/src/services/chunk/cache_reclaim_snapshot/read_cache.inc.rs +++ b/src/services/chunk/cache_reclaim_snapshot/read_cache.inc.rs @@ -176,12 +176,7 @@ impl MatrixReplicaService { return false; } let mut hashes = Vec::with_capacity(chunk.data.len() / page + 1); - let mut from = 0usize; - while from < chunk.data.len() { - let to = (from + page).min(chunk.data.len()); - hashes.push(fnv1a64(&chunk.data[from..to])); - from = to; - } + Self::extend_page_hashes(&chunk.data, 0, &mut hashes); self.page_validation.borrow_mut().insert( page_id.clone(), // `checksum()` returns an owned String; the clone was a second allocation. @@ -190,6 +185,75 @@ impl MatrixReplicaService { true } + /// Hashes of the validation pages of `data` from the page that starts at `from` to the end. + /// `from` is a multiple of the page size, so every hash pushed is of a whole page except + /// possibly the last, which is the partial page at the end of the payload. + fn extend_page_hashes(data: &[u8], from: usize, hashes: &mut Vec) { + let page = CHUNK_VALIDATION_PAGE_BYTES; + debug_assert_eq!(from % page, 0); + let mut from = from; + while from < data.len() { + let to = (from + page).min(data.len()); + hashes.push(fnv1a64(&data[from..to])); + from = to; + } + } + + /// Takes the page's validation memo out, so a write that can prove it only appended may put + /// a corrected one back with `carry_page_validation_over_append`. + fn take_page_validation(&self, page_id: &PageId) -> Option<(u64, String, Vec)> { + self.page_validation.borrow_mut().remove(page_id) + } + + /// A WRITE THAT ONLY APPENDED KEEPS THE MEMO IT TOOK, CORRECTED FOR WHAT IT ADDED. + /// + /// The memo is keyed by (write_sequence, checksum), and an append changes both, so before this + /// every append made the next read start from nothing: `chunk_pages_valid` hashed the whole + /// payload and then every page of it again. A page filled by appends and read after each one + /// therefore cost a read time proportional to the page, which is the shape a log that is + /// followed while it is written has. + /// + /// An append leaves every byte before the old end untouched, so the hash of each page that was + /// FULL at the old end is still the hash of that page. Those are kept. The page that held the + /// old end was partial and its hash is stale, and anything past it is new, so those are hashed + /// from the current bytes. The memo comes out describing the page exactly as a cold build of the + /// new payload would -- a test builds one and compares them -- so the integrity a read checks + /// is the same: a damaged byte fails its page. + /// + /// It carries only a memo that described the page as it was just before this write: + /// `prior_sequence` and `prior_checksum` are that page's, captured before the write changed + /// them. A memo that describes anything else is dropped and the next read builds a fresh one. + fn carry_page_validation_over_append( + &self, + page_id: &PageId, + taken: (u64, String, Vec), + prior_sequence: u64, + prior_checksum: &str, + bytes_before_write: usize, + chunk: &PageState, + ) { + let page = CHUNK_VALIDATION_PAGE_BYTES; + let (sequence, checksum, mut hashes) = taken; + if sequence != prior_sequence || checksum != prior_checksum { + return; + } + // The memo must have been built over exactly the bytes that were here. + if hashes.len() != bytes_before_write.div_ceil(page) || chunk.data.len() < bytes_before_write { + return; + } + let kept_pages = bytes_before_write / page; + hashes.truncate(kept_pages); + Self::extend_page_hashes(&chunk.data, kept_pages * page, &mut hashes); + self.page_validation + .borrow_mut() + .insert(page_id.clone(), (chunk.write_sequence, chunk.checksum(), hashes)); + } + + #[cfg(test)] + pub(crate) fn page_validation_for_test(&self, page_id: &PageId) -> Option<(u64, String, Vec)> { + self.page_validation.borrow().get(page_id).cloned() + } + fn can_insert_read_cache_payload(&self, bytes: u64) -> bool { let max_bytes = self.max_read_cache_bytes.get(); let max_cacheable = self.max_cacheable_read_bytes.get(); diff --git a/src/services/chunk/chunk_writes.inc.rs b/src/services/chunk/chunk_writes.inc.rs index 9011a23f..cef177ca 100644 --- a/src/services/chunk/chunk_writes.inc.rs +++ b/src/services/chunk/chunk_writes.inc.rs @@ -56,6 +56,10 @@ impl MatrixReplicaService { // An append leaves the undo a length, so `make_mut` below copies nothing. self.keep_page_undo_cheap_for_write(&page_id, start_index, end_index); let accounting_before = self.page_accounting(&page_id); + // TAKEN BEFORE THE WRITE, put back after it if the write only appended: see + // `carry_page_validation_over_append`. Nothing between here and the end can fail, so a + // memo taken here is never lost to an early return. + let validation_before_write = self.take_page_validation(&cache_chunk_id); // `make_mut` copies this one chunk's state if a snapshot still names // it, so the copy is per chunk WRITTEN rather than per chunk held. let chunk = std::sync::Arc::make_mut( @@ -67,6 +71,11 @@ impl MatrixReplicaService { // of the bytes that were already here instead of re-reading them. let bytes_before_write = chunk.data.len(); let checksum_before_write = chunk.checksum_fnv1a64; + let sequence_before_write = chunk.write_sequence; + // Only built when there is a memo to compare it with: `checksum()` allocates. + let checksum_text_before_write = validation_before_write + .as_ref() + .map(|_| chunk.checksum()); if end_index > chunk.data.len() { let payload = std::sync::Arc::make_mut(&mut chunk.data); reserve_page_growth(payload, end_index); @@ -108,7 +117,26 @@ impl MatrixReplicaService { } else { clear_chunk_ec_sidecars(chunk); } + // The pure-append test is the one the checksum above uses: the write begins at or past + // the old end, so no byte before the old end changed. + let appended_only = start_index >= bytes_before_write; self.invalidate_chunk_cache(&cache_chunk_id, offset, data.len() as u64); + if appended_only { + if let (Some(taken), Some(checksum_text)) = + (validation_before_write, checksum_text_before_write) + { + if let Some(page) = self.pages.get(&cache_chunk_id) { + self.carry_page_validation_over_append( + &cache_chunk_id, + taken, + sequence_before_write, + &checksum_text, + bytes_before_write, + page, + ); + } + } + } self.reaccount_page(&cache_chunk_id, accounting_before); self.physical_write_count = self.physical_write_count.saturating_add(1); self.physical_write_bytes = self.physical_write_bytes.saturating_add(data.len() as u64);