From da6dd8f6bb1acd689712bf6a65e10e2077948502 Mon Sep 17 00:00:00 2001 From: matrixarkai Date: Fri, 13 Nov 2026 08:38:20 -0800 Subject: [PATCH] The metadata change log is a deque, so a full log drops a record without moving the rest MatrixMetaService keeps its change log as a Vec bounded at 1,024 records. Once it has filled it is always at the bound, and every append drops the oldest record with drain(0..1), which shifts every record the log keeps. That is paid on every put, for ever. A profile of the journal door at 8,000 puts of 1 KiB put 27.7% of its instructions in append_metadata_change and 33% in memmove, and the per-put cost was 181,052 instructions against 124,903 at 1,000 puts: 45% more, from a term that was constant per put once the log was full. The object change feed had exactly this defect and was cured by holding it as a deque; this is the second place it was found. The log is a VecDeque now, with the trim helper that took a Vec replaced by one that takes a deque (nothing else used it). Behaviour is unchanged: the bound, the order and what a snapshot carries are the same, and a snapshot still holds a Vec at its edge. The declaration is pinned by a test that fails if the field goes back to a Vec; it excludes comments and asserts the declaration exists once, so it cannot pass checking nothing. Co-Authored-By: Claude Sonnet 5.5 --- src/lib.rs | 2 +- src/object_store_regression_tests.rs | 29 +++++++++++++++++++ .../retention_validation.inc.rs | 7 ++++- .../maintenance_persistence_changefeed.rs | 12 ++++---- .../mutation_preflight.rs | 2 +- .../prefix_checkpoint/checkpoint_lag.rs | 8 ++--- .../snapshot_lifecycle/export_fingerprint.rs | 2 +- .../snapshot_lifecycle_install.rs | 2 +- .../snapshot_retention.rs | 2 +- .../construction_receipts_construction.rs | 2 +- .../records_leases_shards_readiness.inc.rs | 16 +++++----- .../policy_retention_retention.rs | 2 +- src/services/meta/service_state.inc.rs | 2 +- 13 files changed, 61 insertions(+), 27 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 74cf3b48..b9bead43 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -250,7 +250,7 @@ mod replica_service; pub use replica_service::*; use replica_service::{ select_bounded_chunk_discard_ranges_from, select_bounded_chunk_write_ranges_from, - trim_btree_map_to_retention, trim_vec_front_to_retention, validate_page_route_qos_config, + trim_btree_map_to_retention, trim_deque_front_to_retention, validate_page_route_qos_config, validate_chunk_mutation_qos_config, validate_meta_retention_limit, validate_metadata_admission_policy, validate_metadata_qos_config, validate_metadata_route_cache_config, validate_storage_receipt_retention, diff --git a/src/object_store_regression_tests.rs b/src/object_store_regression_tests.rs index b4d607b9..1fa1ce08 100644 --- a/src/object_store_regression_tests.rs +++ b/src/object_store_regression_tests.rs @@ -3092,6 +3092,35 @@ fn a_page_filled_by_small_appends_reallocates_rarely_and_holds_bounded_slack() { ); } +#[test] +fn the_metadata_change_log_is_a_deque_because_it_drops_one_record_per_append() { + // THE LOG IS ALWAYS AT ITS BOUND once it has filled, and every append then drops its oldest + // record. Held as a `Vec`, that drop shifts every record the log keeps, once per put, for + // ever: 27.7% of the instructions of an 8,000-put profile, 50,000 a put on a log of + // 1,024 records, with the per-put cost 45% above what it was at 1,000 puts. It is a + // deque so the drop moves nothing. The object change feed had the same defect and the same + // cure; this is the second place it was found, so the declaration is pinned. + // + // Comments are excluded and the count is asserted, so a declaration that moved or was + // reworded cannot leave this checking nothing. + const STATE: &str = include_str!("services/meta/service_state.inc.rs"); + let declared: Vec<&str> = STATE + .lines() + .map(str::trim) + .filter(|line| !line.starts_with("//") && line.starts_with("changes:") && line.ends_with(',')) + .collect(); + assert_eq!( + declared.len(), + 1, + "the metadata service should declare its change log exactly once: {declared:?}" + ); + assert!( + declared[0].contains("VecDeque"), + "the metadata change log must be a deque so trimming it moves no records: {}", + declared[0] + ); +} + #[test] fn a_chunk_checksum_is_the_hash_of_its_payload_however_the_write_arrived() { // `write_chunk` resumes the FNV-1a fold over the appended bytes instead of re-hashing the diff --git a/src/services/chunk/cache_reclaim_snapshot/retention_validation.inc.rs b/src/services/chunk/cache_reclaim_snapshot/retention_validation.inc.rs index 86147c86..0cb2d62a 100644 --- a/src/services/chunk/cache_reclaim_snapshot/retention_validation.inc.rs +++ b/src/services/chunk/cache_reclaim_snapshot/retention_validation.inc.rs @@ -73,7 +73,12 @@ pub(crate) fn trim_btree_map_to_retention( overflow as u64 } -pub(crate) fn trim_vec_front_to_retention(items: &mut Vec, retention: usize) -> u64 { +/// Drop the oldest records past `retention`. A deque, because a `Vec` shifts every record it +/// keeps down by one place -- once per append, for ever, on a log that is always at its bound. +pub(crate) fn trim_deque_front_to_retention( + items: &mut std::collections::VecDeque, + retention: usize, +) -> u64 { let overflow = items.len().saturating_sub(retention); if overflow > 0 { items.drain(0..overflow); diff --git a/src/services/meta/maintenance_persistence/maintenance_persistence_changefeed.rs b/src/services/meta/maintenance_persistence/maintenance_persistence_changefeed.rs index 3d7223cb..f47177b8 100644 --- a/src/services/meta/maintenance_persistence/maintenance_persistence_changefeed.rs +++ b/src/services/meta/maintenance_persistence/maintenance_persistence_changefeed.rs @@ -72,12 +72,12 @@ impl MatrixMetaService { )); } let first_retained_change_sequence = - self.changes.first().map_or(0, |change| change.sequence); - let last_retained_change_sequence = self.changes.last().map_or(0, |change| change.sequence); + self.changes.front().map_or(0, |change| change.sequence); + let last_retained_change_sequence = self.changes.back().map_or(0, |change| change.sequence); let first_retained_commit_index = - self.changes.first().map_or(0, |change| change.commit_index); + self.changes.front().map_or(0, |change| change.commit_index); let last_retained_commit_index = - self.changes.last().map_or(0, |change| change.commit_index); + self.changes.back().map_or(0, |change| change.commit_index); let retained_changes_compacted = if first_retained_change_sequence > 0 { after_sequence.saturating_add(1) < first_retained_change_sequence } else { @@ -166,7 +166,7 @@ impl MatrixMetaService { } } report.remaining_changes = retained.len() as u64; - self.changes = retained; + self.changes = retained.into(); if report.compacted_changes > 0 { self.next_commit_index = self.next_commit_index.saturating_add(1); } @@ -320,7 +320,7 @@ impl MatrixMetaService { "prepared metadata changefeed compaction compacted set was incomplete".to_string(), )); } - self.changes = retained; + self.changes = retained.into(); self.next_commit_index = self.next_commit_index.saturating_add(1); Ok(prepared.report) } diff --git a/src/services/meta/maintenance_persistence/mutation_preflight.rs b/src/services/meta/maintenance_persistence/mutation_preflight.rs index 0b97eb38..757f5f65 100644 --- a/src/services/meta/maintenance_persistence/mutation_preflight.rs +++ b/src/services/meta/maintenance_persistence/mutation_preflight.rs @@ -331,7 +331,7 @@ impl MatrixMetaService { } } } - self.changes.push(change); + self.changes.push_back(change); } fn apply_mutation( diff --git a/src/services/meta/maintenance_persistence/prefix_checkpoint/checkpoint_lag.rs b/src/services/meta/maintenance_persistence/prefix_checkpoint/checkpoint_lag.rs index 3fc23800..279ed279 100644 --- a/src/services/meta/maintenance_persistence/prefix_checkpoint/checkpoint_lag.rs +++ b/src/services/meta/maintenance_persistence/prefix_checkpoint/checkpoint_lag.rs @@ -34,16 +34,16 @@ impl MatrixMetaService { target_commit_index.unwrap_or_else(|| self.next_commit_index.saturating_sub(1)); let first_retained_change_sequence = self .changes - .first() + .front() .map_or(self.next_change_sequence, |change| change.sequence); - let last_retained_change_sequence = self.changes.last().map_or(0, |change| change.sequence); + let last_retained_change_sequence = self.changes.back().map_or(0, |change| change.sequence); let first_retained_commit_index = self .changes - .first() + .front() .map_or(target_commit_index, |change| change.commit_index); let last_retained_commit_index = self .changes - .last() + .back() .map_or(target_commit_index, |change| change.commit_index); let mut plan = MetadataChangeCheckpointLagPlan { diff --git a/src/services/meta/maintenance_persistence/snapshot_lifecycle/export_fingerprint.rs b/src/services/meta/maintenance_persistence/snapshot_lifecycle/export_fingerprint.rs index e89cea07..5d784e54 100644 --- a/src/services/meta/maintenance_persistence/snapshot_lifecycle/export_fingerprint.rs +++ b/src/services/meta/maintenance_persistence/snapshot_lifecycle/export_fingerprint.rs @@ -48,7 +48,7 @@ impl MatrixMetaService { transactions: self.transactions.clone(), transaction_receipts: self.transaction_receipts.clone(), change_checkpoints: self.change_checkpoints.clone(), - changes: self.changes.clone(), + changes: self.changes.iter().cloned().collect(), next_commit_index: self.next_commit_index, // THE ROSTER, FROM THE SAME SERVICE STATE THE PLACEMENT EXPORT READS. Two export // functions reading one set of fields is what makes the agreement rule in diff --git a/src/services/meta/maintenance_persistence/snapshot_lifecycle/snapshot_lifecycle_install.rs b/src/services/meta/maintenance_persistence/snapshot_lifecycle/snapshot_lifecycle_install.rs index 5b9cf8dc..7c890cf3 100644 --- a/src/services/meta/maintenance_persistence/snapshot_lifecycle/snapshot_lifecycle_install.rs +++ b/src/services/meta/maintenance_persistence/snapshot_lifecycle/snapshot_lifecycle_install.rs @@ -25,7 +25,7 @@ impl MatrixMetaService { self.transactions = snapshot.transactions; self.transaction_receipts = snapshot.transaction_receipts; self.change_checkpoints = snapshot.change_checkpoints; - self.changes = snapshot.changes; + self.changes = snapshot.changes.into(); self.next_commit_index = snapshot.next_commit_index; self.next_change_sequence = snapshot.next_change_sequence; self.compacted_tombstone_count = snapshot.compacted_tombstone_count; diff --git a/src/services/meta/maintenance_persistence/snapshot_tombstone_retention/snapshot_retention.rs b/src/services/meta/maintenance_persistence/snapshot_tombstone_retention/snapshot_retention.rs index ee72a933..8389b5e1 100644 --- a/src/services/meta/maintenance_persistence/snapshot_tombstone_retention/snapshot_retention.rs +++ b/src/services/meta/maintenance_persistence/snapshot_tombstone_retention/snapshot_retention.rs @@ -39,7 +39,7 @@ impl MatrixMetaService { retained_changes.push(change); } } - self.changes = retained_changes; + self.changes = retained_changes.into(); let mut compacted_batch_receipts = 0u64; if !self.batch_request_receipts.is_empty() { diff --git a/src/services/meta/records_leases_shards/construction_receipts/construction_receipts_construction.rs b/src/services/meta/records_leases_shards/construction_receipts/construction_receipts_construction.rs index 1a7b9f50..e4fd0a37 100644 --- a/src/services/meta/records_leases_shards/construction_receipts/construction_receipts_construction.rs +++ b/src/services/meta/records_leases_shards/construction_receipts/construction_receipts_construction.rs @@ -28,7 +28,7 @@ impl MatrixMetaService { transactions: BTreeMap::new(), transaction_receipts: BTreeMap::new(), change_checkpoints: BTreeMap::new(), - changes: Vec::new(), + changes: std::collections::VecDeque::new(), next_commit_index: 1, next_change_sequence: 1, compacted_tombstone_count: 0, diff --git a/src/services/meta/records_leases_shards/records_leases_shards_readiness.inc.rs b/src/services/meta/records_leases_shards/records_leases_shards_readiness.inc.rs index 9d6b4868..39210315 100644 --- a/src/services/meta/records_leases_shards/records_leases_shards_readiness.inc.rs +++ b/src/services/meta/records_leases_shards/records_leases_shards_readiness.inc.rs @@ -7,7 +7,7 @@ impl MatrixMetaService { pub fn last_applied_metadata_mutation_commit_index(&self) -> u64 { let mut last_commit = self.next_commit_index.saturating_sub(1); - last_commit = last_commit.max(self.changes.last().map_or(0, |change| change.commit_index)); + last_commit = last_commit.max(self.changes.back().map_or(0, |change| change.commit_index)); for receipt in self.batch_request_receipts.values() { last_commit = last_commit.max(receipt.last_commit_index); } @@ -36,7 +36,7 @@ impl MatrixMetaService { pub fn last_applied_metadata_mutation_time_ms(&self) -> u64 { let mut last_update = self .changes - .last() + .back() .map_or(0, |change| change.update_time_ms); for receipt in self.batch_request_receipts.values() { last_update = last_update.max(receipt.now_ms); @@ -79,19 +79,19 @@ impl MatrixMetaService { }; let max_retained_metadata_commit_index = self .changes - .last() + .back() .map(|change| change.commit_index) .unwrap_or(0) .max(self.raft_snapshot_floor_commit_index); let first_retained_change_sequence = - self.changes.first().map_or(0, |change| change.sequence); - let last_retained_change_sequence = self.changes.last().map_or(0, |change| change.sequence); + self.changes.front().map_or(0, |change| change.sequence); + let last_retained_change_sequence = self.changes.back().map_or(0, |change| change.sequence); let first_retained_metadata_commit_index = - self.changes.first().map_or(0, |change| change.commit_index); + self.changes.front().map_or(0, |change| change.commit_index); let last_retained_metadata_commit_index = - self.changes.last().map_or(0, |change| change.commit_index); + self.changes.back().map_or(0, |change| change.commit_index); let retained_mutation_receipts = self.retained_mutation_receipt_count(); - let last_change_sequence = self.changes.last().map_or(0, |change| change.sequence); + let last_change_sequence = self.changes.back().map_or(0, |change| change.sequence); let max_change_checkpoint_lag = self .change_checkpoints .values() diff --git a/src/services/meta/retention_transactions_routes/policy_retention/policy_retention_retention.rs b/src/services/meta/retention_transactions_routes/policy_retention/policy_retention_retention.rs index 6e927798..8a85b9db 100644 --- a/src/services/meta/retention_transactions_routes/policy_retention/policy_retention_retention.rs +++ b/src/services/meta/retention_transactions_routes/policy_retention/policy_retention_retention.rs @@ -21,7 +21,7 @@ impl MatrixMetaService { } fn trim_meta_retention(&mut self) -> MetadataRetentionConfigReport { let trimmed_change_records = - trim_vec_front_to_retention(&mut self.changes, self.change_log_retention_limit); + trim_deque_front_to_retention(&mut self.changes, self.change_log_retention_limit); let trimmed_batch_receipts = trim_btree_map_to_retention( &mut self.batch_request_receipts, self.mutation_receipt_retention_limit, diff --git a/src/services/meta/service_state.inc.rs b/src/services/meta/service_state.inc.rs index a9f6d514..9812cd79 100644 --- a/src/services/meta/service_state.inc.rs +++ b/src/services/meta/service_state.inc.rs @@ -47,7 +47,7 @@ pub struct MatrixMetaService { pub(crate) transactions: BTreeMap, transaction_receipts: BTreeMap, change_checkpoints: BTreeMap, - changes: Vec, + changes: std::collections::VecDeque, pub(crate) next_commit_index: u64, next_change_sequence: u64, compacted_tombstone_count: u64,