diff --git a/src/internal/codec/snapshot_journal_codec/persistent_journal.rs b/src/internal/codec/snapshot_journal_codec/persistent_journal.rs index 7db38e85..d901b529 100644 --- a/src/internal/codec/snapshot_journal_codec/persistent_journal.rs +++ b/src/internal/codec/snapshot_journal_codec/persistent_journal.rs @@ -253,7 +253,7 @@ fn mark_persistent_journal_initialized(path_key: &str) { } } -fn forget_persistent_journal_path(path_key: &str) { +pub(crate) fn forget_persistent_journal_path(path_key: &str) { if let Ok(mut states) = persistent_journal_sync_states().lock() { states.remove(path_key); } diff --git a/src/object_store_regression_tests.rs b/src/object_store_regression_tests.rs index 55e98416..462cf280 100644 --- a/src/object_store_regression_tests.rs +++ b/src/object_store_regression_tests.rs @@ -807,6 +807,51 @@ fn a_repaired_journal_accepts_new_writes_that_survive_a_reopen() { let _ = std::fs::remove_dir_all(root); } +#[test] +fn a_put_whose_journal_write_fails_is_undone_and_leaves_nothing_behind() { + // The journal frame is written inside the put's own transaction, so a failed write has + // to come back through the put's narrow undo. The journal works for the bucket and one + // put, then its path becomes a directory, so the next frame cannot be opened. + let root = std::env::temp_dir().join(format!("mo-journal-failing-put-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&root); + std::fs::create_dir_all(&root).unwrap(); + let path = root.join("store.journal").to_string_lossy().into_owned(); + let mut options = test_options(); + options.persistent_journal_path = path.clone(); + let mut store = MatrixObjectStore::new(options).unwrap(); + store.create_bucket("bucket").unwrap(); + store + .put_object("bucket", "kept", vec![3u8; 2048], "application/octet-stream") + .unwrap(); + let listed_before = store.list_objects("bucket", "", None, 100).unwrap().len(); + assert_eq!(listed_before, 1); + + forget_persistent_journal_path(&path); + std::fs::remove_file(&path).unwrap(); + std::fs::create_dir(&path).unwrap(); + + let error = store + .put_object("bucket", "doomed", vec![7u8; 4096], "application/octet-stream") + .expect_err("the journal cannot be written, so the put must not be acknowledged"); + assert!(!error.to_string().is_empty()); + assert!( + store.get_object("bucket", "doomed").is_err(), + "a put whose journal write failed must not be readable" + ); + assert_eq!( + store.list_objects("bucket", "", None, 100).unwrap().len(), + listed_before, + "a failed put must not leave an object listed" + ); + // The earlier object is untouched, and byte for byte. + assert_eq!(store.get_object("bucket", "kept").unwrap().data, vec![3u8; 2048]); + // A second attempt fails the same way: no frame was left standing for it to write. + assert!(store + .put_object("bucket", "second", vec![1u8; 16], "application/octet-stream") + .is_err()); + let _ = std::fs::remove_dir_all(root); +} + #[test] fn repair_refuses_a_complete_frame_with_a_bad_checksum_and_changes_nothing() { let (root, path, mut bytes) = torn_tail_fixture("badsum"); diff --git a/src/store/core/construction_control/store_construction.rs b/src/store/core/construction_control/store_construction.rs index 7a9ccd57..0e0e9adc 100644 --- a/src/store/core/construction_control/store_construction.rs +++ b/src/store/core/construction_control/store_construction.rs @@ -180,6 +180,7 @@ impl MatrixObjectStore { let mut store = Self { persistent_snapshot_path, persistent_journal_path: persistent_journal_path.clone(), + pending_journal_payload: None, options, buckets: BTreeSet::new(), content_types: BTreeSet::new(), diff --git a/src/store/core/footprint_audit.rs b/src/store/core/footprint_audit.rs index d2ff99e1..542dcb87 100644 --- a/src/store/core/footprint_audit.rs +++ b/src/store/core/footprint_audit.rs @@ -782,6 +782,12 @@ impl MatrixObjectStore { // with nothing excused. let mut store_identity = (self.persistent_snapshot_path.capacity() + self.persistent_journal_path.capacity() + // Held only while a journaled put is applying, so it is empty whenever the + // footprint is read; weighed anyway so the coverage guard has nothing excused. + + self + .pending_journal_payload + .as_ref() + .map_or(0, |frame| frame.capacity()) + self.consensus_read_last_status.borrow().capacity()) as u64; for node_id in &self.raft_voter_node_ids { diff --git a/src/store/core/metadata_object_api/object_put_append.rs b/src/store/core/metadata_object_api/object_put_append.rs index 81e6b6de..335b2e58 100644 --- a/src/store/core/metadata_object_api/object_put_append.rs +++ b/src/store/core/metadata_object_api/object_put_append.rs @@ -32,7 +32,12 @@ impl MatrixObjectStore { options, }); } - let rollback = Some(self.capture_object_mutation_rollback()); + // THE FRAME IS WRITTEN INSIDE THE PUT'S OWN TRANSACTION, not after it. This used to + // capture the whole store first so a failed journal write could be undone, and + // that capture shares every page with the live store: the next write to a page + // then copied it whole. `apply_put_storage_batched` already holds the narrow undo a + // put needs, so the frame goes in before its batch ends and a failure takes the + // path any other failure of the put takes. let journal_payload = Self::build_object_journal_payload( bucket, key, @@ -51,14 +56,21 @@ impl MatrixObjectStore { update_time_ms, options, }; - let metadata = self.apply_command(command)?; - if let Err(err) = self.append_object_journal_payload(&journal_payload) { - if let Some(rollback) = rollback { - self.restore_object_mutation_rollback(rollback); - } - return Err(err); + self.pending_journal_payload = Some(journal_payload); + let outcome = self.apply_command(command); + // Taken whatever happened, so a put that failed before it reached the journal + // cannot leave its frame standing for the next one to write. + let unwritten = self.pending_journal_payload.take(); + debug_assert!( + outcome.is_err() || unwritten.is_none(), + "a put that succeeded left its journal frame unwritten" + ); + // Unreachable while `PutObject` always applies through the batched path, but a + // frame that is silently dropped is an acknowledged write lost on restart. + if let (Ok(_), Some(frame)) = (&outcome, unwritten) { + self.append_object_journal_payload(&frame)?; } - Ok(metadata) + outcome } pub fn append_object( diff --git a/src/store/core/raft_apply_persistence/journal_object_writes.rs b/src/store/core/raft_apply_persistence/journal_object_writes.rs index fac9022f..5fd5ed9d 100644 --- a/src/store/core/raft_apply_persistence/journal_object_writes.rs +++ b/src/store/core/raft_apply_persistence/journal_object_writes.rs @@ -94,6 +94,15 @@ impl MatrixObjectStore { append_persistent_journal_frame(&self.persistent_journal_path, &payload) } + /// Write the frame a put left for its own transaction, if it left one. Takes it, so + /// a payload can be written once and never outlives the put that set it. + fn append_pending_journal_payload(&mut self) -> Result<(), ObjectError> { + match self.pending_journal_payload.take() { + Some(payload) => self.append_object_journal_payload(&payload), + None => Ok(()), + } + } + fn append_batch_put_journal_record( &self, requests: &[ObjectPutRequest], diff --git a/src/store/core/raft_apply_persistence/snapshot_put_copy/put_storage.rs b/src/store/core/raft_apply_persistence/snapshot_put_copy/put_storage.rs index 06dc9762..cdaf3e22 100644 --- a/src/store/core/raft_apply_persistence/snapshot_put_copy/put_storage.rs +++ b/src/store/core/raft_apply_persistence/snapshot_put_copy/put_storage.rs @@ -120,8 +120,11 @@ impl MatrixObjectStore { options, operation, ) { - Ok(metadata) => match self.chunk_service.end_mutation_batch(&batch_id) { - Ok(_) => Ok(metadata), + Ok(metadata) => match self + .append_pending_journal_payload() + .and_then(|()| self.chunk_service.end_mutation_batch(&batch_id).map(|_| ())) + { + Ok(()) => Ok(metadata), Err(err) => { let _ = self.chunk_service.abort_mutation_batch(&batch_id); self.restore_object_mutation_rollback(rollback); diff --git a/src/store/object_store_state.rs b/src/store/object_store_state.rs index e034cfcc..90536b5a 100644 --- a/src/store/object_store_state.rs +++ b/src/store/object_store_state.rs @@ -172,6 +172,12 @@ pub struct MatrixObjectStore { pub(crate) options: StoreOptions, persistent_snapshot_path: String, persistent_journal_path: String, + /// The journal frame for the put now being applied, handed to + /// `apply_put_storage_batched` so it is written INSIDE the put's own transaction. + /// A journal failure then rolls back through the narrow undo the put already + /// holds, where the caller used to capture the whole store beforehand to do it. + /// Taken (never left standing) by the put that set it. + pending_journal_payload: Option>, pub(crate) buckets: BTreeSet, /// One allocation per distinct content type, up to /// `MAX_SHARED_CONTENT_TYPES`. Derived state: it is not in a snapshot, it