From 157bbaf49b4d9d2c919ddeee2d536c0fe3143b3d Mon Sep 17 00:00:00 2001 From: matrixarkai Date: Fri, 13 Nov 2026 09:30:23 -0800 Subject: [PATCH] A journaled append writes its frame inside its own transaction, so an append stops copying its page With a journal configured, append_object_with_options captured the whole store before every append so a failed journal write could be undone. The same defect was fixed for puts, and this is the path TemporalStore's shared store is made of: every write it makes is an append to a growing log object. The capture clones the chunk service, which shares every page with the live store, so each append then found a second holder on the page it was extending and copied the whole page, and cloned every container the store holds. The page being copied was the one the appends were growing, so the cost grew with every append. The frame is now handed to apply_append, which already holds the narrow undo an append needs, and is written after the mutation and before its batch ends. A journal failure aborts the batch and restores through that undo. The pending frame is taken whatever happens, and a frame left unwritten by a successful append is written rather than dropped. Measured with callgrind on the journal door at three replicas, 300-byte appends to one object, instructions per append: 448,272 -> 97,988 at 500 appends and 1,625,833 -> 98,806 at 4,000. Before the change the cost rose 3.6x across that range; after it, 0.8%. Two tests: a failed journal write leaves the object byte-identical at the same length, and 40 appends replay after a reopen to exactly the concatenation. Both fail when the journal write is skipped. Co-Authored-By: Claude Sonnet 5.5 --- src/object_store_regression_tests.rs | 67 +++++++++++++++++++ .../metadata_object_api/object_put_append.rs | 31 ++++++--- .../snapshot_put_copy_append.rs | 9 ++- 3 files changed, 96 insertions(+), 11 deletions(-) diff --git a/src/object_store_regression_tests.rs b/src/object_store_regression_tests.rs index 1fa1ce08..d8d9e7ee 100644 --- a/src/object_store_regression_tests.rs +++ b/src/object_store_regression_tests.rs @@ -852,6 +852,73 @@ fn a_put_whose_journal_write_fails_is_undone_and_leaves_nothing_behind() { let _ = std::fs::remove_dir_all(root); } +#[test] +fn an_append_whose_journal_write_fails_is_undone_and_leaves_the_object_as_it_was() { + // An append writes its journal frame inside its own transaction, as a put does, so a failed + // write has to come back through the append's narrow undo. The object must read back byte for + // byte what it was, at the same length, and a second failing append must find the same. + let root = std::env::temp_dir().join(format!("mo-journal-failing-append-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&root); + std::fs::create_dir_all(&root).unwrap(); + let path = root.join("store.journal").to_string_lossy().into_owned(); + let mut options = test_options(); + options.persistent_journal_path = path.clone(); + let mut store = MatrixObjectStore::new(options).unwrap(); + store.create_bucket("bucket").unwrap(); + store + .append_object("bucket", "wal", b"first".to_vec(), "application/octet-stream") + .unwrap(); + + forget_persistent_journal_path(&path); + std::fs::remove_file(&path).unwrap(); + std::fs::create_dir(&path).unwrap(); + + let error = store + .append_object("bucket", "wal", b"second".to_vec(), "application/octet-stream") + .expect_err("the journal cannot be written, so the append must not be acknowledged"); + assert!(!error.to_string().is_empty()); + let after = store.get_object("bucket", "wal").unwrap(); + assert_eq!(after.data, b"first", "a failed append must not change the object"); + assert_eq!(after.metadata.length, 5, "or its length"); + assert!(store + .append_object("bucket", "wal", b"third".to_vec(), "application/octet-stream") + .is_err()); + assert_eq!(store.get_object("bucket", "wal").unwrap().data, b"first"); + let _ = std::fs::remove_dir_all(root); +} + +#[test] +fn journaled_appends_replay_to_the_object_they_built() { + // Each append's frame is written once, in order, inside its transaction: after a restart the + // journal has to rebuild exactly the concatenation, however many appends went in. + let root = std::env::temp_dir().join(format!("mo-journal-append-replay-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&root); + std::fs::create_dir_all(&root).unwrap(); + let path = root.join("store.journal").to_string_lossy().into_owned(); + let mut options = test_options(); + options.persistent_journal_path = path; + let mut expected: Vec = Vec::new(); + { + let mut store = MatrixObjectStore::new(options.clone()).unwrap(); + store.create_bucket("bucket").unwrap(); + for round in 0..40usize { + let piece: Vec = (0..(7 + round * 3)).map(|i| (round * 31 + i) as u8).collect(); + store + .append_object("bucket", "wal", piece.clone(), "application/octet-stream") + .unwrap(); + expected.extend_from_slice(&piece); + } + assert_eq!(store.get_object("bucket", "wal").unwrap().data, expected); + } + let reopened = MatrixObjectStore::new(options).unwrap(); + assert_eq!( + reopened.get_object("bucket", "wal").unwrap().data, + expected, + "the journal must rebuild the appended object exactly" + ); + let _ = std::fs::remove_dir_all(root); +} + #[test] fn repair_refuses_a_complete_frame_with_a_bad_checksum_and_changes_nothing() { let (root, path, mut bytes) = torn_tail_fixture("badsum"); diff --git a/src/store/core/metadata_object_api/object_put_append.rs b/src/store/core/metadata_object_api/object_put_append.rs index 335b2e58..3b4bca4e 100644 --- a/src/store/core/metadata_object_api/object_put_append.rs +++ b/src/store/core/metadata_object_api/object_put_append.rs @@ -104,7 +104,13 @@ impl MatrixObjectStore { options, ); } - let rollback = Some(self.capture_object_mutation_rollback()); + // THE FRAME IS WRITTEN INSIDE THE APPEND'S OWN TRANSACTION, as a put's is. This captured + // the whole store first so a failed journal write could be undone, and that capture + // shares every page with the live store: each append then copied the page it extends, and + // cloned every container the store holds. Measured on a pure append workload, 448,272 + // instructions an append at 500 appends and 1,625,833 at 4,000, because the page being + // copied was the one the appends were growing. `apply_append` already holds the narrow + // undo an append needs, so the frame goes in before its batch ends. let journal_payload = Self::build_object_journal_payload( bucket, key, @@ -115,21 +121,28 @@ impl MatrixObjectStore { true, None, ); - let metadata = self.apply_append( + self.pending_journal_payload = Some(journal_payload); + let outcome = self.apply_append( bucket.to_string(), key.to_string(), data, content_type, update_time_ms, options, - )?; - if let Err(err) = self.append_object_journal_payload(&journal_payload) { - if let Some(rollback) = rollback { - self.restore_object_mutation_rollback(rollback); - } - return Err(err); + ); + // Taken whatever happened, so an append that failed before it reached the journal + // cannot leave its frame standing for the next write to put. + let unwritten = self.pending_journal_payload.take(); + debug_assert!( + outcome.is_err() || unwritten.is_none(), + "an append that succeeded left its journal frame unwritten" + ); + // Unreachable while an append always applies through `apply_append`, but a frame that is + // silently dropped is an acknowledged write lost on restart. + if let (Ok(_), Some(frame)) = (&outcome, unwritten) { + self.append_object_journal_payload(&frame)?; } - Ok(metadata) + outcome } pub fn append_object_with_placement( diff --git a/src/store/core/raft_apply_persistence/snapshot_put_copy/snapshot_put_copy_append.rs b/src/store/core/raft_apply_persistence/snapshot_put_copy/snapshot_put_copy_append.rs index 1dff8568..73b31fa9 100644 --- a/src/store/core/raft_apply_persistence/snapshot_put_copy/snapshot_put_copy_append.rs +++ b/src/store/core/raft_apply_persistence/snapshot_put_copy/snapshot_put_copy_append.rs @@ -29,8 +29,13 @@ impl MatrixObjectStore { let result = self.apply_append_inner(bucket, key, data, content_type, update_time_ms, options); match result { - Ok(metadata) => match self.chunk_service.end_mutation_batch(&batch_id) { - Ok(_) => { + // THE JOURNAL FRAME, if the caller left one, goes in BEFORE the batch ends, so a + // failed write aborts the batch and restores through this append's own narrow undo. + Ok(metadata) => match self + .append_pending_journal_payload() + .and_then(|()| self.chunk_service.end_mutation_batch(&batch_id).map(|_| ())) + { + Ok(()) => { if self.options.persistent_snapshot_flush_on_commit { self.flush_persistent_snapshot()?; self.flush_object_checkpoint_indexes();