Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 67 additions & 0 deletions src/object_store_regression_tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -852,6 +852,73 @@ fn a_put_whose_journal_write_fails_is_undone_and_leaves_nothing_behind() {
let _ = std::fs::remove_dir_all(root);
}

#[test]
fn an_append_whose_journal_write_fails_is_undone_and_leaves_the_object_as_it_was() {
// An append writes its journal frame inside its own transaction, as a put does, so a failed
// write has to come back through the append's narrow undo. The object must read back byte for
// byte what it was, at the same length, and a second failing append must find the same.
let root = std::env::temp_dir().join(format!("mo-journal-failing-append-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&root);
std::fs::create_dir_all(&root).unwrap();
let path = root.join("store.journal").to_string_lossy().into_owned();
let mut options = test_options();
options.persistent_journal_path = path.clone();
let mut store = MatrixObjectStore::new(options).unwrap();
store.create_bucket("bucket").unwrap();
store
.append_object("bucket", "wal", b"first".to_vec(), "application/octet-stream")
.unwrap();

forget_persistent_journal_path(&path);
std::fs::remove_file(&path).unwrap();
std::fs::create_dir(&path).unwrap();

let error = store
.append_object("bucket", "wal", b"second".to_vec(), "application/octet-stream")
.expect_err("the journal cannot be written, so the append must not be acknowledged");
assert!(!error.to_string().is_empty());
let after = store.get_object("bucket", "wal").unwrap();
assert_eq!(after.data, b"first", "a failed append must not change the object");
assert_eq!(after.metadata.length, 5, "or its length");
assert!(store
.append_object("bucket", "wal", b"third".to_vec(), "application/octet-stream")
.is_err());
assert_eq!(store.get_object("bucket", "wal").unwrap().data, b"first");
let _ = std::fs::remove_dir_all(root);
}

#[test]
fn journaled_appends_replay_to_the_object_they_built() {
// Each append's frame is written once, in order, inside its transaction: after a restart the
// journal has to rebuild exactly the concatenation, however many appends went in.
let root = std::env::temp_dir().join(format!("mo-journal-append-replay-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&root);
std::fs::create_dir_all(&root).unwrap();
let path = root.join("store.journal").to_string_lossy().into_owned();
let mut options = test_options();
options.persistent_journal_path = path;
let mut expected: Vec<u8> = Vec::new();
{
let mut store = MatrixObjectStore::new(options.clone()).unwrap();
store.create_bucket("bucket").unwrap();
for round in 0..40usize {
let piece: Vec<u8> = (0..(7 + round * 3)).map(|i| (round * 31 + i) as u8).collect();
store
.append_object("bucket", "wal", piece.clone(), "application/octet-stream")
.unwrap();
expected.extend_from_slice(&piece);
}
assert_eq!(store.get_object("bucket", "wal").unwrap().data, expected);
}
let reopened = MatrixObjectStore::new(options).unwrap();
assert_eq!(
reopened.get_object("bucket", "wal").unwrap().data,
expected,
"the journal must rebuild the appended object exactly"
);
let _ = std::fs::remove_dir_all(root);
}

#[test]
fn repair_refuses_a_complete_frame_with_a_bad_checksum_and_changes_nothing() {
let (root, path, mut bytes) = torn_tail_fixture("badsum");
Expand Down
31 changes: 22 additions & 9 deletions src/store/core/metadata_object_api/object_put_append.rs
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,13 @@ impl MatrixObjectStore {
options,
);
}
let rollback = Some(self.capture_object_mutation_rollback());
// THE FRAME IS WRITTEN INSIDE THE APPEND'S OWN TRANSACTION, as a put's is. This captured
// the whole store first so a failed journal write could be undone, and that capture
// shares every page with the live store: each append then copied the page it extends, and
// cloned every container the store holds. Measured on a pure append workload, 448,272
// instructions an append at 500 appends and 1,625,833 at 4,000, because the page being
// copied was the one the appends were growing. `apply_append` already holds the narrow
// undo an append needs, so the frame goes in before its batch ends.
let journal_payload = Self::build_object_journal_payload(
bucket,
key,
Expand All @@ -115,21 +121,28 @@ impl MatrixObjectStore {
true,
None,
);
let metadata = self.apply_append(
self.pending_journal_payload = Some(journal_payload);
let outcome = self.apply_append(
bucket.to_string(),
key.to_string(),
data,
content_type,
update_time_ms,
options,
)?;
if let Err(err) = self.append_object_journal_payload(&journal_payload) {
if let Some(rollback) = rollback {
self.restore_object_mutation_rollback(rollback);
}
return Err(err);
);
// Taken whatever happened, so an append that failed before it reached the journal
// cannot leave its frame standing for the next write to put.
let unwritten = self.pending_journal_payload.take();
debug_assert!(
outcome.is_err() || unwritten.is_none(),
"an append that succeeded left its journal frame unwritten"
);
// Unreachable while an append always applies through `apply_append`, but a frame that is
// silently dropped is an acknowledged write lost on restart.
if let (Ok(_), Some(frame)) = (&outcome, unwritten) {
self.append_object_journal_payload(&frame)?;
}
Ok(metadata)
outcome
}

pub fn append_object_with_placement(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,13 @@ impl MatrixObjectStore {
let result =
self.apply_append_inner(bucket, key, data, content_type, update_time_ms, options);
match result {
Ok(metadata) => match self.chunk_service.end_mutation_batch(&batch_id) {
Ok(_) => {
// THE JOURNAL FRAME, if the caller left one, goes in BEFORE the batch ends, so a
// failed write aborts the batch and restores through this append's own narrow undo.
Ok(metadata) => match self
.append_pending_journal_payload()
.and_then(|()| self.chunk_service.end_mutation_batch(&batch_id).map(|_| ()))
{
Ok(()) => {
if self.options.persistent_snapshot_flush_on_commit {
self.flush_persistent_snapshot()?;
self.flush_object_checkpoint_indexes();
Expand Down
Loading