Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -253,7 +253,7 @@ fn mark_persistent_journal_initialized(path_key: &str) {
}
}

fn forget_persistent_journal_path(path_key: &str) {
pub(crate) fn forget_persistent_journal_path(path_key: &str) {
if let Ok(mut states) = persistent_journal_sync_states().lock() {
states.remove(path_key);
}
Expand Down
45 changes: 45 additions & 0 deletions src/object_store_regression_tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -807,6 +807,51 @@ fn a_repaired_journal_accepts_new_writes_that_survive_a_reopen() {
let _ = std::fs::remove_dir_all(root);
}

#[test]
fn a_put_whose_journal_write_fails_is_undone_and_leaves_nothing_behind() {
// The journal frame is written inside the put's own transaction, so a failed write has
// to come back through the put's narrow undo. The journal works for the bucket and one
// put, then its path becomes a directory, so the next frame cannot be opened.
let root = std::env::temp_dir().join(format!("mo-journal-failing-put-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&root);
std::fs::create_dir_all(&root).unwrap();
let path = root.join("store.journal").to_string_lossy().into_owned();
let mut options = test_options();
options.persistent_journal_path = path.clone();
let mut store = MatrixObjectStore::new(options).unwrap();
store.create_bucket("bucket").unwrap();
store
.put_object("bucket", "kept", vec![3u8; 2048], "application/octet-stream")
.unwrap();
let listed_before = store.list_objects("bucket", "", None, 100).unwrap().len();
assert_eq!(listed_before, 1);

forget_persistent_journal_path(&path);
std::fs::remove_file(&path).unwrap();
std::fs::create_dir(&path).unwrap();

let error = store
.put_object("bucket", "doomed", vec![7u8; 4096], "application/octet-stream")
.expect_err("the journal cannot be written, so the put must not be acknowledged");
assert!(!error.to_string().is_empty());
assert!(
store.get_object("bucket", "doomed").is_err(),
"a put whose journal write failed must not be readable"
);
assert_eq!(
store.list_objects("bucket", "", None, 100).unwrap().len(),
listed_before,
"a failed put must not leave an object listed"
);
// The earlier object is untouched, and byte for byte.
assert_eq!(store.get_object("bucket", "kept").unwrap().data, vec![3u8; 2048]);
// A second attempt fails the same way: no frame was left standing for it to write.
assert!(store
.put_object("bucket", "second", vec![1u8; 16], "application/octet-stream")
.is_err());
let _ = std::fs::remove_dir_all(root);
}

#[test]
fn repair_refuses_a_complete_frame_with_a_bad_checksum_and_changes_nothing() {
let (root, path, mut bytes) = torn_tail_fixture("badsum");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -180,6 +180,7 @@ impl MatrixObjectStore {
let mut store = Self {
persistent_snapshot_path,
persistent_journal_path: persistent_journal_path.clone(),
pending_journal_payload: None,
options,
buckets: BTreeSet::new(),
content_types: BTreeSet::new(),
Expand Down
6 changes: 6 additions & 0 deletions src/store/core/footprint_audit.rs
Original file line number Diff line number Diff line change
Expand Up @@ -782,6 +782,12 @@ impl MatrixObjectStore {
// with nothing excused.
let mut store_identity = (self.persistent_snapshot_path.capacity()
+ self.persistent_journal_path.capacity()
// Held only while a journaled put is applying, so it is empty whenever the
// footprint is read; weighed anyway so the coverage guard has nothing excused.
+ self
.pending_journal_payload
.as_ref()
.map_or(0, |frame| frame.capacity())
+ self.consensus_read_last_status.borrow().capacity())
as u64;
for node_id in &self.raft_voter_node_ids {
Expand Down
28 changes: 20 additions & 8 deletions src/store/core/metadata_object_api/object_put_append.rs
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,12 @@ impl MatrixObjectStore {
options,
});
}
let rollback = Some(self.capture_object_mutation_rollback());
// THE FRAME IS WRITTEN INSIDE THE PUT'S OWN TRANSACTION, not after it. This used to
// capture the whole store first so a failed journal write could be undone, and
// that capture shares every page with the live store: the next write to a page
// then copied it whole. `apply_put_storage_batched` already holds the narrow undo a
// put needs, so the frame goes in before its batch ends and a failure takes the
// path any other failure of the put takes.
let journal_payload = Self::build_object_journal_payload(
bucket,
key,
Expand All @@ -51,14 +56,21 @@ impl MatrixObjectStore {
update_time_ms,
options,
};
let metadata = self.apply_command(command)?;
if let Err(err) = self.append_object_journal_payload(&journal_payload) {
if let Some(rollback) = rollback {
self.restore_object_mutation_rollback(rollback);
}
return Err(err);
self.pending_journal_payload = Some(journal_payload);
let outcome = self.apply_command(command);
// Taken whatever happened, so a put that failed before it reached the journal
// cannot leave its frame standing for the next one to write.
let unwritten = self.pending_journal_payload.take();
debug_assert!(
outcome.is_err() || unwritten.is_none(),
"a put that succeeded left its journal frame unwritten"
);
// Unreachable while `PutObject` always applies through the batched path, but a
// frame that is silently dropped is an acknowledged write lost on restart.
if let (Ok(_), Some(frame)) = (&outcome, unwritten) {
self.append_object_journal_payload(&frame)?;
}
Ok(metadata)
outcome
}

pub fn append_object(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,15 @@ impl MatrixObjectStore {
append_persistent_journal_frame(&self.persistent_journal_path, &payload)
}

/// Write the frame a put left for its own transaction, if it left one. Takes it, so
/// a payload can be written once and never outlives the put that set it.
fn append_pending_journal_payload(&mut self) -> Result<(), ObjectError> {
match self.pending_journal_payload.take() {
Some(payload) => self.append_object_journal_payload(&payload),
None => Ok(()),
}
}

fn append_batch_put_journal_record(
&self,
requests: &[ObjectPutRequest],
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -120,8 +120,11 @@ impl MatrixObjectStore {
options,
operation,
) {
Ok(metadata) => match self.chunk_service.end_mutation_batch(&batch_id) {
Ok(_) => Ok(metadata),
Ok(metadata) => match self
.append_pending_journal_payload()
.and_then(|()| self.chunk_service.end_mutation_batch(&batch_id).map(|_| ()))
{
Ok(()) => Ok(metadata),
Err(err) => {
let _ = self.chunk_service.abort_mutation_batch(&batch_id);
self.restore_object_mutation_rollback(rollback);
Expand Down
6 changes: 6 additions & 0 deletions src/store/object_store_state.rs
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,12 @@ pub struct MatrixObjectStore {
pub(crate) options: StoreOptions,
persistent_snapshot_path: String,
persistent_journal_path: String,
/// The journal frame for the put now being applied, handed to
/// `apply_put_storage_batched` so it is written INSIDE the put's own transaction.
/// A journal failure then rolls back through the narrow undo the put already
/// holds, where the caller used to capture the whole store beforehand to do it.
/// Taken (never left standing) by the put that set it.
pending_journal_payload: Option<Vec<u8>>,
pub(crate) buckets: BTreeSet<SharedText>,
/// One allocation per distinct content type, up to
/// `MAX_SHARED_CONTENT_TYPES`. Derived state: it is not in a snapshot, it
Expand Down
Loading