From 38661f88cf77dc87f19a42201826319eccb6499e Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 04:43:00 -0700 Subject: [PATCH 01/49] Add stable durability crash points --- CHANGELOG.md | 4 + docs/formats/segment-store-v1/requirements.md | 15 ++ xtask/src/durability_crash_point.rs | 180 +++++++++++++++++ xtask/src/durability_crash_point_identity.rs | 47 +++++ xtask/src/lib.rs | 8 + .../tests/durability_crash_point_contract.rs | 186 ++++++++++++++++++ 6 files changed, 440 insertions(+) create mode 100644 xtask/src/durability_crash_point.rs create mode 100644 xtask/src/durability_crash_point_identity.rs create mode 100644 xtask/tests/durability_crash_point_contract.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 2b0e9d8..37112c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,10 @@ after its public API and format compatibility policies are established. ### Changed +- Repository crash-matrix tooling now exposes one typed, ordered vocabulary for + `KEEP-CRASH-001` through `KEEP-CRASH-035`. Each identifier is bound to its + segment, catalog, head, recovery-discard, or initialization sequence, and + only record append admits an occurrence counter. - Catalog decoding now verifies the catalog checksum and physical digest before interpreting entry semantics. Corrupt identity-bearing bytes therefore fail at the integrity boundary instead of producing a semantic entry error. diff --git a/docs/formats/segment-store-v1/requirements.md b/docs/formats/segment-store-v1/requirements.md index 60d4ae8..c3da296 100644 --- a/docs/formats/segment-store-v1/requirements.md +++ b/docs/formats/segment-store-v1/requirements.md @@ -82,6 +82,21 @@ recovery. +## Recovery implementation evidence + +Issue #17 implements crash injection and explicit recovery in independently +reviewable slices. The first slice freezes executable crash-point identity and +sequence ownership. It does not claim process-death injection, initialization, +recovery classification, or recovery execution. + + + +| ID | Implemented requirement | Oracle | Executable evidence | Status | +| --- | --- | --- | --- | --- | +| `KEEP-RECOVERY-001` | Crash identifiers `KEEP-CRASH-001` through `KEEP-CRASH-035` form one contiguous typed vocabulary, map to the exact owning protocol sequence, and admit an occurrence counter only for record append | Ordered identifier-and-sequence ledger | `xtask/tests/durability_crash_point_contract.rs` | Implemented in #17 | + + + ## Compatibility and migration The byte grammars, magic values, field widths and order, endianness, kinds, diff --git a/xtask/src/durability_crash_point.rs b/xtask/src/durability_crash_point.rs new file mode 100644 index 0000000..6a0ca81 --- /dev/null +++ b/xtask/src/durability_crash_point.rs @@ -0,0 +1,180 @@ +//! Stable identifiers for deterministic segment-store crash injection. + +/// Durable protocol sequence containing a crash boundary. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum DurabilityCrashSequence { + /// Segment staging, sealing, and immutable-pool publication. + Segment, + /// Catalog staging and immutable-pool publication. + Catalog, + /// Publication-head staging and replacement. + Head, + /// Explicit discard of fingerprint-bound recovery evidence. + RecoveryDiscard, + /// Writer-locked store initialization. + Initialization, +} + +/// One stable process-death boundary in the durable segment-store protocol. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum DurabilityCrashPoint { + /// Exclusively create `staging/current.seg`. + CreateSegmentStage, + /// Write the complete segment header. + WriteSegmentHeader, + /// Append one complete segment record and checksum. + AppendSegmentRecord, + /// Flush the reusable record prefix. + FlushSegmentRecordPrefix, + /// Synchronize the reusable record prefix. + SynchronizeSegmentRecordPrefix, + /// Append the complete segment seal. + AppendSegmentSeal, + /// Flush the sealed segment bytes. + FlushSealedSegment, + /// Synchronize the sealed segment stage. + SynchronizeSealedSegment, + /// Verify and link the segment into the immutable pool. + LinkSegment, + /// Synchronize the segment-pool directory. + SynchronizeSegmentPool, + /// Remove `staging/current.seg`. + RemoveSegmentStage, + /// Synchronize staging after segment-stage removal. + SynchronizeStagingAfterSegment, + /// Exclusively create `staging/current.cat`. + CreateCatalogStage, + /// Write the complete canonical catalog. + WriteCatalog, + /// Flush the complete catalog. + FlushCatalog, + /// Synchronize the catalog stage. + SynchronizeCatalog, + /// Verify and link the catalog into the immutable pool. + LinkCatalog, + /// Synchronize the catalog-pool directory. + SynchronizeCatalogPool, + /// Remove `staging/current.cat`. + RemoveCatalogStage, + /// Synchronize staging after catalog-stage removal. + SynchronizeStagingAfterCatalog, + /// Exclusively create `head.next`. + CreateHeadStage, + /// Write the complete next publication head. + WriteHead, + /// Flush the complete next publication head. + FlushHead, + /// Synchronize `head.next`. + SynchronizeHead, + /// Verify `head.next` and replace `HEAD`. + ReplaceHead, + /// Synchronize the store root after head replacement. + SynchronizeRootAfterHead, + /// Remove a fingerprint-bound segment or catalog recovery stage. + RemoveRecoveryStage, + /// Synchronize staging after recovery-stage removal. + SynchronizeStagingAfterRecovery, + /// Remove fingerprint-bound `head.next` recovery evidence. + RemoveRecoveryHead, + /// Synchronize the store root after recovery-head removal. + SynchronizeRootAfterRecovery, + /// Create or reopen `writer.lock`, then acquire writer authority. + OpenAndLockWriterFile, + /// Create or verify the staging directory. + CreateStagingDirectory, + /// Create or verify the segment-pool directory. + CreateSegmentPoolDirectory, + /// Create or verify the catalog-pool directory. + CreateCatalogPoolDirectory, + /// Synchronize the store root after initialization. + SynchronizeRootAfterInitialization, +} + +impl DurabilityCrashPoint { + /// Every crash boundary in stable protocol order. + pub const ALL: [Self; 35] = [ + Self::CreateSegmentStage, + Self::WriteSegmentHeader, + Self::AppendSegmentRecord, + Self::FlushSegmentRecordPrefix, + Self::SynchronizeSegmentRecordPrefix, + Self::AppendSegmentSeal, + Self::FlushSealedSegment, + Self::SynchronizeSealedSegment, + Self::LinkSegment, + Self::SynchronizeSegmentPool, + Self::RemoveSegmentStage, + Self::SynchronizeStagingAfterSegment, + Self::CreateCatalogStage, + Self::WriteCatalog, + Self::FlushCatalog, + Self::SynchronizeCatalog, + Self::LinkCatalog, + Self::SynchronizeCatalogPool, + Self::RemoveCatalogStage, + Self::SynchronizeStagingAfterCatalog, + Self::CreateHeadStage, + Self::WriteHead, + Self::FlushHead, + Self::SynchronizeHead, + Self::ReplaceHead, + Self::SynchronizeRootAfterHead, + Self::RemoveRecoveryStage, + Self::SynchronizeStagingAfterRecovery, + Self::RemoveRecoveryHead, + Self::SynchronizeRootAfterRecovery, + Self::OpenAndLockWriterFile, + Self::CreateStagingDirectory, + Self::CreateSegmentPoolDirectory, + Self::CreateCatalogPoolDirectory, + Self::SynchronizeRootAfterInitialization, + ]; + + /// Returns the durable protocol sequence containing this boundary. + #[must_use] + pub const fn sequence(self) -> DurabilityCrashSequence { + match self { + Self::CreateSegmentStage + | Self::WriteSegmentHeader + | Self::AppendSegmentRecord + | Self::FlushSegmentRecordPrefix + | Self::SynchronizeSegmentRecordPrefix + | Self::AppendSegmentSeal + | Self::FlushSealedSegment + | Self::SynchronizeSealedSegment + | Self::LinkSegment + | Self::SynchronizeSegmentPool + | Self::RemoveSegmentStage + | Self::SynchronizeStagingAfterSegment => DurabilityCrashSequence::Segment, + Self::CreateCatalogStage + | Self::WriteCatalog + | Self::FlushCatalog + | Self::SynchronizeCatalog + | Self::LinkCatalog + | Self::SynchronizeCatalogPool + | Self::RemoveCatalogStage + | Self::SynchronizeStagingAfterCatalog => DurabilityCrashSequence::Catalog, + Self::CreateHeadStage + | Self::WriteHead + | Self::FlushHead + | Self::SynchronizeHead + | Self::ReplaceHead + | Self::SynchronizeRootAfterHead => DurabilityCrashSequence::Head, + Self::RemoveRecoveryStage + | Self::SynchronizeStagingAfterRecovery + | Self::RemoveRecoveryHead + | Self::SynchronizeRootAfterRecovery => DurabilityCrashSequence::RecoveryDiscard, + Self::OpenAndLockWriterFile + | Self::CreateStagingDirectory + | Self::CreateSegmentPoolDirectory + | Self::CreateCatalogPoolDirectory + | Self::SynchronizeRootAfterInitialization => DurabilityCrashSequence::Initialization, + } + } + + /// Reports whether tests may select a repeated occurrence. + #[must_use] + pub const fn occurrence_counted(self) -> bool { + matches!(self, Self::AppendSegmentRecord) + } +} diff --git a/xtask/src/durability_crash_point_identity.rs b/xtask/src/durability_crash_point_identity.rs new file mode 100644 index 0000000..173afa7 --- /dev/null +++ b/xtask/src/durability_crash_point_identity.rs @@ -0,0 +1,47 @@ +//! This module owns stable text identities for durability crash points. + +use crate::durability_crash_point::DurabilityCrashPoint; + +impl DurabilityCrashPoint { + /// Returns the stable `KEEP-CRASH-NNN` identifier. + #[must_use] + pub const fn identifier(self) -> &'static str { + match self { + Self::CreateSegmentStage => "KEEP-CRASH-001", + Self::WriteSegmentHeader => "KEEP-CRASH-002", + Self::AppendSegmentRecord => "KEEP-CRASH-003", + Self::FlushSegmentRecordPrefix => "KEEP-CRASH-004", + Self::SynchronizeSegmentRecordPrefix => "KEEP-CRASH-005", + Self::AppendSegmentSeal => "KEEP-CRASH-006", + Self::FlushSealedSegment => "KEEP-CRASH-007", + Self::SynchronizeSealedSegment => "KEEP-CRASH-008", + Self::LinkSegment => "KEEP-CRASH-009", + Self::SynchronizeSegmentPool => "KEEP-CRASH-010", + Self::RemoveSegmentStage => "KEEP-CRASH-011", + Self::SynchronizeStagingAfterSegment => "KEEP-CRASH-012", + Self::CreateCatalogStage => "KEEP-CRASH-013", + Self::WriteCatalog => "KEEP-CRASH-014", + Self::FlushCatalog => "KEEP-CRASH-015", + Self::SynchronizeCatalog => "KEEP-CRASH-016", + Self::LinkCatalog => "KEEP-CRASH-017", + Self::SynchronizeCatalogPool => "KEEP-CRASH-018", + Self::RemoveCatalogStage => "KEEP-CRASH-019", + Self::SynchronizeStagingAfterCatalog => "KEEP-CRASH-020", + Self::CreateHeadStage => "KEEP-CRASH-021", + Self::WriteHead => "KEEP-CRASH-022", + Self::FlushHead => "KEEP-CRASH-023", + Self::SynchronizeHead => "KEEP-CRASH-024", + Self::ReplaceHead => "KEEP-CRASH-025", + Self::SynchronizeRootAfterHead => "KEEP-CRASH-026", + Self::RemoveRecoveryStage => "KEEP-CRASH-027", + Self::SynchronizeStagingAfterRecovery => "KEEP-CRASH-028", + Self::RemoveRecoveryHead => "KEEP-CRASH-029", + Self::SynchronizeRootAfterRecovery => "KEEP-CRASH-030", + Self::OpenAndLockWriterFile => "KEEP-CRASH-031", + Self::CreateStagingDirectory => "KEEP-CRASH-032", + Self::CreateSegmentPoolDirectory => "KEEP-CRASH-033", + Self::CreateCatalogPoolDirectory => "KEEP-CRASH-034", + Self::SynchronizeRootAfterInitialization => "KEEP-CRASH-035", + } + } +} diff --git a/xtask/src/lib.rs b/xtask/src/lib.rs index dfd4e8c..5f164b7 100644 --- a/xtask/src/lib.rs +++ b/xtask/src/lib.rs @@ -26,6 +26,11 @@ mod repository_json; pub mod protocol_admission; +#[cfg(feature = "repository-tasks")] +mod durability_crash_point; +#[cfg(feature = "repository-tasks")] +mod durability_crash_point_identity; + #[cfg(test)] #[allow( clippy::redundant_pub_crate, @@ -33,6 +38,9 @@ pub mod protocol_admission; )] mod test_directory; +#[cfg(feature = "repository-tasks")] +pub use durability_crash_point::{DurabilityCrashPoint, DurabilityCrashSequence}; + /// Whether one bounded Golden File Worldline production parser admitted input. #[cfg(feature = "golden-protocol-fuzz")] #[derive(Clone, Copy, Debug, Eq, PartialEq)] diff --git a/xtask/tests/durability_crash_point_contract.rs b/xtask/tests/durability_crash_point_contract.rs new file mode 100644 index 0000000..1945b01 --- /dev/null +++ b/xtask/tests/durability_crash_point_contract.rs @@ -0,0 +1,186 @@ +//! Executable identity contract for deterministic crash injection. + +#![cfg(feature = "repository-tasks")] + +use xtask::{DurabilityCrashPoint, DurabilityCrashSequence}; + +use DurabilityCrashSequence::{Catalog, Head, Initialization, RecoveryDiscard, Segment}; + +const EXPECTED: &[(DurabilityCrashPoint, &str, DurabilityCrashSequence)] = &[ + ( + DurabilityCrashPoint::CreateSegmentStage, + "KEEP-CRASH-001", + Segment, + ), + ( + DurabilityCrashPoint::WriteSegmentHeader, + "KEEP-CRASH-002", + Segment, + ), + ( + DurabilityCrashPoint::AppendSegmentRecord, + "KEEP-CRASH-003", + Segment, + ), + ( + DurabilityCrashPoint::FlushSegmentRecordPrefix, + "KEEP-CRASH-004", + Segment, + ), + ( + DurabilityCrashPoint::SynchronizeSegmentRecordPrefix, + "KEEP-CRASH-005", + Segment, + ), + ( + DurabilityCrashPoint::AppendSegmentSeal, + "KEEP-CRASH-006", + Segment, + ), + ( + DurabilityCrashPoint::FlushSealedSegment, + "KEEP-CRASH-007", + Segment, + ), + ( + DurabilityCrashPoint::SynchronizeSealedSegment, + "KEEP-CRASH-008", + Segment, + ), + (DurabilityCrashPoint::LinkSegment, "KEEP-CRASH-009", Segment), + ( + DurabilityCrashPoint::SynchronizeSegmentPool, + "KEEP-CRASH-010", + Segment, + ), + ( + DurabilityCrashPoint::RemoveSegmentStage, + "KEEP-CRASH-011", + Segment, + ), + ( + DurabilityCrashPoint::SynchronizeStagingAfterSegment, + "KEEP-CRASH-012", + Segment, + ), + ( + DurabilityCrashPoint::CreateCatalogStage, + "KEEP-CRASH-013", + Catalog, + ), + ( + DurabilityCrashPoint::WriteCatalog, + "KEEP-CRASH-014", + Catalog, + ), + ( + DurabilityCrashPoint::FlushCatalog, + "KEEP-CRASH-015", + Catalog, + ), + ( + DurabilityCrashPoint::SynchronizeCatalog, + "KEEP-CRASH-016", + Catalog, + ), + (DurabilityCrashPoint::LinkCatalog, "KEEP-CRASH-017", Catalog), + ( + DurabilityCrashPoint::SynchronizeCatalogPool, + "KEEP-CRASH-018", + Catalog, + ), + ( + DurabilityCrashPoint::RemoveCatalogStage, + "KEEP-CRASH-019", + Catalog, + ), + ( + DurabilityCrashPoint::SynchronizeStagingAfterCatalog, + "KEEP-CRASH-020", + Catalog, + ), + ( + DurabilityCrashPoint::CreateHeadStage, + "KEEP-CRASH-021", + Head, + ), + (DurabilityCrashPoint::WriteHead, "KEEP-CRASH-022", Head), + (DurabilityCrashPoint::FlushHead, "KEEP-CRASH-023", Head), + ( + DurabilityCrashPoint::SynchronizeHead, + "KEEP-CRASH-024", + Head, + ), + (DurabilityCrashPoint::ReplaceHead, "KEEP-CRASH-025", Head), + ( + DurabilityCrashPoint::SynchronizeRootAfterHead, + "KEEP-CRASH-026", + Head, + ), + ( + DurabilityCrashPoint::RemoveRecoveryStage, + "KEEP-CRASH-027", + RecoveryDiscard, + ), + ( + DurabilityCrashPoint::SynchronizeStagingAfterRecovery, + "KEEP-CRASH-028", + RecoveryDiscard, + ), + ( + DurabilityCrashPoint::RemoveRecoveryHead, + "KEEP-CRASH-029", + RecoveryDiscard, + ), + ( + DurabilityCrashPoint::SynchronizeRootAfterRecovery, + "KEEP-CRASH-030", + RecoveryDiscard, + ), + ( + DurabilityCrashPoint::OpenAndLockWriterFile, + "KEEP-CRASH-031", + Initialization, + ), + ( + DurabilityCrashPoint::CreateStagingDirectory, + "KEEP-CRASH-032", + Initialization, + ), + ( + DurabilityCrashPoint::CreateSegmentPoolDirectory, + "KEEP-CRASH-033", + Initialization, + ), + ( + DurabilityCrashPoint::CreateCatalogPoolDirectory, + "KEEP-CRASH-034", + Initialization, + ), + ( + DurabilityCrashPoint::SynchronizeRootAfterInitialization, + "KEEP-CRASH-035", + Initialization, + ), +]; + +#[test] +fn crash_boundaries_have_one_contiguous_stable_vocabulary() { + let actual = + DurabilityCrashPoint::ALL.map(|point| (point, point.identifier(), point.sequence())); + + assert_eq!(actual.as_slice(), EXPECTED); +} + +#[test] +fn only_record_append_selects_an_occurrence() { + let occurrence_counted: Vec<_> = DurabilityCrashPoint::ALL + .into_iter() + .filter(|point| point.occurrence_counted()) + .collect(); + + assert_eq!( + occurrence_counted, + [DurabilityCrashPoint::AppendSegmentRecord] + ); +} From 8ad009d4c61c1424837013f01f74538093b637f0 Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 04:53:37 -0700 Subject: [PATCH 02/49] Order store initialization transitions --- CHANGELOG.md | 4 + docs/formats/segment-store-v1/requirements.md | 7 +- src/adapters/mod.rs | 10 ++ src/adapters/store_initialization.rs | 68 ++++++++++++ src/adapters/store_initialization_error.rs | 37 +++++++ src/adapters/store_initialization_phase.rs | 33 ++++++ src/adapters/store_initialization_receipt.rs | 16 +++ src/adapters/store_initialization_storage.rs | 52 +++++++++ src/lib.rs | 13 ++- tests/store_initialization.rs | 102 ++++++++++++++++++ 10 files changed, 335 insertions(+), 7 deletions(-) create mode 100644 src/adapters/store_initialization.rs create mode 100644 src/adapters/store_initialization_error.rs create mode 100644 src/adapters/store_initialization_phase.rs create mode 100644 src/adapters/store_initialization_receipt.rs create mode 100644 src/adapters/store_initialization_storage.rs create mode 100644 tests/store_initialization.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 37112c1..724d1f0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,10 @@ after its public API and format compatibility policies are established. ### Changed +- Store initialization now exposes one storage-port state machine that admits + the platform before mutation, opens and locks `writer.lock`, admits the three + protocol directories in order, synchronizes the root, and preserves the + exact failed phase without executing later transitions. - Repository crash-matrix tooling now exposes one typed, ordered vocabulary for `KEEP-CRASH-001` through `KEEP-CRASH-035`. Each identifier is bound to its segment, catalog, head, recovery-discard, or initialization sequence, and diff --git a/docs/formats/segment-store-v1/requirements.md b/docs/formats/segment-store-v1/requirements.md index c3da296..3e16e74 100644 --- a/docs/formats/segment-store-v1/requirements.md +++ b/docs/formats/segment-store-v1/requirements.md @@ -86,14 +86,17 @@ recovery. Issue #17 implements crash injection and explicit recovery in independently reviewable slices. The first slice freezes executable crash-point identity and -sequence ownership. It does not claim process-death injection, initialization, -recovery classification, or recovery execution. +sequence ownership. The second slice establishes the ordered initialization +state machine and exact failure phases. It does not claim process-death +injection, production platform admission, recovery classification, or recovery +execution. | ID | Implemented requirement | Oracle | Executable evidence | Status | | --- | --- | --- | --- | --- | | `KEEP-RECOVERY-001` | Crash identifiers `KEEP-CRASH-001` through `KEEP-CRASH-035` form one contiguous typed vocabulary, map to the exact owning protocol sequence, and admit an occurrence counter only for record append | Ordered identifier-and-sequence ledger | `xtask/tests/durability_crash_point_contract.rs` | Implemented in #17 | +| `KEEP-RECOVERY-002` | Initialization admits the platform before mutation, opens and locks the writer file, admits `staging`, `segments`, and `catalogs` in order, and returns a receipt only after root synchronization; every failed operation retains its exact phase and prevents later transitions | Fault-recording initialization port | `src/adapters/store_initialization_tests.rs` | Implemented in #17 | diff --git a/src/adapters/mod.rs b/src/adapters/mod.rs index 184f4fe..941ec78 100644 --- a/src/adapters/mod.rs +++ b/src/adapters/mod.rs @@ -164,6 +164,11 @@ mod segment_write_phase; mod staged_segment; mod storage_profile_id_text; mod storage_profile_id_text_error; +mod store_initialization; +mod store_initialization_error; +mod store_initialization_phase; +mod store_initialization_receipt; +mod store_initialization_storage; mod sync_capable_directory; #[cfg(test)] #[path = "../../tests/support/mod.rs"] @@ -243,6 +248,11 @@ pub use segment_write_error::SegmentWriteError; pub use segment_write_phase::{SegmentDurabilityPhase, SegmentWritePhase}; pub use staged_segment::StagedSegment; pub use storage_profile_id_text_error::StorageProfileIdParseError; +pub use store_initialization::initialize_store; +pub use store_initialization_error::StoreInitializationError; +pub use store_initialization_phase::StoreInitializationPhase; +pub use store_initialization_receipt::StoreInitializationReceipt; +pub use store_initialization_storage::StoreInitializationStorage; pub use writer_lock_acquire_error::WriterLockAcquireError; pub use writer_lock_acquire_phase::WriterLockAcquirePhase; diff --git a/src/adapters/store_initialization.rs b/src/adapters/store_initialization.rs new file mode 100644 index 0000000..7a32484 --- /dev/null +++ b/src/adapters/store_initialization.rs @@ -0,0 +1,68 @@ +//! This module owns ordered segment-store namespace initialization. + +use std::io; + +use super::{ + store_initialization_error::StoreInitializationError, + store_initialization_phase::StoreInitializationPhase, + store_initialization_receipt::StoreInitializationReceipt, + store_initialization_storage::StoreInitializationStorage, +}; + +/// Executes the ordered segment-store initialization protocol. +/// +/// The storage port must admit its platform before any namespace mutation, +/// retain writer authority after opening the lock file, and make every +/// directory operation idempotent by admitting an exact existing directory. +/// The function returns a receipt only after root synchronization succeeds. +/// It is synchronous, performs no internal heap allocation, invokes each port +/// operation at most once, and may block only inside the supplied storage +/// operations. Failure performs no implicit cleanup. +/// +/// # Errors +/// +/// Returns [`StoreInitializationError::Io`] at the first failed phase and does +/// not execute any later phase. +pub fn initialize_store( + storage: &mut impl StoreInitializationStorage, +) -> Result { + phase( + StoreInitializationPhase::AdmitPlatform, + storage.admit_platform(), + )?; + phase( + StoreInitializationPhase::OpenAndLockWriterFile, + storage.open_and_lock_writer_file(), + )?; + admit_directories(storage)?; + phase( + StoreInitializationPhase::SynchronizeRoot, + storage.synchronize_root(), + )?; + Ok(StoreInitializationReceipt::new()) +} + +fn admit_directories( + storage: &mut impl StoreInitializationStorage, +) -> Result<(), StoreInitializationError> { + phase( + StoreInitializationPhase::AdmitStagingDirectory, + storage.admit_staging_directory(), + )?; + phase( + StoreInitializationPhase::AdmitSegmentPoolDirectory, + storage.admit_segment_pool_directory(), + )?; + phase( + StoreInitializationPhase::AdmitCatalogPoolDirectory, + storage.admit_catalog_pool_directory(), + )?; + Ok(()) +} + +fn phase( + phase: StoreInitializationPhase, + result: io::Result<()>, +) -> Result<(), StoreInitializationError> { + result.map_err(|source| StoreInitializationError::Io { phase, source }) +} diff --git a/src/adapters/store_initialization_error.rs b/src/adapters/store_initialization_error.rs new file mode 100644 index 0000000..3516548 --- /dev/null +++ b/src/adapters/store_initialization_error.rs @@ -0,0 +1,37 @@ +//! This module owns typed segment-store initialization failures. + +use std::error::Error; +use std::fmt; +use std::io; + +use super::store_initialization_phase::StoreInitializationPhase; + +/// Failure while executing ordered store initialization. +#[derive(Debug)] +pub enum StoreInitializationError { + /// One initialization phase returned an I/O failure. + Io { + /// Exact failed initialization phase. + phase: StoreInitializationPhase, + /// Underlying storage failure. + source: io::Error, + }, +} + +impl fmt::Display for StoreInitializationError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Io { phase, .. } => { + write!(formatter, "store initialization failed during {phase}") + } + } + } +} + +impl Error for StoreInitializationError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Io { source, .. } => Some(source), + } + } +} diff --git a/src/adapters/store_initialization_phase.rs b/src/adapters/store_initialization_phase.rs new file mode 100644 index 0000000..9090090 --- /dev/null +++ b/src/adapters/store_initialization_phase.rs @@ -0,0 +1,33 @@ +//! This module owns exact segment-store initialization phases. + +use std::fmt; + +/// Exact operation attempted during store initialization. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum StoreInitializationPhase { + /// Prove the platform contract before namespace mutation. + AdmitPlatform, + /// Create or reopen `writer.lock`, then acquire writer authority. + OpenAndLockWriterFile, + /// Create or verify the `staging` directory. + AdmitStagingDirectory, + /// Create or verify the `segments` directory. + AdmitSegmentPoolDirectory, + /// Create or verify the `catalogs` directory. + AdmitCatalogPoolDirectory, + /// Synchronize the store root after all canonical names exist. + SynchronizeRoot, +} + +impl fmt::Display for StoreInitializationPhase { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(match self { + Self::AdmitPlatform => "platform admission", + Self::OpenAndLockWriterFile => "writer-lock admission", + Self::AdmitStagingDirectory => "staging-directory admission", + Self::AdmitSegmentPoolDirectory => "segment-pool admission", + Self::AdmitCatalogPoolDirectory => "catalog-pool admission", + Self::SynchronizeRoot => "root synchronization", + }) + } +} diff --git a/src/adapters/store_initialization_receipt.rs b/src/adapters/store_initialization_receipt.rs new file mode 100644 index 0000000..569847c --- /dev/null +++ b/src/adapters/store_initialization_receipt.rs @@ -0,0 +1,16 @@ +//! This module owns proof of complete store-root initialization. + +/// Proof that ordered store initialization reached root synchronization. +/// +/// Private fields prevent callers from manufacturing a successful receipt. +#[derive(Debug)] +#[must_use] +pub struct StoreInitializationReceipt { + _private: (), +} + +impl StoreInitializationReceipt { + pub(super) const fn new() -> Self { + Self { _private: () } + } +} diff --git a/src/adapters/store_initialization_storage.rs b/src/adapters/store_initialization_storage.rs new file mode 100644 index 0000000..5ab21b1 --- /dev/null +++ b/src/adapters/store_initialization_storage.rs @@ -0,0 +1,52 @@ +//! This module owns the storage port for store initialization. + +use std::io; + +/// Semantic storage operations required by ordered store initialization. +/// +/// Implementations own platform proof, namespace technology, writer +/// authority, and idempotent create-or-admit behavior. The orchestration layer +/// owns only ordering and exact failure attribution. +pub trait StoreInitializationStorage { + /// Proves the complete platform contract without mutating the namespace. + /// + /// # Errors + /// + /// Returns an I/O error when the platform cannot be proved admissible. + fn admit_platform(&mut self) -> io::Result<()>; + + /// Creates or reopens the writer file and retains its exclusive lock. + /// + /// # Errors + /// + /// Returns an I/O error when the writer file or lock cannot be admitted. + fn open_and_lock_writer_file(&mut self) -> io::Result<()>; + + /// Creates or verifies the exact `staging` directory. + /// + /// # Errors + /// + /// Returns an I/O error when the staging directory cannot be admitted. + fn admit_staging_directory(&mut self) -> io::Result<()>; + + /// Creates or verifies the exact `segments` directory. + /// + /// # Errors + /// + /// Returns an I/O error when the segment-pool directory cannot be admitted. + fn admit_segment_pool_directory(&mut self) -> io::Result<()>; + + /// Creates or verifies the exact `catalogs` directory. + /// + /// # Errors + /// + /// Returns an I/O error when the catalog-pool directory cannot be admitted. + fn admit_catalog_pool_directory(&mut self) -> io::Result<()>; + + /// Synchronizes the store root after all canonical names exist. + /// + /// # Errors + /// + /// Returns an I/O error when root synchronization fails. + fn synchronize_root(&mut self) -> io::Result<()>; +} diff --git a/src/lib.rs b/src/lib.rs index 0ff6d0a..64d9db9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -12,10 +12,11 @@ //! deterministic streaming chunk detection, canonical flat-layout identity //! and codecs, a capacity-bounded non-durable reference CAS, and explicit //! immutable-segment writing and verified reading, canonical catalog -//! generations, platform-gated filesystem publication mechanics, and bounded -//! immutable restart snapshots. Production platform admission, store -//! initialization, recovery, retention, and garbage collection APIs remain -//! intentionally absent until their contracts have executable specifications. +//! generations, platform-gated filesystem publication mechanics, bounded +//! immutable restart snapshots, and typed store-initialization orchestration. +//! Production filesystem platform admission, recovery, retention, and garbage +//! collection APIs remain intentionally absent until their contracts have +//! executable specifications. #[cfg(test)] extern crate self as keep; @@ -49,7 +50,9 @@ pub use adapters::{ SegmentRecordLength, SegmentRecordLimit, SegmentRecordLimitError, SegmentRecordPayloadLength, SegmentRecords, SegmentSeal, SegmentSealError, SegmentStage, SegmentStageCreateError, SegmentWriteError, SegmentWritePhase, StagedSegment, StorageProfileIdParseError, - WriterLockAcquireError, WriterLockAcquirePhase, publish_catalog_generation, + StoreInitializationError, StoreInitializationPhase, StoreInitializationReceipt, + StoreInitializationStorage, WriterLockAcquireError, WriterLockAcquirePhase, initialize_store, + publish_catalog_generation, }; pub use blob::{ BlobHashError, BlobHasher, BlobId, BlobLength, BlobReadError, ByteLength, ByteOffset, diff --git a/tests/store_initialization.rs b/tests/store_initialization.rs new file mode 100644 index 0000000..a19b641 --- /dev/null +++ b/tests/store_initialization.rs @@ -0,0 +1,102 @@ +//! Deterministic segment-store initialization laws. + +use std::error::Error; +use std::io; + +use keep::{ + StoreInitializationError, StoreInitializationPhase, StoreInitializationStorage, + initialize_store, +}; + +const PHASES: [StoreInitializationPhase; 6] = [ + StoreInitializationPhase::AdmitPlatform, + StoreInitializationPhase::OpenAndLockWriterFile, + StoreInitializationPhase::AdmitStagingDirectory, + StoreInitializationPhase::AdmitSegmentPoolDirectory, + StoreInitializationPhase::AdmitCatalogPoolDirectory, + StoreInitializationPhase::SynchronizeRoot, +]; + +#[test] +fn initialization_admits_platform_before_every_namespace_transition() -> Result<(), Box> +{ + let mut storage = RecordingStorage::new(None); + + let _receipt = initialize_store(&mut storage)?; + + assert_eq!(storage.attempted, PHASES); + Ok(()) +} + +#[test] +fn initialization_stops_at_and_preserves_every_exact_failure_phase() -> Result<(), Box> { + let mut expected_prefix = Vec::new(); + for phase in PHASES { + expected_prefix.push(phase); + let mut storage = RecordingStorage::new(Some(phase)); + let Err(error) = initialize_store(&mut storage) else { + return Err(format!("initialization ignored injected failure at {phase}").into()); + }; + + match error { + StoreInitializationError::Io { + phase: observed, + source, + } => { + assert_eq!(observed, phase); + assert_eq!(source.kind(), io::ErrorKind::Other); + } + } + assert_eq!(storage.attempted, expected_prefix); + } + Ok(()) +} + +struct RecordingStorage { + attempted: Vec, + fail_at: Option, +} + +impl RecordingStorage { + const fn new(fail_at: Option) -> Self { + Self { + attempted: Vec::new(), + fail_at, + } + } + + fn attempt(&mut self, phase: StoreInitializationPhase) -> io::Result<()> { + self.attempted.push(phase); + if self.fail_at == Some(phase) { + Err(io::Error::other("injected initialization failure")) + } else { + Ok(()) + } + } +} + +impl StoreInitializationStorage for RecordingStorage { + fn admit_platform(&mut self) -> io::Result<()> { + self.attempt(StoreInitializationPhase::AdmitPlatform) + } + + fn open_and_lock_writer_file(&mut self) -> io::Result<()> { + self.attempt(StoreInitializationPhase::OpenAndLockWriterFile) + } + + fn admit_staging_directory(&mut self) -> io::Result<()> { + self.attempt(StoreInitializationPhase::AdmitStagingDirectory) + } + + fn admit_segment_pool_directory(&mut self) -> io::Result<()> { + self.attempt(StoreInitializationPhase::AdmitSegmentPoolDirectory) + } + + fn admit_catalog_pool_directory(&mut self) -> io::Result<()> { + self.attempt(StoreInitializationPhase::AdmitCatalogPoolDirectory) + } + + fn synchronize_root(&mut self) -> io::Result<()> { + self.attempt(StoreInitializationPhase::SynchronizeRoot) + } +} From c60def9b4f4e119a655711ea9671f2241cb80292 Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 05:18:59 -0700 Subject: [PATCH 03/49] Add: Admit Linux ext4 store initialization --- CHANGELOG.md | 11 +- Cargo.lock | 1 + Cargo.toml | 1 + README.md | 26 ++--- .../cap-std-and-cap-fs-ext-4.0.2.md | 27 +++-- docs/formats/segment-store-v1/rationale.md | 18 +++- docs/formats/segment-store-v1/recovery.md | 22 ++-- docs/formats/segment-store-v1/requirements.md | 16 +-- .../filesystem_initialization_namespace.rs | 69 ++++++++++++ .../filesystem_initialization_storage.rs | 100 ++++++++++++++++++ src/adapters/filesystem_platform_admission.rs | 7 +- src/adapters/filesystem_platform_profile.rs | 99 +++++++++++++++++ src/adapters/filesystem_store_initializer.rs | 53 ++++++++++ .../filesystem_store_initializer_tests.rs | 97 +++++++++++++++++ src/adapters/filesystem_writer_lock.rs | 62 ++++++++--- src/adapters/mod.rs | 6 ++ src/adapters/store_initialization_error.rs | 6 ++ src/adapters/writer_lock_acquire_phase.rs | 3 + src/lib.rs | 8 +- tests/store_initialization.rs | 28 +++++ 20 files changed, 592 insertions(+), 68 deletions(-) create mode 100644 src/adapters/filesystem_initialization_namespace.rs create mode 100644 src/adapters/filesystem_initialization_storage.rs create mode 100644 src/adapters/filesystem_platform_profile.rs create mode 100644 src/adapters/filesystem_store_initializer.rs create mode 100644 src/adapters/filesystem_store_initializer_tests.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 724d1f0..619842d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,10 @@ after its public API and format compatibility policies are established. ### Changed +- Production filesystem initialization now admits only the documented writable, + non-casefolded Linux ext4 profile, refuses ambiguous root namespaces before + mutation, completes the canonical directory shape idempotently, retains + writer authority, and returns only after synchronizing the root. - Store initialization now exposes one storage-port state machine that admits the platform before mutation, opens and locks `writer.lock`, admits the three protocol directories in order, synchronizes the root, and preserves the @@ -21,10 +25,9 @@ after its public API and format compatibility policies are established. - Catalog decoding now verifies the catalog checksum and physical digest before interpreting entry semantics. Corrupt identity-bearing bytes therefore fail at the integrity boundary instead of producing a semantic entry error. -- Filesystem catalog publisher construction now consumes an unforgeable - `FilesystemPlatformAdmission`. No public producer exists until crash-tested - initialization can establish the platform contract in issue #17; acquiring - `FilesystemWriterLock` alone no longer authorizes production construction. +- Filesystem catalog publisher construction consumes an unforgeable + `FilesystemPlatformAdmission`; acquiring `FilesystemWriterLock` alone does + not authorize production construction. - Filesystem segment selection now consumes sealed stages through the publisher that created them. Process-local publisher authority prevents an unrelated metadata-equivalent `ClosedSegment` from authorizing retained diff --git a/Cargo.lock b/Cargo.lock index 91eea88..b2e76f4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -328,6 +328,7 @@ dependencies = [ "cap-fs-ext", "cap-std", "divan", + "rustix", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index db24b42..5c4d601 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,6 +16,7 @@ publish = false blake3 = { version = "=1.8.5", default-features = false, features = ["pure", "std"] } cap-fs-ext = { version = "=4.0.2", default-features = false, features = ["std"] } cap-std = { version = "=4.0.2", default-features = false } +rustix = { version = "=1.1.4", default-features = false, features = ["fs", "std"] } [dev-dependencies] allocation-counter = { version = "=0.8.1", default-features = false } diff --git a/README.md b/README.md index bd11fad..3faa98f 100644 --- a/README.md +++ b/README.md @@ -37,8 +37,11 @@ verification. A platform-admitted `FilesystemCatalogPublisher` exclusively creates the fixed `current.seg` stage without truncating existing evidence, and the `FilesystemSegmentStage` lifetime keeps that writer authority borrowed until the writable stage closes. Publisher construction consumes an -unforgeable `FilesystemPlatformAdmission`; no public producer exists until -issue #17 implements crash-tested initialization and platform admission. +unforgeable `FilesystemPlatformAdmission`. On Linux, its public initializer +admits only a writable, non-casefolded ext4 root, refuses unknown or aliased +namespace entries before mutation, creates or verifies the canonical +`writer.lock`, `staging`, `segments`, and `catalogs` shape, and returns only +after root synchronization with the writer lock retained. `FilesystemCatalogPublisher` retains one kernel-managed writer lock and pinned root, staging, segment-pool, and catalog-pool capabilities for the complete @@ -56,16 +59,15 @@ logical reads. The reference CAS is executable evidence for M2 storage laws, not a durable backend. Its committed state is process memory; process death loses it all. -The durable boundary does not yet initialize, platform-admit, or recover a -store root. Acquiring `FilesystemWriterLock` alone cannot construct a -filesystem publisher. Issue #17 must admit the exact existing `writer.lock`, -`staging`, `segments`, and `catalogs` namespace before it can return -`FilesystemPlatformAdmission`. Leftover `head.next`, staged recovery evidence, -unknown namespace entries, and ambiguous crash states remain explicit recovery -work. An absent `HEAD` is admitted for first publication only when both -immutable pools are empty. Retention, complete namespace verification, -compaction, and garbage collection remain planned. Presence in the reference -CAS does not claim retention, crash recovery, or durability. +The durable boundary can initialize and platform-admit a store only under the +documented Linux ext4 contract. Acquiring `FilesystemWriterLock` alone cannot +construct a filesystem publisher. Leftover `head.next`, staged recovery +evidence, and ambiguous crash states remain explicit recovery work. An absent +`HEAD` is admitted for first publication only when both immutable pools are +empty. Crash-injection execution, explicit recovery, retention, complete +namespace verification after initialization, compaction, and garbage +collection remain planned. Presence in the reference CAS does not claim +retention, crash recovery, or durability. ```rust use keep::BlobId; diff --git a/docs/dependencies/cap-std-and-cap-fs-ext-4.0.2.md b/docs/dependencies/cap-std-and-cap-fs-ext-4.0.2.md index 4c7c9f9..26abc36 100644 --- a/docs/dependencies/cap-std-and-cap-fs-ext-4.0.2.md +++ b/docs/dependencies/cap-std-and-cap-fs-ext-4.0.2.md @@ -10,8 +10,9 @@ Keep admits the exactly pinned `cap-std` 4.0.2 and `cap-fs-ext` 4.0.2 packages for the library's segment-store filesystem adapter and behind the `xtask` -crate's `repository-tasks` feature. Rustix 1.1.4 remains admitted only behind -that `xtask` feature. +crate's `repository-tasks` feature. The library also admits Rustix 1.1.4 for +safe Linux filesystem-profile inspection and no-symlink root opening; `xtask` +uses the same exact version behind `repository-tasks`. `cap-std::fs::Dir` pins the admitted repository or corpus directory and opens entries relative to that capability. `cap-fs-ext` supplies no-follow and @@ -26,7 +27,10 @@ The capability packages are present in Keep's published library graph and production filesystem behavior. No dependency-owned type crosses Keep's public API or enters content identities or durable formats. The segment-store writer lock retains capability and file handles behind `FilesystemWriterLock`; its -public acquisition boundary accepts only `std::path::Path`. +public acquisition boundary accepts only `std::path::Path`. The production +initializer uses Rustix's safe `openat2`, `fstatfs`, `fstatvfs`, and ext4 inode +flag APIs to admit only the documented writable, non-casefolded Linux ext4 +profile. The bounded subprocess adapter uses Rustix's safe filesystem API to mark child stdin nonblocking before deadline-bounded input transfer. It uses Rustix's safe @@ -66,11 +70,12 @@ work, and require unsafe code that Keep otherwise forbids. ## Features and resolved graph All three direct dependencies disable default features. Keep enables only -`cap-fs-ext`'s `std` feature and Rustix's `fs`, `process`, and `std` features; -`cap-std` has no enabled feature. The library's capability dependencies are -unconditional because the production segment-store adapter requires them. -The `xtask` declarations remain optional and are activated solely by -`repository-tasks`; Rustix is not a direct library dependency. +`cap-fs-ext`'s `std` feature and Rustix's `fs` and `std` features in the +library; `cap-std` has no enabled feature. The library's filesystem +dependencies are unconditional because the production segment-store adapter +requires them. The `xtask` declarations remain optional and are activated +solely by `repository-tasks`; that feature additionally enables Rustix's +`process` feature. The locked non-Windows graph introduced for this boundary is: @@ -117,9 +122,9 @@ dependencies. ## Failure and recovery boundaries -An open, metadata, read, writer-lock acquisition, descriptor-duplication, -descriptor-flag, child-directory setup, child-spawn, stdin-write, -output-collection, deadline, or cleanup failure is a typed refusal. +An open, metadata, platform-profile, read, writer-lock acquisition, +descriptor-duplication, descriptor-flag, child-directory setup, child-spawn, +stdin-write, output-collection, deadline, or cleanup failure is a typed refusal. Repository tasks never repair, rewrite, or substitute repository data. Repository-task handles exist only for one verification process and carry no durability or recovery semantics. `FilesystemWriterLock` retains the pinned diff --git a/docs/formats/segment-store-v1/rationale.md b/docs/formats/segment-store-v1/rationale.md index 9e0d4c0..9586278 100644 --- a/docs/formats/segment-store-v1/rationale.md +++ b/docs/formats/segment-store-v1/rationale.md @@ -152,11 +152,19 @@ therefore consumes an opaque `FilesystemPlatformAdmission` that owns the acquired lock. The proof is bound to that exact root authority and cannot be constructed from metadata or caller assertion. -Issue #16 exposes no public proof producer. Its crate-internal transition tests -use an explicitly test-only unchecked value, while issue #17 owns the -crash-tested initializer and platform checks that may return production -admission. This staging prevents an incomplete probe from turning successful -syscalls into a durability claim. +The production proof producer admits one deliberately narrow profile: Linux, +writable ext4, a store-root inode without ext4 casefolding, no symbolic link in +the selected path, and successful file and directory synchronization calls. +Initialization then refuses any noncanonical root entry before mutation, +acquires and retains `writer.lock`, completes the three canonical directories +in protocol order, and synchronizes the root before returning the proof. + +The profile treats local single-host mounting as an environmental precondition, +not as a fact derivable from `statfs`. A shared or multiply mounted ext4 volume +is outside the admitted operating contract even if its filesystem magic +matches. Keep can prove the selected kernel and filesystem mechanisms; it +cannot prove that an administrator has not exposed the block device to another +host or that acknowledged synchronization survives dishonest hardware. Rejected alternatives were treating `FilesystemWriterLock` as sufficient, accepting a caller-selected boolean or platform enum, and approving any diff --git a/docs/formats/segment-store-v1/recovery.md b/docs/formats/segment-store-v1/recovery.md index 376a708..1a75187 100644 --- a/docs/formats/segment-store-v1/recovery.md +++ b/docs/formats/segment-store-v1/recovery.md @@ -161,23 +161,25 @@ from an untrusted count before bounds and exact total length agree. ## Platform contract -The initial adapter is supported only when it can prove: +The initial production adapter is supported only on Linux when it proves: - capability-relative no-follow access to regular files and directories; -- case-sensitive, byte-preserving directory names without path aliases; +- an ext4 store root whose inode does not enable ext4 casefolding; +- a writable mount and successful file and directory synchronization calls; - atomic same-filesystem no-clobber hard-link creation; - atomic same-filesystem replacement of one regular file by another; -- file synchronization that covers required data and metadata; - directory synchronization that makes create, link, unlink, rename, and replacement durable; - process-scoped exclusive advisory locking; and -- one host with one writer. - -The adapter refuses network filesystems, shared multi-host mounts, filesystem -types with unknown rename or synchronization semantics, symlinked protocol -paths, and platforms whose directory durability cannot be established. -Windows support is deferred until an adapter and crash harness prove -equivalent semantics. +- one retained writer-lock handle for the writer-authority lifetime. + +The adapter refuses every non-ext4 filesystem, read-only mount, casefolded store +root, symlinked selected path, or platform other than Linux. A single local +host is an explicit deployment precondition: filesystem metadata cannot prove +that an administrator has not exposed one block device to another host. Shared +or multiply mounted ext4 is therefore unsupported even though the adapter +cannot distinguish it from a valid local mount. Windows support is deferred +until an adapter and crash harness prove equivalent semantics. The protocol cannot compensate for hardware or an operating system that acknowledges synchronization without honoring it. Documentation and receipts diff --git a/docs/formats/segment-store-v1/requirements.md b/docs/formats/segment-store-v1/requirements.md index 3e16e74..88a8c13 100644 --- a/docs/formats/segment-store-v1/requirements.md +++ b/docs/formats/segment-store-v1/requirements.md @@ -60,9 +60,9 @@ retention, or garbage collection. Issue #16 implements catalog-generation admission, writer-locked filesystem publication mechanics, and immutable reader snapshots. Production publisher -construction requires `FilesystemPlatformAdmission`, whose initialization and -platform-checked producer remains owned by issue #17 together with explicit -recovery. +construction requires `FilesystemPlatformAdmission`, whose platform-checked +producer is implemented as the initialization slice of issue #17. Explicit +recovery remains separate work. @@ -87,16 +87,18 @@ recovery. Issue #17 implements crash injection and explicit recovery in independently reviewable slices. The first slice freezes executable crash-point identity and sequence ownership. The second slice establishes the ordered initialization -state machine and exact failure phases. It does not claim process-death -injection, production platform admission, recovery classification, or recovery -execution. +state machine and exact failure phases. The third slice binds that state +machine to a fail-closed Linux ext4 adapter and canonical namespace. These +slices do not yet claim process-death injection, recovery classification, or +recovery execution. | ID | Implemented requirement | Oracle | Executable evidence | Status | | --- | --- | --- | --- | --- | | `KEEP-RECOVERY-001` | Crash identifiers `KEEP-CRASH-001` through `KEEP-CRASH-035` form one contiguous typed vocabulary, map to the exact owning protocol sequence, and admit an occurrence counter only for record append | Ordered identifier-and-sequence ledger | `xtask/tests/durability_crash_point_contract.rs` | Implemented in #17 | -| `KEEP-RECOVERY-002` | Initialization admits the platform before mutation, opens and locks the writer file, admits `staging`, `segments`, and `catalogs` in order, and returns a receipt only after root synchronization; every failed operation retains its exact phase and prevents later transitions | Fault-recording initialization port | `src/adapters/store_initialization_tests.rs` | Implemented in #17 | +| `KEEP-RECOVERY-002` | Initialization admits the platform before mutation, opens and locks the writer file, admits `staging`, `segments`, and `catalogs` in order, and returns a receipt only after root synchronization; every failed operation retains its exact phase and prevents later transitions | Fault-recording initialization port | `tests/store_initialization.rs` | Implemented in #17 | +| `KEEP-RECOVERY-003` | Production initialization admits only a writable, non-casefolded Linux ext4 root, refuses any noncanonical root entry before mutation, completes an empty or partial canonical namespace without replacing evidence, excludes a second initializer, and retains writer authority through the synchronized receipt | Capability-relative filesystem fixture and exact platform-profile classifier | `src/adapters/filesystem_store_initializer_tests.rs`, `src/adapters/filesystem_platform_profile.rs`, `tests/store_initialization.rs` | Implemented in #17 | diff --git a/src/adapters/filesystem_initialization_namespace.rs b/src/adapters/filesystem_initialization_namespace.rs new file mode 100644 index 0000000..a6844e2 --- /dev/null +++ b/src/adapters/filesystem_initialization_namespace.rs @@ -0,0 +1,69 @@ +//! This module owns bounded admission of the initialization namespace. + +use std::ffi::OsStr; +use std::io; + +use cap_std::fs::Dir; + +const LOCK_NAME: &str = "writer.lock"; +const STAGING_NAME: &str = "staging"; +const SEGMENTS_NAME: &str = "segments"; +const CATALOGS_NAME: &str = "catalogs"; +const CANONICAL_ENTRY_COUNT: usize = 4; + +pub(super) fn admit(directory: &Dir) -> io::Result<()> { + admit_optional_file(directory, LOCK_NAME)?; + admit_optional_directory(directory, STAGING_NAME)?; + admit_optional_directory(directory, SEGMENTS_NAME)?; + admit_optional_directory(directory, CATALOGS_NAME)?; + admit_membership(directory) +} + +fn admit_optional_file(directory: &Dir, name: &str) -> io::Result<()> { + admit_optional_kind(directory, name, cap_std::fs::FileType::is_file) +} + +fn admit_optional_directory(directory: &Dir, name: &str) -> io::Result<()> { + admit_optional_kind(directory, name, cap_std::fs::FileType::is_dir) +} + +fn admit_optional_kind( + directory: &Dir, + name: &str, + expected: fn(&cap_std::fs::FileType) -> bool, +) -> io::Result<()> { + match directory.symlink_metadata(name) { + Ok(metadata) if expected(&metadata.file_type()) => Ok(()), + Ok(_) => Err(ambiguous_namespace()), + Err(source) if source.kind() == io::ErrorKind::NotFound => Ok(()), + Err(source) => Err(source), + } +} + +fn admit_membership(directory: &Dir) -> io::Result<()> { + let mut observed = 0_usize; + for entry in directory.entries()? { + observed = observed.checked_add(1).ok_or_else(ambiguous_namespace)?; + if observed > CANONICAL_ENTRY_COUNT { + return Err(ambiguous_namespace()); + } + let name = entry?.file_name(); + if !is_canonical(&name) { + return Err(ambiguous_namespace()); + } + } + Ok(()) +} + +fn is_canonical(name: &OsStr) -> bool { + [LOCK_NAME, STAGING_NAME, SEGMENTS_NAME, CATALOGS_NAME] + .into_iter() + .any(|candidate| name == candidate) +} + +fn ambiguous_namespace() -> io::Error { + io::Error::new( + io::ErrorKind::InvalidData, + "store root is not an empty or partial canonical initialization namespace", + ) +} diff --git a/src/adapters/filesystem_initialization_storage.rs b/src/adapters/filesystem_initialization_storage.rs new file mode 100644 index 0000000..06c4184 --- /dev/null +++ b/src/adapters/filesystem_initialization_storage.rs @@ -0,0 +1,100 @@ +//! This module owns concrete filesystem initialization operations. + +use std::io; +use std::path::Path; + +#[cfg(test)] +use cap_std::ambient_authority; +use cap_std::fs::Dir; + +use super::filesystem_initialization_namespace; +use super::filesystem_platform_profile; +use super::sync_capable_directory; +use super::{FilesystemWriterLock, StoreInitializationStorage}; + +const STAGING_NAME: &str = "staging"; +const SEGMENTS_NAME: &str = "segments"; +const CATALOGS_NAME: &str = "catalogs"; + +pub(super) struct FilesystemInitializationStorage { + directory: Dir, + lock: Option, +} + +impl FilesystemInitializationStorage { + pub(super) fn admit(store_root: &Path) -> io::Result { + let directory = filesystem_platform_profile::open(store_root)?; + Ok(Self { + directory, + lock: None, + }) + } + + #[cfg(test)] + pub(super) fn admit_unchecked_for_tests(store_root: &Path) -> io::Result { + let directory = Dir::open_ambient_dir(store_root, ambient_authority())?; + Ok(Self { + directory, + lock: None, + }) + } + + pub(super) fn into_lock(self) -> io::Result { + self.lock.ok_or_else(|| { + io::Error::other("initialization completed without retained writer authority") + }) + } + + fn admit_directory(&self, name: &str) -> io::Result<()> { + if self.lock.is_none() { + return Err(io::Error::other( + "initialization directory mutation requires writer authority", + )); + } + match self.directory.create_dir(name) { + Ok(()) => {} + Err(source) if source.kind() == io::ErrorKind::AlreadyExists => {} + Err(source) => return Err(source), + } + let admitted = sync_capable_directory::open(&self.directory, name)?; + drop(admitted); + Ok(()) + } +} + +impl StoreInitializationStorage for FilesystemInitializationStorage { + fn admit_platform(&mut self) -> io::Result<()> { + filesystem_initialization_namespace::admit(&self.directory) + } + + fn open_and_lock_writer_file(&mut self) -> io::Result<()> { + if self.lock.is_some() { + return Err(io::Error::other("writer authority was already acquired")); + } + let lock = FilesystemWriterLock::initialize_in(self.directory.try_clone()?) + .map_err(io::Error::other)?; + self.lock = Some(lock); + filesystem_initialization_namespace::admit(&self.directory) + } + + fn admit_staging_directory(&mut self) -> io::Result<()> { + self.admit_directory(STAGING_NAME) + } + + fn admit_segment_pool_directory(&mut self) -> io::Result<()> { + self.admit_directory(SEGMENTS_NAME) + } + + fn admit_catalog_pool_directory(&mut self) -> io::Result<()> { + self.admit_directory(CATALOGS_NAME) + } + + fn synchronize_root(&mut self) -> io::Result<()> { + if self.lock.is_none() { + return Err(io::Error::other( + "root synchronization requires writer authority", + )); + } + self.directory.try_clone()?.into_std_file().sync_all() + } +} diff --git a/src/adapters/filesystem_platform_admission.rs b/src/adapters/filesystem_platform_admission.rs index 013533e..a44e8ec 100644 --- a/src/adapters/filesystem_platform_admission.rs +++ b/src/adapters/filesystem_platform_admission.rs @@ -5,14 +5,17 @@ use super::FilesystemWriterLock; /// Exclusive writer authority over a platform-admitted filesystem root. /// /// Fields are private so only Keep's initialization and platform-admission -/// boundary can create production values. That boundary remains intentionally -/// absent until issue #17 supplies its crash-tested implementation. +/// boundary can create production values. #[must_use] pub struct FilesystemPlatformAdmission { lock: FilesystemWriterLock, } impl FilesystemPlatformAdmission { + pub(super) const fn initialized(lock: FilesystemWriterLock) -> Self { + Self { lock } + } + #[cfg(test)] pub(super) const fn unchecked_for_tests(lock: FilesystemWriterLock) -> Self { Self { lock } diff --git a/src/adapters/filesystem_platform_profile.rs b/src/adapters/filesystem_platform_profile.rs new file mode 100644 index 0000000..11a29c6 --- /dev/null +++ b/src/adapters/filesystem_platform_profile.rs @@ -0,0 +1,99 @@ +//! This module owns fail-closed filesystem platform-profile admission. + +use std::io; +use std::path::Path; + +use cap_std::fs::Dir; + +#[cfg(target_os = "linux")] +pub(super) fn open(store_root: &Path) -> io::Result { + use std::fs::File; + + use rustix::fs::{CWD, Mode, OFlags, ResolveFlags, openat2}; + + let descriptor = openat2( + CWD, + store_root, + OFlags::RDONLY | OFlags::DIRECTORY | OFlags::CLOEXEC, + Mode::empty(), + ResolveFlags::NO_MAGICLINKS | ResolveFlags::NO_SYMLINKS, + )?; + let directory = Dir::from_std_file(File::from(descriptor)); + admit_linux_profile(&directory)?; + Ok(directory) +} + +#[cfg(not(target_os = "linux"))] +pub(super) fn open(_store_root: &Path) -> io::Result { + Err(io::Error::new( + io::ErrorKind::Unsupported, + "filesystem initialization currently requires the admitted Linux ext4 profile", + )) +} + +#[cfg(target_os = "linux")] +fn admit_linux_profile(directory: &Dir) -> io::Result<()> { + use rustix::fs::{fstatfs, fstatvfs, ioctl_getflags}; + + let file = directory.try_clone()?.into_std_file(); + let filesystem = fstatfs(&file)?; + let mount = fstatvfs(&file)?; + let inode_flags = ioctl_getflags(&file)?; + admit_linux_properties(filesystem.f_type, mount.f_flag, inode_flags.bits())?; + file.sync_all() +} + +#[cfg(target_os = "linux")] +fn admit_linux_properties( + filesystem_type: rustix::fs::FsWord, + mount_flags: rustix::fs::StatVfsMountFlags, + inode_flags: u32, +) -> io::Result<()> { + // These values are the Linux UAPI ext4 superblock magic and per-directory + // casefold inode flag. Keeping them local makes the admitted profile + // visible at the exact decision boundary. + const EXT4_SUPER_MAGIC: rustix::fs::FsWord = 0x0000_ef53; + const EXT4_CASEFOLD_FLAG: u32 = 0x4000_0000; + + if filesystem_type != EXT4_SUPER_MAGIC + || mount_flags.contains(rustix::fs::StatVfsMountFlags::RDONLY) + || inode_flags & EXT4_CASEFOLD_FLAG != 0 + { + return Err(io::Error::new( + io::ErrorKind::Unsupported, + "store root does not satisfy the admitted local case-sensitive ext4 profile", + )); + } + Ok(()) +} + +#[cfg(all(test, target_os = "linux"))] +mod tests { + use super::admit_linux_properties; + + use rustix::fs::{NFS_SUPER_MAGIC, StatVfsMountFlags}; + + const EXT4_SUPER_MAGIC: rustix::fs::FsWord = 0x0000_ef53; + const EXT4_CASEFOLD_FLAG: u32 = 0x4000_0000; + + #[test] + fn only_writable_case_sensitive_ext4_is_admitted() { + assert!(admit_linux_properties(EXT4_SUPER_MAGIC, StatVfsMountFlags::empty(), 0).is_ok()); + assert!(matches!( + admit_linux_properties( + EXT4_SUPER_MAGIC, + StatVfsMountFlags::empty(), + EXT4_CASEFOLD_FLAG, + ), + Err(ref error) if error.kind() == std::io::ErrorKind::Unsupported + )); + assert!(matches!( + admit_linux_properties(EXT4_SUPER_MAGIC, StatVfsMountFlags::RDONLY, 0), + Err(ref error) if error.kind() == std::io::ErrorKind::Unsupported + )); + assert!(matches!( + admit_linux_properties(NFS_SUPER_MAGIC, StatVfsMountFlags::empty(), 0), + Err(ref error) if error.kind() == std::io::ErrorKind::Unsupported + )); + } +} diff --git a/src/adapters/filesystem_store_initializer.rs b/src/adapters/filesystem_store_initializer.rs new file mode 100644 index 0000000..102f915 --- /dev/null +++ b/src/adapters/filesystem_store_initializer.rs @@ -0,0 +1,53 @@ +//! This module owns production filesystem platform admission and initialization. + +use std::path::Path; + +use super::filesystem_initialization_storage::FilesystemInitializationStorage; +use super::{ + FilesystemPlatformAdmission, StoreInitializationError, StoreInitializationPhase, + initialize_store, +}; + +impl FilesystemPlatformAdmission { + /// Initializes or resumes one admitted filesystem store root. + /// + /// The production adapter currently admits only a writable, + /// case-sensitive Linux ext4 root opened without symbolic links. It + /// validates an empty or partial canonical namespace before mutation, + /// retains the exclusive writer lock, and returns only after synchronizing + /// the complete root namespace. The call is synchronous, allocates no + /// content-sized memory, and may block on filesystem I/O. + /// + /// # Errors + /// + /// Returns [`StoreInitializationError::Io`] at the exact failed phase. + /// Unsupported, read-only, aliased, unknown, or ambiguous platform state is + /// refused before protocol directory creation. + pub fn initialize(store_root: &Path) -> Result { + let storage = FilesystemInitializationStorage::admit(store_root).map_err(|source| { + StoreInitializationError::io(StoreInitializationPhase::AdmitPlatform, source) + })?; + initialize_storage(storage) + } + + #[cfg(test)] + pub(super) fn initialize_unchecked_for_tests( + store_root: &Path, + ) -> Result { + let storage = FilesystemInitializationStorage::admit_unchecked_for_tests(store_root) + .map_err(|source| { + StoreInitializationError::io(StoreInitializationPhase::AdmitPlatform, source) + })?; + initialize_storage(storage) + } +} + +fn initialize_storage( + mut storage: FilesystemInitializationStorage, +) -> Result { + let _receipt = initialize_store(&mut storage)?; + let lock = storage.into_lock().map_err(|source| { + StoreInitializationError::io(StoreInitializationPhase::OpenAndLockWriterFile, source) + })?; + Ok(FilesystemPlatformAdmission::initialized(lock)) +} diff --git a/src/adapters/filesystem_store_initializer_tests.rs b/src/adapters/filesystem_store_initializer_tests.rs new file mode 100644 index 0000000..8c6563b --- /dev/null +++ b/src/adapters/filesystem_store_initializer_tests.rs @@ -0,0 +1,97 @@ +//! Concrete crash-safe filesystem initialization laws. + +use std::error::Error; +use std::fs; + +use super::filesystem_test_sandbox::TestDirectory; +use super::{FilesystemPlatformAdmission, StoreInitializationError, StoreInitializationPhase}; + +const LOCK_NAME: &str = "writer.lock"; +const STAGING_NAME: &str = "staging"; +const SEGMENTS_NAME: &str = "segments"; +const CATALOGS_NAME: &str = "catalogs"; + +#[test] +fn empty_namespace_is_admitted_only_with_the_complete_root_shape() -> Result<(), Box> { + let sandbox = TestDirectory::create("store-initialization-empty")?; + + let admission = FilesystemPlatformAdmission::initialize_unchecked_for_tests(sandbox.path())?; + + assert!(sandbox.path().join(LOCK_NAME).is_file()); + assert!(sandbox.path().join(STAGING_NAME).is_dir()); + assert!(sandbox.path().join(SEGMENTS_NAME).is_dir()); + assert!(sandbox.path().join(CATALOGS_NAME).is_dir()); + drop(admission); + sandbox.remove()?; + Ok(()) +} + +#[test] +fn partial_canonical_namespace_is_completed_without_replacing_evidence() +-> Result<(), Box> { + let sandbox = TestDirectory::create("store-initialization-partial")?; + let retained = b"retained lock evidence"; + fs::write(sandbox.path().join(LOCK_NAME), retained)?; + fs::create_dir(sandbox.path().join(STAGING_NAME))?; + + let admission = FilesystemPlatformAdmission::initialize_unchecked_for_tests(sandbox.path())?; + + assert_eq!(fs::read(sandbox.path().join(LOCK_NAME))?, retained); + assert!(sandbox.path().join(STAGING_NAME).is_dir()); + assert!(sandbox.path().join(SEGMENTS_NAME).is_dir()); + assert!(sandbox.path().join(CATALOGS_NAME).is_dir()); + drop(admission); + sandbox.remove()?; + Ok(()) +} + +#[test] +fn unknown_namespace_refuses_before_writer_file_creation() -> Result<(), Box> { + let sandbox = TestDirectory::create("store-initialization-unknown")?; + fs::write(sandbox.path().join("unknown"), [])?; + + let Err(error) = FilesystemPlatformAdmission::initialize_unchecked_for_tests(sandbox.path()) + else { + return Err("initializer admitted an unknown namespace entry".into()); + }; + + assert!(matches!( + error, + StoreInitializationError::Io { + phase: StoreInitializationPhase::AdmitPlatform, + ref source, + } if source.kind() == std::io::ErrorKind::InvalidData + )); + assert!(!sandbox.path().join(LOCK_NAME).exists()); + sandbox.remove()?; + Ok(()) +} + +#[test] +fn retained_initializer_authority_excludes_a_second_initializer() -> Result<(), Box> { + let sandbox = TestDirectory::create("store-initialization-exclusion")?; + let first = FilesystemPlatformAdmission::initialize_unchecked_for_tests(sandbox.path())?; + + let Err(error) = FilesystemPlatformAdmission::initialize_unchecked_for_tests(sandbox.path()) + else { + return Err("second initializer acquired live writer authority".into()); + }; + + assert!(matches!( + error, + StoreInitializationError::Io { + phase: StoreInitializationPhase::OpenAndLockWriterFile, + ref source, + } if matches!( + source + .get_ref() + .and_then(|nested| nested.downcast_ref::()), + Some(super::WriterLockAcquireError::Busy) + ) + )); + drop(first); + let successor = FilesystemPlatformAdmission::initialize_unchecked_for_tests(sandbox.path())?; + drop(successor); + sandbox.remove()?; + Ok(()) +} diff --git a/src/adapters/filesystem_writer_lock.rs b/src/adapters/filesystem_writer_lock.rs index fc02da6..9911ff1 100644 --- a/src/adapters/filesystem_writer_lock.rs +++ b/src/adapters/filesystem_writer_lock.rs @@ -1,6 +1,7 @@ //! Persistent, capability-relative filesystem writer exclusion. use std::fs::{File, TryLockError}; +use std::io; use std::path::Path; use cap_fs_ext::{FollowSymlinks, OpenOptionsFollowExt, OpenOptionsSyncExt}; @@ -19,7 +20,7 @@ const LOCK_FILE_NAME: &str = "writer.lock"; #[must_use] pub struct FilesystemWriterLock { directory: Dir, - _lock_file: File, + lock_file: File, } impl FilesystemWriterLock { @@ -39,17 +40,23 @@ impl FilesystemWriterLock { Dir::open_ambient_dir(store_root, ambient_authority()).map_err(|source| { WriterLockAcquireError::io(WriterLockAcquirePhase::OpenRoot, source) })?; - let mut options = OpenOptions::new(); - options - .read(true) - .write(true) - .follow(FollowSymlinks::No) - .nonblock(true); - let lock_file = directory - .open_with(LOCK_FILE_NAME, &options) - .map_err(|source| { - WriterLockAcquireError::io(WriterLockAcquirePhase::OpenFile, source) - })?; + let lock_file = open_existing(&directory)?; + Self::acquire(directory, lock_file) + } + + pub(super) fn initialize_in(directory: Dir) -> Result { + let lock_file = open_or_create(&directory)?; + let guard = Self::acquire(directory, lock_file)?; + guard.lock_file.sync_all().map_err(|source| { + WriterLockAcquireError::io(WriterLockAcquirePhase::SynchronizeFile, source) + })?; + Ok(guard) + } + + fn acquire( + directory: Dir, + lock_file: cap_std::fs::File, + ) -> Result { let metadata = lock_file.metadata().map_err(|source| { WriterLockAcquireError::io(WriterLockAcquirePhase::InspectFile, source) })?; @@ -60,7 +67,7 @@ impl FilesystemWriterLock { match lock_file.try_lock() { Ok(()) => Ok(Self { directory, - _lock_file: lock_file, + lock_file, }), Err(TryLockError::WouldBlock) => Err(WriterLockAcquireError::Busy), Err(TryLockError::Error(source)) => Err(WriterLockAcquireError::io( @@ -74,3 +81,32 @@ impl FilesystemWriterLock { self.directory.try_clone() } } + +fn open_or_create(directory: &Dir) -> Result { + let mut options = lock_options(); + options.create_new(true); + match directory.open_with(LOCK_FILE_NAME, &options) { + Ok(file) => Ok(file), + Err(source) if source.kind() == io::ErrorKind::AlreadyExists => open_existing(directory), + Err(source) => Err(WriterLockAcquireError::io( + WriterLockAcquirePhase::OpenFile, + source, + )), + } +} + +fn open_existing(directory: &Dir) -> Result { + directory + .open_with(LOCK_FILE_NAME, &lock_options()) + .map_err(|source| WriterLockAcquireError::io(WriterLockAcquirePhase::OpenFile, source)) +} + +fn lock_options() -> OpenOptions { + let mut options = OpenOptions::new(); + options + .read(true) + .write(true) + .follow(FollowSymlinks::No) + .nonblock(true); + options +} diff --git a/src/adapters/mod.rs b/src/adapters/mod.rs index 941ec78..58a3567 100644 --- a/src/adapters/mod.rs +++ b/src/adapters/mod.rs @@ -74,11 +74,17 @@ mod filesystem_catalog_publisher_tests; mod filesystem_catalog_segment; mod filesystem_catalog_snapshot; mod filesystem_catalog_storage; +mod filesystem_initialization_namespace; +mod filesystem_initialization_storage; mod filesystem_platform_admission; +mod filesystem_platform_profile; mod filesystem_publisher_authority; mod filesystem_segment_stage; #[cfg(test)] mod filesystem_segment_stage_tests; +mod filesystem_store_initializer; +#[cfg(test)] +mod filesystem_store_initializer_tests; #[cfg(test)] #[path = "../../tests/segment_filesystem_stage/sandbox.rs"] mod filesystem_test_sandbox; diff --git a/src/adapters/store_initialization_error.rs b/src/adapters/store_initialization_error.rs index 3516548..a2d357d 100644 --- a/src/adapters/store_initialization_error.rs +++ b/src/adapters/store_initialization_error.rs @@ -18,6 +18,12 @@ pub enum StoreInitializationError { }, } +impl StoreInitializationError { + pub(super) const fn io(phase: StoreInitializationPhase, source: io::Error) -> Self { + Self::Io { phase, source } + } +} + impl fmt::Display for StoreInitializationError { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { match self { diff --git a/src/adapters/writer_lock_acquire_phase.rs b/src/adapters/writer_lock_acquire_phase.rs index 63f3ccd..667094c 100644 --- a/src/adapters/writer_lock_acquire_phase.rs +++ b/src/adapters/writer_lock_acquire_phase.rs @@ -13,6 +13,8 @@ pub enum WriterLockAcquirePhase { InspectFile, /// Acquire the nonblocking exclusive kernel lock. Acquire, + /// Synchronize an initialization-created writer file before admission. + SynchronizeFile, } impl fmt::Display for WriterLockAcquirePhase { @@ -22,6 +24,7 @@ impl fmt::Display for WriterLockAcquirePhase { Self::OpenFile => "file open", Self::InspectFile => "file inspection", Self::Acquire => "kernel acquisition", + Self::SynchronizeFile => "file synchronization", }) } } diff --git a/src/lib.rs b/src/lib.rs index 64d9db9..99b65bc 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -13,10 +13,10 @@ //! and codecs, a capacity-bounded non-durable reference CAS, and explicit //! immutable-segment writing and verified reading, canonical catalog //! generations, platform-gated filesystem publication mechanics, bounded -//! immutable restart snapshots, and typed store-initialization orchestration. -//! Production filesystem platform admission, recovery, retention, and garbage -//! collection APIs remain intentionally absent until their contracts have -//! executable specifications. +//! immutable restart snapshots, typed store-initialization orchestration, and +//! production initialization for the admitted Linux ext4 profile. Recovery, +//! retention, and garbage collection APIs remain intentionally absent until +//! their contracts have executable specifications. #[cfg(test)] extern crate self as keep; diff --git a/tests/store_initialization.rs b/tests/store_initialization.rs index a19b641..e594a95 100644 --- a/tests/store_initialization.rs +++ b/tests/store_initialization.rs @@ -3,11 +3,17 @@ use std::error::Error; use std::io; +#[cfg(not(target_os = "linux"))] +use keep::FilesystemPlatformAdmission; use keep::{ StoreInitializationError, StoreInitializationPhase, StoreInitializationStorage, initialize_store, }; +#[cfg(not(target_os = "linux"))] +#[path = "segment_filesystem_stage/sandbox.rs"] +pub mod sandbox; + const PHASES: [StoreInitializationPhase; 6] = [ StoreInitializationPhase::AdmitPlatform, StoreInitializationPhase::OpenAndLockWriterFile, @@ -52,6 +58,28 @@ fn initialization_stops_at_and_preserves_every_exact_failure_phase() -> Result<( Ok(()) } +#[cfg(not(target_os = "linux"))] +#[test] +fn unsupported_production_platform_refuses_before_namespace_mutation() -> Result<(), Box> +{ + let sandbox = sandbox::TestDirectory::create("store-initialization-unsupported")?; + + let Err(error) = FilesystemPlatformAdmission::initialize(sandbox.path()) else { + return Err("unsupported platform produced filesystem authority".into()); + }; + + assert!(matches!( + error, + StoreInitializationError::Io { + phase: StoreInitializationPhase::AdmitPlatform, + ref source, + } if source.kind() == io::ErrorKind::Unsupported + )); + assert!(std::fs::read_dir(sandbox.path())?.next().is_none()); + sandbox.remove()?; + Ok(()) +} + struct RecordingStorage { attempted: Vec, fail_at: Option, From 5350d9367939c2789401840414ab830384da841a Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 05:21:28 -0700 Subject: [PATCH 04/49] Fix: Refuse replaced writer lock handles --- CHANGELOG.md | 2 + docs/formats/segment-store-v1/recovery.md | 3 +- docs/formats/segment-store-v1/requirements.md | 1 + src/adapters/filesystem_writer_lock.rs | 71 +++++++++++++++++-- src/adapters/filesystem_writer_lock_tests.rs | 65 +++++++++++++++++ src/adapters/writer_lock_acquire_phase.rs | 3 + 6 files changed, 137 insertions(+), 8 deletions(-) create mode 100644 src/adapters/filesystem_writer_lock_tests.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 619842d..e04a138 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,8 @@ after its public API and format compatibility policies are established. non-casefolded Linux ext4 profile, refuses ambiguous root namespaces before mutation, completes the canonical directory shape idempotently, retains writer authority, and returns only after synchronizing the root. +- Writer-lock acquisition now reopens `writer.lock` after kernel locking and + refuses when the resolved entry no longer has the locked device and inode. - Store initialization now exposes one storage-port state machine that admits the platform before mutation, opens and locks `writer.lock`, admits the three protocol directories in order, synchronizes the root, and preserves the diff --git a/docs/formats/segment-store-v1/recovery.md b/docs/formats/segment-store-v1/recovery.md index 1a75187..d704178 100644 --- a/docs/formats/segment-store-v1/recovery.md +++ b/docs/formats/segment-store-v1/recovery.md @@ -171,7 +171,8 @@ The initial production adapter is supported only on Linux when it proves: - directory synchronization that makes create, link, unlink, rename, and replacement durable; - process-scoped exclusive advisory locking; and -- one retained writer-lock handle for the writer-authority lifetime. +- post-acquisition device-and-inode verification of the retained writer-lock + handle. The adapter refuses every non-ext4 filesystem, read-only mount, casefolded store root, symlinked selected path, or platform other than Linux. A single local diff --git a/docs/formats/segment-store-v1/requirements.md b/docs/formats/segment-store-v1/requirements.md index 88a8c13..068fcc7 100644 --- a/docs/formats/segment-store-v1/requirements.md +++ b/docs/formats/segment-store-v1/requirements.md @@ -99,6 +99,7 @@ recovery execution. | `KEEP-RECOVERY-001` | Crash identifiers `KEEP-CRASH-001` through `KEEP-CRASH-035` form one contiguous typed vocabulary, map to the exact owning protocol sequence, and admit an occurrence counter only for record append | Ordered identifier-and-sequence ledger | `xtask/tests/durability_crash_point_contract.rs` | Implemented in #17 | | `KEEP-RECOVERY-002` | Initialization admits the platform before mutation, opens and locks the writer file, admits `staging`, `segments`, and `catalogs` in order, and returns a receipt only after root synchronization; every failed operation retains its exact phase and prevents later transitions | Fault-recording initialization port | `tests/store_initialization.rs` | Implemented in #17 | | `KEEP-RECOVERY-003` | Production initialization admits only a writable, non-casefolded Linux ext4 root, refuses any noncanonical root entry before mutation, completes an empty or partial canonical namespace without replacing evidence, excludes a second initializer, and retains writer authority through the synchronized receipt | Capability-relative filesystem fixture and exact platform-profile classifier | `src/adapters/filesystem_store_initializer_tests.rs`, `src/adapters/filesystem_platform_profile.rs`, `tests/store_initialization.rs` | Implemented in #17 | +| `KEEP-RECOVERY-004` | Writer authority is returned only when the locked handle still has the exact device and inode resolved by the canonical `writer.lock` entry after kernel acquisition | Deterministic lock-entry replacement fixture | `src/adapters/filesystem_writer_lock_tests.rs` | Implemented in #17 | diff --git a/src/adapters/filesystem_writer_lock.rs b/src/adapters/filesystem_writer_lock.rs index 9911ff1..c797cb0 100644 --- a/src/adapters/filesystem_writer_lock.rs +++ b/src/adapters/filesystem_writer_lock.rs @@ -4,14 +4,43 @@ use std::fs::{File, TryLockError}; use std::io; use std::path::Path; -use cap_fs_ext::{FollowSymlinks, OpenOptionsFollowExt, OpenOptionsSyncExt}; +use cap_fs_ext::{FollowSymlinks, MetadataExt, OpenOptionsFollowExt, OpenOptionsSyncExt}; use cap_std::ambient_authority; -use cap_std::fs::{Dir, OpenOptions}; +use cap_std::fs::{Dir, Metadata, OpenOptions}; use super::{WriterLockAcquireError, WriterLockAcquirePhase}; +#[cfg(test)] +#[path = "filesystem_writer_lock_tests.rs"] +mod tests; + const LOCK_FILE_NAME: &str = "writer.lock"; +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct FileIdentity { + device: u64, + inode: u64, +} + +impl FileIdentity { + fn read(file: &cap_std::fs::File) -> Result { + file.metadata() + .map(|metadata| Self::from(&metadata)) + .map_err(|source| { + WriterLockAcquireError::io(WriterLockAcquirePhase::VerifyFileIdentity, source) + }) + } +} + +impl From<&Metadata> for FileIdentity { + fn from(metadata: &Metadata) -> Self { + Self { + device: metadata.dev(), + inode: metadata.ino(), + } + } +} + /// Exclusive kernel-managed writer authority over one pinned store root. /// /// The guard retains both the opened root capability and lock-file handle. @@ -28,7 +57,9 @@ impl FilesystemWriterLock { /// /// The store root is pinned before `writer.lock` is opened relative to it. /// The lock entry must already exist as a regular file and is opened - /// without following symbolic links. + /// without following symbolic links. After nonblocking kernel acquisition, + /// the adapter reopens the directory entry and proves that it still names + /// the locked device and inode before returning authority. /// /// # Errors /// @@ -63,12 +94,16 @@ impl FilesystemWriterLock { if !metadata.is_file() { return Err(WriterLockAcquireError::NotRegular); } + let expected_identity = FileIdentity::from(&metadata); let lock_file = lock_file.into_std(); match lock_file.try_lock() { - Ok(()) => Ok(Self { - directory, - lock_file, - }), + Ok(()) => { + verify_current_identity(&directory, expected_identity)?; + Ok(Self { + directory, + lock_file, + }) + } Err(TryLockError::WouldBlock) => Err(WriterLockAcquireError::Busy), Err(TryLockError::Error(source)) => Err(WriterLockAcquireError::io( WriterLockAcquirePhase::Acquire, @@ -82,6 +117,28 @@ impl FilesystemWriterLock { } } +fn verify_current_identity( + directory: &Dir, + expected: FileIdentity, +) -> Result<(), WriterLockAcquireError> { + let observed_file = directory + .open_with(LOCK_FILE_NAME, &lock_options()) + .map_err(|source| { + WriterLockAcquireError::io(WriterLockAcquirePhase::VerifyFileIdentity, source) + })?; + let observed = FileIdentity::read(&observed_file)?; + if observed == expected { + return Ok(()); + } + Err(WriterLockAcquireError::io( + WriterLockAcquirePhase::VerifyFileIdentity, + io::Error::new( + io::ErrorKind::InvalidData, + "writer.lock changed identity during acquisition", + ), + )) +} + fn open_or_create(directory: &Dir) -> Result { let mut options = lock_options(); options.create_new(true); diff --git a/src/adapters/filesystem_writer_lock_tests.rs b/src/adapters/filesystem_writer_lock_tests.rs new file mode 100644 index 0000000..55e903b --- /dev/null +++ b/src/adapters/filesystem_writer_lock_tests.rs @@ -0,0 +1,65 @@ +//! Replacement-race laws for persistent writer authority. + +use std::error::Error; +use std::fs; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; + +use cap_std::ambient_authority; +use cap_std::fs::Dir; + +use super::{FileIdentity, LOCK_FILE_NAME, open_existing, verify_current_identity}; +use crate::adapters::{WriterLockAcquireError, WriterLockAcquirePhase}; + +static NEXT_DIRECTORY: AtomicU64 = AtomicU64::new(0); + +#[test] +fn replaced_lock_entry_cannot_authorize_the_opened_handle() -> Result<(), Box> { + let sandbox = TestDirectory::create("writer-lock-identity")?; + fs::write(sandbox.path().join(LOCK_FILE_NAME), [])?; + let directory = Dir::open_ambient_dir(sandbox.path(), ambient_authority())?; + let opened = open_existing(&directory)?; + let expected = FileIdentity::read(&opened)?; + + fs::rename( + sandbox.path().join(LOCK_FILE_NAME), + sandbox.path().join("displaced.lock"), + )?; + fs::write(sandbox.path().join(LOCK_FILE_NAME), [])?; + + let error = verify_current_identity(&directory, expected) + .err() + .ok_or("replacement was admitted as the opened lock file")?; + assert!(matches!( + error, + WriterLockAcquireError::Io { + phase: WriterLockAcquirePhase::VerifyFileIdentity, + ref source, + } if source.kind() == std::io::ErrorKind::InvalidData + )); + drop(opened); + sandbox.remove()?; + Ok(()) +} + +struct TestDirectory { + path: PathBuf, +} + +impl TestDirectory { + fn create(name: &str) -> std::io::Result { + let sequence = NEXT_DIRECTORY.fetch_add(1, Ordering::Relaxed); + let path = + std::env::temp_dir().join(format!("keep-{name}-{}-{sequence}", std::process::id())); + fs::create_dir(&path)?; + Ok(Self { path }) + } + + fn path(&self) -> &Path { + &self.path + } + + fn remove(self) -> std::io::Result<()> { + fs::remove_dir_all(self.path) + } +} diff --git a/src/adapters/writer_lock_acquire_phase.rs b/src/adapters/writer_lock_acquire_phase.rs index 667094c..007588f 100644 --- a/src/adapters/writer_lock_acquire_phase.rs +++ b/src/adapters/writer_lock_acquire_phase.rs @@ -13,6 +13,8 @@ pub enum WriterLockAcquirePhase { InspectFile, /// Acquire the nonblocking exclusive kernel lock. Acquire, + /// Verify that the locked handle still names the admitted directory entry. + VerifyFileIdentity, /// Synchronize an initialization-created writer file before admission. SynchronizeFile, } @@ -24,6 +26,7 @@ impl fmt::Display for WriterLockAcquirePhase { Self::OpenFile => "file open", Self::InspectFile => "file inspection", Self::Acquire => "kernel acquisition", + Self::VerifyFileIdentity => "file identity verification", Self::SynchronizeFile => "file synchronization", }) } From 67c2939c0f2c4428cfca032a6503bf54c52fc0c2 Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 05:24:23 -0700 Subject: [PATCH 05/49] Fix: Retain root writer exclusion --- CHANGELOG.md | 2 + .../cap-std-and-cap-fs-ext-4.0.2.md | 10 +-- docs/formats/segment-store-v1/rationale.md | 18 +++-- docs/formats/segment-store-v1/recovery.md | 3 +- docs/formats/segment-store-v1/requirements.md | 2 +- src/adapters/filesystem_writer_lock.rs | 66 +++++++++++-------- src/adapters/writer_lock_acquire_phase.rs | 3 + tests/catalog_writer_lock.rs | 23 +++++++ 8 files changed, 87 insertions(+), 40 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e04a138..160f6f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,8 @@ after its public API and format compatibility policies are established. writer authority, and returns only after synchronizing the root. - Writer-lock acquisition now reopens `writer.lock` after kernel locking and refuses when the resolved entry no longer has the locked device and inode. +- Writer authority now also retains an advisory lock on the pinned store-root + inode, so replacing `writer.lock` cannot split live cooperative authority. - Store initialization now exposes one storage-port state machine that admits the platform before mutation, opens and locks `writer.lock`, admits the three protocol directories in order, synchronizes the root, and preserves the diff --git a/docs/dependencies/cap-std-and-cap-fs-ext-4.0.2.md b/docs/dependencies/cap-std-and-cap-fs-ext-4.0.2.md index 26abc36..f86da98 100644 --- a/docs/dependencies/cap-std-and-cap-fs-ext-4.0.2.md +++ b/docs/dependencies/cap-std-and-cap-fs-ext-4.0.2.md @@ -26,11 +26,11 @@ other ambiguous filesystem state before reading source or protocol bytes. The capability packages are present in Keep's published library graph and production filesystem behavior. No dependency-owned type crosses Keep's public API or enters content identities or durable formats. The segment-store writer -lock retains capability and file handles behind `FilesystemWriterLock`; its -public acquisition boundary accepts only `std::path::Path`. The production -initializer uses Rustix's safe `openat2`, `fstatfs`, `fstatvfs`, and ext4 inode -flag APIs to admit only the documented writable, non-casefolded Linux ext4 -profile. +lock retains the root capability plus root-lock and writer-lock file handles +behind `FilesystemWriterLock`; its public acquisition boundary accepts only +`std::path::Path`. The production initializer uses Rustix's safe `openat2`, +`fstatfs`, `fstatvfs`, and ext4 inode flag APIs to admit only the documented +writable, non-casefolded Linux ext4 profile. The bounded subprocess adapter uses Rustix's safe filesystem API to mark child stdin nonblocking before deadline-bounded input transfer. It uses Rustix's safe diff --git a/docs/formats/segment-store-v1/rationale.md b/docs/formats/segment-store-v1/rationale.md index 9586278..7c03e0e 100644 --- a/docs/formats/segment-store-v1/rationale.md +++ b/docs/formats/segment-store-v1/rationale.md @@ -135,13 +135,17 @@ only after the root-directory sync following head replacement. ## Persistent advisory lock -The writer lock file persists and its contents carry no authority. Successful -exclusive kernel lock acquisition is the only ownership evidence. Process -death releases the lock without requiring presence-based stale-file cleanup. - -This deliberately excludes multi-host and filesystems whose advisory locks do -not provide the required exclusion. Weakening the lock would violate -one-writer publication rather than improve availability. +The writer lock file persists and its contents carry no authority. Writer +ownership requires successful exclusive kernel locks on both the pinned store +root inode and the exact `writer.lock` inode. The root lock prevents a second +cooperative Keep process from acquiring a replacement lock-file inode while +the first writer remains live. Process death releases both locks without +requiring presence-based stale-file cleanup. + +This deliberately excludes uncooperative processes, multi-host operation, and +filesystems whose advisory locks do not provide the required exclusion. +Weakening either lock would violate one-writer publication rather than improve +availability. ## Platform admission before publication diff --git a/docs/formats/segment-store-v1/recovery.md b/docs/formats/segment-store-v1/recovery.md index d704178..32286d7 100644 --- a/docs/formats/segment-store-v1/recovery.md +++ b/docs/formats/segment-store-v1/recovery.md @@ -170,7 +170,8 @@ The initial production adapter is supported only on Linux when it proves: - atomic same-filesystem replacement of one regular file by another; - directory synchronization that makes create, link, unlink, rename, and replacement durable; -- process-scoped exclusive advisory locking; and +- process-scoped exclusive advisory locking on the pinned root and writer file; + and - post-acquisition device-and-inode verification of the retained writer-lock handle. diff --git a/docs/formats/segment-store-v1/requirements.md b/docs/formats/segment-store-v1/requirements.md index 068fcc7..7d71066 100644 --- a/docs/formats/segment-store-v1/requirements.md +++ b/docs/formats/segment-store-v1/requirements.md @@ -74,7 +74,7 @@ recovery remains separate work. | `KEEP-CATALOG-004` | Every catalog location equals a verified top-level record span in the exact named segment; construction and admission require every supplied segment to be referenced, and admission scans each referenced segment once | Bounded grouped lookup plan and golden artifacts | `tests/catalog_encoding.rs`, `tests/catalog_locations.rs` | Implemented in #16 | | `KEEP-CATALOG-005` | Publication admits only the exact expected successor and reports expected and observed generation and digest on staleness | Generation transition model | `tests/catalog_transition.rs` | Implemented in #16 | | `KEEP-CATALOG-006` | A reader retains one complete catalog generation and never combines it with a concurrent head | Immutable snapshot model | `tests/catalog_snapshot.rs` | Implemented in #16 | -| `KEEP-CATALOG-007` | One persistent kernel-managed writer lock excludes a second writer without deleting or replacing the lock file; the lock alone cannot construct a publisher without platform admission | Two-handle lock model and construction architecture law | `tests/catalog_writer_lock.rs`, `tests/catalog_filesystem_publication/directory_laws.rs` | Implemented in #16 | +| `KEEP-CATALOG-007` | Retained kernel locks on the pinned store root and persistent writer file exclude a second cooperative writer even if the directory entry is replaced; neither lock is deleted on release, and lock ownership alone cannot construct a publisher without platform admission | Multi-handle lock model, replacement fixture, and construction architecture law | `tests/catalog_writer_lock.rs`, `tests/catalog_filesystem_publication/directory_laws.rs` | Implemented in #16; replacement-hardened in #17 | | `KEEP-CATALOG-008` | Segment, catalog, and head publication follows the documented synchronization order; retained fixed-name recovery state refuses before mutation; an absent head requires empty immutable pools; retry of an already-current candidate performs no publication mutation and re-synchronizes the root | Fault-recording port and filesystem fixtures | `tests/catalog_publication.rs`, `tests/catalog_filesystem_publication.rs` | Implemented in #16 | | `KEEP-CATALOG-009` | Restart loading refuses corrupt, unsupported, noncanonical, dangling, and conflicting catalog state | Corruption matrix | `tests/catalog_restart.rs` | Implemented in #16 | | `KEEP-CATALOG-010` | Model-based transitions and lookups agree with a deterministic `BTreeMap` catalog | Boring reference catalog | `tests/catalog_model.rs` | Implemented in #16 | diff --git a/src/adapters/filesystem_writer_lock.rs b/src/adapters/filesystem_writer_lock.rs index c797cb0..b0ee0de 100644 --- a/src/adapters/filesystem_writer_lock.rs +++ b/src/adapters/filesystem_writer_lock.rs @@ -43,41 +43,44 @@ impl From<&Metadata> for FileIdentity { /// Exclusive kernel-managed writer authority over one pinned store root. /// -/// The guard retains both the opened root capability and lock-file handle. -/// Dropping it closes the handle and releases the process-scoped kernel lock; -/// it never deletes, renames, truncates, or replaces `writer.lock`. +/// The guard retains the opened root capability plus exclusive advisory locks +/// on the root inode and `writer.lock`. Dropping it closes both lock handles; +/// it never deletes, renames, truncates, or replaces protocol state. #[must_use] pub struct FilesystemWriterLock { directory: Dir, + _root_lock_file: File, lock_file: File, } impl FilesystemWriterLock { /// Tries to acquire exclusive writer authority without blocking. /// - /// The store root is pinned before `writer.lock` is opened relative to it. - /// The lock entry must already exist as a regular file and is opened - /// without following symbolic links. After nonblocking kernel acquisition, - /// the adapter reopens the directory entry and proves that it still names - /// the locked device and inode before returning authority. + /// The store root is pinned and nonblocking-locked before `writer.lock` is + /// opened relative to it. The lock entry must already exist as a regular + /// file and is opened without following symbolic links. After nonblocking + /// file locking, the adapter reopens the directory entry and proves that it + /// still names the locked device and inode before returning authority. /// /// # Errors /// /// Returns [`WriterLockAcquireError::Busy`] when another handle or process - /// owns the lock. Other failures preserve their exact acquisition phase and - /// I/O source. A missing lock file is never created by this operation. + /// owns either lock. Other failures preserve their exact acquisition phase + /// and I/O source. A missing lock file is never created by this operation. pub fn try_acquire(store_root: &Path) -> Result { let directory = Dir::open_ambient_dir(store_root, ambient_authority()).map_err(|source| { WriterLockAcquireError::io(WriterLockAcquirePhase::OpenRoot, source) })?; + let root_lock_file = acquire_root(&directory)?; let lock_file = open_existing(&directory)?; - Self::acquire(directory, lock_file) + Self::acquire(directory, root_lock_file, lock_file) } pub(super) fn initialize_in(directory: Dir) -> Result { + let root_lock_file = acquire_root(&directory)?; let lock_file = open_or_create(&directory)?; - let guard = Self::acquire(directory, lock_file)?; + let guard = Self::acquire(directory, root_lock_file, lock_file)?; guard.lock_file.sync_all().map_err(|source| { WriterLockAcquireError::io(WriterLockAcquirePhase::SynchronizeFile, source) })?; @@ -86,6 +89,7 @@ impl FilesystemWriterLock { fn acquire( directory: Dir, + root_lock_file: File, lock_file: cap_std::fs::File, ) -> Result { let metadata = lock_file.metadata().map_err(|source| { @@ -96,20 +100,13 @@ impl FilesystemWriterLock { } let expected_identity = FileIdentity::from(&metadata); let lock_file = lock_file.into_std(); - match lock_file.try_lock() { - Ok(()) => { - verify_current_identity(&directory, expected_identity)?; - Ok(Self { - directory, - lock_file, - }) - } - Err(TryLockError::WouldBlock) => Err(WriterLockAcquireError::Busy), - Err(TryLockError::Error(source)) => Err(WriterLockAcquireError::io( - WriterLockAcquirePhase::Acquire, - source, - )), - } + acquire_lock(&lock_file, WriterLockAcquirePhase::Acquire)?; + verify_current_identity(&directory, expected_identity)?; + Ok(Self { + directory, + _root_lock_file: root_lock_file, + lock_file, + }) } pub(super) fn clone_directory(&self) -> std::io::Result { @@ -117,6 +114,23 @@ impl FilesystemWriterLock { } } +fn acquire_root(directory: &Dir) -> Result { + let file = directory + .try_clone() + .map_err(|source| WriterLockAcquireError::io(WriterLockAcquirePhase::AcquireRoot, source))? + .into_std_file(); + acquire_lock(&file, WriterLockAcquirePhase::AcquireRoot)?; + Ok(file) +} + +fn acquire_lock(file: &File, phase: WriterLockAcquirePhase) -> Result<(), WriterLockAcquireError> { + match file.try_lock() { + Ok(()) => Ok(()), + Err(TryLockError::WouldBlock) => Err(WriterLockAcquireError::Busy), + Err(TryLockError::Error(source)) => Err(WriterLockAcquireError::io(phase, source)), + } +} + fn verify_current_identity( directory: &Dir, expected: FileIdentity, diff --git a/src/adapters/writer_lock_acquire_phase.rs b/src/adapters/writer_lock_acquire_phase.rs index 007588f..7fa9320 100644 --- a/src/adapters/writer_lock_acquire_phase.rs +++ b/src/adapters/writer_lock_acquire_phase.rs @@ -7,6 +7,8 @@ use std::fmt; pub enum WriterLockAcquirePhase { /// Pin the caller-selected store root. OpenRoot, + /// Acquire the nonblocking exclusive lock on the pinned root inode. + AcquireRoot, /// Open `writer.lock` relative to the pinned root without following links. OpenFile, /// Verify that the opened lock handle names a regular file. @@ -23,6 +25,7 @@ impl fmt::Display for WriterLockAcquirePhase { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { formatter.write_str(match self { Self::OpenRoot => "root open", + Self::AcquireRoot => "root kernel acquisition", Self::OpenFile => "file open", Self::InspectFile => "file inspection", Self::Acquire => "kernel acquisition", diff --git a/tests/catalog_writer_lock.rs b/tests/catalog_writer_lock.rs index c354dfd..47ca0c4 100644 --- a/tests/catalog_writer_lock.rs +++ b/tests/catalog_writer_lock.rs @@ -33,6 +33,29 @@ fn one_persistent_lock_excludes_every_second_writer() -> Result<(), Box Result<(), Box> { + let sandbox = initialized_lock("writer-lock-replacement")?; + let first = FilesystemWriterLock::try_acquire(sandbox.path())?; + fs::rename( + sandbox.path().join(LOCK_NAME), + sandbox.path().join("displaced.lock"), + )?; + fs::write(sandbox.path().join(LOCK_NAME), RETAINED_EVIDENCE)?; + + let error = require_error( + FilesystemWriterLock::try_acquire(sandbox.path()), + "replacement lock entry split live writer authority", + )?; + assert!(matches!(error, WriterLockAcquireError::Busy)); + + drop(first); + let successor = FilesystemWriterLock::try_acquire(sandbox.path())?; + drop(successor); + sandbox.remove()?; + Ok(()) +} + #[test] fn missing_lock_evidence_is_never_created_by_acquisition() -> Result<(), Box> { let sandbox = TestDirectory::create("writer-lock-missing")?; From 0f0af60f582fdc9927864571e5738041b7873185 Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 05:36:44 -0700 Subject: [PATCH 06/49] Add: Bound deterministic recovery inventory --- CHANGELOG.md | 4 + README.md | 6 +- docs/formats/segment-store-v1/recovery.md | 6 + docs/formats/segment-store-v1/requirements.md | 1 + src/adapters/mod.rs | 14 ++ src/adapters/recovery_entry_name.rs | 71 ++++++++ src/adapters/recovery_inventory.rs | 164 ++++++++++++++++++ src/adapters/recovery_inventory_error.rs | 114 ++++++++++++ src/adapters/recovery_inventory_limit.rs | 73 ++++++++ src/adapters/recovery_inventory_operation.rs | 21 +++ src/adapters/recovery_inventory_storage.rs | 30 ++++ src/adapters/recovery_namespace.rs | 27 +++ src/lib.rs | 11 +- tests/recovery_inventory.rs | 57 ++++++ tests/recovery_inventory/inventory_double.rs | 65 +++++++ tests/recovery_inventory/refusal_laws.rs | 149 ++++++++++++++++ 16 files changed, 808 insertions(+), 5 deletions(-) create mode 100644 src/adapters/recovery_entry_name.rs create mode 100644 src/adapters/recovery_inventory.rs create mode 100644 src/adapters/recovery_inventory_error.rs create mode 100644 src/adapters/recovery_inventory_limit.rs create mode 100644 src/adapters/recovery_inventory_operation.rs create mode 100644 src/adapters/recovery_inventory_storage.rs create mode 100644 src/adapters/recovery_namespace.rs create mode 100644 tests/recovery_inventory.rs create mode 100644 tests/recovery_inventory/inventory_double.rs create mode 100644 tests/recovery_inventory/refusal_laws.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 160f6f2..6806ea0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,10 @@ after its public API and format compatibility policies are established. refuses when the resolved entry no longer has the locked device and inode. - Writer authority now also retains an advisory lock on the pinned store-root inode, so replacing `writer.lock` cannot split live cooperative authority. +- Recovery inventory now counts the root and three protocol directories before + retaining names, enforces the configurable protocol-bounded entry ceiling, + refuses count drift and duplicates exactly, and returns deterministic + namespace-and-raw-byte ordering through a read-only storage port. - Store initialization now exposes one storage-port state machine that admits the platform before mutation, opens and locks `writer.lock`, admits the three protocol directories in order, synchronizes the root, and preserves the diff --git a/README.md b/README.md index 3faa98f..659327e 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,11 @@ documented Linux ext4 contract. Acquiring `FilesystemWriterLock` alone cannot construct a filesystem publisher. Leftover `head.next`, staged recovery evidence, and ambiguous crash states remain explicit recovery work. An absent `HEAD` is admitted for first publication only when both immutable pools are -empty. Crash-injection execution, explicit recovery, retention, complete +empty. The public storage-independent recovery inventory counts all four +protocol namespaces before retaining names, applies a configurable ceiling no +greater than 2,097,152 entries, and returns duplicate-free deterministic raw +name order; its concrete filesystem reader and artifact classification remain +planned. Crash-injection execution, explicit recovery, retention, complete namespace verification after initialization, compaction, and garbage collection remain planned. Presence in the reference CAS does not claim retention, crash recovery, or durability. diff --git a/docs/formats/segment-store-v1/recovery.md b/docs/formats/segment-store-v1/recovery.md index 32286d7..2b6fad5 100644 --- a/docs/formats/segment-store-v1/recovery.md +++ b/docs/formats/segment-store-v1/recovery.md @@ -22,6 +22,12 @@ verifies complete content before classification. Unknown names, symlinks, conflicting canonical coordinates, multiple fixed-name stages, or a head that cannot be proven atomic are unrecoverable ambiguity. +The current public `read_recovery_inventory` slice implements this fixed-order +count-before-retain orchestration through a read-only storage port. It enforces +the configured and protocol ceilings, count stability, duplicate refusal, and +namespace-plus-raw-byte ordering. The concrete filesystem inventory adapter and +artifact classification remain unimplemented. + The sole admissible duplicate digest is one fixed staging name and its exact digest-derived pool name after a link transition. Recovery admits that pair only after complete byte-for-byte verification proves the stage, pool entry, diff --git a/docs/formats/segment-store-v1/requirements.md b/docs/formats/segment-store-v1/requirements.md index 7d71066..09da8ae 100644 --- a/docs/formats/segment-store-v1/requirements.md +++ b/docs/formats/segment-store-v1/requirements.md @@ -100,6 +100,7 @@ recovery execution. | `KEEP-RECOVERY-002` | Initialization admits the platform before mutation, opens and locks the writer file, admits `staging`, `segments`, and `catalogs` in order, and returns a receipt only after root synchronization; every failed operation retains its exact phase and prevents later transitions | Fault-recording initialization port | `tests/store_initialization.rs` | Implemented in #17 | | `KEEP-RECOVERY-003` | Production initialization admits only a writable, non-casefolded Linux ext4 root, refuses any noncanonical root entry before mutation, completes an empty or partial canonical namespace without replacing evidence, excludes a second initializer, and retains writer authority through the synchronized receipt | Capability-relative filesystem fixture and exact platform-profile classifier | `src/adapters/filesystem_store_initializer_tests.rs`, `src/adapters/filesystem_platform_profile.rs`, `tests/store_initialization.rs` | Implemented in #17 | | `KEEP-RECOVERY-004` | Writer authority is returned only when the locked handle still has the exact device and inode resolved by the canonical `writer.lock` entry after kernel acquisition | Deterministic lock-entry replacement fixture | `src/adapters/filesystem_writer_lock_tests.rs` | Implemented in #17 | +| `KEEP-RECOVERY-005` | Recovery counts the root and three protocol directories in fixed order before retaining names, refuses at the configured or protocol entry ceiling with the exact observed-at-least count, then returns one duplicate-free inventory sorted by namespace and raw name bytes | Fault-recording inventory port | `tests/recovery_inventory.rs` | Implemented in #17 | diff --git a/src/adapters/mod.rs b/src/adapters/mod.rs index 58a3567..bec8f9d 100644 --- a/src/adapters/mod.rs +++ b/src/adapters/mod.rs @@ -111,6 +111,13 @@ mod publication_head_decode_error; mod publication_head_decode_error_display; mod publication_head_decoder; mod publication_head_encoder; +mod recovery_entry_name; +mod recovery_inventory; +mod recovery_inventory_error; +mod recovery_inventory_limit; +mod recovery_inventory_operation; +mod recovery_inventory_storage; +mod recovery_namespace; mod sealed_segment; mod segment_digest; mod segment_digest_builder; @@ -228,6 +235,13 @@ pub use layout_id_binary_error::LayoutIdBinaryParseError; pub use layout_id_text_error::LayoutIdTextParseError; pub use layout_record::CanonicalLayoutRecord; pub use publication_head_decode_error::PublicationHeadDecodeError; +pub use recovery_entry_name::{RecoveryEntryName, RecoveryEntryNameError}; +pub use recovery_inventory::{RecoveryInventory, RecoveryInventoryEntry, read_recovery_inventory}; +pub use recovery_inventory_error::RecoveryInventoryError; +pub use recovery_inventory_limit::{RecoveryInventoryLimit, RecoveryInventoryLimitError}; +pub use recovery_inventory_operation::RecoveryInventoryOperation; +pub use recovery_inventory_storage::RecoveryInventoryStorage; +pub use recovery_namespace::RecoveryNamespace; pub use sealed_segment::SealedSegment; pub use segment_digest::SegmentDigest; pub use segment_header::SegmentHeader; diff --git a/src/adapters/recovery_entry_name.rs b/src/adapters/recovery_entry_name.rs new file mode 100644 index 0000000..5c65076 --- /dev/null +++ b/src/adapters/recovery_entry_name.rs @@ -0,0 +1,71 @@ +//! This module owns one bounded-inventory path-component spelling. + +use std::error::Error; +use std::fmt; + +/// One raw path-component name observed during recovery inventory. +/// +/// The value may be a noncanonical protocol name so later classification can +/// report ambiguity. Construction rejects only spellings that cannot identify +/// one child entry. The owned allocation is bounded by the admitted filesystem +/// component and the inventory entry-count limit. +#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)] +pub struct RecoveryEntryName { + bytes: Vec, +} + +impl RecoveryEntryName { + /// Admits one raw, nonempty child-entry spelling. + /// + /// # Errors + /// + /// Returns [`RecoveryEntryNameError`] for an empty name, NUL, path + /// separator, or dot component. + pub fn new(bytes: Vec) -> Result { + if bytes.is_empty() { + return Err(RecoveryEntryNameError::Empty); + } + if bytes.contains(&0) { + return Err(RecoveryEntryNameError::Nul); + } + if bytes.contains(&b'/') { + return Err(RecoveryEntryNameError::PathSeparator); + } + if bytes == b"." || bytes == b".." { + return Err(RecoveryEntryNameError::DotComponent); + } + Ok(Self { bytes }) + } + + /// Returns the exact raw name bytes. + #[must_use] + pub fn as_bytes(&self) -> &[u8] { + &self.bytes + } +} + +/// Why a recovery entry name cannot identify one child entry. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum RecoveryEntryNameError { + /// The name is empty. + Empty, + /// The name contains NUL. + Nul, + /// The name contains a path separator. + PathSeparator, + /// The name is `.` or `..`. + DotComponent, +} + +impl fmt::Display for RecoveryEntryNameError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(match self { + Self::Empty => "recovery entry name is empty", + Self::Nul => "recovery entry name contains NUL", + Self::PathSeparator => "recovery entry name contains a path separator", + Self::DotComponent => "recovery entry name is a dot component", + }) + } +} + +impl Error for RecoveryEntryNameError {} diff --git a/src/adapters/recovery_inventory.rs b/src/adapters/recovery_inventory.rs new file mode 100644 index 0000000..e93d901 --- /dev/null +++ b/src/adapters/recovery_inventory.rs @@ -0,0 +1,164 @@ +//! This module owns bounded, deterministic recovery inventory orchestration. + +use super::{ + RecoveryEntryName, RecoveryInventoryError, RecoveryInventoryLimit, RecoveryInventoryOperation, + RecoveryInventoryStorage, RecoveryNamespace, +}; + +const NAMESPACES: [RecoveryNamespace; 4] = [ + RecoveryNamespace::Root, + RecoveryNamespace::Staging, + RecoveryNamespace::Segments, + RecoveryNamespace::Catalogs, +]; + +/// One namespace-qualified entry in a recovery inventory. +#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)] +pub struct RecoveryInventoryEntry { + namespace: RecoveryNamespace, + name: RecoveryEntryName, +} + +impl RecoveryInventoryEntry { + /// Returns the owning protocol namespace. + #[must_use] + pub const fn namespace(&self) -> RecoveryNamespace { + self.namespace + } + + /// Returns the exact raw name. + #[must_use] + pub const fn name(&self) -> &RecoveryEntryName { + &self.name + } +} + +/// One duplicate-free, deterministically ordered recovery inventory. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct RecoveryInventory { + entries: Vec, +} + +impl RecoveryInventory { + /// Returns entries ordered by namespace and raw name bytes. + #[must_use] + pub fn entries(&self) -> &[RecoveryInventoryEntry] { + &self.entries + } +} + +/// Counts every namespace before retaining and sorting any entry name. +/// +/// The operation is synchronous and may block on storage I/O. Its allocation +/// is bounded by `limit` and the admitted entry-name component lengths. It +/// never writes, repairs, deletes, or substitutes storage state. +/// +/// # Errors +/// +/// Returns [`RecoveryInventoryError`] on storage refusal, entry-limit excess, +/// count drift, duplicate names, or address-space incompatibility. +pub fn read_recovery_inventory( + storage: &mut impl RecoveryInventoryStorage, + limit: RecoveryInventoryLimit, +) -> Result { + let counts = count_namespaces(storage, limit)?; + read_namespaces(storage, counts) +} + +fn count_namespaces( + storage: &mut impl RecoveryInventoryStorage, + limit: RecoveryInventoryLimit, +) -> Result<[u64; 4], RecoveryInventoryError> { + let mut counts = [0_u64; 4]; + let mut total = 0_u64; + for (count_slot, namespace) in counts.iter_mut().zip(NAMESPACES) { + let count = storage.count_entries(namespace).map_err(|source| { + RecoveryInventoryError::io(namespace, RecoveryInventoryOperation::Count, source) + })?; + let remaining = limit + .get() + .checked_sub(total) + .ok_or(RecoveryInventoryError::AddressSpace { observed: total })?; + if count > remaining { + let observed_at_least = + limit + .get() + .checked_add(1) + .ok_or_else(|| RecoveryInventoryError::AddressSpace { + observed: limit.get(), + })?; + return Err(RecoveryInventoryError::EntryLimit { + maximum: limit.get(), + observed_at_least, + }); + } + total = total + .checked_add(count) + .ok_or(RecoveryInventoryError::AddressSpace { observed: count })?; + *count_slot = count; + } + Ok(counts) +} + +fn read_namespaces( + storage: &mut impl RecoveryInventoryStorage, + counts: [u64; 4], +) -> Result { + let total = counts.into_iter().try_fold(0_u64, |total, count| { + total + .checked_add(count) + .ok_or(RecoveryInventoryError::AddressSpace { observed: count }) + })?; + let capacity = usize::try_from(total) + .map_err(|_| RecoveryInventoryError::AddressSpace { observed: total })?; + let mut entries = Vec::with_capacity(capacity); + for (namespace, counted) in NAMESPACES.into_iter().zip(counts) { + let names = storage + .read_entry_names(namespace, counted) + .map_err(|source| { + RecoveryInventoryError::io(namespace, RecoveryInventoryOperation::ReadNames, source) + })?; + admit_names(&mut entries, namespace, counted, names)?; + } + entries.sort_unstable(); + refuse_duplicate(&entries)?; + Ok(RecoveryInventory { entries }) +} + +fn admit_names( + entries: &mut Vec, + namespace: RecoveryNamespace, + counted: u64, + names: Vec, +) -> Result<(), RecoveryInventoryError> { + let observed = u64::try_from(names.len()) + .map_err(|_| RecoveryInventoryError::AddressSpace { observed: counted })?; + if observed != counted { + return Err(RecoveryInventoryError::Changed { + namespace, + counted, + observed, + }); + } + entries.extend( + names + .into_iter() + .map(|name| RecoveryInventoryEntry { namespace, name }), + ); + Ok(()) +} + +fn refuse_duplicate(entries: &[RecoveryInventoryEntry]) -> Result<(), RecoveryInventoryError> { + for pair in entries.windows(2) { + let [left, right] = pair else { + continue; + }; + if left == right { + return Err(RecoveryInventoryError::Duplicate { + namespace: left.namespace, + name: left.name.clone(), + }); + } + } + Ok(()) +} diff --git a/src/adapters/recovery_inventory_error.rs b/src/adapters/recovery_inventory_error.rs new file mode 100644 index 0000000..a9eae72 --- /dev/null +++ b/src/adapters/recovery_inventory_error.rs @@ -0,0 +1,114 @@ +//! This module owns bounded recovery-inventory failures. + +use std::error::Error; +use std::fmt; +use std::io; + +use super::{RecoveryEntryName, RecoveryInventoryOperation, RecoveryNamespace}; + +/// Why read-only recovery inventory could not produce one exact snapshot. +#[derive(Debug)] +pub enum RecoveryInventoryError { + /// A storage operation failed. + Io { + /// Namespace being inspected. + namespace: RecoveryNamespace, + /// Exact failed operation. + operation: RecoveryInventoryOperation, + /// Underlying storage refusal. + source: io::Error, + }, + /// Counting exceeded the configured entry ceiling. + EntryLimit { + /// Admitted maximum. + maximum: u64, + /// Smallest count proved before stopping. + observed_at_least: u64, + }, + /// A namespace changed between count and name reads. + Changed { + /// Namespace that changed. + namespace: RecoveryNamespace, + /// Previously observed count. + counted: u64, + /// Count returned by the bounded name read. + observed: u64, + }, + /// One namespace returned the same raw name more than once. + Duplicate { + /// Namespace containing the duplicate. + namespace: RecoveryNamespace, + /// Exact duplicate raw name. + name: RecoveryEntryName, + }, + /// The admitted count cannot fit the current address space. + AddressSpace { + /// Entry count that could not be represented. + observed: u64, + }, +} + +impl RecoveryInventoryError { + pub(super) const fn io( + namespace: RecoveryNamespace, + operation: RecoveryInventoryOperation, + source: io::Error, + ) -> Self { + Self::Io { + namespace, + operation, + source, + } + } +} + +impl fmt::Display for RecoveryInventoryError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Io { + namespace, + operation, + source, + } => write!( + formatter, + "recovery inventory {operation} failed in {namespace}: {source}" + ), + Self::EntryLimit { + maximum, + observed_at_least, + } => write!( + formatter, + "recovery inventory exceeds {maximum} entries; observed at least {observed_at_least}" + ), + Self::Changed { + namespace, + counted, + observed, + } => write!( + formatter, + "{namespace} changed during recovery inventory: counted {counted}, observed {observed}" + ), + Self::Duplicate { namespace, name } => write!( + formatter, + "{namespace} returned duplicate recovery name {:?}", + name.as_bytes() + ), + Self::AddressSpace { observed } => write!( + formatter, + "recovery inventory count {observed} does not fit the address space" + ), + } + } +} + +impl Error for RecoveryInventoryError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Io { source, .. } => Some(source), + Self::EntryLimit { .. } + | Self::Changed { .. } + | Self::Duplicate { .. } + | Self::AddressSpace { .. } => None, + } + } +} diff --git a/src/adapters/recovery_inventory_limit.rs b/src/adapters/recovery_inventory_limit.rs new file mode 100644 index 0000000..fe5d2a8 --- /dev/null +++ b/src/adapters/recovery_inventory_limit.rs @@ -0,0 +1,73 @@ +//! This module owns the recovery inventory entry-count bound. + +use std::error::Error; +use std::fmt; + +/// Maximum entries retained by one complete recovery inventory. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct RecoveryInventoryLimit { + maximum: u64, +} + +impl RecoveryInventoryLimit { + /// Protocol-wide maximum inventory entry count. + pub const PROTOCOL_MAXIMUM: u64 = 2_097_152; + + /// Admits a configured maximum no greater than the protocol ceiling. + /// + /// Zero is valid and admits only an empty inventory. + /// + /// # Errors + /// + /// Returns [`RecoveryInventoryLimitError`] when `maximum` exceeds the + /// protocol ceiling. + pub const fn new(maximum: u64) -> Result { + if maximum <= Self::PROTOCOL_MAXIMUM { + Ok(Self { maximum }) + } else { + Err(RecoveryInventoryLimitError::AboveProtocolMaximum { + requested: maximum, + maximum: Self::PROTOCOL_MAXIMUM, + }) + } + } + + /// Returns the protocol-wide maximum. + #[must_use] + pub const fn protocol_maximum() -> Self { + Self { + maximum: Self::PROTOCOL_MAXIMUM, + } + } + + /// Returns the admitted maximum. + #[must_use] + pub const fn get(self) -> u64 { + self.maximum + } +} + +/// Why a recovery inventory limit is invalid. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum RecoveryInventoryLimitError { + /// The requested limit exceeds the protocol ceiling. + AboveProtocolMaximum { + /// Caller-requested limit. + requested: u64, + /// Protocol maximum. + maximum: u64, + }, +} + +impl fmt::Display for RecoveryInventoryLimitError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::AboveProtocolMaximum { requested, maximum } => write!( + formatter, + "recovery inventory limit {requested} exceeds protocol maximum {maximum}" + ), + } + } +} + +impl Error for RecoveryInventoryLimitError {} diff --git a/src/adapters/recovery_inventory_operation.rs b/src/adapters/recovery_inventory_operation.rs new file mode 100644 index 0000000..65c81ed --- /dev/null +++ b/src/adapters/recovery_inventory_operation.rs @@ -0,0 +1,21 @@ +//! This module owns recovery-inventory storage operation identity. + +use std::fmt; + +/// Exact storage operation attempted during recovery inventory. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum RecoveryInventoryOperation { + /// Count entries without retaining their names. + Count, + /// Read validated raw entry names after count admission. + ReadNames, +} + +impl fmt::Display for RecoveryInventoryOperation { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(match self { + Self::Count => "entry count", + Self::ReadNames => "entry-name read", + }) + } +} diff --git a/src/adapters/recovery_inventory_storage.rs b/src/adapters/recovery_inventory_storage.rs new file mode 100644 index 0000000..2c9b113 --- /dev/null +++ b/src/adapters/recovery_inventory_storage.rs @@ -0,0 +1,30 @@ +//! This module owns the recovery-inventory storage port. + +use std::io; + +use super::{RecoveryEntryName, RecoveryNamespace}; + +/// Read-only storage capabilities required to inventory recovery evidence. +/// +/// Implementations must count without retaining names. After count admission, +/// `read_entry_names` must stop and refuse if it observes more than +/// `expected_count`; callers independently verify the returned exact count. +pub trait RecoveryInventoryStorage { + /// Counts one namespace without retaining entry names. + /// + /// # Errors + /// + /// Returns the underlying storage refusal. + fn count_entries(&mut self, namespace: RecoveryNamespace) -> io::Result; + + /// Reads at most the previously observed number of validated raw names. + /// + /// # Errors + /// + /// Returns the underlying storage, validation, or drift refusal. + fn read_entry_names( + &mut self, + namespace: RecoveryNamespace, + expected_count: u64, + ) -> io::Result>; +} diff --git a/src/adapters/recovery_namespace.rs b/src/adapters/recovery_namespace.rs new file mode 100644 index 0000000..fe687f7 --- /dev/null +++ b/src/adapters/recovery_namespace.rs @@ -0,0 +1,27 @@ +//! This module owns recovery-inventory namespace identity. + +use std::fmt; + +/// One protocol-owned directory scanned during recovery inventory. +#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] +pub enum RecoveryNamespace { + /// The store root. + Root, + /// The fixed-name staging directory. + Staging, + /// The immutable segment pool. + Segments, + /// The immutable catalog pool. + Catalogs, +} + +impl fmt::Display for RecoveryNamespace { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(match self { + Self::Root => "store root", + Self::Staging => "staging", + Self::Segments => "segments", + Self::Catalogs => "catalogs", + }) + } +} diff --git a/src/lib.rs b/src/lib.rs index 99b65bc..82fa4c5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -43,16 +43,19 @@ pub use adapters::{ FilesystemCatalogPublisher, FilesystemCatalogSnapshot, FilesystemPlatformAdmission, FilesystemSegmentStage, FilesystemWriterLock, LayoutDecodeError, LayoutDecodePolicy, LayoutEncodeError, LayoutIdBinaryParseError, LayoutIdTextParseError, - PublicationHeadDecodeError, SealedSegment, SegmentDigest, SegmentDurabilityPhase, - SegmentHeader, SegmentHeaderError, SegmentPublication, SegmentPublicationError, - SegmentReadError, SegmentReadPolicy, SegmentRecordAdmissionError, SegmentRecordChecksum, + PublicationHeadDecodeError, RecoveryEntryName, RecoveryEntryNameError, RecoveryInventory, + RecoveryInventoryEntry, RecoveryInventoryError, RecoveryInventoryLimit, + RecoveryInventoryLimitError, RecoveryInventoryOperation, RecoveryInventoryStorage, + RecoveryNamespace, SealedSegment, SegmentDigest, SegmentDurabilityPhase, SegmentHeader, + SegmentHeaderError, SegmentPublication, SegmentPublicationError, SegmentReadError, + SegmentReadPolicy, SegmentRecordAdmissionError, SegmentRecordChecksum, SegmentRecordDecodeError, SegmentRecordHeader, SegmentRecordHeaderError, SegmentRecordIdentity, SegmentRecordLength, SegmentRecordLimit, SegmentRecordLimitError, SegmentRecordPayloadLength, SegmentRecords, SegmentSeal, SegmentSealError, SegmentStage, SegmentStageCreateError, SegmentWriteError, SegmentWritePhase, StagedSegment, StorageProfileIdParseError, StoreInitializationError, StoreInitializationPhase, StoreInitializationReceipt, StoreInitializationStorage, WriterLockAcquireError, WriterLockAcquirePhase, initialize_store, - publish_catalog_generation, + publish_catalog_generation, read_recovery_inventory, }; pub use blob::{ BlobHashError, BlobHasher, BlobId, BlobLength, BlobReadError, ByteLength, ByteOffset, diff --git a/tests/recovery_inventory.rs b/tests/recovery_inventory.rs new file mode 100644 index 0000000..886592f --- /dev/null +++ b/tests/recovery_inventory.rs @@ -0,0 +1,57 @@ +//! Bounded, deterministic recovery-inventory laws. + +#[path = "recovery_inventory/inventory_double.rs"] +pub mod inventory_double; +#[path = "recovery_inventory/refusal_laws.rs"] +mod refusal_laws; + +use std::error::Error; + +use inventory_double::{InventoryCall, InventoryDouble}; +use keep::{RecoveryEntryName, RecoveryInventoryLimit, RecoveryNamespace, read_recovery_inventory}; + +#[test] +fn every_count_precedes_name_retention_in_fixed_namespace_order() -> Result<(), Box> { + let names = [ + vec![name(b"writer.lock")?, name(b"HEAD")?], + vec![name(b"current.seg")?], + vec![], + vec![name(b"catalog-2")?], + ]; + let mut storage = InventoryDouble::new([2, 1, 0, 1], names); + + let inventory = + read_recovery_inventory(&mut storage, RecoveryInventoryLimit::protocol_maximum())?; + + assert_eq!( + storage.calls(), + &[ + InventoryCall::Count(RecoveryNamespace::Root), + InventoryCall::Count(RecoveryNamespace::Staging), + InventoryCall::Count(RecoveryNamespace::Segments), + InventoryCall::Count(RecoveryNamespace::Catalogs), + InventoryCall::Read(RecoveryNamespace::Root, 2), + InventoryCall::Read(RecoveryNamespace::Staging, 1), + InventoryCall::Read(RecoveryNamespace::Segments, 0), + InventoryCall::Read(RecoveryNamespace::Catalogs, 1), + ] + ); + assert_eq!( + inventory + .entries() + .iter() + .map(|entry| (entry.namespace(), entry.name().as_bytes())) + .collect::>(), + vec![ + (RecoveryNamespace::Root, b"HEAD".as_slice()), + (RecoveryNamespace::Root, b"writer.lock".as_slice()), + (RecoveryNamespace::Staging, b"current.seg".as_slice()), + (RecoveryNamespace::Catalogs, b"catalog-2".as_slice()), + ] + ); + Ok(()) +} + +pub(crate) fn name(bytes: &[u8]) -> Result> { + Ok(RecoveryEntryName::new(bytes.to_vec())?) +} diff --git a/tests/recovery_inventory/inventory_double.rs b/tests/recovery_inventory/inventory_double.rs new file mode 100644 index 0000000..e9c9f3d --- /dev/null +++ b/tests/recovery_inventory/inventory_double.rs @@ -0,0 +1,65 @@ +//! Fault-recording recovery inventory port. + +use std::io; + +use keep::{RecoveryEntryName, RecoveryInventoryStorage, RecoveryNamespace}; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +/// One observed inventory-port call. +pub enum InventoryCall { + /// Count the selected namespace. + Count(RecoveryNamespace), + /// Read the selected namespace with the admitted expected count. + Read(RecoveryNamespace, u64), +} + +/// Deterministic recovery-inventory storage double. +pub struct InventoryDouble { + counts: [u64; 4], + names: [Vec; 4], + calls: Vec, +} + +impl InventoryDouble { + pub(crate) const fn new(counts: [u64; 4], names: [Vec; 4]) -> Self { + Self { + counts, + names, + calls: Vec::new(), + } + } + + pub(crate) fn calls(&self) -> &[InventoryCall] { + &self.calls + } +} + +impl RecoveryInventoryStorage for InventoryDouble { + fn count_entries(&mut self, namespace: RecoveryNamespace) -> io::Result { + self.calls.push(InventoryCall::Count(namespace)); + let [root, staging, segments, catalogs] = self.counts; + Ok(match namespace { + RecoveryNamespace::Root => root, + RecoveryNamespace::Staging => staging, + RecoveryNamespace::Segments => segments, + RecoveryNamespace::Catalogs => catalogs, + }) + } + + fn read_entry_names( + &mut self, + namespace: RecoveryNamespace, + expected_count: u64, + ) -> io::Result> { + self.calls + .push(InventoryCall::Read(namespace, expected_count)); + let [root, staging, segments, catalogs] = &self.names; + Ok(match namespace { + RecoveryNamespace::Root => root, + RecoveryNamespace::Staging => staging, + RecoveryNamespace::Segments => segments, + RecoveryNamespace::Catalogs => catalogs, + } + .clone()) + } +} diff --git a/tests/recovery_inventory/refusal_laws.rs b/tests/recovery_inventory/refusal_laws.rs new file mode 100644 index 0000000..7a553ea --- /dev/null +++ b/tests/recovery_inventory/refusal_laws.rs @@ -0,0 +1,149 @@ +//! Exact bounded-inventory refusal laws. + +use std::error::Error; +use std::io; + +use keep::{ + RecoveryEntryNameError, RecoveryInventoryError, RecoveryInventoryLimit, + RecoveryInventoryLimitError, RecoveryInventoryOperation, RecoveryInventoryStorage, + RecoveryNamespace, read_recovery_inventory, +}; + +use super::inventory_double::{InventoryCall, InventoryDouble}; +use super::name; + +#[test] +fn configured_limit_refuses_before_any_name_is_retained() -> Result<(), Box> { + let mut storage = InventoryDouble::new([2, 2, 0, 0], empty_names()); + let limit = RecoveryInventoryLimit::new(3)?; + let Err(error) = read_recovery_inventory(&mut storage, limit) else { + return Err("inventory exceeded its configured entry limit".into()); + }; + + assert!(matches!( + error, + RecoveryInventoryError::EntryLimit { + maximum: 3, + observed_at_least: 4, + } + )); + assert_eq!( + storage.calls(), + &[ + InventoryCall::Count(RecoveryNamespace::Root), + InventoryCall::Count(RecoveryNamespace::Staging), + ] + ); + Ok(()) +} + +#[test] +fn changed_entry_count_is_an_exact_refusal() -> Result<(), Box> { + let mut names = empty_names(); + names[0] = vec![name(b"HEAD")?, name(b"writer.lock")?]; + let mut storage = InventoryDouble::new([1, 0, 0, 0], names); + let Err(error) = + read_recovery_inventory(&mut storage, RecoveryInventoryLimit::protocol_maximum()) + else { + return Err("inventory admitted a count that changed before reading".into()); + }; + + assert!(matches!( + error, + RecoveryInventoryError::Changed { + namespace: RecoveryNamespace::Root, + counted: 1, + observed: 2, + } + )); + Ok(()) +} + +#[test] +fn duplicate_names_are_refused_after_deterministic_sorting() -> Result<(), Box> { + let mut names = empty_names(); + names[2] = vec![name(b"segment-a")?, name(b"segment-a")?]; + let mut storage = InventoryDouble::new([0, 0, 2, 0], names); + let Err(error) = + read_recovery_inventory(&mut storage, RecoveryInventoryLimit::protocol_maximum()) + else { + return Err("inventory admitted a duplicate namespace entry".into()); + }; + + assert!(matches!( + error, + RecoveryInventoryError::Duplicate { + namespace: RecoveryNamespace::Segments, + ref name, + } if name.as_bytes() == b"segment-a" + )); + Ok(()) +} + +#[test] +fn limits_and_entry_names_refuse_out_of_contract_values() { + assert!(matches!( + RecoveryInventoryLimit::new(2_097_153), + Err(RecoveryInventoryLimitError::AboveProtocolMaximum { + requested: 2_097_153, + maximum: 2_097_152, + }) + )); + assert!(matches!( + keep::RecoveryEntryName::new(Vec::new()), + Err(RecoveryEntryNameError::Empty) + )); + assert!(matches!( + keep::RecoveryEntryName::new(b"../HEAD".to_vec()), + Err(RecoveryEntryNameError::PathSeparator) + )); + assert!(matches!( + keep::RecoveryEntryName::new(vec![b'H', 0, b'D']), + Err(RecoveryEntryNameError::Nul) + )); +} + +#[test] +fn storage_refusal_preserves_namespace_operation_and_source() -> Result<(), Box> { + let mut storage = CountFailure; + let Err(error) = + read_recovery_inventory(&mut storage, RecoveryInventoryLimit::protocol_maximum()) + else { + return Err("inventory discarded a storage count failure".into()); + }; + + assert!(matches!( + error, + RecoveryInventoryError::Io { + namespace: RecoveryNamespace::Root, + operation: RecoveryInventoryOperation::Count, + ref source, + } if source.kind() == io::ErrorKind::PermissionDenied + )); + Ok(()) +} + +fn empty_names() -> [Vec; 4] { + std::array::from_fn(|_| Vec::new()) +} + +struct CountFailure; + +impl RecoveryInventoryStorage for CountFailure { + fn count_entries(&mut self, _namespace: RecoveryNamespace) -> io::Result { + Err(io::Error::new( + io::ErrorKind::PermissionDenied, + "injected count refusal", + )) + } + + fn read_entry_names( + &mut self, + _namespace: RecoveryNamespace, + _expected_count: u64, + ) -> io::Result> { + Err(io::Error::other( + "name reads are unreachable after count refusal", + )) + } +} From b2dffe8f12a83b9eaca5b1da154b66d1e7cd6bb6 Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 05:38:54 -0700 Subject: [PATCH 07/49] Fix: Stop recovery counts at the ceiling --- CHANGELOG.md | 3 ++- src/adapters/recovery_inventory.rs | 8 +++++--- src/adapters/recovery_inventory_storage.rs | 11 ++++++----- tests/recovery_inventory.rs | 8 ++++---- tests/recovery_inventory/inventory_double.rs | 6 +++--- tests/recovery_inventory/refusal_laws.rs | 6 +++--- 6 files changed, 23 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6806ea0..97aa225 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,7 +20,8 @@ after its public API and format compatibility policies are established. inode, so replacing `writer.lock` cannot split live cooperative authority. - Recovery inventory now counts the root and three protocol directories before retaining names, enforces the configurable protocol-bounded entry ceiling, - refuses count drift and duplicates exactly, and returns deterministic + stops namespace counting at the first globally excessive entry, refuses + count drift and duplicates exactly, and returns deterministic namespace-and-raw-byte ordering through a read-only storage port. - Store initialization now exposes one storage-port state machine that admits the platform before mutation, opens and locks `writer.lock`, admits the three diff --git a/src/adapters/recovery_inventory.rs b/src/adapters/recovery_inventory.rs index e93d901..d20d9ba 100644 --- a/src/adapters/recovery_inventory.rs +++ b/src/adapters/recovery_inventory.rs @@ -72,13 +72,15 @@ fn count_namespaces( let mut counts = [0_u64; 4]; let mut total = 0_u64; for (count_slot, namespace) in counts.iter_mut().zip(NAMESPACES) { - let count = storage.count_entries(namespace).map_err(|source| { - RecoveryInventoryError::io(namespace, RecoveryInventoryOperation::Count, source) - })?; let remaining = limit .get() .checked_sub(total) .ok_or(RecoveryInventoryError::AddressSpace { observed: total })?; + let count = storage + .count_entries(namespace, remaining) + .map_err(|source| { + RecoveryInventoryError::io(namespace, RecoveryInventoryOperation::Count, source) + })?; if count > remaining { let observed_at_least = limit diff --git a/src/adapters/recovery_inventory_storage.rs b/src/adapters/recovery_inventory_storage.rs index 2c9b113..a250092 100644 --- a/src/adapters/recovery_inventory_storage.rs +++ b/src/adapters/recovery_inventory_storage.rs @@ -6,16 +6,17 @@ use super::{RecoveryEntryName, RecoveryNamespace}; /// Read-only storage capabilities required to inventory recovery evidence. /// -/// Implementations must count without retaining names. After count admission, -/// `read_entry_names` must stop and refuse if it observes more than -/// `expected_count`; callers independently verify the returned exact count. +/// Implementations must count without retaining names and stop after observing +/// `remaining + 1` entries. After count admission, `read_entry_names` must stop +/// and refuse if it observes more than `expected_count`; callers independently +/// verify the returned exact count. pub trait RecoveryInventoryStorage { - /// Counts one namespace without retaining entry names. + /// Counts one namespace up to the remaining global budget plus one. /// /// # Errors /// /// Returns the underlying storage refusal. - fn count_entries(&mut self, namespace: RecoveryNamespace) -> io::Result; + fn count_entries(&mut self, namespace: RecoveryNamespace, remaining: u64) -> io::Result; /// Reads at most the previously observed number of validated raw names. /// diff --git a/tests/recovery_inventory.rs b/tests/recovery_inventory.rs index 886592f..8c9966e 100644 --- a/tests/recovery_inventory.rs +++ b/tests/recovery_inventory.rs @@ -26,10 +26,10 @@ fn every_count_precedes_name_retention_in_fixed_namespace_order() -> Result<(), assert_eq!( storage.calls(), &[ - InventoryCall::Count(RecoveryNamespace::Root), - InventoryCall::Count(RecoveryNamespace::Staging), - InventoryCall::Count(RecoveryNamespace::Segments), - InventoryCall::Count(RecoveryNamespace::Catalogs), + InventoryCall::Count(RecoveryNamespace::Root, 2_097_152), + InventoryCall::Count(RecoveryNamespace::Staging, 2_097_150), + InventoryCall::Count(RecoveryNamespace::Segments, 2_097_149), + InventoryCall::Count(RecoveryNamespace::Catalogs, 2_097_149), InventoryCall::Read(RecoveryNamespace::Root, 2), InventoryCall::Read(RecoveryNamespace::Staging, 1), InventoryCall::Read(RecoveryNamespace::Segments, 0), diff --git a/tests/recovery_inventory/inventory_double.rs b/tests/recovery_inventory/inventory_double.rs index e9c9f3d..cfad988 100644 --- a/tests/recovery_inventory/inventory_double.rs +++ b/tests/recovery_inventory/inventory_double.rs @@ -8,7 +8,7 @@ use keep::{RecoveryEntryName, RecoveryInventoryStorage, RecoveryNamespace}; /// One observed inventory-port call. pub enum InventoryCall { /// Count the selected namespace. - Count(RecoveryNamespace), + Count(RecoveryNamespace, u64), /// Read the selected namespace with the admitted expected count. Read(RecoveryNamespace, u64), } @@ -35,8 +35,8 @@ impl InventoryDouble { } impl RecoveryInventoryStorage for InventoryDouble { - fn count_entries(&mut self, namespace: RecoveryNamespace) -> io::Result { - self.calls.push(InventoryCall::Count(namespace)); + fn count_entries(&mut self, namespace: RecoveryNamespace, remaining: u64) -> io::Result { + self.calls.push(InventoryCall::Count(namespace, remaining)); let [root, staging, segments, catalogs] = self.counts; Ok(match namespace { RecoveryNamespace::Root => root, diff --git a/tests/recovery_inventory/refusal_laws.rs b/tests/recovery_inventory/refusal_laws.rs index 7a553ea..7b632a1 100644 --- a/tests/recovery_inventory/refusal_laws.rs +++ b/tests/recovery_inventory/refusal_laws.rs @@ -30,8 +30,8 @@ fn configured_limit_refuses_before_any_name_is_retained() -> Result<(), Box [Vec; 4] { struct CountFailure; impl RecoveryInventoryStorage for CountFailure { - fn count_entries(&mut self, _namespace: RecoveryNamespace) -> io::Result { + fn count_entries(&mut self, _namespace: RecoveryNamespace, _remaining: u64) -> io::Result { Err(io::Error::new( io::ErrorKind::PermissionDenied, "injected count refusal", From 13122de1a59a6a96d6015c36a56b3b2b81b64539 Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 05:51:50 -0700 Subject: [PATCH 08/49] Add: Read bounded filesystem recovery inventory --- CHANGELOG.md | 4 + README.md | 11 +- docs/formats/segment-store-v1/recovery.md | 7 +- docs/formats/segment-store-v1/requirements.md | 1 + .../filesystem_recovery_inventory_reader.rs | 137 +++++++++++++++ .../filesystem_recovery_inventory_scan.rs | 73 ++++++++ .../filesystem_recovery_inventory_tests.rs | 156 ++++++++++++++++++ src/adapters/filesystem_recovery_namespace.rs | 87 ++++++++++ src/adapters/mod.rs | 6 + src/adapters/recovery_inventory_operation.rs | 6 + src/lib.rs | 8 +- tests/recovery_inventory.rs | 31 ++++ 12 files changed, 516 insertions(+), 11 deletions(-) create mode 100644 src/adapters/filesystem_recovery_inventory_reader.rs create mode 100644 src/adapters/filesystem_recovery_inventory_scan.rs create mode 100644 src/adapters/filesystem_recovery_inventory_tests.rs create mode 100644 src/adapters/filesystem_recovery_namespace.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 97aa225..639a3dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,10 @@ after its public API and format compatibility policies are established. stops namespace counting at the first globally excessive entry, refuses count drift and duplicates exactly, and returns deterministic namespace-and-raw-byte ordering through a read-only storage port. +- Filesystem recovery inventory now pins the root and all three protocol + directories without following links, verifies child-directory identity + before and after bounded scanning, and preserves raw Linux entry-name bytes + without mutating protocol state. - Store initialization now exposes one storage-port state machine that admits the platform before mutation, opens and locks `writer.lock`, admits the three protocol directories in order, synchronizes the root, and preserves the diff --git a/README.md b/README.md index 659327e..42b4a61 100644 --- a/README.md +++ b/README.md @@ -67,11 +67,12 @@ evidence, and ambiguous crash states remain explicit recovery work. An absent empty. The public storage-independent recovery inventory counts all four protocol namespaces before retaining names, applies a configurable ceiling no greater than 2,097,152 entries, and returns duplicate-free deterministic raw -name order; its concrete filesystem reader and artifact classification remain -planned. Crash-injection execution, explicit recovery, retention, complete -namespace verification after initialization, compaction, and garbage -collection remain planned. Presence in the reference CAS does not claim -retention, crash recovery, or durability. +name order. `FilesystemRecoveryInventoryReader` implements that contract with +pinned, no-follow namespace capabilities and pre/post identity verification on +the admitted Linux ext4 profile. Artifact classification remains planned. +Crash-injection execution, explicit recovery, retention, compaction, and +garbage collection remain planned. Presence in the reference CAS does not +claim retention, crash recovery, or durability. ```rust use keep::BlobId; diff --git a/docs/formats/segment-store-v1/recovery.md b/docs/formats/segment-store-v1/recovery.md index 2b6fad5..f38c8c4 100644 --- a/docs/formats/segment-store-v1/recovery.md +++ b/docs/formats/segment-store-v1/recovery.md @@ -25,8 +25,11 @@ cannot be proven atomic are unrecoverable ambiguity. The current public `read_recovery_inventory` slice implements this fixed-order count-before-retain orchestration through a read-only storage port. It enforces the configured and protocol ceilings, count stability, duplicate refusal, and -namespace-plus-raw-byte ordering. The concrete filesystem inventory adapter and -artifact classification remain unimplemented. +namespace-plus-raw-byte ordering. `FilesystemRecoveryInventoryReader` pins the +root and three no-follow child directories, bounds each scan by the remaining +global budget, preserves raw Linux name bytes, and verifies child-directory +identity before and after inventory. Artifact classification remains +unimplemented. The sole admissible duplicate digest is one fixed staging name and its exact digest-derived pool name after a link transition. Recovery admits that pair diff --git a/docs/formats/segment-store-v1/requirements.md b/docs/formats/segment-store-v1/requirements.md index 09da8ae..76f7ab9 100644 --- a/docs/formats/segment-store-v1/requirements.md +++ b/docs/formats/segment-store-v1/requirements.md @@ -101,6 +101,7 @@ recovery execution. | `KEEP-RECOVERY-003` | Production initialization admits only a writable, non-casefolded Linux ext4 root, refuses any noncanonical root entry before mutation, completes an empty or partial canonical namespace without replacing evidence, excludes a second initializer, and retains writer authority through the synchronized receipt | Capability-relative filesystem fixture and exact platform-profile classifier | `src/adapters/filesystem_store_initializer_tests.rs`, `src/adapters/filesystem_platform_profile.rs`, `tests/store_initialization.rs` | Implemented in #17 | | `KEEP-RECOVERY-004` | Writer authority is returned only when the locked handle still has the exact device and inode resolved by the canonical `writer.lock` entry after kernel acquisition | Deterministic lock-entry replacement fixture | `src/adapters/filesystem_writer_lock_tests.rs` | Implemented in #17 | | `KEEP-RECOVERY-005` | Recovery counts the root and three protocol directories in fixed order before retaining names, refuses at the configured or protocol entry ceiling with the exact observed-at-least count, then returns one duplicate-free inventory sorted by namespace and raw name bytes | Fault-recording inventory port | `tests/recovery_inventory.rs` | Implemented in #17 | +| `KEEP-RECOVERY-006` | Filesystem inventory pins the admitted root and protocol directories without following links, verifies child-directory identity before and after scanning, stops each count at the remaining global budget plus one, preserves raw Linux entry-name bytes, and performs no protocol mutation | Capability-relative filesystem fixture | `src/adapters/filesystem_recovery_inventory_tests.rs`, `tests/recovery_inventory.rs` | Implemented in #17 | diff --git a/src/adapters/filesystem_recovery_inventory_reader.rs b/src/adapters/filesystem_recovery_inventory_reader.rs new file mode 100644 index 0000000..f636262 --- /dev/null +++ b/src/adapters/filesystem_recovery_inventory_reader.rs @@ -0,0 +1,137 @@ +//! This module owns capability-relative filesystem recovery inventory. + +use std::io; +use std::path::Path; + +#[cfg(test)] +use cap_std::ambient_authority; +use cap_std::fs::Dir; + +use super::{ + RecoveryEntryName, RecoveryInventory, RecoveryInventoryError, RecoveryInventoryLimit, + RecoveryInventoryOperation, RecoveryInventoryStorage, RecoveryNamespace, + filesystem_platform_profile, filesystem_recovery_inventory_scan, + filesystem_recovery_namespace::PinnedRecoveryDirectory, read_recovery_inventory, +}; + +const STAGING_NAME: &str = "staging"; +const SEGMENTS_NAME: &str = "segments"; +const CATALOGS_NAME: &str = "catalogs"; + +/// A pinned, read-only view of the four recovery inventory namespaces. +/// +/// Opening performs no protocol mutation. The production constructor admits +/// the same Linux ext4 profile as store initialization and pins all three child +/// directories without following symbolic links. +#[must_use] +pub struct FilesystemRecoveryInventoryReader { + root: Dir, + staging: PinnedRecoveryDirectory, + segments: PinnedRecoveryDirectory, + catalogs: PinnedRecoveryDirectory, +} + +impl FilesystemRecoveryInventoryReader { + /// Opens one initialized store for read-only recovery inventory. + /// + /// The call is synchronous, allocates no content-sized memory, and may + /// block on filesystem I/O. + /// + /// # Errors + /// + /// Returns [`RecoveryInventoryError::Io`] with the exact namespace and + /// open phase for an unsupported platform or missing, replaced, linked, or + /// non-directory protocol namespace. + pub fn open(store_root: &Path) -> Result { + let root = filesystem_platform_profile::open(store_root).map_err(|source| { + RecoveryInventoryError::io( + RecoveryNamespace::Root, + RecoveryInventoryOperation::OpenNamespace, + source, + ) + })?; + Self::from_root(root) + } + + #[cfg(test)] + pub(super) fn open_unchecked_for_tests( + store_root: &Path, + ) -> Result { + let root = Dir::open_ambient_dir(store_root, ambient_authority()).map_err(|source| { + RecoveryInventoryError::io( + RecoveryNamespace::Root, + RecoveryInventoryOperation::OpenNamespace, + source, + ) + })?; + Self::from_root(root) + } + + fn from_root(root: Dir) -> Result { + let staging = + PinnedRecoveryDirectory::open(&root, RecoveryNamespace::Staging, STAGING_NAME)?; + let segments = + PinnedRecoveryDirectory::open(&root, RecoveryNamespace::Segments, SEGMENTS_NAME)?; + let catalogs = + PinnedRecoveryDirectory::open(&root, RecoveryNamespace::Catalogs, CATALOGS_NAME)?; + Ok(Self { + root, + staging, + segments, + catalogs, + }) + } + + /// Reads one bounded, deterministic inventory without mutating protocol + /// state. + /// + /// The call is synchronous and may block on directory I/O. Peak allocation + /// includes the final `limit`-bounded inventory plus one temporary + /// `limit + 1`-bounded namespace table used to prove count drift. + /// + /// # Errors + /// + /// Returns [`RecoveryInventoryError`] on namespace replacement, storage + /// refusal, entry-limit excess, count drift, or duplicate names. + pub fn read( + &mut self, + limit: RecoveryInventoryLimit, + ) -> Result { + self.verify_namespaces()?; + let inventory = read_recovery_inventory(self, limit)?; + self.verify_namespaces()?; + Ok(inventory) + } + + fn verify_namespaces(&self) -> Result<(), RecoveryInventoryError> { + self.staging.verify(&self.root)?; + self.segments.verify(&self.root)?; + self.catalogs.verify(&self.root) + } + + const fn directory(&self, namespace: RecoveryNamespace) -> &Dir { + match namespace { + RecoveryNamespace::Root => &self.root, + RecoveryNamespace::Staging => self.staging.directory(), + RecoveryNamespace::Segments => self.segments.directory(), + RecoveryNamespace::Catalogs => self.catalogs.directory(), + } + } +} + +impl RecoveryInventoryStorage for FilesystemRecoveryInventoryReader { + fn count_entries(&mut self, namespace: RecoveryNamespace, remaining: u64) -> io::Result { + filesystem_recovery_inventory_scan::count_entries(self.directory(namespace), remaining) + } + + fn read_entry_names( + &mut self, + namespace: RecoveryNamespace, + expected_count: u64, + ) -> io::Result> { + filesystem_recovery_inventory_scan::read_entry_names( + self.directory(namespace), + expected_count, + ) + } +} diff --git a/src/adapters/filesystem_recovery_inventory_scan.rs b/src/adapters/filesystem_recovery_inventory_scan.rs new file mode 100644 index 0000000..7077be7 --- /dev/null +++ b/src/adapters/filesystem_recovery_inventory_scan.rs @@ -0,0 +1,73 @@ +//! This module owns bounded filesystem recovery name scanning. + +use std::io; + +use cap_std::fs::Dir; + +use super::{RecoveryEntryName, RecoveryInventoryLimit}; + +pub(super) fn count_entries(directory: &Dir, remaining: u64) -> io::Result { + if remaining > RecoveryInventoryLimit::PROTOCOL_MAXIMUM { + return Err(invalid_input( + "recovery count budget exceeds protocol maximum", + )); + } + let ceiling = remaining + .checked_add(1) + .ok_or_else(|| invalid_input("recovery count budget cannot admit a drift witness"))?; + let mut observed = 0_u64; + for entry in directory.entries()? { + let _entry = entry?; + observed = observed + .checked_add(1) + .ok_or_else(|| invalid_input("recovery entry count overflowed"))?; + if observed == ceiling { + break; + } + } + Ok(observed) +} + +pub(super) fn read_entry_names( + directory: &Dir, + expected_count: u64, +) -> io::Result> { + if expected_count > RecoveryInventoryLimit::PROTOCOL_MAXIMUM { + return Err(invalid_input( + "recovery expected count exceeds protocol maximum", + )); + } + let expected = usize::try_from(expected_count) + .map_err(|_| invalid_input("recovery expected count does not fit the address space"))?; + let capacity = expected + .checked_add(1) + .ok_or_else(|| invalid_input("recovery name capacity overflowed"))?; + let mut names = Vec::with_capacity(capacity); + for entry in directory.entries()? { + names.push(entry_name(&entry?)?); + if names.len() == capacity { + break; + } + } + Ok(names) +} + +#[cfg(unix)] +fn entry_name(entry: &cap_std::fs::DirEntry) -> io::Result { + use std::os::unix::ffi::OsStrExt; + + RecoveryEntryName::new(entry.file_name().as_bytes().to_vec()) + .map_err(|source| io::Error::new(io::ErrorKind::InvalidData, source)) +} + +#[cfg(not(unix))] +fn entry_name(_entry: &cap_std::fs::DirEntry) -> io::Result { + Err(io::Error::new( + io::ErrorKind::Unsupported, + "raw recovery entry names currently require a Unix platform", + )) +} + +fn invalid_input(message: &'static str) -> io::Error { + io::Error::new(io::ErrorKind::InvalidInput, message) +} diff --git a/src/adapters/filesystem_recovery_inventory_tests.rs b/src/adapters/filesystem_recovery_inventory_tests.rs new file mode 100644 index 0000000..e19c204 --- /dev/null +++ b/src/adapters/filesystem_recovery_inventory_tests.rs @@ -0,0 +1,156 @@ +//! Capability-relative filesystem recovery-inventory laws. + +use std::error::Error; +#[cfg(target_os = "linux")] +use std::ffi::OsString; +use std::fs; +#[cfg(target_os = "linux")] +use std::os::unix::ffi::OsStringExt; + +use super::filesystem_test_sandbox::TestDirectory; +use super::{ + FilesystemRecoveryInventoryReader, RecoveryInventoryError, RecoveryInventoryLimit, + RecoveryInventoryOperation, RecoveryNamespace, +}; + +const LOCK_NAME: &str = "writer.lock"; +const STAGING_NAME: &str = "staging"; +const SEGMENTS_NAME: &str = "segments"; +const CATALOGS_NAME: &str = "catalogs"; + +#[test] +fn filesystem_inventory_returns_deterministic_name_order() -> Result<(), Box> { + let sandbox = initialized_namespace("recovery-inventory-order")?; + let name = b"segment-z"; + fs::write( + sandbox.path().join(SEGMENTS_NAME).join("segment-z"), + b"retained evidence", + )?; + let mut reader = FilesystemRecoveryInventoryReader::open_unchecked_for_tests(sandbox.path())?; + + let inventory = reader.read(RecoveryInventoryLimit::protocol_maximum())?; + + assert!(inventory.entries().windows(2).all(|pair| { + let [left, right] = pair else { + return false; + }; + left < right + })); + assert!(inventory.entries().iter().any(|entry| { + entry.namespace() == RecoveryNamespace::Segments && entry.name().as_bytes() == name + })); + assert_eq!( + fs::read(sandbox.path().join(SEGMENTS_NAME).join("segment-z"))?, + b"retained evidence" + ); + drop(reader); + sandbox.remove()?; + Ok(()) +} + +#[cfg(target_os = "linux")] +#[test] +fn filesystem_inventory_preserves_raw_linux_name_bytes() -> Result<(), Box> { + let sandbox = initialized_namespace("recovery-inventory-raw")?; + let raw_name = vec![b's', b'e', b'g', 0x80]; + fs::write( + sandbox + .path() + .join(SEGMENTS_NAME) + .join(OsString::from_vec(raw_name.clone())), + [], + )?; + let mut reader = FilesystemRecoveryInventoryReader::open_unchecked_for_tests(sandbox.path())?; + + let inventory = reader.read(RecoveryInventoryLimit::protocol_maximum())?; + + assert!(inventory.entries().iter().any(|entry| { + entry.namespace() == RecoveryNamespace::Segments && entry.name().as_bytes() == raw_name + })); + drop(reader); + sandbox.remove()?; + Ok(()) +} + +#[test] +fn filesystem_count_stops_at_the_first_global_excess_entry() -> Result<(), Box> { + let sandbox = initialized_namespace("recovery-inventory-limit")?; + fs::write(sandbox.path().join(SEGMENTS_NAME).join("one"), [])?; + let mut reader = FilesystemRecoveryInventoryReader::open_unchecked_for_tests(sandbox.path())?; + let Err(error) = reader.read(RecoveryInventoryLimit::new(4)?) else { + return Err("filesystem inventory exceeded the global entry ceiling".into()); + }; + + assert!(matches!( + error, + RecoveryInventoryError::EntryLimit { + maximum: 4, + observed_at_least: 5, + } + )); + drop(reader); + sandbox.remove()?; + Ok(()) +} + +#[test] +fn filesystem_inventory_never_follows_protocol_directory_links() -> Result<(), Box> { + use std::os::unix::fs::symlink; + + let sandbox = TestDirectory::create("recovery-inventory-symlink")?; + fs::write(sandbox.path().join(LOCK_NAME), [])?; + fs::create_dir(sandbox.path().join("target"))?; + symlink("target", sandbox.path().join(STAGING_NAME))?; + fs::create_dir(sandbox.path().join(SEGMENTS_NAME))?; + fs::create_dir(sandbox.path().join(CATALOGS_NAME))?; + + let Err(error) = FilesystemRecoveryInventoryReader::open_unchecked_for_tests(sandbox.path()) + else { + return Err("filesystem inventory followed a staging-directory link".into()); + }; + assert!(matches!( + error, + RecoveryInventoryError::Io { + namespace: RecoveryNamespace::Staging, + operation: RecoveryInventoryOperation::OpenNamespace, + .. + } + )); + sandbox.remove()?; + Ok(()) +} + +#[test] +fn replaced_protocol_directory_refuses_the_pinned_inventory() -> Result<(), Box> { + let sandbox = initialized_namespace("recovery-inventory-replacement")?; + let mut reader = FilesystemRecoveryInventoryReader::open_unchecked_for_tests(sandbox.path())?; + fs::rename( + sandbox.path().join(STAGING_NAME), + sandbox.path().join("displaced-staging"), + )?; + fs::create_dir(sandbox.path().join(STAGING_NAME))?; + + let Err(error) = reader.read(RecoveryInventoryLimit::protocol_maximum()) else { + return Err("filesystem inventory admitted a replaced staging directory".into()); + }; + assert!(matches!( + error, + RecoveryInventoryError::Io { + namespace: RecoveryNamespace::Staging, + operation: RecoveryInventoryOperation::VerifyNamespace, + ref source, + } if source.kind() == std::io::ErrorKind::InvalidData + )); + drop(reader); + sandbox.remove()?; + Ok(()) +} + +fn initialized_namespace(name: &str) -> Result> { + let sandbox = TestDirectory::create(name)?; + fs::write(sandbox.path().join(LOCK_NAME), [])?; + fs::create_dir(sandbox.path().join(STAGING_NAME))?; + fs::create_dir(sandbox.path().join(SEGMENTS_NAME))?; + fs::create_dir(sandbox.path().join(CATALOGS_NAME))?; + Ok(sandbox) +} diff --git a/src/adapters/filesystem_recovery_namespace.rs b/src/adapters/filesystem_recovery_namespace.rs new file mode 100644 index 0000000..95dc0ce --- /dev/null +++ b/src/adapters/filesystem_recovery_namespace.rs @@ -0,0 +1,87 @@ +//! This module owns pinned recovery namespace identity. + +use std::io; + +use cap_fs_ext::MetadataExt; +use cap_std::fs::{Dir, Metadata}; + +use super::{ + RecoveryInventoryError, RecoveryInventoryOperation, RecoveryNamespace, sync_capable_directory, +}; + +pub(super) struct PinnedRecoveryDirectory { + namespace: RecoveryNamespace, + name: &'static str, + identity: DirectoryIdentity, + directory: Dir, +} + +impl PinnedRecoveryDirectory { + pub(super) fn open( + root: &Dir, + namespace: RecoveryNamespace, + name: &'static str, + ) -> Result { + let directory = sync_capable_directory::open(root, name).map_err(|source| { + RecoveryInventoryError::io(namespace, RecoveryInventoryOperation::OpenNamespace, source) + })?; + let identity = DirectoryIdentity::read(&directory).map_err(|source| { + RecoveryInventoryError::io(namespace, RecoveryInventoryOperation::OpenNamespace, source) + })?; + Ok(Self { + namespace, + name, + identity, + directory, + }) + } + + pub(super) fn verify(&self, root: &Dir) -> Result<(), RecoveryInventoryError> { + let metadata = root.symlink_metadata(self.name).map_err(|source| { + RecoveryInventoryError::io( + self.namespace, + RecoveryInventoryOperation::VerifyNamespace, + source, + ) + })?; + let observed = DirectoryIdentity::from(&metadata); + if metadata.is_dir() && observed == self.identity { + return Ok(()); + } + Err(RecoveryInventoryError::io( + self.namespace, + RecoveryInventoryOperation::VerifyNamespace, + io::Error::new( + io::ErrorKind::InvalidData, + "recovery namespace changed identity after it was pinned", + ), + )) + } + + pub(super) const fn directory(&self) -> &Dir { + &self.directory + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct DirectoryIdentity { + device: u64, + inode: u64, +} + +impl DirectoryIdentity { + fn read(directory: &Dir) -> io::Result { + directory + .dir_metadata() + .map(|metadata| Self::from(&metadata)) + } +} + +impl From<&Metadata> for DirectoryIdentity { + fn from(metadata: &Metadata) -> Self { + Self { + device: metadata.dev(), + inode: metadata.ino(), + } + } +} diff --git a/src/adapters/mod.rs b/src/adapters/mod.rs index bec8f9d..dc907fd 100644 --- a/src/adapters/mod.rs +++ b/src/adapters/mod.rs @@ -79,6 +79,11 @@ mod filesystem_initialization_storage; mod filesystem_platform_admission; mod filesystem_platform_profile; mod filesystem_publisher_authority; +mod filesystem_recovery_inventory_reader; +mod filesystem_recovery_inventory_scan; +#[cfg(all(test, unix))] +mod filesystem_recovery_inventory_tests; +mod filesystem_recovery_namespace; mod filesystem_segment_stage; #[cfg(test)] mod filesystem_segment_stage_tests; @@ -226,6 +231,7 @@ pub use filesystem_catalog_publication_error::FilesystemCatalogPublicationError; pub use filesystem_catalog_publisher::FilesystemCatalogPublisher; pub use filesystem_catalog_snapshot::FilesystemCatalogSnapshot; pub use filesystem_platform_admission::FilesystemPlatformAdmission; +pub use filesystem_recovery_inventory_reader::FilesystemRecoveryInventoryReader; pub use filesystem_segment_stage::FilesystemSegmentStage; pub use filesystem_writer_lock::FilesystemWriterLock; pub use layout_decode_error::LayoutDecodeError; diff --git a/src/adapters/recovery_inventory_operation.rs b/src/adapters/recovery_inventory_operation.rs index 65c81ed..54cb899 100644 --- a/src/adapters/recovery_inventory_operation.rs +++ b/src/adapters/recovery_inventory_operation.rs @@ -5,6 +5,10 @@ use std::fmt; /// Exact storage operation attempted during recovery inventory. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum RecoveryInventoryOperation { + /// Open and pin a protocol namespace. + OpenNamespace, + /// Verify that a pinned namespace still resolves from the root. + VerifyNamespace, /// Count entries without retaining their names. Count, /// Read validated raw entry names after count admission. @@ -14,6 +18,8 @@ pub enum RecoveryInventoryOperation { impl fmt::Display for RecoveryInventoryOperation { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { formatter.write_str(match self { + Self::OpenNamespace => "namespace open", + Self::VerifyNamespace => "namespace verification", Self::Count => "entry count", Self::ReadNames => "entry-name read", }) diff --git a/src/lib.rs b/src/lib.rs index 82fa4c5..01fa6ce 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -41,10 +41,10 @@ pub use adapters::{ CatalogSuccessor, CatalogTransitionError, ChecksummedCatalog, ChecksummedPublicationHead, ChecksummedSegmentRecord, ClosedSegment, FilesystemCatalogPublicationError, FilesystemCatalogPublisher, FilesystemCatalogSnapshot, FilesystemPlatformAdmission, - FilesystemSegmentStage, FilesystemWriterLock, LayoutDecodeError, LayoutDecodePolicy, - LayoutEncodeError, LayoutIdBinaryParseError, LayoutIdTextParseError, - PublicationHeadDecodeError, RecoveryEntryName, RecoveryEntryNameError, RecoveryInventory, - RecoveryInventoryEntry, RecoveryInventoryError, RecoveryInventoryLimit, + FilesystemRecoveryInventoryReader, FilesystemSegmentStage, FilesystemWriterLock, + LayoutDecodeError, LayoutDecodePolicy, LayoutEncodeError, LayoutIdBinaryParseError, + LayoutIdTextParseError, PublicationHeadDecodeError, RecoveryEntryName, RecoveryEntryNameError, + RecoveryInventory, RecoveryInventoryEntry, RecoveryInventoryError, RecoveryInventoryLimit, RecoveryInventoryLimitError, RecoveryInventoryOperation, RecoveryInventoryStorage, RecoveryNamespace, SealedSegment, SegmentDigest, SegmentDurabilityPhase, SegmentHeader, SegmentHeaderError, SegmentPublication, SegmentPublicationError, SegmentReadError, diff --git a/tests/recovery_inventory.rs b/tests/recovery_inventory.rs index 8c9966e..e6aa684 100644 --- a/tests/recovery_inventory.rs +++ b/tests/recovery_inventory.rs @@ -4,11 +4,20 @@ pub mod inventory_double; #[path = "recovery_inventory/refusal_laws.rs"] mod refusal_laws; +#[cfg(not(target_os = "linux"))] +#[path = "segment_filesystem_stage/sandbox.rs"] +pub mod sandbox; use std::error::Error; +#[cfg(not(target_os = "linux"))] +use std::fs; use inventory_double::{InventoryCall, InventoryDouble}; +#[cfg(not(target_os = "linux"))] +use keep::{FilesystemRecoveryInventoryReader, RecoveryInventoryError, RecoveryInventoryOperation}; use keep::{RecoveryEntryName, RecoveryInventoryLimit, RecoveryNamespace, read_recovery_inventory}; +#[cfg(not(target_os = "linux"))] +use sandbox::TestDirectory; #[test] fn every_count_precedes_name_retention_in_fixed_namespace_order() -> Result<(), Box> { @@ -52,6 +61,28 @@ fn every_count_precedes_name_retention_in_fixed_namespace_order() -> Result<(), Ok(()) } +#[cfg(not(target_os = "linux"))] +#[test] +fn unsupported_platform_refuses_before_filesystem_inventory_mutation() -> Result<(), Box> +{ + let sandbox = TestDirectory::create("recovery-inventory-unsupported")?; + let Err(error) = FilesystemRecoveryInventoryReader::open(sandbox.path()) else { + return Err("filesystem inventory admitted an unsupported platform".into()); + }; + + assert!(matches!( + error, + RecoveryInventoryError::Io { + namespace: RecoveryNamespace::Root, + operation: RecoveryInventoryOperation::OpenNamespace, + ref source, + } if source.kind() == std::io::ErrorKind::Unsupported + )); + assert_eq!(fs::read_dir(sandbox.path())?.count(), 0); + sandbox.remove()?; + Ok(()) +} + pub(crate) fn name(bytes: &[u8]) -> Result> { Ok(RecoveryEntryName::new(bytes.to_vec())?) } From 0042423abdaefeffcbbc9cff760e15ada878195c Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 06:08:32 -0700 Subject: [PATCH 09/49] Add: Classify canonical recovery names --- docs/formats/segment-store-v1/recovery.md | 13 +- docs/formats/segment-store-v1/requirements.md | 6 +- src/adapters/mod.rs | 13 ++ src/adapters/recovery_entry_role.rs | 46 ++++++ src/adapters/recovery_inventory.rs | 8 ++ src/adapters/recovery_name_classification.rs | 136 ++++++++++++++++++ .../recovery_name_classification_error.rs | 79 ++++++++++ src/adapters/recovery_name_manifest.rs | 61 ++++++++ src/adapters/recovery_pool_name.rs | 81 +++++++++++ src/adapters/recovery_pool_name_error.rs | 67 +++++++++ src/adapters/recovery_required_entry.rs | 27 ++++ src/lib.rs | 31 ++-- tests/recovery_name_classification.rs | 80 +++++++++++ .../grammar_laws.rs | 107 ++++++++++++++ .../refusal_laws.rs | 97 +++++++++++++ tests/recovery_name_classification_memory.rs | 55 +++++++ 16 files changed, 886 insertions(+), 21 deletions(-) create mode 100644 src/adapters/recovery_entry_role.rs create mode 100644 src/adapters/recovery_name_classification.rs create mode 100644 src/adapters/recovery_name_classification_error.rs create mode 100644 src/adapters/recovery_name_manifest.rs create mode 100644 src/adapters/recovery_pool_name.rs create mode 100644 src/adapters/recovery_pool_name_error.rs create mode 100644 src/adapters/recovery_required_entry.rs create mode 100644 tests/recovery_name_classification.rs create mode 100644 tests/recovery_name_classification/grammar_laws.rs create mode 100644 tests/recovery_name_classification/refusal_laws.rs create mode 100644 tests/recovery_name_classification_memory.rs diff --git a/docs/formats/segment-store-v1/recovery.md b/docs/formats/segment-store-v1/recovery.md index f38c8c4..cfcdec4 100644 --- a/docs/formats/segment-store-v1/recovery.md +++ b/docs/formats/segment-store-v1/recovery.md @@ -18,9 +18,10 @@ count of `2,097,153`, not a host-order-dependent exact total. A configured limit may be lower but never higher. After the count is admitted, recovery sorts names by raw canonical bytes and -verifies complete content before classification. Unknown names, symlinks, -conflicting canonical coordinates, multiple fixed-name stages, or a head that -cannot be proven atomic are unrecoverable ambiguity. +classifies the namespace grammar before opening artifact bytes. Unknown names, +symlinks, noncanonical pool coordinates, or multiple fixed-name stages are +unrecoverable ambiguity. Content verification then classifies each artifact; +a head that cannot be proven atomic is also unrecoverable ambiguity. The current public `read_recovery_inventory` slice implements this fixed-order count-before-retain orchestration through a read-only storage port. It enforces @@ -28,8 +29,10 @@ the configured and protocol ceilings, count stability, duplicate refusal, and namespace-plus-raw-byte ordering. `FilesystemRecoveryInventoryReader` pins the root and three no-follow child directories, bounds each scan by the remaining global budget, preserves raw Linux name bytes, and verifies child-directory -identity before and after inventory. Artifact classification remains -unimplemented. +identity before and after inventory. `classify_recovery_names` requires the +four initialized root entries, types each fixed name and immutable-pool +coordinate, and refuses an unknown or conflicting name without artifact I/O. +Content classification remains unimplemented. The sole admissible duplicate digest is one fixed staging name and its exact digest-derived pool name after a link transition. Recovery admits that pair diff --git a/docs/formats/segment-store-v1/requirements.md b/docs/formats/segment-store-v1/requirements.md index 76f7ab9..af28869 100644 --- a/docs/formats/segment-store-v1/requirements.md +++ b/docs/formats/segment-store-v1/requirements.md @@ -89,8 +89,9 @@ reviewable slices. The first slice freezes executable crash-point identity and sequence ownership. The second slice establishes the ordered initialization state machine and exact failure phases. The third slice binds that state machine to a fail-closed Linux ext4 adapter and canonical namespace. These -slices do not yet claim process-death injection, recovery classification, or -recovery execution. +slices now also classify canonical recovery names before opening artifact +bytes. They do not yet claim content classification, process-death injection, +or recovery execution. @@ -102,6 +103,7 @@ recovery execution. | `KEEP-RECOVERY-004` | Writer authority is returned only when the locked handle still has the exact device and inode resolved by the canonical `writer.lock` entry after kernel acquisition | Deterministic lock-entry replacement fixture | `src/adapters/filesystem_writer_lock_tests.rs` | Implemented in #17 | | `KEEP-RECOVERY-005` | Recovery counts the root and three protocol directories in fixed order before retaining names, refuses at the configured or protocol entry ceiling with the exact observed-at-least count, then returns one duplicate-free inventory sorted by namespace and raw name bytes | Fault-recording inventory port | `tests/recovery_inventory.rs` | Implemented in #17 | | `KEEP-RECOVERY-006` | Filesystem inventory pins the admitted root and protocol directories without following links, verifies child-directory identity before and after scanning, stops each count at the remaining global budget plus one, preserves raw Linux entry-name bytes, and performs no protocol mutation | Capability-relative filesystem fixture | `src/adapters/filesystem_recovery_inventory_tests.rs`, `tests/recovery_inventory.rs` | Implemented in #17 | +| `KEEP-RECOVERY-007` | Name classification requires the four initialized root entries, admits only fixed protocol names and canonical pool coordinates in their owning namespaces, refuses simultaneous fixed recovery stages before artifact reads, and moves a refused raw name without duplicating its allocation | Canonical-name matrix and allocation counter | `tests/recovery_name_classification.rs`, `tests/recovery_name_classification_memory.rs` | Implemented in #17 | diff --git a/src/adapters/mod.rs b/src/adapters/mod.rs index dc907fd..9032fdf 100644 --- a/src/adapters/mod.rs +++ b/src/adapters/mod.rs @@ -117,12 +117,19 @@ mod publication_head_decode_error_display; mod publication_head_decoder; mod publication_head_encoder; mod recovery_entry_name; +mod recovery_entry_role; mod recovery_inventory; mod recovery_inventory_error; mod recovery_inventory_limit; mod recovery_inventory_operation; mod recovery_inventory_storage; +mod recovery_name_classification; +mod recovery_name_classification_error; +mod recovery_name_manifest; mod recovery_namespace; +mod recovery_pool_name; +mod recovery_pool_name_error; +mod recovery_required_entry; mod sealed_segment; mod segment_digest; mod segment_digest_builder; @@ -242,12 +249,18 @@ pub use layout_id_text_error::LayoutIdTextParseError; pub use layout_record::CanonicalLayoutRecord; pub use publication_head_decode_error::PublicationHeadDecodeError; pub use recovery_entry_name::{RecoveryEntryName, RecoveryEntryNameError}; +pub use recovery_entry_role::RecoveryEntryRole; pub use recovery_inventory::{RecoveryInventory, RecoveryInventoryEntry, read_recovery_inventory}; pub use recovery_inventory_error::RecoveryInventoryError; pub use recovery_inventory_limit::{RecoveryInventoryLimit, RecoveryInventoryLimitError}; pub use recovery_inventory_operation::RecoveryInventoryOperation; pub use recovery_inventory_storage::RecoveryInventoryStorage; +pub use recovery_name_classification::classify_recovery_names; +pub use recovery_name_classification_error::RecoveryNameClassificationError; +pub use recovery_name_manifest::{RecoveryNameManifest, RecoveryNamedEntry}; pub use recovery_namespace::RecoveryNamespace; +pub use recovery_pool_name_error::RecoveryPoolNameError; +pub use recovery_required_entry::RecoveryRequiredEntry; pub use sealed_segment::SealedSegment; pub use segment_digest::SegmentDigest; pub use segment_header::SegmentHeader; diff --git a/src/adapters/recovery_entry_role.rs b/src/adapters/recovery_entry_role.rs new file mode 100644 index 0000000..53932f1 --- /dev/null +++ b/src/adapters/recovery_entry_role.rs @@ -0,0 +1,46 @@ +//! This module owns semantic recovery namespace roles. + +use super::SegmentDigest; +use crate::{CatalogDigest, CatalogGeneration}; + +/// Semantic role selected by one canonical recovery entry name. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum RecoveryEntryRole { + /// Persistent writer-lock file. + WriterLock, + /// Canonical staging directory. + StagingDirectory, + /// Canonical immutable segment-pool directory. + SegmentPoolDirectory, + /// Canonical immutable catalog-pool directory. + CatalogPoolDirectory, + /// Current publication head. + CurrentHead, + /// Candidate next publication head. + NextHeadStage, + /// Fixed segment staging file. + SegmentStage, + /// Fixed catalog staging file. + CatalogStage, + /// Digest-addressed immutable segment. + ImmutableSegment { + /// Physical segment digest parsed from the name. + digest: SegmentDigest, + }, + /// Generation-and-digest-addressed immutable catalog. + ImmutableCatalog { + /// Catalog generation parsed from the name. + generation: CatalogGeneration, + /// Physical catalog digest parsed from the name. + digest: CatalogDigest, + }, +} + +impl RecoveryEntryRole { + pub(super) const fn is_stage(self) -> bool { + matches!( + self, + Self::NextHeadStage | Self::SegmentStage | Self::CatalogStage + ) + } +} diff --git a/src/adapters/recovery_inventory.rs b/src/adapters/recovery_inventory.rs index d20d9ba..ba58212 100644 --- a/src/adapters/recovery_inventory.rs +++ b/src/adapters/recovery_inventory.rs @@ -31,6 +31,10 @@ impl RecoveryInventoryEntry { pub const fn name(&self) -> &RecoveryEntryName { &self.name } + + pub(super) fn into_parts(self) -> (RecoveryNamespace, RecoveryEntryName) { + (self.namespace, self.name) + } } /// One duplicate-free, deterministically ordered recovery inventory. @@ -45,6 +49,10 @@ impl RecoveryInventory { pub fn entries(&self) -> &[RecoveryInventoryEntry] { &self.entries } + + pub(super) fn into_entries(self) -> Vec { + self.entries + } } /// Counts every namespace before retaining and sorting any entry name. diff --git a/src/adapters/recovery_name_classification.rs b/src/adapters/recovery_name_classification.rs new file mode 100644 index 0000000..81a8204 --- /dev/null +++ b/src/adapters/recovery_name_classification.rs @@ -0,0 +1,136 @@ +//! This module owns deterministic recovery entry-name classification. + +use super::{ + RecoveryEntryName, RecoveryEntryRole, RecoveryInventory, RecoveryNameClassificationError, + RecoveryNameManifest, RecoveryNamedEntry, RecoveryNamespace, RecoveryRequiredEntry, + recovery_pool_name, +}; + +/// Classifies one complete inventory without opening artifact bytes. +/// +/// The operation preserves inventory order and allocates at most one manifest +/// entry per inventory entry. It rejects unknown names, missing initialized +/// root entries, noncanonical pool coordinates, and simultaneous fixed stages. +/// +/// # Errors +/// +/// Returns [`RecoveryNameClassificationError`] at the first deterministic +/// namespace, grammar, required-entry, or stage-conflict failure. +pub fn classify_recovery_names( + inventory: RecoveryInventory, +) -> Result { + let mut required = RequiredEntries::default(); + let mut first_stage = None; + let mut named = Vec::with_capacity(inventory.entries().len()); + for entry in inventory.into_entries() { + let (namespace, name) = entry.into_parts(); + let role = match classify(namespace, &name) { + Ok(role) => role, + Err(NameFailure::Unexpected) => { + return Err(RecoveryNameClassificationError::Unexpected { namespace, name }); + } + Err(NameFailure::Pool(source)) => { + return Err(RecoveryNameClassificationError::PoolName { + namespace, + name, + source, + }); + } + }; + required.observe(role); + if role.is_stage() { + if let Some(first) = first_stage { + return Err(RecoveryNameClassificationError::ConflictingStages { + first, + second: role, + }); + } + first_stage = Some(role); + } + named.push(RecoveryNamedEntry::new(namespace, name, role)); + } + if let Some(required) = required.first_missing() { + return Err(RecoveryNameClassificationError::Missing { required }); + } + Ok(RecoveryNameManifest::new(named)) +} + +fn classify( + namespace: RecoveryNamespace, + name: &RecoveryEntryName, +) -> Result { + match namespace { + RecoveryNamespace::Root => classify_root(name), + RecoveryNamespace::Staging => classify_staging(name), + RecoveryNamespace::Segments => recovery_pool_name::segment(name) + .map(|digest| RecoveryEntryRole::ImmutableSegment { digest }) + .map_err(NameFailure::Pool), + RecoveryNamespace::Catalogs => recovery_pool_name::catalog(name) + .map(|(generation, digest)| RecoveryEntryRole::ImmutableCatalog { generation, digest }) + .map_err(NameFailure::Pool), + } +} + +fn classify_root(name: &RecoveryEntryName) -> Result { + match name.as_bytes() { + b"writer.lock" => Ok(RecoveryEntryRole::WriterLock), + b"staging" => Ok(RecoveryEntryRole::StagingDirectory), + b"segments" => Ok(RecoveryEntryRole::SegmentPoolDirectory), + b"catalogs" => Ok(RecoveryEntryRole::CatalogPoolDirectory), + b"HEAD" => Ok(RecoveryEntryRole::CurrentHead), + b"head.next" => Ok(RecoveryEntryRole::NextHeadStage), + _ => Err(NameFailure::Unexpected), + } +} + +fn classify_staging(name: &RecoveryEntryName) -> Result { + match name.as_bytes() { + b"current.seg" => Ok(RecoveryEntryRole::SegmentStage), + b"current.cat" => Ok(RecoveryEntryRole::CatalogStage), + _ => Err(NameFailure::Unexpected), + } +} + +enum NameFailure { + Unexpected, + Pool(super::RecoveryPoolNameError), +} + +#[derive(Default)] +struct RequiredEntries(u8); + +impl RequiredEntries { + const WRITER_LOCK: u8 = 1 << 0; + const STAGING: u8 = 1 << 1; + const SEGMENTS: u8 = 1 << 2; + const CATALOGS: u8 = 1 << 3; + + const fn observe(&mut self, role: RecoveryEntryRole) { + match role { + RecoveryEntryRole::WriterLock => self.0 |= Self::WRITER_LOCK, + RecoveryEntryRole::StagingDirectory => self.0 |= Self::STAGING, + RecoveryEntryRole::SegmentPoolDirectory => self.0 |= Self::SEGMENTS, + RecoveryEntryRole::CatalogPoolDirectory => self.0 |= Self::CATALOGS, + RecoveryEntryRole::CurrentHead + | RecoveryEntryRole::NextHeadStage + | RecoveryEntryRole::SegmentStage + | RecoveryEntryRole::CatalogStage + | RecoveryEntryRole::ImmutableSegment { .. } + | RecoveryEntryRole::ImmutableCatalog { .. } => {} + } + } + + const fn first_missing(&self) -> Option { + if self.0 & Self::WRITER_LOCK == 0 { + Some(RecoveryRequiredEntry::WriterLock) + } else if self.0 & Self::STAGING == 0 { + Some(RecoveryRequiredEntry::StagingDirectory) + } else if self.0 & Self::SEGMENTS == 0 { + Some(RecoveryRequiredEntry::SegmentPoolDirectory) + } else if self.0 & Self::CATALOGS == 0 { + Some(RecoveryRequiredEntry::CatalogPoolDirectory) + } else { + None + } + } +} diff --git a/src/adapters/recovery_name_classification_error.rs b/src/adapters/recovery_name_classification_error.rs new file mode 100644 index 0000000..d0c2fd9 --- /dev/null +++ b/src/adapters/recovery_name_classification_error.rs @@ -0,0 +1,79 @@ +//! This module owns recovery name-classification failures. + +use std::error::Error; +use std::fmt; + +use super::{ + RecoveryEntryName, RecoveryEntryRole, RecoveryNamespace, RecoveryPoolNameError, + RecoveryRequiredEntry, +}; + +/// Why a recovery inventory cannot name one unambiguous protocol state. +#[derive(Debug)] +pub enum RecoveryNameClassificationError { + /// A fixed-name namespace contains an unknown entry. + Unexpected { + /// Owning namespace. + namespace: RecoveryNamespace, + /// Exact unexpected name. + name: RecoveryEntryName, + }, + /// An immutable-pool name is noncanonical. + PoolName { + /// Owning immutable-pool namespace. + namespace: RecoveryNamespace, + /// Exact invalid name. + name: RecoveryEntryName, + /// Exact grammar refusal. + source: RecoveryPoolNameError, + }, + /// An initialized root entry is absent. + Missing { + /// Required absent entry. + required: RecoveryRequiredEntry, + }, + /// More than one fixed recovery stage exists. + ConflictingStages { + /// First stage in deterministic inventory order. + first: RecoveryEntryRole, + /// Second stage in deterministic inventory order. + second: RecoveryEntryRole, + }, +} + +impl fmt::Display for RecoveryNameClassificationError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Unexpected { namespace, name } => write!( + formatter, + "unexpected recovery name {:?} in {namespace}", + name.as_bytes() + ), + Self::PoolName { + namespace, + name, + source, + } => write!( + formatter, + "invalid recovery pool name {:?} in {namespace}: {source}", + name.as_bytes() + ), + Self::Missing { required } => { + write!(formatter, "initialized recovery root is missing {required}") + } + Self::ConflictingStages { first, second } => write!( + formatter, + "recovery inventory contains conflicting stages {first:?} and {second:?}" + ), + } + } +} + +impl Error for RecoveryNameClassificationError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::PoolName { source, .. } => Some(source), + Self::Unexpected { .. } | Self::Missing { .. } | Self::ConflictingStages { .. } => None, + } + } +} diff --git a/src/adapters/recovery_name_manifest.rs b/src/adapters/recovery_name_manifest.rs new file mode 100644 index 0000000..277cf40 --- /dev/null +++ b/src/adapters/recovery_name_manifest.rs @@ -0,0 +1,61 @@ +//! This module owns canonical recovery name manifests. + +use super::{RecoveryEntryName, RecoveryEntryRole, RecoveryNamespace}; + +/// One namespace entry paired with its canonical semantic role. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct RecoveryNamedEntry { + namespace: RecoveryNamespace, + name: RecoveryEntryName, + role: RecoveryEntryRole, +} + +impl RecoveryNamedEntry { + pub(super) const fn new( + namespace: RecoveryNamespace, + name: RecoveryEntryName, + role: RecoveryEntryRole, + ) -> Self { + Self { + namespace, + name, + role, + } + } + + /// Returns the owning namespace. + #[must_use] + pub const fn namespace(&self) -> RecoveryNamespace { + self.namespace + } + + /// Returns the exact raw entry name. + #[must_use] + pub const fn name(&self) -> &RecoveryEntryName { + &self.name + } + + /// Returns the canonical semantic role. + #[must_use] + pub const fn role(&self) -> RecoveryEntryRole { + self.role + } +} + +/// One complete canonical namespace manifest awaiting content classification. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct RecoveryNameManifest { + entries: Vec, +} + +impl RecoveryNameManifest { + pub(super) const fn new(entries: Vec) -> Self { + Self { entries } + } + + /// Returns entries in inventory namespace-and-name order. + #[must_use] + pub fn entries(&self) -> &[RecoveryNamedEntry] { + &self.entries + } +} diff --git a/src/adapters/recovery_pool_name.rs b/src/adapters/recovery_pool_name.rs new file mode 100644 index 0000000..751ddc3 --- /dev/null +++ b/src/adapters/recovery_pool_name.rs @@ -0,0 +1,81 @@ +//! This module owns canonical immutable-pool name parsing. + +use super::lower_hex::{LowerHexError, decode_digest_32}; +use super::{RecoveryEntryName, RecoveryPoolNameError, SegmentDigest}; +use crate::{CatalogDigest, CatalogGeneration}; + +const SEGMENT_NAME_LENGTH: usize = 68; +const CATALOG_NAME_LENGTH: usize = 85; +const GENERATION_LENGTH: usize = 16; + +pub(super) fn segment(name: &RecoveryEntryName) -> Result { + let bytes = name.as_bytes(); + require_length(bytes, SEGMENT_NAME_LENGTH)?; + let digest = bytes + .strip_suffix(b".seg") + .ok_or(RecoveryPoolNameError::WrongSuffix)?; + decode_digest(digest).map(SegmentDigest::from_validated) +} + +pub(super) fn catalog( + name: &RecoveryEntryName, +) -> Result<(CatalogGeneration, CatalogDigest), RecoveryPoolNameError> { + let bytes = name.as_bytes(); + require_length(bytes, CATALOG_NAME_LENGTH)?; + let stem = bytes + .strip_suffix(b".cat") + .ok_or(RecoveryPoolNameError::WrongSuffix)?; + let generation_bytes = stem + .get(..GENERATION_LENGTH) + .ok_or(RecoveryPoolNameError::WrongSeparator)?; + if stem.get(GENERATION_LENGTH) != Some(&b'-') { + return Err(RecoveryPoolNameError::WrongSeparator); + } + let digest_bytes = stem + .get(GENERATION_LENGTH + 1..) + .ok_or(RecoveryPoolNameError::WrongSeparator)?; + let generation = decode_generation(generation_bytes)?; + let digest = CatalogDigest::from_validated(decode_digest(digest_bytes)?); + Ok((generation, digest)) +} + +const fn require_length(bytes: &[u8], expected: usize) -> Result<(), RecoveryPoolNameError> { + if bytes.len() == expected { + Ok(()) + } else { + Err(RecoveryPoolNameError::WrongLength { + expected, + observed: bytes.len(), + }) + } +} + +fn decode_generation(bytes: &[u8]) -> Result { + if bytes + .iter() + .copied() + .any(|byte| matches!(byte, b'A'..=b'F')) + { + return Err(RecoveryPoolNameError::UppercaseGeneration); + } + if !bytes.iter().all(u8::is_ascii_hexdigit) { + return Err(RecoveryPoolNameError::InvalidGenerationAlphabet); + } + let text = + std::str::from_utf8(bytes).map_err(|_| RecoveryPoolNameError::InvalidGenerationAlphabet)?; + let value = u64::from_str_radix(text, 16) + .map_err(|_| RecoveryPoolNameError::InvalidGenerationAlphabet)?; + CatalogGeneration::new(value).map_err(|_| RecoveryPoolNameError::ZeroGeneration) +} + +fn decode_digest(bytes: &[u8]) -> Result<[u8; 32], RecoveryPoolNameError> { + let text = + std::str::from_utf8(bytes).map_err(|_| RecoveryPoolNameError::InvalidDigestAlphabet)?; + decode_digest_32(text).map_err(|source| match source { + LowerHexError::WrongLength { expected, observed } => { + RecoveryPoolNameError::DigestLength { expected, observed } + } + LowerHexError::Uppercase => RecoveryPoolNameError::UppercaseDigest, + LowerHexError::InvalidAlphabet => RecoveryPoolNameError::InvalidDigestAlphabet, + }) +} diff --git a/src/adapters/recovery_pool_name_error.rs b/src/adapters/recovery_pool_name_error.rs new file mode 100644 index 0000000..5c7024b --- /dev/null +++ b/src/adapters/recovery_pool_name_error.rs @@ -0,0 +1,67 @@ +//! This module owns immutable-pool name parsing failures. + +use std::error::Error; +use std::fmt; + +/// Why an immutable-pool entry name is noncanonical. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum RecoveryPoolNameError { + /// The complete name has the wrong width. + WrongLength { + /// Required byte width. + expected: usize, + /// Observed byte width. + observed: usize, + }, + /// The artifact suffix is not exact. + WrongSuffix, + /// The catalog generation separator is not `-`. + WrongSeparator, + /// The catalog generation contains uppercase hexadecimal. + UppercaseGeneration, + /// The catalog generation contains a non-hexadecimal byte. + InvalidGenerationAlphabet, + /// Catalog generation zero is forbidden. + ZeroGeneration, + /// The digest has the wrong width. + DigestLength { + /// Required byte width. + expected: usize, + /// Observed byte width. + observed: usize, + }, + /// The digest contains uppercase hexadecimal. + UppercaseDigest, + /// The digest contains a non-hexadecimal byte. + InvalidDigestAlphabet, +} + +impl fmt::Display for RecoveryPoolNameError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::WrongLength { expected, observed } => { + write!(formatter, "pool name length {observed} is not {expected}") + } + Self::WrongSuffix => formatter.write_str("pool name suffix is noncanonical"), + Self::WrongSeparator => { + formatter.write_str("catalog pool generation separator is not '-'") + } + Self::UppercaseGeneration => { + formatter.write_str("catalog pool generation uses uppercase hexadecimal") + } + Self::InvalidGenerationAlphabet => { + formatter.write_str("catalog pool generation is not hexadecimal") + } + Self::ZeroGeneration => formatter.write_str("catalog pool generation is zero"), + Self::DigestLength { expected, observed } => { + write!(formatter, "pool digest length {observed} is not {expected}") + } + Self::UppercaseDigest => formatter.write_str("pool digest uses uppercase hexadecimal"), + Self::InvalidDigestAlphabet => { + formatter.write_str("pool digest is not lowercase hexadecimal") + } + } + } +} + +impl Error for RecoveryPoolNameError {} diff --git a/src/adapters/recovery_required_entry.rs b/src/adapters/recovery_required_entry.rs new file mode 100644 index 0000000..1639c98 --- /dev/null +++ b/src/adapters/recovery_required_entry.rs @@ -0,0 +1,27 @@ +//! This module owns required initialized recovery-root entries. + +use std::fmt; + +/// One fixed entry required in every initialized store root. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum RecoveryRequiredEntry { + /// Persistent `writer.lock`. + WriterLock, + /// `staging` directory. + StagingDirectory, + /// `segments` directory. + SegmentPoolDirectory, + /// `catalogs` directory. + CatalogPoolDirectory, +} + +impl fmt::Display for RecoveryRequiredEntry { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(match self { + Self::WriterLock => "writer.lock", + Self::StagingDirectory => "staging", + Self::SegmentPoolDirectory => "segments", + Self::CatalogPoolDirectory => "catalogs", + }) + } +} diff --git a/src/lib.rs b/src/lib.rs index 01fa6ce..8ba069f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -14,9 +14,10 @@ //! immutable-segment writing and verified reading, canonical catalog //! generations, platform-gated filesystem publication mechanics, bounded //! immutable restart snapshots, typed store-initialization orchestration, and -//! production initialization for the admitted Linux ext4 profile. Recovery, -//! retention, and garbage collection APIs remain intentionally absent until -//! their contracts have executable specifications. +//! production initialization for the admitted Linux ext4 profile. Recovery +//! inventory and name classification are read-only; explicit recovery +//! execution, retention, and garbage collection APIs remain intentionally +//! absent until their contracts have executable specifications. #[cfg(test)] extern crate self as keep; @@ -44,17 +45,19 @@ pub use adapters::{ FilesystemRecoveryInventoryReader, FilesystemSegmentStage, FilesystemWriterLock, LayoutDecodeError, LayoutDecodePolicy, LayoutEncodeError, LayoutIdBinaryParseError, LayoutIdTextParseError, PublicationHeadDecodeError, RecoveryEntryName, RecoveryEntryNameError, - RecoveryInventory, RecoveryInventoryEntry, RecoveryInventoryError, RecoveryInventoryLimit, - RecoveryInventoryLimitError, RecoveryInventoryOperation, RecoveryInventoryStorage, - RecoveryNamespace, SealedSegment, SegmentDigest, SegmentDurabilityPhase, SegmentHeader, - SegmentHeaderError, SegmentPublication, SegmentPublicationError, SegmentReadError, - SegmentReadPolicy, SegmentRecordAdmissionError, SegmentRecordChecksum, - SegmentRecordDecodeError, SegmentRecordHeader, SegmentRecordHeaderError, SegmentRecordIdentity, - SegmentRecordLength, SegmentRecordLimit, SegmentRecordLimitError, SegmentRecordPayloadLength, - SegmentRecords, SegmentSeal, SegmentSealError, SegmentStage, SegmentStageCreateError, - SegmentWriteError, SegmentWritePhase, StagedSegment, StorageProfileIdParseError, - StoreInitializationError, StoreInitializationPhase, StoreInitializationReceipt, - StoreInitializationStorage, WriterLockAcquireError, WriterLockAcquirePhase, initialize_store, + RecoveryEntryRole, RecoveryInventory, RecoveryInventoryEntry, RecoveryInventoryError, + RecoveryInventoryLimit, RecoveryInventoryLimitError, RecoveryInventoryOperation, + RecoveryInventoryStorage, RecoveryNameClassificationError, RecoveryNameManifest, + RecoveryNamedEntry, RecoveryNamespace, RecoveryPoolNameError, RecoveryRequiredEntry, + SealedSegment, SegmentDigest, SegmentDurabilityPhase, SegmentHeader, SegmentHeaderError, + SegmentPublication, SegmentPublicationError, SegmentReadError, SegmentReadPolicy, + SegmentRecordAdmissionError, SegmentRecordChecksum, SegmentRecordDecodeError, + SegmentRecordHeader, SegmentRecordHeaderError, SegmentRecordIdentity, SegmentRecordLength, + SegmentRecordLimit, SegmentRecordLimitError, SegmentRecordPayloadLength, SegmentRecords, + SegmentSeal, SegmentSealError, SegmentStage, SegmentStageCreateError, SegmentWriteError, + SegmentWritePhase, StagedSegment, StorageProfileIdParseError, StoreInitializationError, + StoreInitializationPhase, StoreInitializationReceipt, StoreInitializationStorage, + WriterLockAcquireError, WriterLockAcquirePhase, classify_recovery_names, initialize_store, publish_catalog_generation, read_recovery_inventory, }; pub use blob::{ diff --git a/tests/recovery_name_classification.rs b/tests/recovery_name_classification.rs new file mode 100644 index 0000000..9d616e8 --- /dev/null +++ b/tests/recovery_name_classification.rs @@ -0,0 +1,80 @@ +//! Deterministic recovery namespace classification laws. + +#[path = "recovery_name_classification/grammar_laws.rs"] +mod grammar_laws; +#[path = "recovery_inventory/inventory_double.rs"] +pub mod inventory_double; +#[path = "recovery_name_classification/refusal_laws.rs"] +mod refusal_laws; + +use std::error::Error; + +use inventory_double::InventoryDouble; +use keep::{ + RecoveryEntryName, RecoveryEntryRole, RecoveryInventory, RecoveryInventoryLimit, + classify_recovery_names, read_recovery_inventory, +}; + +#[test] +fn canonical_names_produce_exact_typed_roles() -> Result<(), Box> { + let segment_name = format!("{}.seg", "00".repeat(32)); + let catalog_name = format!("{:016x}-{}.cat", 1_u64, "11".repeat(32)); + let inventory = inventory([ + names(&["writer.lock", "staging", "segments", "catalogs", "HEAD"])?, + names(&["current.seg"])?, + vec![name(segment_name.as_bytes())?], + vec![name(catalog_name.as_bytes())?], + ])?; + + let manifest = classify_recovery_names(inventory)?; + + assert!( + manifest + .entries() + .iter() + .any(|entry| matches!(entry.role(), RecoveryEntryRole::CurrentHead)) + ); + assert!( + manifest + .entries() + .iter() + .any(|entry| matches!(entry.role(), RecoveryEntryRole::SegmentStage)) + ); + assert!(manifest.entries().iter().any(|entry| matches!( + entry.role(), + RecoveryEntryRole::ImmutableSegment { digest } + if digest.as_bytes() == &[0_u8; 32] + ))); + assert!(manifest.entries().iter().any(|entry| matches!( + entry.role(), + RecoveryEntryRole::ImmutableCatalog { generation, digest } + if generation.get() == 1 && digest.as_bytes() == &[0x11_u8; 32] + ))); + Ok(()) +} + +pub(crate) fn inventory( + names: [Vec; 4], +) -> Result> { + let counts = names.each_ref().map(|entries| { + u64::try_from(entries.len()).map_err(|_| "test inventory count does not fit u64") + }); + let [root, staging, segments, catalogs] = counts; + let mut storage = InventoryDouble::new([root?, staging?, segments?, catalogs?], names); + let inventory = + read_recovery_inventory(&mut storage, RecoveryInventoryLimit::protocol_maximum())?; + assert_eq!(storage.calls().len(), 8); + Ok(inventory) +} + +pub(crate) fn initialized_root() -> Result, Box> { + names(&["writer.lock", "staging", "segments", "catalogs"]) +} + +pub(crate) fn names(values: &[&str]) -> Result, Box> { + values.iter().map(|value| name(value.as_bytes())).collect() +} + +pub(crate) fn name(bytes: &[u8]) -> Result> { + Ok(RecoveryEntryName::new(bytes.to_vec())?) +} diff --git a/tests/recovery_name_classification/grammar_laws.rs b/tests/recovery_name_classification/grammar_laws.rs new file mode 100644 index 0000000..82877f2 --- /dev/null +++ b/tests/recovery_name_classification/grammar_laws.rs @@ -0,0 +1,107 @@ +//! Canonical immutable-pool recovery-name grammar laws. + +use std::error::Error; + +use keep::{ + RecoveryNameClassificationError, RecoveryNamespace, RecoveryPoolNameError, + classify_recovery_names, +}; + +use super::{initialized_root, inventory, name}; + +#[test] +fn segment_name_grammar_returns_exact_refusals() -> Result<(), Box> { + let wrong_suffix = format!("{}.cat", "00".repeat(32)); + let uppercase = format!("{}.seg", "AA".repeat(32)); + let invalid = format!("{}g.seg", "0".repeat(63)); + + assert_eq!( + pool_error(RecoveryNamespace::Segments, b"short")?, + RecoveryPoolNameError::WrongLength { + expected: 68, + observed: 5, + } + ); + assert_eq!( + pool_error(RecoveryNamespace::Segments, wrong_suffix.as_bytes())?, + RecoveryPoolNameError::WrongSuffix + ); + assert_eq!( + pool_error(RecoveryNamespace::Segments, uppercase.as_bytes())?, + RecoveryPoolNameError::UppercaseDigest + ); + assert_eq!( + pool_error(RecoveryNamespace::Segments, invalid.as_bytes())?, + RecoveryPoolNameError::InvalidDigestAlphabet + ); + Ok(()) +} + +#[test] +fn catalog_name_grammar_returns_exact_refusals() -> Result<(), Box> { + let digest = "00".repeat(32); + let cases = [ + ( + format!("{:016x}_{}.cat", 1_u64, digest), + RecoveryPoolNameError::WrongSeparator, + ), + ( + format!("000000000000000A-{digest}.cat"), + RecoveryPoolNameError::UppercaseGeneration, + ), + ( + format!("000000000000000G-{digest}.cat"), + RecoveryPoolNameError::InvalidGenerationAlphabet, + ), + ( + format!("0000000000000000-{digest}.cat"), + RecoveryPoolNameError::ZeroGeneration, + ), + ( + format!("0000000000000001-{}.cat", "AA".repeat(32)), + RecoveryPoolNameError::UppercaseDigest, + ), + ( + format!("0000000000000001-{}g.cat", "0".repeat(63)), + RecoveryPoolNameError::InvalidDigestAlphabet, + ), + ]; + + for (candidate, expected) in cases { + assert_eq!( + pool_error(RecoveryNamespace::Catalogs, candidate.as_bytes())?, + expected + ); + } + Ok(()) +} + +fn pool_error( + namespace: RecoveryNamespace, + candidate: &[u8], +) -> Result> { + let entries = match namespace { + RecoveryNamespace::Segments => [ + initialized_root()?, + Vec::new(), + vec![name(candidate)?], + Vec::new(), + ], + RecoveryNamespace::Catalogs => [ + initialized_root()?, + Vec::new(), + Vec::new(), + vec![name(candidate)?], + ], + RecoveryNamespace::Root | RecoveryNamespace::Staging => { + return Err("pool grammar test requires an immutable namespace".into()); + } + }; + let Err(error) = classify_recovery_names(inventory(entries)?) else { + return Err("name classification admitted a noncanonical pool name".into()); + }; + match error { + RecoveryNameClassificationError::PoolName { source, .. } => Ok(source), + unexpected => Err(format!("unexpected name-classification error: {unexpected}").into()), + } +} diff --git a/tests/recovery_name_classification/refusal_laws.rs b/tests/recovery_name_classification/refusal_laws.rs new file mode 100644 index 0000000..53a2fef --- /dev/null +++ b/tests/recovery_name_classification/refusal_laws.rs @@ -0,0 +1,97 @@ +//! Exact recovery-name ambiguity laws. + +use std::error::Error; + +use keep::{ + RecoveryEntryRole, RecoveryNameClassificationError, RecoveryNamespace, RecoveryPoolNameError, + RecoveryRequiredEntry, classify_recovery_names, +}; + +use super::{initialized_root, inventory, name, names}; + +#[test] +fn unknown_name_is_refused_with_its_exact_namespace_and_bytes() -> Result<(), Box> { + let Err(error) = classify_recovery_names(inventory([ + initialized_root()?, + Vec::new(), + names(&["not-a-segment"])?, + Vec::new(), + ])?) else { + return Err("name classification admitted an unknown segment entry".into()); + }; + + assert!(matches!( + error, + RecoveryNameClassificationError::PoolName { + namespace: RecoveryNamespace::Segments, + ref name, + source: RecoveryPoolNameError::WrongLength { + expected: 68, + observed: 13, + }, + } if name.as_bytes() == b"not-a-segment" + )); + Ok(()) +} + +#[test] +fn missing_initialized_root_entry_is_an_exact_refusal() -> Result<(), Box> { + let Err(error) = classify_recovery_names(inventory([ + names(&["staging", "segments", "catalogs"])?, + Vec::new(), + Vec::new(), + Vec::new(), + ])?) else { + return Err("name classification admitted a missing writer lock".into()); + }; + + assert!(matches!( + error, + RecoveryNameClassificationError::Missing { + required: RecoveryRequiredEntry::WriterLock, + } + )); + Ok(()) +} + +#[test] +fn uppercase_pool_digest_is_noncanonical() -> Result<(), Box> { + let Err(error) = classify_recovery_names(inventory([ + initialized_root()?, + Vec::new(), + vec![name(format!("{}.seg", "AA".repeat(32)).as_bytes())?], + Vec::new(), + ])?) else { + return Err("name classification admitted uppercase pool identity".into()); + }; + + assert!(matches!( + error, + RecoveryNameClassificationError::PoolName { + source: RecoveryPoolNameError::UppercaseDigest, + .. + } + )); + Ok(()) +} + +#[test] +fn simultaneous_fixed_stages_are_unrecoverable_ambiguity() -> Result<(), Box> { + let Err(error) = classify_recovery_names(inventory([ + initialized_root()?, + names(&["current.seg", "current.cat"])?, + Vec::new(), + Vec::new(), + ])?) else { + return Err("name classification admitted simultaneous fixed stages".into()); + }; + + assert!(matches!( + error, + RecoveryNameClassificationError::ConflictingStages { + first: RecoveryEntryRole::CatalogStage, + second: RecoveryEntryRole::SegmentStage, + } + )); + Ok(()) +} diff --git a/tests/recovery_name_classification_memory.rs b/tests/recovery_name_classification_memory.rs new file mode 100644 index 0000000..b622af3 --- /dev/null +++ b/tests/recovery_name_classification_memory.rs @@ -0,0 +1,55 @@ +//! Isolated heap-allocation evidence for recovery name classification. + +#[path = "recovery_inventory/inventory_double.rs"] +pub mod inventory_double; + +use std::error::Error; + +use allocation_counter::measure; +use inventory_double::InventoryDouble; +use keep::{ + RecoveryEntryName, RecoveryInventoryLimit, RecoveryNameClassificationError, + classify_recovery_names, read_recovery_inventory, +}; + +#[test] +fn refusal_moves_the_exact_name_without_an_extra_allocation() -> Result<(), Box> { + let names = [ + recovery_names(&["writer.lock", "staging", "segments", "catalogs", "unknown"])?, + Vec::new(), + Vec::new(), + Vec::new(), + ]; + let mut storage = InventoryDouble::new([5, 0, 0, 0], names); + let inventory = + read_recovery_inventory(&mut storage, RecoveryInventoryLimit::protocol_maximum())?; + assert_eq!(storage.calls().len(), 8); + let mut input = Some(inventory); + let mut result = None; + + let allocations = measure(|| { + if let Some(inventory) = input.take() { + result = Some(classify_recovery_names(inventory)); + } + }); + let error = match result.ok_or("name classification did not run")? { + Ok(_manifest) => return Err("unknown recovery name was admitted".into()), + Err(error) => error, + }; + + assert!(matches!( + error, + RecoveryNameClassificationError::Unexpected { ref name, .. } + if name.as_bytes() == b"unknown" + )); + assert_eq!(allocations.count_total, 1); + assert_eq!(allocations.count_max, 1); + Ok(()) +} + +fn recovery_names(values: &[&str]) -> Result, Box> { + values + .iter() + .map(|value| Ok(RecoveryEntryName::new(value.as_bytes().to_vec())?)) + .collect() +} From 7df129b47a212eb77da04a641d18072c7be4ea98 Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 06:24:56 -0700 Subject: [PATCH 10/49] Add: Fingerprint bounded recovery stages --- docs/formats/segment-store-v1/recovery.md | 6 +- docs/formats/segment-store-v1/requirements.md | 1 + src/adapters/framed_blake3.rs | 33 ++- src/adapters/mod.rs | 18 ++ src/adapters/recovery_stage.rs | 39 ++++ src/adapters/recovery_stage_evidence.rs | 42 ++++ src/adapters/recovery_stage_fingerprint.rs | 25 +++ .../recovery_stage_fingerprint_algorithm.rs | 18 ++ .../recovery_stage_fingerprint_error.rs | 125 +++++++++++ src/adapters/recovery_stage_fingerprinter.rs | 133 ++++++++++++ src/adapters/recovery_stage_length.rs | 18 ++ src/adapters/recovery_stage_metadata.rs | 49 +++++ src/adapters/recovery_stage_metadata_error.rs | 37 ++++ src/lib.rs | 28 +-- tests/recovery_stage_fingerprint.rs | 43 ++++ .../recovery_stage_fingerprint/reader_laws.rs | 199 ++++++++++++++++++ tests/recovery_stage_fingerprint_memory.rs | 27 +++ 17 files changed, 821 insertions(+), 20 deletions(-) create mode 100644 src/adapters/recovery_stage.rs create mode 100644 src/adapters/recovery_stage_evidence.rs create mode 100644 src/adapters/recovery_stage_fingerprint.rs create mode 100644 src/adapters/recovery_stage_fingerprint_algorithm.rs create mode 100644 src/adapters/recovery_stage_fingerprint_error.rs create mode 100644 src/adapters/recovery_stage_fingerprinter.rs create mode 100644 src/adapters/recovery_stage_length.rs create mode 100644 src/adapters/recovery_stage_metadata.rs create mode 100644 src/adapters/recovery_stage_metadata_error.rs create mode 100644 tests/recovery_stage_fingerprint.rs create mode 100644 tests/recovery_stage_fingerprint/reader_laws.rs create mode 100644 tests/recovery_stage_fingerprint_memory.rs diff --git a/docs/formats/segment-store-v1/recovery.md b/docs/formats/segment-store-v1/recovery.md index cfcdec4..6727d65 100644 --- a/docs/formats/segment-store-v1/recovery.md +++ b/docs/formats/segment-store-v1/recovery.md @@ -32,7 +32,11 @@ global budget, preserves raw Linux name bytes, and verifies child-directory identity before and after inventory. `classify_recovery_names` requires the four initialized root entries, types each fixed name and immutable-pool coordinate, and refuses an unknown or conflicting name without artifact I/O. -Content classification remains unimplemented. +`fingerprint_recovery_stage` then reads a fixed stage through a zero-allocation +bounded stream, refuses metadata or observed bytes above the name-selected +maximum, and returns its exact observed length and +`KEEP:RECOVERY:STAGE\0` fingerprint. Semantic content classification remains +unimplemented. The sole admissible duplicate digest is one fixed staging name and its exact digest-derived pool name after a link transition. Recovery admits that pair diff --git a/docs/formats/segment-store-v1/requirements.md b/docs/formats/segment-store-v1/requirements.md index af28869..109d833 100644 --- a/docs/formats/segment-store-v1/requirements.md +++ b/docs/formats/segment-store-v1/requirements.md @@ -104,6 +104,7 @@ or recovery execution. | `KEEP-RECOVERY-005` | Recovery counts the root and three protocol directories in fixed order before retaining names, refuses at the configured or protocol entry ceiling with the exact observed-at-least count, then returns one duplicate-free inventory sorted by namespace and raw name bytes | Fault-recording inventory port | `tests/recovery_inventory.rs` | Implemented in #17 | | `KEEP-RECOVERY-006` | Filesystem inventory pins the admitted root and protocol directories without following links, verifies child-directory identity before and after scanning, stops each count at the remaining global budget plus one, preserves raw Linux entry-name bytes, and performs no protocol mutation | Capability-relative filesystem fixture | `src/adapters/filesystem_recovery_inventory_tests.rs`, `tests/recovery_inventory.rs` | Implemented in #17 | | `KEEP-RECOVERY-007` | Name classification requires the four initialized root entries, admits only fixed protocol names and canonical pool coordinates in their owning namespaces, refuses simultaneous fixed recovery stages before artifact reads, and moves a refused raw name without duplicating its allocation | Canonical-name matrix and allocation counter | `tests/recovery_name_classification.rs`, `tests/recovery_name_classification_memory.rs` | Implemented in #17 | +| `KEEP-RECOVERY-008` | Stage evidence is fingerprinted through a zero-allocation bounded streaming reader under the named recovery domain; metadata and observed bytes cannot exceed the name-selected protocol maximum, and failures retain exact stage and offset | Independent framing oracle, adversarial reader matrix, and allocation counter | `tests/recovery_stage_fingerprint.rs`, `tests/recovery_stage_fingerprint_memory.rs` | Implemented in #17 | diff --git a/src/adapters/framed_blake3.rs b/src/adapters/framed_blake3.rs index f40d7ba..127a55c 100644 --- a/src/adapters/framed_blake3.rs +++ b/src/adapters/framed_blake3.rs @@ -6,13 +6,32 @@ const VERSION: u16 = 1; const ALGORITHM: u8 = 1; pub(super) fn hash(domain: &[u8], parts: &[&[u8]], length: u64) -> [u8; 32] { - let mut hasher = Hasher::new(); - hasher.update(domain); - hasher.update(&VERSION.to_be_bytes()); - hasher.update(&[ALGORITHM]); + let mut state = State::new(domain); for part in parts { - hasher.update(part); + state.update(part); + } + state.finalize(length) +} + +pub(super) struct State { + hasher: Hasher, +} + +impl State { + pub(super) fn new(domain: &[u8]) -> Self { + let mut hasher = Hasher::new(); + hasher.update(domain); + hasher.update(&VERSION.to_be_bytes()); + hasher.update(&[ALGORITHM]); + Self { hasher } + } + + pub(super) fn update(&mut self, bytes: &[u8]) { + self.hasher.update(bytes); + } + + pub(super) fn finalize(mut self, length: u64) -> [u8; 32] { + self.hasher.update(&length.to_be_bytes()); + *self.hasher.finalize().as_bytes() } - hasher.update(&length.to_be_bytes()); - *hasher.finalize().as_bytes() } diff --git a/src/adapters/mod.rs b/src/adapters/mod.rs index 9032fdf..37bc3ce 100644 --- a/src/adapters/mod.rs +++ b/src/adapters/mod.rs @@ -130,6 +130,15 @@ mod recovery_namespace; mod recovery_pool_name; mod recovery_pool_name_error; mod recovery_required_entry; +mod recovery_stage; +mod recovery_stage_evidence; +mod recovery_stage_fingerprint; +mod recovery_stage_fingerprint_algorithm; +mod recovery_stage_fingerprint_error; +mod recovery_stage_fingerprinter; +mod recovery_stage_length; +mod recovery_stage_metadata; +mod recovery_stage_metadata_error; mod sealed_segment; mod segment_digest; mod segment_digest_builder; @@ -261,6 +270,15 @@ pub use recovery_name_manifest::{RecoveryNameManifest, RecoveryNamedEntry}; pub use recovery_namespace::RecoveryNamespace; pub use recovery_pool_name_error::RecoveryPoolNameError; pub use recovery_required_entry::RecoveryRequiredEntry; +pub use recovery_stage::RecoveryStage; +pub use recovery_stage_evidence::RecoveryStageEvidence; +pub use recovery_stage_fingerprint::RecoveryStageFingerprint; +pub use recovery_stage_fingerprint_algorithm::RecoveryStageFingerprintAlgorithm; +pub use recovery_stage_fingerprint_error::RecoveryStageFingerprintError; +pub use recovery_stage_fingerprinter::fingerprint_recovery_stage; +pub use recovery_stage_length::RecoveryStageLength; +pub use recovery_stage_metadata::RecoveryStageMetadata; +pub use recovery_stage_metadata_error::RecoveryStageMetadataError; pub use sealed_segment::SealedSegment; pub use segment_digest::SegmentDigest; pub use segment_header::SegmentHeader; diff --git a/src/adapters/recovery_stage.rs b/src/adapters/recovery_stage.rs new file mode 100644 index 0000000..d334537 --- /dev/null +++ b/src/adapters/recovery_stage.rs @@ -0,0 +1,39 @@ +//! This module owns fixed recovery-stage identities and bounds. + +use std::fmt; + +use super::segment_header; +use crate::CatalogLength; + +/// One fixed mutable artifact retained for explicit recovery. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum RecoveryStage { + /// `staging/current.seg`. + Segment, + /// `staging/current.cat`. + Catalog, + /// Root `head.next`. + NextHead, +} + +impl RecoveryStage { + /// Returns the name-selected version-1 maximum byte length. + #[must_use] + pub const fn maximum_length(self) -> u64 { + match self { + Self::Segment => segment_header::MAXIMUM_SEGMENT_LENGTH, + Self::Catalog => CatalogLength::MAXIMUM.get(), + Self::NextHead => 128_u64, + } + } +} + +impl fmt::Display for RecoveryStage { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(match self { + Self::Segment => "current.seg", + Self::Catalog => "current.cat", + Self::NextHead => "head.next", + }) + } +} diff --git a/src/adapters/recovery_stage_evidence.rs b/src/adapters/recovery_stage_evidence.rs new file mode 100644 index 0000000..15bc1d8 --- /dev/null +++ b/src/adapters/recovery_stage_evidence.rs @@ -0,0 +1,42 @@ +//! This module owns immutable recovery-stage evidence. + +use super::{RecoveryStage, RecoveryStageFingerprint, RecoveryStageLength}; + +/// Fingerprint-bound evidence for one complete observed fixed stage. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct RecoveryStageEvidence { + stage: RecoveryStage, + length: RecoveryStageLength, + fingerprint: RecoveryStageFingerprint, +} + +impl RecoveryStageEvidence { + pub(super) const fn new( + stage: RecoveryStage, + length: RecoveryStageLength, + fingerprint: RecoveryStageFingerprint, + ) -> Self { + Self { + stage, + length, + fingerprint, + } + } + + /// Returns the exact fixed stage. + #[must_use] + pub const fn stage(self) -> RecoveryStage { + self.stage + } + + /// Returns the complete observed byte length. + pub const fn length(self) -> RecoveryStageLength { + self.length + } + + /// Returns the domain-separated digest of the observed bytes. + pub const fn fingerprint(self) -> RecoveryStageFingerprint { + self.fingerprint + } +} diff --git a/src/adapters/recovery_stage_fingerprint.rs b/src/adapters/recovery_stage_fingerprint.rs new file mode 100644 index 0000000..dcbbd85 --- /dev/null +++ b/src/adapters/recovery_stage_fingerprint.rs @@ -0,0 +1,25 @@ +//! This module owns domain-separated recovery-stage fingerprints. + +use super::RecoveryStageFingerprintAlgorithm; + +/// Exact digest of one bounded observed recovery stage. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct RecoveryStageFingerprint([u8; 32]); + +impl RecoveryStageFingerprint { + pub(super) const fn from_validated(bytes: [u8; 32]) -> Self { + Self(bytes) + } + + /// Returns the version-1 algorithm coordinate. + pub const fn algorithm(self) -> RecoveryStageFingerprintAlgorithm { + RecoveryStageFingerprintAlgorithm::FramedBlake3V1 + } + + /// Returns the exact 32-byte digest. + #[must_use] + pub const fn as_bytes(&self) -> &[u8; 32] { + &self.0 + } +} diff --git a/src/adapters/recovery_stage_fingerprint_algorithm.rs b/src/adapters/recovery_stage_fingerprint_algorithm.rs new file mode 100644 index 0000000..82f49e2 --- /dev/null +++ b/src/adapters/recovery_stage_fingerprint_algorithm.rs @@ -0,0 +1,18 @@ +//! This module owns the recovery-stage fingerprint algorithm coordinate. + +/// Registered recovery-stage fingerprint algorithm. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum RecoveryStageFingerprintAlgorithm { + /// Version-1 framed BLAKE3-256. + FramedBlake3V1, +} + +impl RecoveryStageFingerprintAlgorithm { + /// Returns the canonical wire coordinate. + #[must_use] + pub const fn code(self) -> u8 { + match self { + Self::FramedBlake3V1 => 1, + } + } +} diff --git a/src/adapters/recovery_stage_fingerprint_error.rs b/src/adapters/recovery_stage_fingerprint_error.rs new file mode 100644 index 0000000..d3b5c98 --- /dev/null +++ b/src/adapters/recovery_stage_fingerprint_error.rs @@ -0,0 +1,125 @@ +//! This module owns recovery-stage fingerprinting failures. + +use std::error::Error; +use std::fmt; +use std::io; + +use super::RecoveryStage; + +/// Why one fixed recovery stage could not produce bounded exact evidence. +#[derive(Debug)] +pub enum RecoveryStageFingerprintError { + /// The stream produced at least one byte above its maximum. + EvidenceOversized { + /// Fixed stage being observed. + stage: RecoveryStage, + /// Name-selected maximum. + maximum: u64, + /// Bounded lower limit on the observed byte count. + observed_at_least: u64, + }, + /// A reader reported more bytes than it was offered. + ReaderContract { + /// Fixed stage being observed. + stage: RecoveryStage, + /// Exact stream offset before the call. + offset: u64, + /// Buffer bytes offered to the reader. + offered: usize, + /// Byte count reported by the reader. + observed: usize, + }, + /// A bounded platform length could not be represented. + PlatformLength { + /// Fixed stage being observed. + stage: RecoveryStage, + /// Bounded value that could not be represented. + observed: u64, + }, + /// The fixed stack buffer width could not be represented as `u64`. + PlatformBufferLength { + /// Fixed stage being observed. + stage: RecoveryStage, + /// Stack-buffer byte width. + observed: usize, + }, + /// Stream offset arithmetic overflowed. + LengthOverflow { + /// Fixed stage being observed. + stage: RecoveryStage, + /// Offset before the overflowing addition. + offset: u64, + /// Reported increment. + increment: u64, + }, + /// The underlying reader failed. + Read { + /// Fixed stage being observed. + stage: RecoveryStage, + /// Exact stream offset before the failed call. + offset: u64, + /// Underlying failure. + source: io::Error, + }, +} + +impl fmt::Display for RecoveryStageFingerprintError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::EvidenceOversized { + stage, + maximum, + observed_at_least, + } => write!( + formatter, + "{stage} produced at least {observed_at_least} bytes above maximum {maximum}" + ), + Self::ReaderContract { + stage, + offset, + offered, + observed, + } => write!( + formatter, + "{stage} reader reported {observed} bytes from {offered} offered at offset {offset}" + ), + Self::PlatformLength { stage, observed } => write!( + formatter, + "{stage} bounded read length {observed} does not fit this platform" + ), + Self::PlatformBufferLength { stage, observed } => write!( + formatter, + "{stage} buffer length {observed} does not fit the stream coordinate" + ), + Self::LengthOverflow { + stage, + offset, + increment, + } => write!( + formatter, + "{stage} stream offset {offset} overflows with increment {increment}" + ), + Self::Read { + stage, + offset, + source, + } => write!( + formatter, + "{stage} read failed at offset {offset}: {source}" + ), + } + } +} + +impl Error for RecoveryStageFingerprintError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Read { source, .. } => Some(source), + Self::EvidenceOversized { .. } + | Self::ReaderContract { .. } + | Self::PlatformLength { .. } + | Self::PlatformBufferLength { .. } + | Self::LengthOverflow { .. } => None, + } + } +} diff --git a/src/adapters/recovery_stage_fingerprinter.rs b/src/adapters/recovery_stage_fingerprinter.rs new file mode 100644 index 0000000..56ba5e2 --- /dev/null +++ b/src/adapters/recovery_stage_fingerprinter.rs @@ -0,0 +1,133 @@ +//! This module owns bounded streaming recovery-stage observation. + +use std::io::{self, Read}; + +use super::{ + RecoveryStage, RecoveryStageEvidence, RecoveryStageFingerprint, RecoveryStageFingerprintError, + RecoveryStageLength, RecoveryStageMetadata, framed_blake3, +}; + +const DOMAIN: &[u8] = b"KEEP:RECOVERY:STAGE\0"; +const BUFFER_LENGTH: usize = 8_192; + +/// Reads and fingerprints one complete fixed recovery stage without retaining +/// its bytes. +/// +/// The reader is offered at most the name-selected maximum plus one byte. The +/// returned evidence binds the exact observed length and version-1 +/// domain-separated digest. +/// +/// # Errors +/// +/// Returns [`RecoveryStageFingerprintError`] for oversized stream evidence, a +/// broken reader contract, checked length failure, or underlying I/O failure. +pub fn fingerprint_recovery_stage( + metadata: RecoveryStageMetadata, + mut reader: R, +) -> Result { + let stage = metadata.stage(); + let maximum = stage.maximum_length(); + read_bounded(stage, maximum, &mut reader) +} + +fn read_bounded( + stage: RecoveryStage, + maximum: u64, + reader: &mut R, +) -> Result { + let limit = maximum + .checked_add(1) + .ok_or(RecoveryStageFingerprintError::LengthOverflow { + stage, + offset: maximum, + increment: 1, + })?; + let mut fingerprint_state = framed_blake3::State::new(DOMAIN); + let mut buffer = [0_u8; BUFFER_LENGTH]; + let buffer_length = u64::try_from(BUFFER_LENGTH).map_err(|_| { + RecoveryStageFingerprintError::PlatformBufferLength { + stage, + observed: BUFFER_LENGTH, + } + })?; + let mut offset = 0_u64; + loop { + let remaining = + limit + .checked_sub(offset) + .ok_or(RecoveryStageFingerprintError::LengthOverflow { + stage, + offset, + increment: 0, + })?; + let offered_u64 = remaining.min(buffer_length); + let offered = usize::try_from(offered_u64).map_err(|_| { + RecoveryStageFingerprintError::PlatformLength { + stage, + observed: offered_u64, + } + })?; + let target = + buffer + .get_mut(..offered) + .ok_or(RecoveryStageFingerprintError::PlatformLength { + stage, + observed: offered_u64, + })?; + let count = match reader.read(target) { + Ok(0) => break, + Ok(count) => count, + Err(source) if source.kind() == io::ErrorKind::Interrupted => continue, + Err(source) => { + return Err(RecoveryStageFingerprintError::Read { + stage, + offset, + source, + }); + } + }; + if count > offered { + return Err(RecoveryStageFingerprintError::ReaderContract { + stage, + offset, + offered, + observed: count, + }); + } + let increment = + u64::try_from(count).map_err(|_| RecoveryStageFingerprintError::PlatformLength { + stage, + observed: offered_u64, + })?; + let observed = + offset + .checked_add(increment) + .ok_or(RecoveryStageFingerprintError::LengthOverflow { + stage, + offset, + increment, + })?; + if observed > maximum { + return Err(RecoveryStageFingerprintError::EvidenceOversized { + stage, + maximum, + observed_at_least: observed, + }); + } + let bytes = buffer + .get(..count) + .ok_or(RecoveryStageFingerprintError::ReaderContract { + stage, + offset, + offered, + observed: count, + })?; + fingerprint_state.update(bytes); + offset = observed; + } + Ok(RecoveryStageEvidence::new( + stage, + RecoveryStageLength::from_validated(offset), + RecoveryStageFingerprint::from_validated(fingerprint_state.finalize(offset)), + )) +} diff --git a/src/adapters/recovery_stage_length.rs b/src/adapters/recovery_stage_length.rs new file mode 100644 index 0000000..7aafa82 --- /dev/null +++ b/src/adapters/recovery_stage_length.rs @@ -0,0 +1,18 @@ +//! This module owns validated recovery-stage byte lengths. + +/// Exact bounded byte length of one observed fixed recovery stage. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct RecoveryStageLength(u64); + +impl RecoveryStageLength { + pub(super) const fn from_validated(value: u64) -> Self { + Self(value) + } + + /// Returns the exact byte count. + #[must_use] + pub const fn get(self) -> u64 { + self.0 + } +} diff --git a/src/adapters/recovery_stage_metadata.rs b/src/adapters/recovery_stage_metadata.rs new file mode 100644 index 0000000..59cb967 --- /dev/null +++ b/src/adapters/recovery_stage_metadata.rs @@ -0,0 +1,49 @@ +//! This module owns admitted recovery-stage metadata. + +use super::{RecoveryStage, RecoveryStageLength, RecoveryStageMetadataError}; + +/// Name-selected fixed stage with an admitted metadata byte length. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct RecoveryStageMetadata { + stage: RecoveryStage, + length: RecoveryStageLength, +} + +impl RecoveryStageMetadata { + /// Admits one metadata byte length under its fixed-name protocol maximum. + /// + /// # Errors + /// + /// Returns [`RecoveryStageMetadataError`] when `observed` exceeds the + /// maximum selected by `stage`. + pub const fn new( + stage: RecoveryStage, + observed: u64, + ) -> Result { + let maximum = stage.maximum_length(); + if observed > maximum { + Err(RecoveryStageMetadataError::Oversized { + stage, + maximum, + observed, + }) + } else { + Ok(Self { + stage, + length: RecoveryStageLength::from_validated(observed), + }) + } + } + + /// Returns the fixed stage selected by the canonical name. + #[must_use] + pub const fn stage(self) -> RecoveryStage { + self.stage + } + + /// Returns the admitted metadata byte length. + pub const fn length(self) -> RecoveryStageLength { + self.length + } +} diff --git a/src/adapters/recovery_stage_metadata_error.rs b/src/adapters/recovery_stage_metadata_error.rs new file mode 100644 index 0000000..298c6e8 --- /dev/null +++ b/src/adapters/recovery_stage_metadata_error.rs @@ -0,0 +1,37 @@ +//! This module owns recovery-stage metadata admission failures. + +use std::error::Error; +use std::fmt; + +use super::RecoveryStage; + +/// Why fixed recovery-stage metadata could not be admitted. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum RecoveryStageMetadataError { + /// Metadata reports a stage above its name-selected maximum. + Oversized { + /// Fixed stage being observed. + stage: RecoveryStage, + /// Name-selected maximum. + maximum: u64, + /// Metadata byte length. + observed: u64, + }, +} + +impl fmt::Display for RecoveryStageMetadataError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Oversized { + stage, + maximum, + observed, + } => write!( + formatter, + "{stage} metadata length {observed} exceeds maximum {maximum}" + ), + } + } +} + +impl Error for RecoveryStageMetadataError {} diff --git a/src/lib.rs b/src/lib.rs index 8ba069f..5bae430 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -15,9 +15,10 @@ //! generations, platform-gated filesystem publication mechanics, bounded //! immutable restart snapshots, typed store-initialization orchestration, and //! production initialization for the admitted Linux ext4 profile. Recovery -//! inventory and name classification are read-only; explicit recovery -//! execution, retention, and garbage collection APIs remain intentionally -//! absent until their contracts have executable specifications. +//! inventory, name classification, and bounded stage fingerprinting are +//! read-only; semantic recovery planning, execution, retention, and garbage +//! collection APIs remain intentionally absent until their contracts have +//! executable specifications. #[cfg(test)] extern crate self as keep; @@ -49,15 +50,18 @@ pub use adapters::{ RecoveryInventoryLimit, RecoveryInventoryLimitError, RecoveryInventoryOperation, RecoveryInventoryStorage, RecoveryNameClassificationError, RecoveryNameManifest, RecoveryNamedEntry, RecoveryNamespace, RecoveryPoolNameError, RecoveryRequiredEntry, - SealedSegment, SegmentDigest, SegmentDurabilityPhase, SegmentHeader, SegmentHeaderError, - SegmentPublication, SegmentPublicationError, SegmentReadError, SegmentReadPolicy, - SegmentRecordAdmissionError, SegmentRecordChecksum, SegmentRecordDecodeError, - SegmentRecordHeader, SegmentRecordHeaderError, SegmentRecordIdentity, SegmentRecordLength, - SegmentRecordLimit, SegmentRecordLimitError, SegmentRecordPayloadLength, SegmentRecords, - SegmentSeal, SegmentSealError, SegmentStage, SegmentStageCreateError, SegmentWriteError, - SegmentWritePhase, StagedSegment, StorageProfileIdParseError, StoreInitializationError, - StoreInitializationPhase, StoreInitializationReceipt, StoreInitializationStorage, - WriterLockAcquireError, WriterLockAcquirePhase, classify_recovery_names, initialize_store, + RecoveryStage, RecoveryStageEvidence, RecoveryStageFingerprint, + RecoveryStageFingerprintAlgorithm, RecoveryStageFingerprintError, RecoveryStageLength, + RecoveryStageMetadata, RecoveryStageMetadataError, SealedSegment, SegmentDigest, + SegmentDurabilityPhase, SegmentHeader, SegmentHeaderError, SegmentPublication, + SegmentPublicationError, SegmentReadError, SegmentReadPolicy, SegmentRecordAdmissionError, + SegmentRecordChecksum, SegmentRecordDecodeError, SegmentRecordHeader, SegmentRecordHeaderError, + SegmentRecordIdentity, SegmentRecordLength, SegmentRecordLimit, SegmentRecordLimitError, + SegmentRecordPayloadLength, SegmentRecords, SegmentSeal, SegmentSealError, SegmentStage, + SegmentStageCreateError, SegmentWriteError, SegmentWritePhase, StagedSegment, + StorageProfileIdParseError, StoreInitializationError, StoreInitializationPhase, + StoreInitializationReceipt, StoreInitializationStorage, WriterLockAcquireError, + WriterLockAcquirePhase, classify_recovery_names, fingerprint_recovery_stage, initialize_store, publish_catalog_generation, read_recovery_inventory, }; pub use blob::{ diff --git a/tests/recovery_stage_fingerprint.rs b/tests/recovery_stage_fingerprint.rs new file mode 100644 index 0000000..b481dc7 --- /dev/null +++ b/tests/recovery_stage_fingerprint.rs @@ -0,0 +1,43 @@ +//! Bounded recovery-stage fingerprint laws. + +#[path = "recovery_stage_fingerprint/reader_laws.rs"] +mod reader_laws; + +use std::error::Error; +use std::io::Cursor; + +use keep::{RecoveryStage, RecoveryStageMetadata, fingerprint_recovery_stage}; + +const DOMAIN: &[u8] = b"KEEP:RECOVERY:STAGE\0"; + +#[test] +fn every_stage_uses_its_exact_protocol_maximum() { + assert_eq!(RecoveryStage::Segment.maximum_length(), 1_073_741_824); + assert_eq!(RecoveryStage::Catalog.maximum_length(), 167_772_352); + assert_eq!(RecoveryStage::NextHead.maximum_length(), 128); +} + +#[test] +fn exact_bytes_match_an_independent_framed_blake3_oracle() -> Result<(), Box> { + for bytes in [b"".as_slice(), b"recovery evidence".as_slice()] { + let metadata = + RecoveryStageMetadata::new(RecoveryStage::Segment, u64::try_from(bytes.len())?)?; + let evidence = fingerprint_recovery_stage(metadata, Cursor::new(bytes))?; + + assert_eq!(evidence.stage(), RecoveryStage::Segment); + assert_eq!(evidence.length().get(), u64::try_from(bytes.len())?); + assert_eq!(evidence.fingerprint().algorithm().code(), 1); + assert_eq!(evidence.fingerprint().as_bytes(), &oracle(bytes)?); + } + Ok(()) +} + +fn oracle(bytes: &[u8]) -> Result<[u8; 32], Box> { + let mut hasher = blake3::Hasher::new(); + hasher.update(DOMAIN); + hasher.update(&1_u16.to_be_bytes()); + hasher.update(&[1_u8]); + hasher.update(bytes); + hasher.update(&u64::try_from(bytes.len())?.to_be_bytes()); + Ok(*hasher.finalize().as_bytes()) +} diff --git a/tests/recovery_stage_fingerprint/reader_laws.rs b/tests/recovery_stage_fingerprint/reader_laws.rs new file mode 100644 index 0000000..439c0fe --- /dev/null +++ b/tests/recovery_stage_fingerprint/reader_laws.rs @@ -0,0 +1,199 @@ +//! Adversarial recovery-stage reader laws. + +use std::error::Error; +use std::io::{self, Cursor, Read}; + +use keep::{ + RecoveryStage, RecoveryStageFingerprintError, RecoveryStageMetadata, + RecoveryStageMetadataError, fingerprint_recovery_stage, +}; + +#[test] +fn oversized_metadata_refuses_before_reading() -> Result<(), Box> { + let reader = CountingReader::new(Cursor::new(Vec::::new())); + let observed = RecoveryStage::NextHead + .maximum_length() + .checked_add(1) + .ok_or("test maximum overflow")?; + + let Err(error) = RecoveryStageMetadata::new(RecoveryStage::NextHead, observed) else { + return Err("oversized metadata was admitted".into()); + }; + + assert!(matches!( + error, + RecoveryStageMetadataError::Oversized { + stage: RecoveryStage::NextHead, + maximum: 128, + observed: 129, + } + )); + assert_eq!(reader.calls, 0); + Ok(()) +} + +#[test] +fn maximum_plus_one_stream_refuses_at_the_first_excess_byte() -> Result<(), Box> { + let bytes = vec![0_u8; 129]; + let metadata = RecoveryStageMetadata::new(RecoveryStage::NextHead, 128)?; + let Err(error) = fingerprint_recovery_stage(metadata, Cursor::new(bytes)) else { + return Err("oversized stream was admitted".into()); + }; + + assert!(matches!( + error, + RecoveryStageFingerprintError::EvidenceOversized { + stage: RecoveryStage::NextHead, + maximum: 128, + observed_at_least: 129, + } + )); + Ok(()) +} + +#[test] +fn interrupted_and_short_reads_preserve_the_fingerprint() -> Result<(), Box> { + let bytes = b"partition-independent stage evidence"; + let metadata = RecoveryStageMetadata::new(RecoveryStage::Catalog, u64::try_from(bytes.len())?)?; + let expected = fingerprint_recovery_stage(metadata, Cursor::new(bytes))?; + let reader = InterruptedReader::new(bytes); + + let observed = fingerprint_recovery_stage(metadata, reader)?; + + assert_eq!(observed, expected); + Ok(()) +} + +#[test] +fn read_failure_retains_stage_offset_and_source() -> Result<(), Box> { + let reader = FailingReader::new(b"prefix", 6); + let metadata = RecoveryStageMetadata::new(RecoveryStage::Segment, 6)?; + let Err(error) = fingerprint_recovery_stage(metadata, reader) else { + return Err("failing stage reader was admitted".into()); + }; + + assert!(matches!( + error, + RecoveryStageFingerprintError::Read { + stage: RecoveryStage::Segment, + offset: 6, + ref source, + } if source.kind() == io::ErrorKind::PermissionDenied + )); + Ok(()) +} + +#[test] +fn overreported_read_count_is_an_exact_contract_refusal() -> Result<(), Box> { + let metadata = RecoveryStageMetadata::new(RecoveryStage::NextHead, 0)?; + let Err(error) = fingerprint_recovery_stage(metadata, OverreportingReader) else { + return Err("overreporting stage reader was admitted".into()); + }; + + assert!(matches!( + error, + RecoveryStageFingerprintError::ReaderContract { + stage: RecoveryStage::NextHead, + offset: 0, + offered: 129, + observed: 130, + } + )); + Ok(()) +} + +struct CountingReader { + inner: R, + calls: usize, +} + +impl CountingReader { + const fn new(inner: R) -> Self { + Self { inner, calls: 0 } + } +} + +impl Read for CountingReader { + fn read(&mut self, buffer: &mut [u8]) -> io::Result { + self.calls = self + .calls + .checked_add(1) + .ok_or_else(|| io::Error::other("test call count overflow"))?; + self.inner.read(buffer) + } +} + +struct InterruptedReader<'a> { + bytes: &'a [u8], + offset: usize, + interrupt: bool, +} + +impl<'a> InterruptedReader<'a> { + const fn new(bytes: &'a [u8]) -> Self { + Self { + bytes, + offset: 0, + interrupt: true, + } + } +} + +impl Read for InterruptedReader<'_> { + fn read(&mut self, buffer: &mut [u8]) -> io::Result { + if self.interrupt { + self.interrupt = false; + return Err(io::ErrorKind::Interrupted.into()); + } + self.interrupt = true; + let remaining = self.bytes.get(self.offset..).unwrap_or_default(); + let count = remaining.len().min(3).min(buffer.len()); + let target = buffer + .get_mut(..count) + .ok_or_else(|| io::Error::other("test buffer range missing"))?; + let source = remaining + .get(..count) + .ok_or_else(|| io::Error::other("test source range missing"))?; + target.copy_from_slice(source); + self.offset = self + .offset + .checked_add(count) + .ok_or_else(|| io::Error::other("test offset overflow"))?; + Ok(count) + } +} + +struct FailingReader<'a> { + prefix: Cursor<&'a [u8]>, + failure_offset: u64, +} + +impl<'a> FailingReader<'a> { + const fn new(prefix: &'a [u8], failure_offset: u64) -> Self { + Self { + prefix: Cursor::new(prefix), + failure_offset, + } + } +} + +impl Read for FailingReader<'_> { + fn read(&mut self, buffer: &mut [u8]) -> io::Result { + if self.prefix.position() == self.failure_offset { + Err(io::ErrorKind::PermissionDenied.into()) + } else { + self.prefix.read(buffer) + } + } +} + +struct OverreportingReader; + +impl Read for OverreportingReader { + fn read(&mut self, buffer: &mut [u8]) -> io::Result { + buffer + .len() + .checked_add(1) + .ok_or_else(|| io::Error::other("test overreport overflow")) + } +} diff --git a/tests/recovery_stage_fingerprint_memory.rs b/tests/recovery_stage_fingerprint_memory.rs new file mode 100644 index 0000000..2b36a0d --- /dev/null +++ b/tests/recovery_stage_fingerprint_memory.rs @@ -0,0 +1,27 @@ +//! Isolated heap-allocation evidence for recovery-stage fingerprinting. + +use std::error::Error; +use std::io::Cursor; + +use allocation_counter::{AllocationInfo, measure}; +use keep::{RecoveryStage, RecoveryStageMetadata, fingerprint_recovery_stage}; + +#[test] +fn fingerprinting_retains_no_stage_bytes_and_allocates_nothing() -> Result<(), Box> { + let bytes = [0x5a_u8; 16_384]; + let length = u64::try_from(bytes.len())?; + let metadata = RecoveryStageMetadata::new(RecoveryStage::Segment, length)?; + let mut result = None; + + let allocations = measure(|| { + result = Some(fingerprint_recovery_stage( + metadata, + Cursor::new(bytes.as_slice()), + )); + }); + let evidence = result.ok_or("stage fingerprint measurement did not run")??; + + assert_eq!(evidence.length().get(), length); + assert_eq!(allocations, AllocationInfo::default()); + Ok(()) +} From 8596ce4ca4bcbfbb5332131f7fb2b9406020b2da Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 06:44:07 -0700 Subject: [PATCH 11/49] Add: Bind recovery stages to pinned filesystems --- CHANGELOG.md | 4 + README.md | 5 +- docs/formats/segment-store-v1/recovery.md | 7 +- docs/formats/segment-store-v1/requirements.md | 1 + .../filesystem_recovery_inventory_reader.rs | 72 +++++- src/adapters/filesystem_recovery_stage.rs | 142 ++++++++++++ .../filesystem_recovery_stage_error.rs | 161 ++++++++++++++ .../filesystem_recovery_stage_tests.rs | 208 ++++++++++++++++++ src/adapters/mod.rs | 7 + src/adapters/recovery_stage.rs | 14 +- src/lib.rs | 39 ++-- 11 files changed, 630 insertions(+), 30 deletions(-) create mode 100644 src/adapters/filesystem_recovery_stage.rs create mode 100644 src/adapters/filesystem_recovery_stage_error.rs create mode 100644 src/adapters/filesystem_recovery_stage_tests.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 639a3dc..dd4fea7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,10 @@ after its public API and format compatibility policies are established. directories without following links, verifies child-directory identity before and after bounded scanning, and preserves raw Linux entry-name bytes without mutating protocol state. +- Fixed recovery stages can now be fingerprinted relative to the pinned + recovery inventory capability. Observation admits only regular files, never + follows links, streams under the name-selected bound, and refuses entry + replacement or length drift without mutating protocol state. - Store initialization now exposes one storage-port state machine that admits the platform before mutation, opens and locks `writer.lock`, admits the three protocol directories in order, synchronizes the root, and preserves the diff --git a/README.md b/README.md index 42b4a61..9b11a1d 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,10 @@ protocol namespaces before retaining names, applies a configurable ceiling no greater than 2,097,152 entries, and returns duplicate-free deterministic raw name order. `FilesystemRecoveryInventoryReader` implements that contract with pinned, no-follow namespace capabilities and pre/post identity verification on -the admitted Linux ext4 profile. Artifact classification remains planned. +the admitted Linux ext4 profile. Its bounded stage-fingerprint operation opens +fixed stages relative to those capabilities, refuses links and nonregular +files, and verifies entry identity and length after reading. Semantic artifact +classification remains planned. Crash-injection execution, explicit recovery, retention, compaction, and garbage collection remain planned. Presence in the reference CAS does not claim retention, crash recovery, or durability. diff --git a/docs/formats/segment-store-v1/recovery.md b/docs/formats/segment-store-v1/recovery.md index 6727d65..2ac2395 100644 --- a/docs/formats/segment-store-v1/recovery.md +++ b/docs/formats/segment-store-v1/recovery.md @@ -35,8 +35,11 @@ coordinate, and refuses an unknown or conflicting name without artifact I/O. `fingerprint_recovery_stage` then reads a fixed stage through a zero-allocation bounded stream, refuses metadata or observed bytes above the name-selected maximum, and returns its exact observed length and -`KEEP:RECOVERY:STAGE\0` fingerprint. Semantic content classification remains -unimplemented. +`KEEP:RECOVERY:STAGE\0` fingerprint. +`FilesystemRecoveryInventoryReader::fingerprint_stage` binds that stream to +the pinned root or staging capability, opens without following links, admits +only regular files, and refuses entry replacement or length drift after +reading. Semantic content classification remains unimplemented. The sole admissible duplicate digest is one fixed staging name and its exact digest-derived pool name after a link transition. Recovery admits that pair diff --git a/docs/formats/segment-store-v1/requirements.md b/docs/formats/segment-store-v1/requirements.md index 109d833..2e05cdc 100644 --- a/docs/formats/segment-store-v1/requirements.md +++ b/docs/formats/segment-store-v1/requirements.md @@ -105,6 +105,7 @@ or recovery execution. | `KEEP-RECOVERY-006` | Filesystem inventory pins the admitted root and protocol directories without following links, verifies child-directory identity before and after scanning, stops each count at the remaining global budget plus one, preserves raw Linux entry-name bytes, and performs no protocol mutation | Capability-relative filesystem fixture | `src/adapters/filesystem_recovery_inventory_tests.rs`, `tests/recovery_inventory.rs` | Implemented in #17 | | `KEEP-RECOVERY-007` | Name classification requires the four initialized root entries, admits only fixed protocol names and canonical pool coordinates in their owning namespaces, refuses simultaneous fixed recovery stages before artifact reads, and moves a refused raw name without duplicating its allocation | Canonical-name matrix and allocation counter | `tests/recovery_name_classification.rs`, `tests/recovery_name_classification_memory.rs` | Implemented in #17 | | `KEEP-RECOVERY-008` | Stage evidence is fingerprinted through a zero-allocation bounded streaming reader under the named recovery domain; metadata and observed bytes cannot exceed the name-selected protocol maximum, and failures retain exact stage and offset | Independent framing oracle, adversarial reader matrix, and allocation counter | `tests/recovery_stage_fingerprint.rs`, `tests/recovery_stage_fingerprint_memory.rs` | Implemented in #17 | +| `KEEP-RECOVERY-009` | Filesystem stage observation uses the pinned inventory capability, never follows a fixed-stage link, admits only regular files, and refuses entry replacement or length drift after bounded fingerprinting | Capability-relative replacement fixtures | `src/adapters/filesystem_recovery_stage_tests.rs` | Implemented in #17 | diff --git a/src/adapters/filesystem_recovery_inventory_reader.rs b/src/adapters/filesystem_recovery_inventory_reader.rs index f636262..cf296f1 100644 --- a/src/adapters/filesystem_recovery_inventory_reader.rs +++ b/src/adapters/filesystem_recovery_inventory_reader.rs @@ -8,10 +8,12 @@ use cap_std::ambient_authority; use cap_std::fs::Dir; use super::{ - RecoveryEntryName, RecoveryInventory, RecoveryInventoryError, RecoveryInventoryLimit, - RecoveryInventoryOperation, RecoveryInventoryStorage, RecoveryNamespace, + FilesystemRecoveryStageError, RecoveryEntryName, RecoveryInventory, RecoveryInventoryError, + RecoveryInventoryLimit, RecoveryInventoryOperation, RecoveryInventoryStorage, + RecoveryNamespace, RecoveryStage, RecoveryStageEvidence, RecoveryStageNamespacePhase, filesystem_platform_profile, filesystem_recovery_inventory_scan, - filesystem_recovery_namespace::PinnedRecoveryDirectory, read_recovery_inventory, + filesystem_recovery_namespace::PinnedRecoveryDirectory, filesystem_recovery_stage, + read_recovery_inventory, }; const STAGING_NAME: &str = "staging"; @@ -103,12 +105,67 @@ impl FilesystemRecoveryInventoryReader { Ok(inventory) } + /// Produces bounded exact evidence for one fixed recovery stage. + /// + /// Opens relative to the pinned root or staging capability without + /// following links, admits only a regular file, streams under the + /// name-selected bound, then verifies the handle, entry, and namespaces. + /// The synchronous call allocates no content-sized memory, may block on + /// filesystem I/O, and performs no protocol mutation. + /// + /// # Errors + /// + /// Returns [`FilesystemRecoveryStageError`] on namespace drift, link or + /// file refusal, oversized evidence, entry replacement, length drift, or + /// underlying I/O failure. + pub fn fingerprint_stage( + &self, + stage: RecoveryStage, + ) -> Result { + self.verify_stage_namespaces(stage, RecoveryStageNamespacePhase::BeforeObservation)?; + let evidence = filesystem_recovery_stage::fingerprint(self.stage_directory(stage), stage)?; + self.verify_stage_namespaces(stage, RecoveryStageNamespacePhase::AfterObservation)?; + Ok(evidence) + } + + #[cfg(test)] + pub(super) fn fingerprint_stage_with( + &self, + stage: RecoveryStage, + after_open: F, + ) -> Result + where + F: FnOnce(), + { + self.verify_stage_namespaces(stage, RecoveryStageNamespacePhase::BeforeObservation)?; + let evidence = filesystem_recovery_stage::fingerprint_with( + self.stage_directory(stage), + stage, + after_open, + )?; + self.verify_stage_namespaces(stage, RecoveryStageNamespacePhase::AfterObservation)?; + Ok(evidence) + } + fn verify_namespaces(&self) -> Result<(), RecoveryInventoryError> { self.staging.verify(&self.root)?; self.segments.verify(&self.root)?; self.catalogs.verify(&self.root) } + fn verify_stage_namespaces( + &self, + stage: RecoveryStage, + phase: RecoveryStageNamespacePhase, + ) -> Result<(), FilesystemRecoveryStageError> { + self.verify_namespaces() + .map_err(|source| FilesystemRecoveryStageError::Namespace { + stage, + phase, + source, + }) + } + const fn directory(&self, namespace: RecoveryNamespace) -> &Dir { match namespace { RecoveryNamespace::Root => &self.root, @@ -117,6 +174,15 @@ impl FilesystemRecoveryInventoryReader { RecoveryNamespace::Catalogs => self.catalogs.directory(), } } + + const fn stage_directory(&self, stage: RecoveryStage) -> &Dir { + match stage { + RecoveryStage::Segment | RecoveryStage::Catalog => { + self.directory(RecoveryNamespace::Staging) + } + RecoveryStage::NextHead => self.directory(RecoveryNamespace::Root), + } + } } impl RecoveryInventoryStorage for FilesystemRecoveryInventoryReader { diff --git a/src/adapters/filesystem_recovery_stage.rs b/src/adapters/filesystem_recovery_stage.rs new file mode 100644 index 0000000..f977a1d --- /dev/null +++ b/src/adapters/filesystem_recovery_stage.rs @@ -0,0 +1,142 @@ +//! This module owns pinned filesystem recovery-stage observation. + +use cap_fs_ext::{FollowSymlinks, MetadataExt, OpenOptionsFollowExt, OpenOptionsSyncExt}; +use cap_std::fs::{Dir, File, Metadata, OpenOptions}; + +use super::{ + FilesystemRecoveryStageError, RecoveryStage, RecoveryStageEvidence, RecoveryStageLength, + RecoveryStageMetadata, fingerprint_recovery_stage, +}; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct FileIdentity { + device: u64, + inode: u64, +} + +impl From<&Metadata> for FileIdentity { + fn from(metadata: &Metadata) -> Self { + Self { + device: metadata.dev(), + inode: metadata.ino(), + } + } +} + +struct AdmittedStage { + identity: FileIdentity, + metadata: RecoveryStageMetadata, +} + +pub(super) fn fingerprint( + directory: &Dir, + stage: RecoveryStage, +) -> Result { + observe(directory, stage, || {}) +} + +#[cfg(test)] +pub(super) fn fingerprint_with( + directory: &Dir, + stage: RecoveryStage, + after_open: F, +) -> Result +where + F: FnOnce(), +{ + observe(directory, stage, after_open) +} + +fn observe( + directory: &Dir, + stage: RecoveryStage, + after_open: F, +) -> Result +where + F: FnOnce(), +{ + let mut file = open_stage(directory, stage)?; + let admitted = admit_stage(&file, stage)?; + after_open(); + let evidence = fingerprint_recovery_stage(admitted.metadata, &mut file) + .map_err(|source| FilesystemRecoveryStageError::Fingerprint { stage, source })?; + verify_length(stage, admitted.metadata.length(), evidence.length().get())?; + verify_opened_handle(&file, stage, &admitted)?; + verify_current_entry(directory, stage, &admitted)?; + Ok(evidence) +} + +fn open_stage(directory: &Dir, stage: RecoveryStage) -> Result { + directory + .open_with(stage.file_name(), &read_options()) + .map_err(|source| FilesystemRecoveryStageError::Open { stage, source }) +} + +fn admit_stage( + file: &File, + stage: RecoveryStage, +) -> Result { + let metadata = file + .metadata() + .map_err(|source| FilesystemRecoveryStageError::Inspect { stage, source })?; + if !metadata.is_file() { + return Err(FilesystemRecoveryStageError::NonRegular { stage }); + } + let identity = FileIdentity::from(&metadata); + let metadata = RecoveryStageMetadata::new(stage, metadata.len()) + .map_err(|source| FilesystemRecoveryStageError::MetadataAdmission { stage, source })?; + Ok(AdmittedStage { identity, metadata }) +} + +fn verify_opened_handle( + file: &File, + stage: RecoveryStage, + admitted: &AdmittedStage, +) -> Result<(), FilesystemRecoveryStageError> { + let metadata = file + .metadata() + .map_err(|source| FilesystemRecoveryStageError::Inspect { stage, source })?; + if !metadata.is_file() || FileIdentity::from(&metadata) != admitted.identity { + return Err(FilesystemRecoveryStageError::Replaced { stage }); + } + verify_length(stage, admitted.metadata.length(), metadata.len()) +} + +fn verify_current_entry( + directory: &Dir, + stage: RecoveryStage, + admitted: &AdmittedStage, +) -> Result<(), FilesystemRecoveryStageError> { + let file = directory + .open_with(stage.file_name(), &read_options()) + .map_err(|source| FilesystemRecoveryStageError::VerifyEntry { stage, source })?; + let metadata = file + .metadata() + .map_err(|source| FilesystemRecoveryStageError::VerifyEntry { stage, source })?; + if !metadata.is_file() || FileIdentity::from(&metadata) != admitted.identity { + return Err(FilesystemRecoveryStageError::Replaced { stage }); + } + verify_length(stage, admitted.metadata.length(), metadata.len()) +} + +const fn verify_length( + stage: RecoveryStage, + expected: RecoveryStageLength, + observed: u64, +) -> Result<(), FilesystemRecoveryStageError> { + if observed == expected.get() { + Ok(()) + } else { + Err(FilesystemRecoveryStageError::LengthChanged { + stage, + expected, + observed, + }) + } +} + +fn read_options() -> OpenOptions { + let mut options = OpenOptions::new(); + options.read(true).follow(FollowSymlinks::No).nonblock(true); + options +} diff --git a/src/adapters/filesystem_recovery_stage_error.rs b/src/adapters/filesystem_recovery_stage_error.rs new file mode 100644 index 0000000..1305e4a --- /dev/null +++ b/src/adapters/filesystem_recovery_stage_error.rs @@ -0,0 +1,161 @@ +//! This module owns capability-relative recovery-stage observation failures. + +use std::error::Error; +use std::fmt; +use std::io; + +use super::{ + RecoveryInventoryError, RecoveryStage, RecoveryStageFingerprintError, RecoveryStageLength, + RecoveryStageMetadataError, +}; + +/// Why one filesystem recovery stage could not produce exact evidence. +#[derive(Debug)] +pub enum FilesystemRecoveryStageError { + /// A pinned recovery namespace failed verification. + Namespace { + /// Fixed stage being observed. + stage: RecoveryStage, + /// Whether verification failed before or after observation. + phase: RecoveryStageNamespacePhase, + /// Underlying namespace refusal. + source: RecoveryInventoryError, + }, + /// The fixed stage could not be opened without following links. + Open { + /// Fixed stage being observed. + stage: RecoveryStage, + /// Underlying open failure. + source: io::Error, + }, + /// Metadata could not be read from an opened stage handle. + Inspect { + /// Fixed stage being observed. + stage: RecoveryStage, + /// Underlying metadata failure. + source: io::Error, + }, + /// The fixed stage exists but is not a regular file. + NonRegular { + /// Fixed stage being observed. + stage: RecoveryStage, + }, + /// The stage metadata length exceeds its name-selected maximum. + MetadataAdmission { + /// Fixed stage being observed. + stage: RecoveryStage, + /// Exact metadata-admission refusal. + source: RecoveryStageMetadataError, + }, + /// Bounded streaming fingerprinting failed. + Fingerprint { + /// Fixed stage being observed. + stage: RecoveryStage, + /// Exact streaming refusal. + source: RecoveryStageFingerprintError, + }, + /// The fixed stage entry could not be reopened for identity verification. + VerifyEntry { + /// Fixed stage being observed. + stage: RecoveryStage, + /// Underlying verification-open failure. + source: io::Error, + }, + /// The fixed stage entry no longer names the opened file. + Replaced { + /// Fixed stage being observed. + stage: RecoveryStage, + }, + /// The file length changed while evidence was collected. + LengthChanged { + /// Fixed stage being observed. + stage: RecoveryStage, + /// Length admitted before reading. + expected: RecoveryStageLength, + /// Length observed during or after reading. + observed: u64, + }, +} + +/// Namespace-verification position around stage observation. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum RecoveryStageNamespacePhase { + /// Verification before opening the stage. + BeforeObservation, + /// Verification after entry and handle verification. + AfterObservation, +} + +impl fmt::Display for FilesystemRecoveryStageError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Namespace { + stage, + phase, + source, + } => write!( + formatter, + "{stage} namespace verification failed {phase}: {source}" + ), + Self::Open { stage, source } => { + write!(formatter, "failed to open recovery stage {stage}: {source}") + } + Self::Inspect { stage, source } => { + write!( + formatter, + "failed to inspect recovery stage {stage}: {source}" + ) + } + Self::NonRegular { stage } => { + write!(formatter, "recovery stage {stage} is not a regular file") + } + Self::MetadataAdmission { stage, source, .. } => write!( + formatter, + "recovery stage {stage} metadata was refused: {source}" + ), + Self::Fingerprint { stage, source, .. } => write!( + formatter, + "recovery stage {stage} fingerprint failed: {source}" + ), + Self::VerifyEntry { stage, source } => write!( + formatter, + "failed to verify recovery stage entry {stage}: {source}" + ), + Self::Replaced { stage } => { + write!(formatter, "recovery stage {stage} changed file identity") + } + Self::LengthChanged { + stage, + expected, + observed, + } => write!( + formatter, + "recovery stage {stage} changed length from {} to {observed}", + expected.get() + ), + } + } +} + +impl fmt::Display for RecoveryStageNamespacePhase { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(match self { + Self::BeforeObservation => "before observation", + Self::AfterObservation => "after observation", + }) + } +} + +impl Error for FilesystemRecoveryStageError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Namespace { source, .. } => Some(source), + Self::Open { source, .. } + | Self::Inspect { source, .. } + | Self::VerifyEntry { source, .. } => Some(source), + Self::MetadataAdmission { source, .. } => Some(source), + Self::Fingerprint { source, .. } => Some(source), + Self::NonRegular { .. } | Self::Replaced { .. } | Self::LengthChanged { .. } => None, + } + } +} diff --git a/src/adapters/filesystem_recovery_stage_tests.rs b/src/adapters/filesystem_recovery_stage_tests.rs new file mode 100644 index 0000000..f12bf54 --- /dev/null +++ b/src/adapters/filesystem_recovery_stage_tests.rs @@ -0,0 +1,208 @@ +//! Capability-relative filesystem recovery-stage evidence laws. + +use std::error::Error; +use std::fs; + +use super::{ + FilesystemRecoveryInventoryReader, FilesystemRecoveryStageError, RecoveryStage, + RecoveryStageMetadataError, filesystem_test_sandbox::TestDirectory, +}; + +#[test] +fn pinned_stage_is_fingerprinted_without_mutation() -> Result<(), Box> { + let fixture = StageFixture::new("recovery-stage-fingerprint")?; + let cases: [(RecoveryStage, &[u8]); 3] = [ + (RecoveryStage::Segment, b"retained segment evidence"), + (RecoveryStage::Catalog, b"retained catalog evidence"), + (RecoveryStage::NextHead, b"retained head evidence"), + ]; + for (stage, bytes) in cases { + fs::write(fixture.stage_path(stage), bytes)?; + } + let reader = fixture.reader()?; + + for (stage, bytes) in cases { + let evidence = reader.fingerprint_stage(stage)?; + assert_eq!(evidence.stage(), stage); + assert_eq!(evidence.length().get(), u64::try_from(bytes.len())?); + assert_eq!(fs::read(fixture.stage_path(stage))?, bytes); + } + drop(reader); + fixture.remove()?; + Ok(()) +} + +#[test] +fn fixed_stage_symbolic_link_is_never_followed() -> Result<(), Box> { + use std::os::unix::fs::symlink; + + let fixture = StageFixture::new("recovery-stage-symlink")?; + let target = fixture.root().join("outside"); + fs::write(&target, b"outside bytes")?; + symlink(&target, fixture.segment_path())?; + let reader = fixture.reader()?; + + let Err(error) = reader.fingerprint_stage(RecoveryStage::Segment) else { + return Err("symbolic stage was followed".into()); + }; + + assert!(matches!( + error, + FilesystemRecoveryStageError::Open { + stage: RecoveryStage::Segment, + .. + } + )); + drop(reader); + fixture.remove()?; + Ok(()) +} + +#[test] +fn non_regular_stage_is_refused_before_fingerprinting() -> Result<(), Box> { + let fixture = StageFixture::new("recovery-stage-non-regular")?; + fs::create_dir(fixture.segment_path())?; + let reader = fixture.reader()?; + + let Err(error) = reader.fingerprint_stage(RecoveryStage::Segment) else { + return Err("directory stage was admitted".into()); + }; + + assert!(matches!( + error, + FilesystemRecoveryStageError::NonRegular { + stage: RecoveryStage::Segment, + } + )); + drop(reader); + fixture.remove()?; + Ok(()) +} + +#[test] +fn oversized_sparse_stage_refuses_from_metadata() -> Result<(), Box> { + let fixture = StageFixture::new("recovery-stage-oversized")?; + let file = fs::File::create(fixture.segment_path())?; + let observed = RecoveryStage::Segment + .maximum_length() + .checked_add(1) + .ok_or("test segment maximum overflow")?; + file.set_len(observed)?; + let reader = fixture.reader()?; + + let Err(error) = reader.fingerprint_stage(RecoveryStage::Segment) else { + return Err("oversized sparse stage was admitted".into()); + }; + + assert!(matches!( + error, + FilesystemRecoveryStageError::MetadataAdmission { + stage: RecoveryStage::Segment, + source: RecoveryStageMetadataError::Oversized { + observed: actual, + .. + }, + } if actual == observed + )); + drop(reader); + fixture.remove()?; + Ok(()) +} + +#[test] +fn replaced_stage_entry_refuses_the_opened_evidence() -> Result<(), Box> { + let fixture = StageFixture::new("recovery-stage-replaced")?; + fs::write(fixture.segment_path(), b"original")?; + let replacement_path = fixture.segment_path(); + let retained_path = fixture.root().join("retained-stage"); + let reader = fixture.reader()?; + + let mut hook_result = Ok(()); + let result = reader.fingerprint_stage_with(RecoveryStage::Segment, || { + hook_result = fs::rename(&replacement_path, &retained_path) + .and_then(|()| fs::write(&replacement_path, b"replacement")); + }); + hook_result?; + let Err(error) = result else { + return Err("replaced stage entry was admitted".into()); + }; + + assert!(matches!( + error, + FilesystemRecoveryStageError::Replaced { + stage: RecoveryStage::Segment, + } + )); + drop(reader); + fixture.remove()?; + Ok(()) +} + +#[test] +fn stage_length_drift_refuses_the_streamed_evidence() -> Result<(), Box> { + let fixture = StageFixture::new("recovery-stage-length-drift")?; + fs::write(fixture.segment_path(), b"old")?; + let path = fixture.segment_path(); + let reader = fixture.reader()?; + + let mut hook_result = Ok(()); + let result = reader.fingerprint_stage_with(RecoveryStage::Segment, || { + hook_result = fs::write(&path, b"new length"); + }); + hook_result?; + let Err(error) = result else { + return Err("length-drifted stage evidence was admitted".into()); + }; + + assert!(matches!( + error, + FilesystemRecoveryStageError::LengthChanged { + stage: RecoveryStage::Segment, + expected, + observed: 10, + } if expected.get() == 3 + )); + drop(reader); + fixture.remove()?; + Ok(()) +} + +struct StageFixture { + directory: TestDirectory, +} + +impl StageFixture { + fn new(name: &str) -> Result> { + let directory = TestDirectory::create(name)?; + for name in ["staging", "segments", "catalogs"] { + fs::create_dir(directory.path().join(name))?; + } + Ok(Self { directory }) + } + + fn root(&self) -> &std::path::Path { + self.directory.path() + } + + fn segment_path(&self) -> std::path::PathBuf { + self.stage_path(RecoveryStage::Segment) + } + + fn stage_path(&self, stage: RecoveryStage) -> std::path::PathBuf { + match stage { + RecoveryStage::Segment => self.root().join("staging/current.seg"), + RecoveryStage::Catalog => self.root().join("staging/current.cat"), + RecoveryStage::NextHead => self.root().join("head.next"), + } + } + + fn reader(&self) -> Result> { + Ok(FilesystemRecoveryInventoryReader::open_unchecked_for_tests( + self.root(), + )?) + } + + fn remove(self) -> std::io::Result<()> { + self.directory.remove() + } +} diff --git a/src/adapters/mod.rs b/src/adapters/mod.rs index 37bc3ce..8be451b 100644 --- a/src/adapters/mod.rs +++ b/src/adapters/mod.rs @@ -84,6 +84,10 @@ mod filesystem_recovery_inventory_scan; #[cfg(all(test, unix))] mod filesystem_recovery_inventory_tests; mod filesystem_recovery_namespace; +mod filesystem_recovery_stage; +mod filesystem_recovery_stage_error; +#[cfg(all(test, unix))] +mod filesystem_recovery_stage_tests; mod filesystem_segment_stage; #[cfg(test)] mod filesystem_segment_stage_tests; @@ -248,6 +252,9 @@ pub use filesystem_catalog_publisher::FilesystemCatalogPublisher; pub use filesystem_catalog_snapshot::FilesystemCatalogSnapshot; pub use filesystem_platform_admission::FilesystemPlatformAdmission; pub use filesystem_recovery_inventory_reader::FilesystemRecoveryInventoryReader; +pub use filesystem_recovery_stage_error::{ + FilesystemRecoveryStageError, RecoveryStageNamespacePhase, +}; pub use filesystem_segment_stage::FilesystemSegmentStage; pub use filesystem_writer_lock::FilesystemWriterLock; pub use layout_decode_error::LayoutDecodeError; diff --git a/src/adapters/recovery_stage.rs b/src/adapters/recovery_stage.rs index d334537..dacd428 100644 --- a/src/adapters/recovery_stage.rs +++ b/src/adapters/recovery_stage.rs @@ -17,6 +17,14 @@ pub enum RecoveryStage { } impl RecoveryStage { + pub(super) const fn file_name(self) -> &'static str { + match self { + Self::Segment => "current.seg", + Self::Catalog => "current.cat", + Self::NextHead => "head.next", + } + } + /// Returns the name-selected version-1 maximum byte length. #[must_use] pub const fn maximum_length(self) -> u64 { @@ -30,10 +38,6 @@ impl RecoveryStage { impl fmt::Display for RecoveryStage { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter.write_str(match self { - Self::Segment => "current.seg", - Self::Catalog => "current.cat", - Self::NextHead => "head.next", - }) + formatter.write_str(self.file_name()) } } diff --git a/src/lib.rs b/src/lib.rs index 5bae430..ad84dbf 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -43,25 +43,26 @@ pub use adapters::{ CatalogSuccessor, CatalogTransitionError, ChecksummedCatalog, ChecksummedPublicationHead, ChecksummedSegmentRecord, ClosedSegment, FilesystemCatalogPublicationError, FilesystemCatalogPublisher, FilesystemCatalogSnapshot, FilesystemPlatformAdmission, - FilesystemRecoveryInventoryReader, FilesystemSegmentStage, FilesystemWriterLock, - LayoutDecodeError, LayoutDecodePolicy, LayoutEncodeError, LayoutIdBinaryParseError, - LayoutIdTextParseError, PublicationHeadDecodeError, RecoveryEntryName, RecoveryEntryNameError, - RecoveryEntryRole, RecoveryInventory, RecoveryInventoryEntry, RecoveryInventoryError, - RecoveryInventoryLimit, RecoveryInventoryLimitError, RecoveryInventoryOperation, - RecoveryInventoryStorage, RecoveryNameClassificationError, RecoveryNameManifest, - RecoveryNamedEntry, RecoveryNamespace, RecoveryPoolNameError, RecoveryRequiredEntry, - RecoveryStage, RecoveryStageEvidence, RecoveryStageFingerprint, - RecoveryStageFingerprintAlgorithm, RecoveryStageFingerprintError, RecoveryStageLength, - RecoveryStageMetadata, RecoveryStageMetadataError, SealedSegment, SegmentDigest, - SegmentDurabilityPhase, SegmentHeader, SegmentHeaderError, SegmentPublication, - SegmentPublicationError, SegmentReadError, SegmentReadPolicy, SegmentRecordAdmissionError, - SegmentRecordChecksum, SegmentRecordDecodeError, SegmentRecordHeader, SegmentRecordHeaderError, - SegmentRecordIdentity, SegmentRecordLength, SegmentRecordLimit, SegmentRecordLimitError, - SegmentRecordPayloadLength, SegmentRecords, SegmentSeal, SegmentSealError, SegmentStage, - SegmentStageCreateError, SegmentWriteError, SegmentWritePhase, StagedSegment, - StorageProfileIdParseError, StoreInitializationError, StoreInitializationPhase, - StoreInitializationReceipt, StoreInitializationStorage, WriterLockAcquireError, - WriterLockAcquirePhase, classify_recovery_names, fingerprint_recovery_stage, initialize_store, + FilesystemRecoveryInventoryReader, FilesystemRecoveryStageError, FilesystemSegmentStage, + FilesystemWriterLock, LayoutDecodeError, LayoutDecodePolicy, LayoutEncodeError, + LayoutIdBinaryParseError, LayoutIdTextParseError, PublicationHeadDecodeError, + RecoveryEntryName, RecoveryEntryNameError, RecoveryEntryRole, RecoveryInventory, + RecoveryInventoryEntry, RecoveryInventoryError, RecoveryInventoryLimit, + RecoveryInventoryLimitError, RecoveryInventoryOperation, RecoveryInventoryStorage, + RecoveryNameClassificationError, RecoveryNameManifest, RecoveryNamedEntry, RecoveryNamespace, + RecoveryPoolNameError, RecoveryRequiredEntry, RecoveryStage, RecoveryStageEvidence, + RecoveryStageFingerprint, RecoveryStageFingerprintAlgorithm, RecoveryStageFingerprintError, + RecoveryStageLength, RecoveryStageMetadata, RecoveryStageMetadataError, + RecoveryStageNamespacePhase, SealedSegment, SegmentDigest, SegmentDurabilityPhase, + SegmentHeader, SegmentHeaderError, SegmentPublication, SegmentPublicationError, + SegmentReadError, SegmentReadPolicy, SegmentRecordAdmissionError, SegmentRecordChecksum, + SegmentRecordDecodeError, SegmentRecordHeader, SegmentRecordHeaderError, SegmentRecordIdentity, + SegmentRecordLength, SegmentRecordLimit, SegmentRecordLimitError, SegmentRecordPayloadLength, + SegmentRecords, SegmentSeal, SegmentSealError, SegmentStage, SegmentStageCreateError, + SegmentWriteError, SegmentWritePhase, StagedSegment, StorageProfileIdParseError, + StoreInitializationError, StoreInitializationPhase, StoreInitializationReceipt, + StoreInitializationStorage, WriterLockAcquireError, WriterLockAcquirePhase, + classify_recovery_names, fingerprint_recovery_stage, initialize_store, publish_catalog_generation, read_recovery_inventory, }; pub use blob::{ From 17fa15fcc4f735b5d5fd8c4e290e86579159c286 Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 09:37:42 -0700 Subject: [PATCH 12/49] Add: Classify retained segment stages --- CHANGELOG.md | 4 + README.md | 6 +- docs/formats/segment-store-v1/recovery.md | 6 +- docs/formats/segment-store-v1/requirements.md | 6 +- src/adapters/mod.rs | 8 + src/adapters/recovery_segment_classifier.rs | 198 ++++++++++++++++++ src/adapters/recovery_segment_stage.rs | 42 ++++ src/adapters/recovery_segment_stage_error.rs | 73 +++++++ src/adapters/recovery_segment_truncation.rs | 45 ++++ src/lib.rs | 17 +- tests/recovery_segment_classification.rs | 31 +++ .../refusal_laws.rs | 138 ++++++++++++ .../state_laws.rs | 144 +++++++++++++ 13 files changed, 705 insertions(+), 13 deletions(-) create mode 100644 src/adapters/recovery_segment_classifier.rs create mode 100644 src/adapters/recovery_segment_stage.rs create mode 100644 src/adapters/recovery_segment_stage_error.rs create mode 100644 src/adapters/recovery_segment_truncation.rs create mode 100644 tests/recovery_segment_classification.rs create mode 100644 tests/recovery_segment_classification/refusal_laws.rs create mode 100644 tests/recovery_segment_classification/state_laws.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index dd4fea7..686d3c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,10 @@ after its public API and format compatibility policies are established. recovery inventory capability. Observation admits only regular files, never follows links, streams under the name-selected bound, and refuses entry replacement or length drift without mutating protocol state. +- Complete caller-supplied segment-stage bytes now classify as a validated + reusable prefix, a complete admitted immutable segment, or an exact + truncation. Complete-looking corruption, duplicate identities, and + caller-policy excess remain typed refusals. - Store initialization now exposes one storage-port state machine that admits the platform before mutation, opens and locks `writer.lock`, admits the three protocol directories in order, synchronizes the root, and preserves the diff --git a/README.md b/README.md index 9b11a1d..bfae74f 100644 --- a/README.md +++ b/README.md @@ -71,8 +71,10 @@ name order. `FilesystemRecoveryInventoryReader` implements that contract with pinned, no-follow namespace capabilities and pre/post identity verification on the admitted Linux ext4 profile. Its bounded stage-fingerprint operation opens fixed stages relative to those capabilities, refuses links and nonregular -files, and verifies entry identity and length after reading. Semantic artifact -classification remains planned. +files, and verifies entry identity and length after reading. Complete +caller-supplied segment-stage bytes can be classified as a reusable prefix, +complete admitted segment, or exact truncation. Catalog, head, and +filesystem-streaming semantic classification remain planned. Crash-injection execution, explicit recovery, retention, compaction, and garbage collection remain planned. Presence in the reference CAS does not claim retention, crash recovery, or durability. diff --git a/docs/formats/segment-store-v1/recovery.md b/docs/formats/segment-store-v1/recovery.md index 2ac2395..13d121e 100644 --- a/docs/formats/segment-store-v1/recovery.md +++ b/docs/formats/segment-store-v1/recovery.md @@ -39,7 +39,11 @@ maximum, and returns its exact observed length and `FilesystemRecoveryInventoryReader::fingerprint_stage` binds that stream to the pinned root or staging capability, opens without following links, admits only regular files, and refuses entry replacement or length drift after -reading. Semantic content classification remains unimplemented. +reading. `classify_recovery_segment_stage` classifies complete caller-supplied +stage bytes as a validated reusable prefix, a complete admitted segment, or an +exact truncation and preserves complete-looking corruption as a typed refusal. +Catalog-stage, head-stage, and filesystem-streaming semantic classification +remain unimplemented. The sole admissible duplicate digest is one fixed staging name and its exact digest-derived pool name after a link transition. Recovery admits that pair diff --git a/docs/formats/segment-store-v1/requirements.md b/docs/formats/segment-store-v1/requirements.md index 2e05cdc..df1dbb3 100644 --- a/docs/formats/segment-store-v1/requirements.md +++ b/docs/formats/segment-store-v1/requirements.md @@ -90,8 +90,9 @@ sequence ownership. The second slice establishes the ordered initialization state machine and exact failure phases. The third slice binds that state machine to a fail-closed Linux ext4 adapter and canonical namespace. These slices now also classify canonical recovery names before opening artifact -bytes. They do not yet claim content classification, process-death injection, -or recovery execution. +bytes and classify complete caller-supplied segment-stage bytes. They do not +yet claim catalog or head content classification, process-death injection, or +recovery execution. @@ -106,6 +107,7 @@ or recovery execution. | `KEEP-RECOVERY-007` | Name classification requires the four initialized root entries, admits only fixed protocol names and canonical pool coordinates in their owning namespaces, refuses simultaneous fixed recovery stages before artifact reads, and moves a refused raw name without duplicating its allocation | Canonical-name matrix and allocation counter | `tests/recovery_name_classification.rs`, `tests/recovery_name_classification_memory.rs` | Implemented in #17 | | `KEEP-RECOVERY-008` | Stage evidence is fingerprinted through a zero-allocation bounded streaming reader under the named recovery domain; metadata and observed bytes cannot exceed the name-selected protocol maximum, and failures retain exact stage and offset | Independent framing oracle, adversarial reader matrix, and allocation counter | `tests/recovery_stage_fingerprint.rs`, `tests/recovery_stage_fingerprint_memory.rs` | Implemented in #17 | | `KEEP-RECOVERY-009` | Filesystem stage observation uses the pinned inventory capability, never follows a fixed-stage link, admits only regular files, and refuses entry replacement or length drift after bounded fingerprinting | Capability-relative replacement fixtures | `src/adapters/filesystem_recovery_stage_tests.rs` | Implemented in #17 | +| `KEEP-RECOVERY-010` | Whole-byte segment-stage classification distinguishes a validated reusable prefix, a complete admitted immutable segment, and exact header, record, or seal truncation; complete-looking corruption, duplicates, and resource-limit excess remain typed refusals | Canonical prefix and corruption matrix | `tests/recovery_segment_classification.rs`, `tests/recovery_segment_classification/*.rs` | Implemented in #17 | diff --git a/src/adapters/mod.rs b/src/adapters/mod.rs index 8be451b..bcd30ee 100644 --- a/src/adapters/mod.rs +++ b/src/adapters/mod.rs @@ -134,6 +134,10 @@ mod recovery_namespace; mod recovery_pool_name; mod recovery_pool_name_error; mod recovery_required_entry; +mod recovery_segment_classifier; +mod recovery_segment_stage; +mod recovery_segment_stage_error; +mod recovery_segment_truncation; mod recovery_stage; mod recovery_stage_evidence; mod recovery_stage_fingerprint; @@ -277,6 +281,10 @@ pub use recovery_name_manifest::{RecoveryNameManifest, RecoveryNamedEntry}; pub use recovery_namespace::RecoveryNamespace; pub use recovery_pool_name_error::RecoveryPoolNameError; pub use recovery_required_entry::RecoveryRequiredEntry; +pub use recovery_segment_classifier::classify_recovery_segment_stage; +pub use recovery_segment_stage::{RecoverySegmentStage, ReusableRecoverySegment}; +pub use recovery_segment_stage_error::RecoverySegmentStageError; +pub use recovery_segment_truncation::RecoverySegmentTruncation; pub use recovery_stage::RecoveryStage; pub use recovery_stage_evidence::RecoveryStageEvidence; pub use recovery_stage_fingerprint::RecoveryStageFingerprint; diff --git a/src/adapters/recovery_segment_classifier.rs b/src/adapters/recovery_segment_classifier.rs new file mode 100644 index 0000000..0e679f5 --- /dev/null +++ b/src/adapters/recovery_segment_classifier.rs @@ -0,0 +1,198 @@ +//! This module owns whole-byte semantic classification of `current.seg`. + +use super::{ + AdmittedSegment, RecoverySegmentStage, RecoverySegmentStageError, RecoverySegmentTruncation, + RecoveryStage, RecoveryStageMetadata, ReusableRecoverySegment, SegmentHeader, SegmentReadError, + SegmentReadPolicy, SegmentSeal, segment_identity_index, segment_record_cursor_decode, + segment_record_header, segment_seal, +}; + +/// Classifies one complete caller-supplied segment-stage byte sequence. +/// +/// The input must contain all currently observed `current.seg` bytes. The call +/// performs no I/O and does not retain or copy content bytes. Reusable and +/// complete states allocate a duplicate-identity index bounded by +/// `policy.record_limit()`. +/// +/// # Errors +/// +/// Returns [`RecoverySegmentStageError`] for oversized input, complete-looking +/// corruption, duplicate identities, resource refusal, arithmetic failure, or +/// an unsupported format coordinate. Known incomplete boundaries are returned +/// as [`RecoverySegmentStage::Truncated`]. +pub fn classify_recovery_segment_stage( + encoded: &[u8], + policy: SegmentReadPolicy, +) -> Result, RecoverySegmentStageError> { + let observed = + u64::try_from(encoded.len()).map_err(|_| RecoverySegmentStageError::AddressSpace { + observed: encoded.len(), + })?; + let metadata = RecoveryStageMetadata::new(RecoveryStage::Segment, observed) + .map_err(|source| RecoverySegmentStageError::Metadata { source })?; + let Some(header_bytes) = encoded.get(..SegmentHeader::ENCODED_LENGTH) else { + return Ok(RecoverySegmentStage::Truncated( + RecoverySegmentTruncation::Header { + required: SegmentHeader::ENCODED_LENGTH, + observed: encoded.len(), + }, + )); + }; + SegmentHeader::decode(header_bytes) + .map_err(|source| RecoverySegmentStageError::Header { source })?; + classify_tail(encoded, metadata.length(), policy) +} + +fn classify_tail( + encoded: &[u8], + length: super::RecoveryStageLength, + policy: SegmentReadPolicy, +) -> Result, RecoverySegmentStageError> { + let records = encoded.get(SegmentHeader::ENCODED_LENGTH..).ok_or( + RecoverySegmentStageError::AddressSpace { + observed: encoded.len(), + }, + )?; + let initial_offset = u64::try_from(SegmentHeader::ENCODED_LENGTH).map_err(|_| { + RecoverySegmentStageError::AddressSpace { + observed: SegmentHeader::ENCODED_LENGTH, + } + })?; + let mut cursor = ReusableCursor::new(records, initial_offset); + loop { + if cursor.remaining.is_empty() { + return admit_reusable(records, cursor.record_index, length, policy); + } + if is_seal_candidate(cursor.remaining) { + if cursor.remaining.len() < SegmentSeal::ENCODED_LENGTH { + return Ok(RecoverySegmentStage::Truncated( + RecoverySegmentTruncation::Seal { + offset: cursor.offset, + required: SegmentSeal::ENCODED_LENGTH, + observed: cursor.remaining.len(), + }, + )); + } + return AdmittedSegment::decode(encoded, policy) + .map(RecoverySegmentStage::Complete) + .map_err(|source| RecoverySegmentStageError::Complete { source }); + } + match cursor.advance(policy) { + Ok(()) => {} + Err(source) => return classify_cursor_error(source), + } + } +} + +fn admit_reusable<'a>( + records: &[u8], + record_count: u32, + length: super::RecoveryStageLength, + policy: SegmentReadPolicy, +) -> Result, RecoverySegmentStageError> { + segment_identity_index::validate(records, record_count, policy) + .map_err(|source| RecoverySegmentStageError::Record { source })?; + Ok(RecoverySegmentStage::Reusable( + ReusableRecoverySegment::new(record_count, length), + )) +} + +const fn classify_cursor_error( + source: SegmentReadError, +) -> Result, RecoverySegmentStageError> { + match source { + SegmentReadError::RecordHeaderTruncated { + record_index, + offset, + required, + observed, + } => Ok(RecoverySegmentStage::Truncated( + RecoverySegmentTruncation::TailHeader { + record_index, + offset, + required, + observed, + }, + )), + SegmentReadError::RecordTruncated { + record_index, + offset, + expected, + observed, + } => Ok(RecoverySegmentStage::Truncated( + RecoverySegmentTruncation::Record { + record_index, + offset, + expected, + observed, + }, + )), + source => Err(RecoverySegmentStageError::Record { source }), + } +} + +fn is_seal_candidate(remaining: &[u8]) -> bool { + has_magic(remaining, segment_seal::MAGIC) + || (remaining.len() == SegmentSeal::ENCODED_LENGTH + && !has_magic(remaining, segment_record_header::MAGIC)) +} + +fn has_magic(remaining: &[u8], expected: [u8; 16]) -> bool { + let Some(magic) = remaining.first_chunk::<16>() else { + return false; + }; + *magic == expected +} + +struct ReusableCursor<'a> { + remaining: &'a [u8], + record_index: u32, + offset: u64, +} + +impl<'a> ReusableCursor<'a> { + const fn new(remaining: &'a [u8], offset: u64) -> Self { + Self { + remaining, + record_index: 0, + offset, + } + } + + fn advance(&mut self, policy: SegmentReadPolicy) -> Result<(), SegmentReadError> { + let observed = + self.record_index + .checked_add(1) + .ok_or(SegmentReadError::RecordIndexArithmetic { + record_index: self.record_index, + })?; + let maximum = policy.record_limit().get(); + if observed > maximum { + return Err(SegmentReadError::RecordCountLimit { maximum, observed }); + } + let decoded = segment_record_cursor_decode::decode( + self.remaining, + self.record_index, + self.offset, + policy, + )?; + self.remaining = + self.remaining + .get(decoded.host_length..) + .ok_or(SegmentReadError::RecordTruncated { + record_index: self.record_index, + offset: self.offset, + expected: decoded.record_length, + observed: self.remaining.len(), + })?; + self.offset = self.offset.checked_add(decoded.record_length).ok_or( + SegmentReadError::OffsetArithmetic { + record_index: self.record_index, + offset: self.offset, + record_length: decoded.record_length, + }, + )?; + self.record_index = observed; + Ok(()) + } +} diff --git a/src/adapters/recovery_segment_stage.rs b/src/adapters/recovery_segment_stage.rs new file mode 100644 index 0000000..2e2b968 --- /dev/null +++ b/src/adapters/recovery_segment_stage.rs @@ -0,0 +1,42 @@ +//! This module owns admitted recovery states for one segment stage. + +use super::{AdmittedSegment, RecoverySegmentTruncation, RecoveryStageLength}; + +/// Semantic state of complete caller-supplied `current.seg` bytes. +#[must_use] +pub enum RecoverySegmentStage<'a> { + /// Header plus zero or more complete admitted records, without a seal. + Reusable(ReusableRecoverySegment), + /// Fully admitted immutable segment, including its terminal seal. + Complete(AdmittedSegment<'a>), + /// Incomplete bytes whose exact missing boundary is known. + Truncated(RecoverySegmentTruncation), +} + +/// Validated reusable segment prefix retained for explicit recovery. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct ReusableRecoverySegment { + record_count: u32, + length: RecoveryStageLength, +} + +impl ReusableRecoverySegment { + pub(super) const fn new(record_count: u32, length: RecoveryStageLength) -> Self { + Self { + record_count, + length, + } + } + + /// Returns the exact number of complete admitted records. + #[must_use] + pub const fn record_count(self) -> u32 { + self.record_count + } + + /// Returns the complete validated prefix length. + pub const fn length(self) -> RecoveryStageLength { + self.length + } +} diff --git a/src/adapters/recovery_segment_stage_error.rs b/src/adapters/recovery_segment_stage_error.rs new file mode 100644 index 0000000..5e4bf57 --- /dev/null +++ b/src/adapters/recovery_segment_stage_error.rs @@ -0,0 +1,73 @@ +//! This module owns recovery segment-stage classification failures. + +use std::error::Error; +use std::fmt; + +use super::{RecoveryStageMetadataError, SegmentHeaderError, SegmentReadError}; + +/// Why complete supplied `current.seg` bytes could not be classified lawfully. +#[derive(Debug)] +pub enum RecoverySegmentStageError { + /// The caller-supplied slice length cannot fit the protocol coordinate. + AddressSpace { + /// Host byte count that could not be represented. + observed: usize, + }, + /// The complete stage exceeds the segment-stage protocol maximum. + Metadata { + /// Exact metadata-admission refusal. + source: RecoveryStageMetadataError, + }, + /// A complete fixed segment header is corrupt or unsupported. + Header { + /// Exact header refusal. + source: SegmentHeaderError, + }, + /// A complete-looking record or reusable-prefix invariant was refused. + Record { + /// Exact record or prefix refusal. + source: SegmentReadError, + }, + /// A complete-looking sealed segment was refused. + Complete { + /// Exact immutable-segment refusal. + source: SegmentReadError, + }, +} + +impl fmt::Display for RecoverySegmentStageError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::AddressSpace { observed } => write!( + formatter, + "segment-stage length {observed} does not fit the protocol coordinate" + ), + Self::Metadata { source } => { + write!(formatter, "segment-stage metadata was refused: {source}") + } + Self::Header { source } => { + write!(formatter, "segment-stage header was refused: {source}") + } + Self::Record { source } => { + write!( + formatter, + "segment-stage record prefix was refused: {source}" + ) + } + Self::Complete { source } => { + write!(formatter, "complete segment stage was refused: {source}") + } + } + } +} + +impl Error for RecoverySegmentStageError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Metadata { source } => Some(source), + Self::Header { source } => Some(source), + Self::Record { source } | Self::Complete { source } => Some(source), + Self::AddressSpace { .. } => None, + } + } +} diff --git a/src/adapters/recovery_segment_truncation.rs b/src/adapters/recovery_segment_truncation.rs new file mode 100644 index 0000000..c1a19f8 --- /dev/null +++ b/src/adapters/recovery_segment_truncation.rs @@ -0,0 +1,45 @@ +//! This module owns exact segment-stage truncation coordinates. + +/// Known incomplete boundary in one `current.seg` byte sequence. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum RecoverySegmentTruncation { + /// The fixed segment header is incomplete. + Header { + /// Required fixed-header byte count. + required: usize, + /// Supplied byte count. + observed: usize, + }, + /// The next tail header is too short to classify as record or seal. + TailHeader { + /// Zero-based next record position. + record_index: u32, + /// Physical stage byte offset. + offset: u64, + /// Required record-header byte count. + required: usize, + /// Remaining tail byte count. + observed: usize, + }, + /// A valid record header declares more bytes than remain. + Record { + /// Zero-based record position. + record_index: u32, + /// Physical stage byte offset. + offset: u64, + /// Declared complete record byte count. + expected: u64, + /// Remaining record byte count. + observed: usize, + }, + /// A recognized terminal seal is incomplete. + Seal { + /// Physical stage byte offset. + offset: u64, + /// Required fixed-seal byte count. + required: usize, + /// Remaining seal byte count. + observed: usize, + }, +} diff --git a/src/lib.rs b/src/lib.rs index ad84dbf..f383a27 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -50,20 +50,21 @@ pub use adapters::{ RecoveryInventoryEntry, RecoveryInventoryError, RecoveryInventoryLimit, RecoveryInventoryLimitError, RecoveryInventoryOperation, RecoveryInventoryStorage, RecoveryNameClassificationError, RecoveryNameManifest, RecoveryNamedEntry, RecoveryNamespace, - RecoveryPoolNameError, RecoveryRequiredEntry, RecoveryStage, RecoveryStageEvidence, - RecoveryStageFingerprint, RecoveryStageFingerprintAlgorithm, RecoveryStageFingerprintError, - RecoveryStageLength, RecoveryStageMetadata, RecoveryStageMetadataError, - RecoveryStageNamespacePhase, SealedSegment, SegmentDigest, SegmentDurabilityPhase, - SegmentHeader, SegmentHeaderError, SegmentPublication, SegmentPublicationError, - SegmentReadError, SegmentReadPolicy, SegmentRecordAdmissionError, SegmentRecordChecksum, + RecoveryPoolNameError, RecoveryRequiredEntry, RecoverySegmentStage, RecoverySegmentStageError, + RecoverySegmentTruncation, RecoveryStage, RecoveryStageEvidence, RecoveryStageFingerprint, + RecoveryStageFingerprintAlgorithm, RecoveryStageFingerprintError, RecoveryStageLength, + RecoveryStageMetadata, RecoveryStageMetadataError, RecoveryStageNamespacePhase, + ReusableRecoverySegment, SealedSegment, SegmentDigest, SegmentDurabilityPhase, SegmentHeader, + SegmentHeaderError, SegmentPublication, SegmentPublicationError, SegmentReadError, + SegmentReadPolicy, SegmentRecordAdmissionError, SegmentRecordChecksum, SegmentRecordDecodeError, SegmentRecordHeader, SegmentRecordHeaderError, SegmentRecordIdentity, SegmentRecordLength, SegmentRecordLimit, SegmentRecordLimitError, SegmentRecordPayloadLength, SegmentRecords, SegmentSeal, SegmentSealError, SegmentStage, SegmentStageCreateError, SegmentWriteError, SegmentWritePhase, StagedSegment, StorageProfileIdParseError, StoreInitializationError, StoreInitializationPhase, StoreInitializationReceipt, StoreInitializationStorage, WriterLockAcquireError, WriterLockAcquirePhase, - classify_recovery_names, fingerprint_recovery_stage, initialize_store, - publish_catalog_generation, read_recovery_inventory, + classify_recovery_names, classify_recovery_segment_stage, fingerprint_recovery_stage, + initialize_store, publish_catalog_generation, read_recovery_inventory, }; pub use blob::{ BlobHashError, BlobHasher, BlobId, BlobLength, BlobReadError, ByteLength, ByteOffset, diff --git a/tests/recovery_segment_classification.rs b/tests/recovery_segment_classification.rs new file mode 100644 index 0000000..4f8d21c --- /dev/null +++ b/tests/recovery_segment_classification.rs @@ -0,0 +1,31 @@ +//! Public recovery segment-stage classification laws. + +#[path = "recovery_segment_classification/refusal_laws.rs"] +mod refusal_laws; +#[path = "recovery_segment_classification/state_laws.rs"] +mod state_laws; +mod support; + +use std::error::Error; + +use keep::{LayoutEntryLimit, SegmentReadPolicy, SegmentRecordLimit}; +use support::decode_hex; + +const EMPTY_SEGMENT_HEX: &str = include_str!("../conformance/segment-store/v1/empty-segment.hex"); +const ONE_ZERO_SEGMENT_HEX: &str = + include_str!("../conformance/segment-store/v1/one-zero-segment.hex"); +const HEADER_LENGTH: usize = 64; +const RECORD_END: usize = 209; +const SEAL_LENGTH: usize = 128; + +const fn maximum_policy() -> SegmentReadPolicy { + SegmentReadPolicy::new(SegmentRecordLimit::MAXIMUM, LayoutEntryLimit::MAXIMUM) +} + +fn segment_bytes(hex: &str) -> Result, Box> { + decode_hex( + hex.strip_suffix('\n') + .ok_or("segment fixture must end in one LF")?, + ) + .map_err(Into::into) +} diff --git a/tests/recovery_segment_classification/refusal_laws.rs b/tests/recovery_segment_classification/refusal_laws.rs new file mode 100644 index 0000000..ac4dad8 --- /dev/null +++ b/tests/recovery_segment_classification/refusal_laws.rs @@ -0,0 +1,138 @@ +//! Complete-looking corruption and bounded-resource refusal laws. + +use std::error::Error; + +use keep::{ + LayoutEntryLimit, RecoverySegmentStageError, SegmentReadError, SegmentReadPolicy, + SegmentRecordLimit, classify_recovery_segment_stage, +}; + +use super::{HEADER_LENGTH, ONE_ZERO_SEGMENT_HEX, RECORD_END, maximum_policy, segment_bytes}; + +#[test] +fn complete_invalid_header_is_a_typed_refusal() -> Result<(), Box> { + let mut encoded = segment_bytes(ONE_ZERO_SEGMENT_HEX)?; + let byte = encoded.first_mut().ok_or("missing header byte")?; + *byte ^= 1; + + let error = classify_recovery_segment_stage(&encoded, maximum_policy()) + .err() + .ok_or("corrupt header was classified as lawful")?; + + assert!(matches!(error, RecoverySegmentStageError::Header { .. })); + Ok(()) +} + +#[test] +fn complete_invalid_record_is_a_typed_refusal() -> Result<(), Box> { + let mut encoded = segment_bytes(ONE_ZERO_SEGMENT_HEX)?; + let checksum_byte = encoded + .get_mut(RECORD_END.checked_sub(1).ok_or("checksum underflow")?) + .ok_or("missing record checksum")?; + *checksum_byte ^= 1; + + let error = classify_recovery_segment_stage(&encoded, maximum_policy()) + .err() + .ok_or("corrupt record was classified as lawful")?; + + assert!(matches!( + error, + RecoverySegmentStageError::Record { + source: SegmentReadError::RecordDecode { + record_index: 0, + offset: 64, + .. + }, + } + )); + Ok(()) +} + +#[test] +fn complete_invalid_seal_is_a_typed_refusal() -> Result<(), Box> { + let mut encoded = segment_bytes(ONE_ZERO_SEGMENT_HEX)?; + let seal_byte = encoded.last_mut().ok_or("missing seal checksum")?; + *seal_byte ^= 1; + + let error = classify_recovery_segment_stage(&encoded, maximum_policy()) + .err() + .ok_or("corrupt seal was classified as lawful")?; + + assert!(matches!( + error, + RecoverySegmentStageError::Complete { + source: SegmentReadError::Seal { .. }, + } + )); + Ok(()) +} + +#[test] +fn fixed_width_tail_with_invalid_seal_magic_stays_a_seal_refusal() -> Result<(), Box> { + let mut encoded = segment_bytes(ONE_ZERO_SEGMENT_HEX)?; + let magic_byte = encoded.get_mut(RECORD_END).ok_or("missing seal magic")?; + *magic_byte ^= 1; + + let error = classify_recovery_segment_stage(&encoded, maximum_policy()) + .err() + .ok_or("invalid seal magic was classified as lawful")?; + + assert!(matches!( + error, + RecoverySegmentStageError::Complete { + source: SegmentReadError::Seal { .. }, + } + )); + Ok(()) +} + +#[test] +fn duplicate_reusable_records_are_never_resumable() -> Result<(), Box> { + let complete = segment_bytes(ONE_ZERO_SEGMENT_HEX)?; + let header = complete.get(..HEADER_LENGTH).ok_or("missing header")?; + let record = complete + .get(HEADER_LENGTH..RECORD_END) + .ok_or("missing record")?; + let mut encoded = Vec::with_capacity( + HEADER_LENGTH + .checked_add(record.len().checked_mul(2).ok_or("record overflow")?) + .ok_or("stage overflow")?, + ); + encoded.extend_from_slice(header); + encoded.extend_from_slice(record); + encoded.extend_from_slice(record); + + let error = classify_recovery_segment_stage(&encoded, maximum_policy()) + .err() + .ok_or("duplicate record stage was reusable")?; + + assert!(matches!( + error, + RecoverySegmentStageError::Record { + source: SegmentReadError::DuplicateRecordIdentity { .. }, + } + )); + Ok(()) +} + +#[test] +fn reusable_record_count_obeys_the_caller_limit() -> Result<(), Box> { + let complete = segment_bytes(ONE_ZERO_SEGMENT_HEX)?; + let encoded = complete.get(..RECORD_END).ok_or("missing record prefix")?; + let policy = SegmentReadPolicy::new(SegmentRecordLimit::new(0)?, LayoutEntryLimit::MAXIMUM); + + let error = classify_recovery_segment_stage(encoded, policy) + .err() + .ok_or("record above caller limit was reusable")?; + + assert!(matches!( + error, + RecoverySegmentStageError::Record { + source: SegmentReadError::RecordCountLimit { + maximum: 0, + observed: 1, + }, + } + )); + Ok(()) +} diff --git a/tests/recovery_segment_classification/state_laws.rs b/tests/recovery_segment_classification/state_laws.rs new file mode 100644 index 0000000..2e56570 --- /dev/null +++ b/tests/recovery_segment_classification/state_laws.rs @@ -0,0 +1,144 @@ +//! Lawful reusable, complete, and truncated stage states. + +use std::error::Error; + +use keep::{RecoverySegmentStage, RecoverySegmentTruncation, classify_recovery_segment_stage}; + +use super::{ + EMPTY_SEGMENT_HEX, HEADER_LENGTH, ONE_ZERO_SEGMENT_HEX, RECORD_END, SEAL_LENGTH, + maximum_policy, segment_bytes, +}; + +#[test] +fn canonical_header_is_a_reusable_empty_stage() -> Result<(), Box> { + let complete = segment_bytes(EMPTY_SEGMENT_HEX)?; + let encoded = complete.get(..HEADER_LENGTH).ok_or("missing header")?; + + let RecoverySegmentStage::Reusable(stage) = + classify_recovery_segment_stage(encoded, maximum_policy())? + else { + return Err("canonical empty prefix was not reusable".into()); + }; + + assert_eq!(stage.record_count(), 0); + assert_eq!(stage.length().get(), u64::try_from(HEADER_LENGTH)?); + Ok(()) +} + +#[test] +fn canonical_record_prefix_is_reusable_without_rewriting() -> Result<(), Box> { + let complete = segment_bytes(ONE_ZERO_SEGMENT_HEX)?; + let encoded = complete.get(..RECORD_END).ok_or("missing record prefix")?; + + let RecoverySegmentStage::Reusable(stage) = + classify_recovery_segment_stage(encoded, maximum_policy())? + else { + return Err("canonical record prefix was not reusable".into()); + }; + + assert_eq!(stage.record_count(), 1); + assert_eq!(stage.length().get(), u64::try_from(RECORD_END)?); + Ok(()) +} + +#[test] +fn canonical_sealed_stage_is_a_complete_segment() -> Result<(), Box> { + let encoded = segment_bytes(ONE_ZERO_SEGMENT_HEX)?; + + let RecoverySegmentStage::Complete(segment) = + classify_recovery_segment_stage(&encoded, maximum_policy())? + else { + return Err("canonical sealed stage was not complete".into()); + }; + + assert_eq!(segment.encoded(), encoded); + assert_eq!(segment.record_count(), 1); + Ok(()) +} + +#[test] +fn partial_fixed_header_is_exactly_truncated() -> Result<(), Box> { + let complete = segment_bytes(EMPTY_SEGMENT_HEX)?; + let observed = HEADER_LENGTH.checked_sub(1).ok_or("header underflow")?; + let encoded = complete.get(..observed).ok_or("missing partial header")?; + + let state = classify_recovery_segment_stage(encoded, maximum_policy())?; + + assert!(matches!( + state, + RecoverySegmentStage::Truncated(RecoverySegmentTruncation::Header { + required: HEADER_LENGTH, + observed: actual, + }) if actual == observed + )); + Ok(()) +} + +#[test] +fn partial_record_header_is_exactly_truncated() -> Result<(), Box> { + let complete = segment_bytes(ONE_ZERO_SEGMENT_HEX)?; + let tail_length = 100_usize; + let observed_end = HEADER_LENGTH + .checked_add(tail_length) + .ok_or("record-header prefix overflow")?; + let encoded = complete + .get(..observed_end) + .ok_or("missing partial record header")?; + + let state = classify_recovery_segment_stage(encoded, maximum_policy())?; + + assert!(matches!( + state, + RecoverySegmentStage::Truncated(RecoverySegmentTruncation::TailHeader { + record_index: 0, + offset: 64, + required: 112, + observed: actual, + }) if actual == tail_length + )); + Ok(()) +} + +#[test] +fn partial_record_body_is_exactly_truncated() -> Result<(), Box> { + let complete = segment_bytes(ONE_ZERO_SEGMENT_HEX)?; + let observed_end = RECORD_END.checked_sub(1).ok_or("record underflow")?; + let encoded = complete + .get(..observed_end) + .ok_or("missing partial record")?; + + let state = classify_recovery_segment_stage(encoded, maximum_policy())?; + + assert!(matches!( + state, + RecoverySegmentStage::Truncated(RecoverySegmentTruncation::Record { + record_index: 0, + offset: 64, + expected: 145, + observed: 144, + }) + )); + Ok(()) +} + +#[test] +fn partial_seal_is_exactly_truncated() -> Result<(), Box> { + let complete = segment_bytes(ONE_ZERO_SEGMENT_HEX)?; + let partial_seal = SEAL_LENGTH.checked_div(2).ok_or("seal divisor")?; + let observed_end = RECORD_END + .checked_add(partial_seal) + .ok_or("seal prefix overflow")?; + let encoded = complete.get(..observed_end).ok_or("missing partial seal")?; + + let state = classify_recovery_segment_stage(encoded, maximum_policy())?; + + assert!(matches!( + state, + RecoverySegmentStage::Truncated(RecoverySegmentTruncation::Seal { + offset: 209, + required: SEAL_LENGTH, + observed: actual, + }) if actual == partial_seal + )); + Ok(()) +} From 02f761b97eb0aaf1d9d037f2f1502f0120876399 Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 09:47:22 -0700 Subject: [PATCH 13/49] Add: Classify retained publication stages --- CHANGELOG.md | 4 + README.md | 6 +- docs/formats/segment-store-v1/recovery.md | 5 +- docs/formats/segment-store-v1/requirements.md | 5 +- src/adapters/catalog_decoder.rs | 2 +- src/adapters/catalog_header_decoder.rs | 10 ++ src/adapters/mod.rs | 12 +++ src/adapters/recovery_catalog_stage.rs | 24 +++++ src/adapters/recovery_catalog_stage_error.rs | 61 +++++++++++ src/adapters/recovery_next_head_stage.rs | 17 +++ .../recovery_next_head_stage_error.rs | 53 +++++++++ .../recovery_publication_stage_classifier.rs | 88 +++++++++++++++ src/lib.rs | 14 +-- ...covery_publication_stage_classification.rs | 24 +++++ .../catalog_laws.rs | 102 ++++++++++++++++++ .../next_head_laws.rs | 82 ++++++++++++++ 16 files changed, 496 insertions(+), 13 deletions(-) create mode 100644 src/adapters/recovery_catalog_stage.rs create mode 100644 src/adapters/recovery_catalog_stage_error.rs create mode 100644 src/adapters/recovery_next_head_stage.rs create mode 100644 src/adapters/recovery_next_head_stage_error.rs create mode 100644 src/adapters/recovery_publication_stage_classifier.rs create mode 100644 tests/recovery_publication_stage_classification.rs create mode 100644 tests/recovery_publication_stage_classification/catalog_laws.rs create mode 100644 tests/recovery_publication_stage_classification/next_head_laws.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 686d3c7..14a69c9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,10 @@ after its public API and format compatibility policies are established. reusable prefix, a complete admitted immutable segment, or an exact truncation. Complete-looking corruption, duplicate identities, and caller-policy excess remain typed refusals. +- Complete caller-supplied catalog and candidate-head stages now distinguish + exact fixed-header, declared-body, or fixed-width truncation from canonical + bytes. Complete-looking corruption and oversized stages remain typed + refusals without claiming transitive catalog reachability. - Store initialization now exposes one storage-port state machine that admits the platform before mutation, opens and locks `writer.lock`, admits the three protocol directories in order, synchronizes the root, and preserves the diff --git a/README.md b/README.md index bfae74f..6b98296 100644 --- a/README.md +++ b/README.md @@ -73,8 +73,10 @@ the admitted Linux ext4 profile. Its bounded stage-fingerprint operation opens fixed stages relative to those capabilities, refuses links and nonregular files, and verifies entry identity and length after reading. Complete caller-supplied segment-stage bytes can be classified as a reusable prefix, -complete admitted segment, or exact truncation. Catalog, head, and -filesystem-streaming semantic classification remain planned. +complete admitted segment, or exact truncation. Catalog and next-head stages +likewise distinguish exact truncation from complete canonical bytes. +Transitive publication-view admission and filesystem-streaming semantic +classification remain planned. Crash-injection execution, explicit recovery, retention, compaction, and garbage collection remain planned. Presence in the reference CAS does not claim retention, crash recovery, or durability. diff --git a/docs/formats/segment-store-v1/recovery.md b/docs/formats/segment-store-v1/recovery.md index 13d121e..21c4af6 100644 --- a/docs/formats/segment-store-v1/recovery.md +++ b/docs/formats/segment-store-v1/recovery.md @@ -42,8 +42,9 @@ only regular files, and refuses entry replacement or length drift after reading. `classify_recovery_segment_stage` classifies complete caller-supplied stage bytes as a validated reusable prefix, a complete admitted segment, or an exact truncation and preserves complete-looking corruption as a typed refusal. -Catalog-stage, head-stage, and filesystem-streaming semantic classification -remain unimplemented. +Catalog- and next-head-stage classifiers likewise distinguish exact truncation +from complete canonical bytes. Transitive publication-view admission and +filesystem-streaming semantic classification remain unimplemented. The sole admissible duplicate digest is one fixed staging name and its exact digest-derived pool name after a link transition. Recovery admits that pair diff --git a/docs/formats/segment-store-v1/requirements.md b/docs/formats/segment-store-v1/requirements.md index df1dbb3..37899c5 100644 --- a/docs/formats/segment-store-v1/requirements.md +++ b/docs/formats/segment-store-v1/requirements.md @@ -90,8 +90,8 @@ sequence ownership. The second slice establishes the ordered initialization state machine and exact failure phases. The third slice binds that state machine to a fail-closed Linux ext4 adapter and canonical namespace. These slices now also classify canonical recovery names before opening artifact -bytes and classify complete caller-supplied segment-stage bytes. They do not -yet claim catalog or head content classification, process-death injection, or +bytes and classify complete caller-supplied fixed-stage bytes. They do not yet +claim transitive publication-view admission, process-death injection, or recovery execution. @@ -108,6 +108,7 @@ recovery execution. | `KEEP-RECOVERY-008` | Stage evidence is fingerprinted through a zero-allocation bounded streaming reader under the named recovery domain; metadata and observed bytes cannot exceed the name-selected protocol maximum, and failures retain exact stage and offset | Independent framing oracle, adversarial reader matrix, and allocation counter | `tests/recovery_stage_fingerprint.rs`, `tests/recovery_stage_fingerprint_memory.rs` | Implemented in #17 | | `KEEP-RECOVERY-009` | Filesystem stage observation uses the pinned inventory capability, never follows a fixed-stage link, admits only regular files, and refuses entry replacement or length drift after bounded fingerprinting | Capability-relative replacement fixtures | `src/adapters/filesystem_recovery_stage_tests.rs` | Implemented in #17 | | `KEEP-RECOVERY-010` | Whole-byte segment-stage classification distinguishes a validated reusable prefix, a complete admitted immutable segment, and exact header, record, or seal truncation; complete-looking corruption, duplicates, and resource-limit excess remain typed refusals | Canonical prefix and corruption matrix | `tests/recovery_segment_classification.rs`, `tests/recovery_segment_classification/*.rs` | Implemented in #17 | +| `KEEP-RECOVERY-011` | Whole-byte catalog and next-head stage classification distinguishes exact fixed-header, declared-body, and fixed-width truncation from complete canonical bytes; complete-looking corruption and oversize remain typed format or metadata refusals | Canonical publication-artifact truncation and corruption matrix | `tests/recovery_publication_stage_classification.rs`, `tests/recovery_publication_stage_classification/*.rs` | Implemented in #17 | diff --git a/src/adapters/catalog_decoder.rs b/src/adapters/catalog_decoder.rs index b1127f1..1903ac1 100644 --- a/src/adapters/catalog_decoder.rs +++ b/src/adapters/catalog_decoder.rs @@ -25,7 +25,7 @@ pub(super) fn decode(encoded: &[u8]) -> Result, CatalogDe )) } -fn validate_header( +pub(super) fn validate_header( fields: &catalog_header_decoder::DecodedCatalogHeader, ) -> Result { validate_fixed_fields(fields)?; diff --git a/src/adapters/catalog_header_decoder.rs b/src/adapters/catalog_header_decoder.rs index 55c0e8a..5e6174c 100644 --- a/src/adapters/catalog_header_decoder.rs +++ b/src/adapters/catalog_header_decoder.rs @@ -30,6 +30,16 @@ pub(super) fn decode(encoded: &[u8]) -> Result Result { + if encoded.len() < HEADER_LENGTH_BYTES { + return Err(CatalogDecodeError::MinimumLength { + minimum: HEADER_LENGTH_BYTES, + observed: encoded.len(), + }); + } Ok(DecodedCatalogHeader { magic: read_array(encoded, 0)?, version: u16::from_be_bytes(read_array(encoded, 16)?), diff --git a/src/adapters/mod.rs b/src/adapters/mod.rs index bcd30ee..87c327e 100644 --- a/src/adapters/mod.rs +++ b/src/adapters/mod.rs @@ -120,6 +120,8 @@ mod publication_head_decode_error; mod publication_head_decode_error_display; mod publication_head_decoder; mod publication_head_encoder; +mod recovery_catalog_stage; +mod recovery_catalog_stage_error; mod recovery_entry_name; mod recovery_entry_role; mod recovery_inventory; @@ -131,8 +133,11 @@ mod recovery_name_classification; mod recovery_name_classification_error; mod recovery_name_manifest; mod recovery_namespace; +mod recovery_next_head_stage; +mod recovery_next_head_stage_error; mod recovery_pool_name; mod recovery_pool_name_error; +mod recovery_publication_stage_classifier; mod recovery_required_entry; mod recovery_segment_classifier; mod recovery_segment_stage; @@ -268,6 +273,8 @@ pub use layout_id_binary_error::LayoutIdBinaryParseError; pub use layout_id_text_error::LayoutIdTextParseError; pub use layout_record::CanonicalLayoutRecord; pub use publication_head_decode_error::PublicationHeadDecodeError; +pub use recovery_catalog_stage::RecoveryCatalogStage; +pub use recovery_catalog_stage_error::RecoveryCatalogStageError; pub use recovery_entry_name::{RecoveryEntryName, RecoveryEntryNameError}; pub use recovery_entry_role::RecoveryEntryRole; pub use recovery_inventory::{RecoveryInventory, RecoveryInventoryEntry, read_recovery_inventory}; @@ -279,7 +286,12 @@ pub use recovery_name_classification::classify_recovery_names; pub use recovery_name_classification_error::RecoveryNameClassificationError; pub use recovery_name_manifest::{RecoveryNameManifest, RecoveryNamedEntry}; pub use recovery_namespace::RecoveryNamespace; +pub use recovery_next_head_stage::RecoveryNextHeadStage; +pub use recovery_next_head_stage_error::RecoveryNextHeadStageError; pub use recovery_pool_name_error::RecoveryPoolNameError; +pub use recovery_publication_stage_classifier::{ + classify_recovery_catalog_stage, classify_recovery_next_head_stage, +}; pub use recovery_required_entry::RecoveryRequiredEntry; pub use recovery_segment_classifier::classify_recovery_segment_stage; pub use recovery_segment_stage::{RecoverySegmentStage, ReusableRecoverySegment}; diff --git a/src/adapters/recovery_catalog_stage.rs b/src/adapters/recovery_catalog_stage.rs new file mode 100644 index 0000000..924414d --- /dev/null +++ b/src/adapters/recovery_catalog_stage.rs @@ -0,0 +1,24 @@ +//! This module owns admitted recovery states for one catalog stage. + +use super::ChecksummedCatalog; + +/// Semantic state of complete caller-supplied `current.cat` bytes. +#[must_use] +pub enum RecoveryCatalogStage<'a> { + /// The fixed catalog header is incomplete. + HeaderTruncated { + /// Required fixed-header byte count. + required: usize, + /// Supplied byte count. + observed: usize, + }, + /// The admitted header declares more bytes than were supplied. + BodyTruncated { + /// Declared canonical catalog byte count. + expected: u64, + /// Supplied byte count. + observed: usize, + }, + /// Fully framing-, checksum-, digest-, and entry-verified catalog. + Complete(ChecksummedCatalog<'a>), +} diff --git a/src/adapters/recovery_catalog_stage_error.rs b/src/adapters/recovery_catalog_stage_error.rs new file mode 100644 index 0000000..81990fe --- /dev/null +++ b/src/adapters/recovery_catalog_stage_error.rs @@ -0,0 +1,61 @@ +//! This module owns recovery catalog-stage classification failures. + +use std::error::Error; +use std::fmt; + +use super::{CatalogDecodeError, RecoveryStageMetadataError}; + +/// Why complete supplied `current.cat` bytes could not be classified lawfully. +#[derive(Debug)] +pub enum RecoveryCatalogStageError { + /// The caller-supplied slice length cannot fit the protocol coordinate. + AddressSpace { + /// Host byte count that could not be represented. + observed: usize, + }, + /// The complete stage exceeds the catalog-stage protocol maximum. + Metadata { + /// Exact metadata-admission refusal. + source: RecoveryStageMetadataError, + }, + /// The complete fixed catalog header was refused. + Header { + /// Exact catalog-header refusal. + source: CatalogDecodeError, + }, + /// Complete-looking catalog bytes were refused. + Complete { + /// Exact canonical catalog refusal. + source: CatalogDecodeError, + }, +} + +impl fmt::Display for RecoveryCatalogStageError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::AddressSpace { observed } => write!( + formatter, + "catalog-stage length {observed} does not fit the protocol coordinate" + ), + Self::Metadata { source } => { + write!(formatter, "catalog-stage metadata was refused: {source}") + } + Self::Header { source } => { + write!(formatter, "catalog-stage header was refused: {source}") + } + Self::Complete { source } => { + write!(formatter, "complete catalog stage was refused: {source}") + } + } + } +} + +impl Error for RecoveryCatalogStageError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Metadata { source } => Some(source), + Self::Header { source } | Self::Complete { source } => Some(source), + Self::AddressSpace { .. } => None, + } + } +} diff --git a/src/adapters/recovery_next_head_stage.rs b/src/adapters/recovery_next_head_stage.rs new file mode 100644 index 0000000..f098b1f --- /dev/null +++ b/src/adapters/recovery_next_head_stage.rs @@ -0,0 +1,17 @@ +//! This module owns admitted recovery states for `head.next`. + +use super::ChecksummedPublicationHead; + +/// Semantic state of complete caller-supplied `head.next` bytes. +#[must_use] +pub enum RecoveryNextHeadStage<'a> { + /// The fixed-width publication head is incomplete. + Truncated { + /// Required publication-head byte count. + required: usize, + /// Supplied byte count. + observed: usize, + }, + /// Fully framing- and checksum-verified publication head. + Complete(ChecksummedPublicationHead<'a>), +} diff --git a/src/adapters/recovery_next_head_stage_error.rs b/src/adapters/recovery_next_head_stage_error.rs new file mode 100644 index 0000000..3faa555 --- /dev/null +++ b/src/adapters/recovery_next_head_stage_error.rs @@ -0,0 +1,53 @@ +//! This module owns candidate-head stage classification failures. + +use std::error::Error; +use std::fmt; + +use super::{PublicationHeadDecodeError, RecoveryStageMetadataError}; + +/// Why complete supplied `head.next` bytes could not be classified lawfully. +#[derive(Debug)] +pub enum RecoveryNextHeadStageError { + /// The caller-supplied slice length cannot fit the protocol coordinate. + AddressSpace { + /// Host byte count that could not be represented. + observed: usize, + }, + /// The complete stage exceeds the fixed candidate-head width. + Metadata { + /// Exact metadata-admission refusal. + source: RecoveryStageMetadataError, + }, + /// Complete-looking candidate-head bytes were refused. + Complete { + /// Exact canonical publication-head refusal. + source: PublicationHeadDecodeError, + }, +} + +impl fmt::Display for RecoveryNextHeadStageError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::AddressSpace { observed } => write!( + formatter, + "next-head length {observed} does not fit the protocol coordinate" + ), + Self::Metadata { source } => { + write!(formatter, "next-head metadata was refused: {source}") + } + Self::Complete { source } => { + write!(formatter, "complete next-head stage was refused: {source}") + } + } + } +} + +impl Error for RecoveryNextHeadStageError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Metadata { source } => Some(source), + Self::Complete { source } => Some(source), + Self::AddressSpace { .. } => None, + } + } +} diff --git a/src/adapters/recovery_publication_stage_classifier.rs b/src/adapters/recovery_publication_stage_classifier.rs new file mode 100644 index 0000000..60371c1 --- /dev/null +++ b/src/adapters/recovery_publication_stage_classifier.rs @@ -0,0 +1,88 @@ +//! This module owns whole-byte classification of staged publication metadata. + +use super::{ + ChecksummedCatalog, ChecksummedPublicationHead, RecoveryCatalogStage, + RecoveryCatalogStageError, RecoveryNextHeadStage, RecoveryNextHeadStageError, RecoveryStage, + RecoveryStageMetadata, catalog_decoder, catalog_header_decoder, publication_head_decoder, +}; + +/// Classifies one complete caller-supplied catalog-stage byte sequence. +/// +/// The input must contain all currently observed `current.cat` bytes. The call +/// performs no I/O, allocation, or content copy. +/// +/// # Errors +/// +/// Returns [`RecoveryCatalogStageError`] for oversized input, a complete +/// invalid header, or complete-looking canonical catalog refusal. Known +/// incomplete boundaries are returned as truncation states. +pub fn classify_recovery_catalog_stage( + encoded: &[u8], +) -> Result, RecoveryCatalogStageError> { + let observed = catalog_metadata_length(encoded)?; + if encoded.len() < catalog_header_decoder::HEADER_LENGTH_BYTES { + return Ok(RecoveryCatalogStage::HeaderTruncated { + required: catalog_header_decoder::HEADER_LENGTH_BYTES, + observed: encoded.len(), + }); + } + let fields = catalog_header_decoder::decode_header(encoded) + .map_err(|source| RecoveryCatalogStageError::Header { source })?; + let metadata = catalog_decoder::validate_header(&fields) + .map_err(|source| RecoveryCatalogStageError::Header { source })?; + if observed < metadata.length().get() { + return Ok(RecoveryCatalogStage::BodyTruncated { + expected: metadata.length().get(), + observed: encoded.len(), + }); + } + ChecksummedCatalog::decode(encoded) + .map(RecoveryCatalogStage::Complete) + .map_err(|source| RecoveryCatalogStageError::Complete { source }) +} + +/// Classifies one complete caller-supplied candidate-head byte sequence. +/// +/// The input must contain all currently observed `head.next` bytes. The call +/// performs no I/O, allocation, or content copy. +/// +/// # Errors +/// +/// Returns [`RecoveryNextHeadStageError`] for oversized input or a +/// complete-looking canonical publication-head refusal. Short input is +/// returned as a truncation state. +pub fn classify_recovery_next_head_stage( + encoded: &[u8], +) -> Result, RecoveryNextHeadStageError> { + admit_next_head_metadata(encoded)?; + if encoded.len() < publication_head_decoder::ENCODED_LENGTH { + return Ok(RecoveryNextHeadStage::Truncated { + required: publication_head_decoder::ENCODED_LENGTH, + observed: encoded.len(), + }); + } + ChecksummedPublicationHead::decode(encoded) + .map(RecoveryNextHeadStage::Complete) + .map_err(|source| RecoveryNextHeadStageError::Complete { source }) +} + +fn catalog_metadata_length(encoded: &[u8]) -> Result { + let observed = + u64::try_from(encoded.len()).map_err(|_| RecoveryCatalogStageError::AddressSpace { + observed: encoded.len(), + })?; + RecoveryStageMetadata::new(RecoveryStage::Catalog, observed) + .map(RecoveryStageMetadata::length) + .map(super::RecoveryStageLength::get) + .map_err(|source| RecoveryCatalogStageError::Metadata { source }) +} + +fn admit_next_head_metadata(encoded: &[u8]) -> Result<(), RecoveryNextHeadStageError> { + let observed = + u64::try_from(encoded.len()).map_err(|_| RecoveryNextHeadStageError::AddressSpace { + observed: encoded.len(), + })?; + RecoveryStageMetadata::new(RecoveryStage::NextHead, observed) + .map(|_metadata| ()) + .map_err(|source| RecoveryNextHeadStageError::Metadata { source }) +} diff --git a/src/lib.rs b/src/lib.rs index f383a27..8a7fcf5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -46,10 +46,11 @@ pub use adapters::{ FilesystemRecoveryInventoryReader, FilesystemRecoveryStageError, FilesystemSegmentStage, FilesystemWriterLock, LayoutDecodeError, LayoutDecodePolicy, LayoutEncodeError, LayoutIdBinaryParseError, LayoutIdTextParseError, PublicationHeadDecodeError, - RecoveryEntryName, RecoveryEntryNameError, RecoveryEntryRole, RecoveryInventory, - RecoveryInventoryEntry, RecoveryInventoryError, RecoveryInventoryLimit, - RecoveryInventoryLimitError, RecoveryInventoryOperation, RecoveryInventoryStorage, - RecoveryNameClassificationError, RecoveryNameManifest, RecoveryNamedEntry, RecoveryNamespace, + RecoveryCatalogStage, RecoveryCatalogStageError, RecoveryEntryName, RecoveryEntryNameError, + RecoveryEntryRole, RecoveryInventory, RecoveryInventoryEntry, RecoveryInventoryError, + RecoveryInventoryLimit, RecoveryInventoryLimitError, RecoveryInventoryOperation, + RecoveryInventoryStorage, RecoveryNameClassificationError, RecoveryNameManifest, + RecoveryNamedEntry, RecoveryNamespace, RecoveryNextHeadStage, RecoveryNextHeadStageError, RecoveryPoolNameError, RecoveryRequiredEntry, RecoverySegmentStage, RecoverySegmentStageError, RecoverySegmentTruncation, RecoveryStage, RecoveryStageEvidence, RecoveryStageFingerprint, RecoveryStageFingerprintAlgorithm, RecoveryStageFingerprintError, RecoveryStageLength, @@ -63,8 +64,9 @@ pub use adapters::{ SegmentWriteError, SegmentWritePhase, StagedSegment, StorageProfileIdParseError, StoreInitializationError, StoreInitializationPhase, StoreInitializationReceipt, StoreInitializationStorage, WriterLockAcquireError, WriterLockAcquirePhase, - classify_recovery_names, classify_recovery_segment_stage, fingerprint_recovery_stage, - initialize_store, publish_catalog_generation, read_recovery_inventory, + classify_recovery_catalog_stage, classify_recovery_names, classify_recovery_next_head_stage, + classify_recovery_segment_stage, fingerprint_recovery_stage, initialize_store, + publish_catalog_generation, read_recovery_inventory, }; pub use blob::{ BlobHashError, BlobHasher, BlobId, BlobLength, BlobReadError, ByteLength, ByteOffset, diff --git a/tests/recovery_publication_stage_classification.rs b/tests/recovery_publication_stage_classification.rs new file mode 100644 index 0000000..b833196 --- /dev/null +++ b/tests/recovery_publication_stage_classification.rs @@ -0,0 +1,24 @@ +//! Public recovery catalog- and next-head-stage classification laws. + +#[path = "recovery_publication_stage_classification/catalog_laws.rs"] +mod catalog_laws; +#[path = "recovery_publication_stage_classification/next_head_laws.rs"] +mod next_head_laws; +mod support; + +use std::error::Error; + +use support::decode_hex; + +const CATALOG_HEX: &str = include_str!("../conformance/segment-store/v1/one-zero-catalog.hex"); +const HEAD_HEX: &str = include_str!("../conformance/segment-store/v1/one-zero-head.hex"); +const CATALOG_HEADER_LENGTH: usize = 128; +const HEAD_LENGTH: usize = 128; + +fn fixture(hex: &str) -> Result, Box> { + decode_hex( + hex.strip_suffix('\n') + .ok_or("publication fixture must end in one LF")?, + ) + .map_err(Into::into) +} diff --git a/tests/recovery_publication_stage_classification/catalog_laws.rs b/tests/recovery_publication_stage_classification/catalog_laws.rs new file mode 100644 index 0000000..5dbe5ef --- /dev/null +++ b/tests/recovery_publication_stage_classification/catalog_laws.rs @@ -0,0 +1,102 @@ +//! Catalog-stage truncation and complete-admission laws. + +use std::error::Error; + +use keep::{ + CatalogDecodeError, RecoveryCatalogStage, RecoveryCatalogStageError, + classify_recovery_catalog_stage, +}; + +use super::{CATALOG_HEADER_LENGTH, CATALOG_HEX, fixture}; + +#[test] +fn canonical_catalog_stage_is_complete() -> Result<(), Box> { + let encoded = fixture(CATALOG_HEX)?; + + let RecoveryCatalogStage::Complete(catalog) = classify_recovery_catalog_stage(&encoded)? else { + return Err("canonical catalog stage was not complete".into()); + }; + + assert_eq!(catalog.encoded(), encoded); + assert_eq!(catalog.generation().get(), 1); + Ok(()) +} + +#[test] +fn partial_catalog_header_is_exactly_truncated() -> Result<(), Box> { + let complete = fixture(CATALOG_HEX)?; + let observed = CATALOG_HEADER_LENGTH + .checked_sub(1) + .ok_or("catalog header underflow")?; + let encoded = complete + .get(..observed) + .ok_or("missing partial catalog header")?; + + let state = classify_recovery_catalog_stage(encoded)?; + + assert!(matches!( + state, + RecoveryCatalogStage::HeaderTruncated { + required: CATALOG_HEADER_LENGTH, + observed: actual, + } if actual == observed + )); + Ok(()) +} + +#[test] +fn partial_declared_catalog_body_is_exactly_truncated() -> Result<(), Box> { + let complete = fixture(CATALOG_HEX)?; + let observed = complete.len().checked_sub(1).ok_or("catalog underflow")?; + let encoded = complete.get(..observed).ok_or("missing partial catalog")?; + + let state = classify_recovery_catalog_stage(encoded)?; + + assert!(matches!( + state, + RecoveryCatalogStage::BodyTruncated { + expected: 352, + observed: actual, + } if actual == observed + )); + Ok(()) +} + +#[test] +fn complete_invalid_catalog_header_is_a_header_refusal() -> Result<(), Box> { + let mut encoded = fixture(CATALOG_HEX)?; + let byte = encoded.first_mut().ok_or("missing catalog header")?; + *byte ^= 1; + + let error = classify_recovery_catalog_stage(&encoded) + .err() + .ok_or("invalid catalog header was classified as lawful")?; + + assert!(matches!( + error, + RecoveryCatalogStageError::Header { + source: CatalogDecodeError::InvalidMagic { .. }, + } + )); + Ok(()) +} + +#[test] +fn complete_invalid_catalog_checksum_is_a_complete_refusal() -> Result<(), Box> { + let mut encoded = fixture(CATALOG_HEX)?; + let byte = encoded.last_mut().ok_or("missing catalog checksum")?; + *byte ^= 1; + + let error = classify_recovery_catalog_stage(&encoded) + .err() + .ok_or("invalid catalog checksum was classified as lawful")?; + + assert!(matches!( + error, + RecoveryCatalogStageError::Complete { + source: CatalogDecodeError::DigestMismatch { .. } + | CatalogDecodeError::ChecksumMismatch { .. }, + } + )); + Ok(()) +} diff --git a/tests/recovery_publication_stage_classification/next_head_laws.rs b/tests/recovery_publication_stage_classification/next_head_laws.rs new file mode 100644 index 0000000..b8692af --- /dev/null +++ b/tests/recovery_publication_stage_classification/next_head_laws.rs @@ -0,0 +1,82 @@ +//! Candidate-head truncation, admission, and refusal laws. + +use std::error::Error; + +use keep::{ + PublicationHeadDecodeError, RecoveryNextHeadStage, RecoveryNextHeadStageError, RecoveryStage, + RecoveryStageMetadataError, classify_recovery_next_head_stage, +}; + +use super::{HEAD_HEX, HEAD_LENGTH, fixture}; + +#[test] +fn canonical_next_head_stage_is_complete() -> Result<(), Box> { + let encoded = fixture(HEAD_HEX)?; + + let RecoveryNextHeadStage::Complete(head) = classify_recovery_next_head_stage(&encoded)? else { + return Err("canonical next-head stage was not complete".into()); + }; + + assert_eq!(head.encoded(), encoded); + assert_eq!(head.generation().get(), 1); + Ok(()) +} + +#[test] +fn partial_next_head_is_exactly_truncated() -> Result<(), Box> { + let complete = fixture(HEAD_HEX)?; + let observed = HEAD_LENGTH.checked_sub(1).ok_or("head underflow")?; + let encoded = complete.get(..observed).ok_or("missing partial head")?; + + let state = classify_recovery_next_head_stage(encoded)?; + + assert!(matches!( + state, + RecoveryNextHeadStage::Truncated { + required: HEAD_LENGTH, + observed: actual, + } if actual == observed + )); + Ok(()) +} + +#[test] +fn oversized_next_head_is_refused_before_decoding() -> Result<(), Box> { + let mut encoded = fixture(HEAD_HEX)?; + encoded.push(0); + + let error = classify_recovery_next_head_stage(&encoded) + .err() + .ok_or("oversized next head was classified as lawful")?; + + assert!(matches!( + error, + RecoveryNextHeadStageError::Metadata { + source: RecoveryStageMetadataError::Oversized { + stage: RecoveryStage::NextHead, + maximum: 128, + observed: 129, + }, + } + )); + Ok(()) +} + +#[test] +fn complete_invalid_next_head_is_a_typed_refusal() -> Result<(), Box> { + let mut encoded = fixture(HEAD_HEX)?; + let byte = encoded.last_mut().ok_or("missing head checksum")?; + *byte ^= 1; + + let error = classify_recovery_next_head_stage(&encoded) + .err() + .ok_or("invalid next head was classified as lawful")?; + + assert!(matches!( + error, + RecoveryNextHeadStageError::Complete { + source: PublicationHeadDecodeError::ChecksumMismatch { .. }, + } + )); + Ok(()) +} From b5856f792b2c5446199e3c132b477eb787f4e59a Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 10:02:56 -0700 Subject: [PATCH 14/49] Add: Bind recovery stage assessments --- CHANGELOG.md | 4 + README.md | 6 +- docs/formats/segment-store-v1/recovery.md | 5 + docs/formats/segment-store-v1/requirements.md | 8 +- src/adapters/admitted_recovery_stage_bytes.rs | 33 ++++++ src/adapters/mod.rs | 12 ++ src/adapters/recovery_stage_assessment.rs | 42 +++++++ .../recovery_stage_assessment_error.rs | 52 +++++++++ src/adapters/recovery_stage_assessor.rs | 37 ++++++ src/adapters/recovery_stage_byte_admission.rs | 56 ++++++++++ .../recovery_stage_byte_admission_error.rs | 105 ++++++++++++++++++ src/lib.rs | 66 +++++------ tests/recovery_stage_assessment.rs | 39 +++++++ .../admission_laws.rs | 86 ++++++++++++++ .../assessment_laws.rs | 84 ++++++++++++++ 15 files changed, 598 insertions(+), 37 deletions(-) create mode 100644 src/adapters/admitted_recovery_stage_bytes.rs create mode 100644 src/adapters/recovery_stage_assessment.rs create mode 100644 src/adapters/recovery_stage_assessment_error.rs create mode 100644 src/adapters/recovery_stage_assessor.rs create mode 100644 src/adapters/recovery_stage_byte_admission.rs create mode 100644 src/adapters/recovery_stage_byte_admission_error.rs create mode 100644 tests/recovery_stage_assessment.rs create mode 100644 tests/recovery_stage_assessment/admission_laws.rs create mode 100644 tests/recovery_stage_assessment/assessment_laws.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 14a69c9..615c478 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,10 @@ after its public API and format compatibility policies are established. exact fixed-header, declared-body, or fixed-width truncation from canonical bytes. Complete-looking corruption and oversized stages remain typed refusals without claiming transitive catalog reachability. +- Read-only recovery assessment now admits materialized stage bytes only when + their canonical-name stage, exact length, and recomputed versioned + fingerprint equal prior observation evidence, then dispatches through the + stage-selected semantic classifier. - Store initialization now exposes one storage-port state machine that admits the platform before mutation, opens and locks `writer.lock`, admits the three protocol directories in order, synchronizes the root, and preserves the diff --git a/README.md b/README.md index 6b98296..d80a027 100644 --- a/README.md +++ b/README.md @@ -75,8 +75,10 @@ files, and verifies entry identity and length after reading. Complete caller-supplied segment-stage bytes can be classified as a reusable prefix, complete admitted segment, or exact truncation. Catalog and next-head stages likewise distinguish exact truncation from complete canonical bytes. -Transitive publication-view admission and filesystem-streaming semantic -classification remain planned. +Materialized bytes enter read-only semantic assessment only after their stage, +length, and recomputed fingerprint match prior observation evidence. +Transitive publication-view admission and filesystem-streaming classification +remain planned. Crash-injection execution, explicit recovery, retention, compaction, and garbage collection remain planned. Presence in the reference CAS does not claim retention, crash recovery, or durability. diff --git a/docs/formats/segment-store-v1/recovery.md b/docs/formats/segment-store-v1/recovery.md index 21c4af6..5c80ef9 100644 --- a/docs/formats/segment-store-v1/recovery.md +++ b/docs/formats/segment-store-v1/recovery.md @@ -45,6 +45,11 @@ exact truncation and preserves complete-looking corruption as a typed refusal. Catalog- and next-head-stage classifiers likewise distinguish exact truncation from complete canonical bytes. Transitive publication-view admission and filesystem-streaming semantic classification remain unimplemented. +`admit_recovery_stage_bytes` first requires the canonical-name stage, exact +length, and recomputed stage fingerprint to match prior observation evidence; +only `assess_recovery_stage` may dispatch those admitted bytes to a semantic +classifier. Matching evidence does not convert corrupt bytes into lawful +content. The sole admissible duplicate digest is one fixed staging name and its exact digest-derived pool name after a link transition. Recovery admits that pair diff --git a/docs/formats/segment-store-v1/requirements.md b/docs/formats/segment-store-v1/requirements.md index 37899c5..09e185f 100644 --- a/docs/formats/segment-store-v1/requirements.md +++ b/docs/formats/segment-store-v1/requirements.md @@ -90,9 +90,10 @@ sequence ownership. The second slice establishes the ordered initialization state machine and exact failure phases. The third slice binds that state machine to a fail-closed Linux ext4 adapter and canonical namespace. These slices now also classify canonical recovery names before opening artifact -bytes and classify complete caller-supplied fixed-stage bytes. They do not yet -claim transitive publication-view admission, process-death injection, or -recovery execution. +bytes, bind materialized bytes back to prior stage evidence, and dispatch +complete caller-supplied fixed-stage bytes through their name-selected +classifiers. They do not yet claim transitive publication-view admission, +process-death injection, or recovery execution. @@ -109,6 +110,7 @@ recovery execution. | `KEEP-RECOVERY-009` | Filesystem stage observation uses the pinned inventory capability, never follows a fixed-stage link, admits only regular files, and refuses entry replacement or length drift after bounded fingerprinting | Capability-relative replacement fixtures | `src/adapters/filesystem_recovery_stage_tests.rs` | Implemented in #17 | | `KEEP-RECOVERY-010` | Whole-byte segment-stage classification distinguishes a validated reusable prefix, a complete admitted immutable segment, and exact header, record, or seal truncation; complete-looking corruption, duplicates, and resource-limit excess remain typed refusals | Canonical prefix and corruption matrix | `tests/recovery_segment_classification.rs`, `tests/recovery_segment_classification/*.rs` | Implemented in #17 | | `KEEP-RECOVERY-011` | Whole-byte catalog and next-head stage classification distinguishes exact fixed-header, declared-body, and fixed-width truncation from complete canonical bytes; complete-looking corruption and oversize remain typed format or metadata refusals | Canonical publication-artifact truncation and corruption matrix | `tests/recovery_publication_stage_classification.rs`, `tests/recovery_publication_stage_classification/*.rs` | Implemented in #17 | +| `KEEP-RECOVERY-012` | Read-only semantic assessment admits materialized stage bytes only when the canonical-name stage, exact observed length, and `KEEP:RECOVERY:STAGE\0` fingerprint equal prior evidence, then dispatches through the name-selected segment, catalog, or next-head classifier | Evidence-binding mutation matrix and canonical stage assessments | `tests/recovery_stage_assessment.rs`, `tests/recovery_stage_assessment/*.rs` | Implemented in #17 | diff --git a/src/adapters/admitted_recovery_stage_bytes.rs b/src/adapters/admitted_recovery_stage_bytes.rs new file mode 100644 index 0000000..4520d41 --- /dev/null +++ b/src/adapters/admitted_recovery_stage_bytes.rs @@ -0,0 +1,33 @@ +//! This module owns fingerprint-bound materialized recovery-stage bytes. + +use super::{RecoveryStage, RecoveryStageEvidence}; + +/// Complete materialized bytes proven equal to prior stage evidence. +#[must_use] +pub struct AdmittedRecoveryStageBytes<'a> { + evidence: RecoveryStageEvidence, + encoded: &'a [u8], +} + +impl<'a> AdmittedRecoveryStageBytes<'a> { + pub(super) const fn new(evidence: RecoveryStageEvidence, encoded: &'a [u8]) -> Self { + Self { evidence, encoded } + } + + /// Returns the canonical-name-selected fixed stage. + #[must_use] + pub const fn stage(&self) -> RecoveryStage { + self.evidence.stage() + } + + /// Returns the exact previously observed evidence. + pub const fn evidence(&self) -> RecoveryStageEvidence { + self.evidence + } + + /// Returns the complete fingerprint-matched bytes. + #[must_use] + pub const fn encoded(&self) -> &'a [u8] { + self.encoded + } +} diff --git a/src/adapters/mod.rs b/src/adapters/mod.rs index 87c327e..a1ded6a 100644 --- a/src/adapters/mod.rs +++ b/src/adapters/mod.rs @@ -6,6 +6,7 @@ //! policy, physical location, namespace publication, recovery, or retention. mod admitted_catalog; +mod admitted_recovery_stage_bytes; mod admitted_segment; mod admitted_segment_record; mod blob_id_binary; @@ -144,6 +145,11 @@ mod recovery_segment_stage; mod recovery_segment_stage_error; mod recovery_segment_truncation; mod recovery_stage; +mod recovery_stage_assessment; +mod recovery_stage_assessment_error; +mod recovery_stage_assessor; +mod recovery_stage_byte_admission; +mod recovery_stage_byte_admission_error; mod recovery_stage_evidence; mod recovery_stage_fingerprint; mod recovery_stage_fingerprint_algorithm; @@ -224,6 +230,7 @@ mod writer_lock_acquire_error; mod writer_lock_acquire_phase; pub use admitted_catalog::AdmittedCatalog; +pub use admitted_recovery_stage_bytes::AdmittedRecoveryStageBytes; pub use admitted_segment::AdmittedSegment; pub use admitted_segment_record::AdmittedSegmentRecord; pub use blob_id_binary_error::BlobIdBinaryParseError; @@ -298,6 +305,11 @@ pub use recovery_segment_stage::{RecoverySegmentStage, ReusableRecoverySegment}; pub use recovery_segment_stage_error::RecoverySegmentStageError; pub use recovery_segment_truncation::RecoverySegmentTruncation; pub use recovery_stage::RecoveryStage; +pub use recovery_stage_assessment::RecoveryStageAssessment; +pub use recovery_stage_assessment_error::RecoveryStageAssessmentError; +pub use recovery_stage_assessor::assess_recovery_stage; +pub use recovery_stage_byte_admission::admit_recovery_stage_bytes; +pub use recovery_stage_byte_admission_error::RecoveryStageByteAdmissionError; pub use recovery_stage_evidence::RecoveryStageEvidence; pub use recovery_stage_fingerprint::RecoveryStageFingerprint; pub use recovery_stage_fingerprint_algorithm::RecoveryStageFingerprintAlgorithm; diff --git a/src/adapters/recovery_stage_assessment.rs b/src/adapters/recovery_stage_assessment.rs new file mode 100644 index 0000000..fd1b35e --- /dev/null +++ b/src/adapters/recovery_stage_assessment.rs @@ -0,0 +1,42 @@ +//! This module owns name-selected semantic recovery-stage assessments. + +use super::{ + RecoveryCatalogStage, RecoveryNextHeadStage, RecoverySegmentStage, RecoveryStageEvidence, +}; + +/// Read-only semantic assessment of fingerprint-bound fixed-stage bytes. +#[must_use] +pub enum RecoveryStageAssessment<'a> { + /// `staging/current.seg` semantic state. + Segment { + /// Exact evidence matched before semantic classification. + evidence: RecoveryStageEvidence, + /// Validated segment-stage semantic state. + state: RecoverySegmentStage<'a>, + }, + /// `staging/current.cat` semantic state. + Catalog { + /// Exact evidence matched before semantic classification. + evidence: RecoveryStageEvidence, + /// Validated catalog-stage semantic state. + state: RecoveryCatalogStage<'a>, + }, + /// Root `head.next` semantic state. + NextHead { + /// Exact evidence matched before semantic classification. + evidence: RecoveryStageEvidence, + /// Validated next-head semantic state. + state: RecoveryNextHeadStage<'a>, + }, +} + +impl RecoveryStageAssessment<'_> { + /// Returns the exact evidence matched before semantic classification. + pub const fn evidence(&self) -> RecoveryStageEvidence { + match self { + Self::Segment { evidence, .. } + | Self::Catalog { evidence, .. } + | Self::NextHead { evidence, .. } => *evidence, + } + } +} diff --git a/src/adapters/recovery_stage_assessment_error.rs b/src/adapters/recovery_stage_assessment_error.rs new file mode 100644 index 0000000..0434d38 --- /dev/null +++ b/src/adapters/recovery_stage_assessment_error.rs @@ -0,0 +1,52 @@ +//! This module owns semantic recovery-stage assessment failures. + +use std::error::Error; +use std::fmt; + +use super::{RecoveryCatalogStageError, RecoveryNextHeadStageError, RecoverySegmentStageError}; + +/// Why fingerprint-bound stage bytes had no lawful semantic assessment. +#[derive(Debug)] +pub enum RecoveryStageAssessmentError { + /// Segment-stage classification failed. + Segment { + /// Exact segment-stage refusal. + source: RecoverySegmentStageError, + }, + /// Catalog-stage classification failed. + Catalog { + /// Exact catalog-stage refusal. + source: RecoveryCatalogStageError, + }, + /// Candidate-head classification failed. + NextHead { + /// Exact next-head refusal. + source: RecoveryNextHeadStageError, + }, +} + +impl fmt::Display for RecoveryStageAssessmentError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Segment { source } => { + write!(formatter, "segment-stage assessment failed: {source}") + } + Self::Catalog { source } => { + write!(formatter, "catalog-stage assessment failed: {source}") + } + Self::NextHead { source } => { + write!(formatter, "next-head assessment failed: {source}") + } + } + } +} + +impl Error for RecoveryStageAssessmentError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Segment { source } => Some(source), + Self::Catalog { source } => Some(source), + Self::NextHead { source } => Some(source), + } + } +} diff --git a/src/adapters/recovery_stage_assessor.rs b/src/adapters/recovery_stage_assessor.rs new file mode 100644 index 0000000..f8ec4fe --- /dev/null +++ b/src/adapters/recovery_stage_assessor.rs @@ -0,0 +1,37 @@ +//! This module owns name-selected dispatch of admitted recovery-stage bytes. + +use super::{ + AdmittedRecoveryStageBytes, RecoveryStage, RecoveryStageAssessment, + RecoveryStageAssessmentError, SegmentReadPolicy, classify_recovery_catalog_stage, + classify_recovery_next_head_stage, classify_recovery_segment_stage, +}; + +/// Classifies fingerprint-bound bytes through their only lawful stage grammar. +/// +/// The call performs no I/O or content copy. Segment assessment may allocate a +/// duplicate-identity index bounded by `segment_policy.record_limit()`; +/// catalog and candidate-head assessment allocate nothing. +/// +/// # Errors +/// +/// Returns [`RecoveryStageAssessmentError`] with the exact name-selected +/// semantic classifier refusal. +pub fn assess_recovery_stage<'a>( + admitted: &AdmittedRecoveryStageBytes<'a>, + segment_policy: SegmentReadPolicy, +) -> Result, RecoveryStageAssessmentError> { + let evidence = admitted.evidence(); + match admitted.stage() { + RecoveryStage::Segment => { + classify_recovery_segment_stage(admitted.encoded(), segment_policy) + .map(|state| RecoveryStageAssessment::Segment { evidence, state }) + .map_err(|source| RecoveryStageAssessmentError::Segment { source }) + } + RecoveryStage::Catalog => classify_recovery_catalog_stage(admitted.encoded()) + .map(|state| RecoveryStageAssessment::Catalog { evidence, state }) + .map_err(|source| RecoveryStageAssessmentError::Catalog { source }), + RecoveryStage::NextHead => classify_recovery_next_head_stage(admitted.encoded()) + .map(|state| RecoveryStageAssessment::NextHead { evidence, state }) + .map_err(|source| RecoveryStageAssessmentError::NextHead { source }), + } +} diff --git a/src/adapters/recovery_stage_byte_admission.rs b/src/adapters/recovery_stage_byte_admission.rs new file mode 100644 index 0000000..9f86c8a --- /dev/null +++ b/src/adapters/recovery_stage_byte_admission.rs @@ -0,0 +1,56 @@ +//! This module owns admission of materialized bytes against prior stage evidence. + +use super::{ + AdmittedRecoveryStageBytes, RecoveryStage, RecoveryStageByteAdmissionError, + RecoveryStageEvidence, RecoveryStageMetadata, fingerprint_recovery_stage, +}; + +/// Admits complete materialized bytes only when they equal prior stage evidence. +/// +/// The call performs no I/O or allocation. It checks the canonical-name stage +/// and exact length before recomputing the versioned bounded fingerprint. +/// +/// # Errors +/// +/// Returns [`RecoveryStageByteAdmissionError`] on stage, length, protocol +/// maximum, or fingerprint disagreement. +pub fn admit_recovery_stage_bytes( + expected_stage: RecoveryStage, + evidence: RecoveryStageEvidence, + encoded: &[u8], +) -> Result, RecoveryStageByteAdmissionError> { + if evidence.stage() != expected_stage { + return Err(RecoveryStageByteAdmissionError::StageMismatch { + expected: expected_stage, + observed: evidence.stage(), + }); + } + let observed = u64::try_from(encoded.len()).map_err(|_| { + RecoveryStageByteAdmissionError::AddressSpace { + observed: encoded.len(), + } + })?; + if evidence.length().get() != observed { + return Err(RecoveryStageByteAdmissionError::LengthMismatch { + stage: expected_stage, + expected: evidence.length(), + observed, + }); + } + let metadata = RecoveryStageMetadata::new(expected_stage, observed) + .map_err(|source| RecoveryStageByteAdmissionError::Metadata { source })?; + let recomputed = fingerprint_recovery_stage(metadata, encoded).map_err(|source| { + RecoveryStageByteAdmissionError::Fingerprint { + stage: expected_stage, + source, + } + })?; + if recomputed.fingerprint() != evidence.fingerprint() { + return Err(RecoveryStageByteAdmissionError::FingerprintMismatch { + stage: expected_stage, + expected: evidence.fingerprint(), + observed: recomputed.fingerprint(), + }); + } + Ok(AdmittedRecoveryStageBytes::new(evidence, encoded)) +} diff --git a/src/adapters/recovery_stage_byte_admission_error.rs b/src/adapters/recovery_stage_byte_admission_error.rs new file mode 100644 index 0000000..5a2df95 --- /dev/null +++ b/src/adapters/recovery_stage_byte_admission_error.rs @@ -0,0 +1,105 @@ +//! This module owns recovery-stage byte-admission failures. + +use std::error::Error; +use std::fmt; + +use super::{ + RecoveryStage, RecoveryStageFingerprint, RecoveryStageFingerprintError, RecoveryStageLength, + RecoveryStageMetadataError, +}; + +/// Why materialized bytes did not match prior fixed-stage evidence. +#[derive(Debug)] +pub enum RecoveryStageByteAdmissionError { + /// The canonical-name-selected stage differs from the evidence. + StageMismatch { + /// Stage selected by the canonical inventory name. + expected: RecoveryStage, + /// Stage carried by the prior evidence. + observed: RecoveryStage, + }, + /// The supplied slice length cannot fit the protocol coordinate. + AddressSpace { + /// Host byte count that could not be represented. + observed: usize, + }, + /// The supplied byte length differs from prior evidence. + LengthMismatch { + /// Fixed stage being admitted. + stage: RecoveryStage, + /// Previously observed exact length. + expected: RecoveryStageLength, + /// Supplied byte count. + observed: u64, + }, + /// The supplied length violates the selected stage's protocol maximum. + Metadata { + /// Exact metadata-admission refusal. + source: RecoveryStageMetadataError, + }, + /// Recomputing the bounded versioned fingerprint failed. + Fingerprint { + /// Fixed stage being admitted. + stage: RecoveryStage, + /// Exact fingerprinting refusal. + source: RecoveryStageFingerprintError, + }, + /// The supplied bytes differ from the prior observation. + FingerprintMismatch { + /// Fixed stage being admitted. + stage: RecoveryStage, + /// Previously observed fingerprint. + expected: RecoveryStageFingerprint, + /// Fingerprint of the supplied complete bytes. + observed: RecoveryStageFingerprint, + }, +} + +impl fmt::Display for RecoveryStageByteAdmissionError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::StageMismatch { expected, observed } => write!( + formatter, + "recovery stage evidence names {observed}, expected {expected}" + ), + Self::AddressSpace { observed } => write!( + formatter, + "recovery stage length {observed} does not fit the protocol coordinate" + ), + Self::LengthMismatch { + stage, + expected, + observed, + } => write!( + formatter, + "{stage} length {observed} differs from observed length {}", + expected.get() + ), + Self::Metadata { source } => { + write!(formatter, "recovery stage metadata was refused: {source}") + } + Self::Fingerprint { stage, source } => { + write!( + formatter, + "{stage} fingerprint recomputation failed: {source}" + ) + } + Self::FingerprintMismatch { stage, .. } => { + write!(formatter, "{stage} fingerprint differs from prior evidence") + } + } + } +} + +impl Error for RecoveryStageByteAdmissionError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Metadata { source } => Some(source), + Self::Fingerprint { source, .. } => Some(source), + Self::StageMismatch { .. } + | Self::AddressSpace { .. } + | Self::LengthMismatch { .. } + | Self::FingerprintMismatch { .. } => None, + } + } +} diff --git a/src/lib.rs b/src/lib.rs index 8a7fcf5..5cba013 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -32,38 +32,40 @@ mod profile; mod reference; pub use adapters::{ - AdmittedCatalog, AdmittedSegment, AdmittedSegmentRecord, BlobIdBinaryParseError, - BlobIdTextParseError, CanonicalCatalog, CanonicalLayoutRecord, CanonicalPublicationHead, - CatalogAdmissionError, CatalogAllocationPhase, CatalogDecodeError, CatalogEncodeError, - CatalogEntryDecodeError, CatalogPublicationError, CatalogPublicationExpectation, - CatalogPublicationOutcome, CatalogPublicationPhase, CatalogPublicationReadiness, - CatalogPublicationReceipt, CatalogPublicationStorage, CatalogRestartArtifact, - CatalogRestartByteLimit, CatalogRestartByteLimitError, CatalogRestartError, - CatalogRestartPhase, CatalogRestartPolicy, CatalogSnapshot, CatalogSnapshotError, - CatalogSuccessor, CatalogTransitionError, ChecksummedCatalog, ChecksummedPublicationHead, - ChecksummedSegmentRecord, ClosedSegment, FilesystemCatalogPublicationError, - FilesystemCatalogPublisher, FilesystemCatalogSnapshot, FilesystemPlatformAdmission, - FilesystemRecoveryInventoryReader, FilesystemRecoveryStageError, FilesystemSegmentStage, - FilesystemWriterLock, LayoutDecodeError, LayoutDecodePolicy, LayoutEncodeError, - LayoutIdBinaryParseError, LayoutIdTextParseError, PublicationHeadDecodeError, - RecoveryCatalogStage, RecoveryCatalogStageError, RecoveryEntryName, RecoveryEntryNameError, - RecoveryEntryRole, RecoveryInventory, RecoveryInventoryEntry, RecoveryInventoryError, - RecoveryInventoryLimit, RecoveryInventoryLimitError, RecoveryInventoryOperation, - RecoveryInventoryStorage, RecoveryNameClassificationError, RecoveryNameManifest, - RecoveryNamedEntry, RecoveryNamespace, RecoveryNextHeadStage, RecoveryNextHeadStageError, - RecoveryPoolNameError, RecoveryRequiredEntry, RecoverySegmentStage, RecoverySegmentStageError, - RecoverySegmentTruncation, RecoveryStage, RecoveryStageEvidence, RecoveryStageFingerprint, - RecoveryStageFingerprintAlgorithm, RecoveryStageFingerprintError, RecoveryStageLength, - RecoveryStageMetadata, RecoveryStageMetadataError, RecoveryStageNamespacePhase, - ReusableRecoverySegment, SealedSegment, SegmentDigest, SegmentDurabilityPhase, SegmentHeader, - SegmentHeaderError, SegmentPublication, SegmentPublicationError, SegmentReadError, - SegmentReadPolicy, SegmentRecordAdmissionError, SegmentRecordChecksum, - SegmentRecordDecodeError, SegmentRecordHeader, SegmentRecordHeaderError, SegmentRecordIdentity, - SegmentRecordLength, SegmentRecordLimit, SegmentRecordLimitError, SegmentRecordPayloadLength, - SegmentRecords, SegmentSeal, SegmentSealError, SegmentStage, SegmentStageCreateError, - SegmentWriteError, SegmentWritePhase, StagedSegment, StorageProfileIdParseError, - StoreInitializationError, StoreInitializationPhase, StoreInitializationReceipt, - StoreInitializationStorage, WriterLockAcquireError, WriterLockAcquirePhase, + AdmittedCatalog, AdmittedRecoveryStageBytes, AdmittedSegment, AdmittedSegmentRecord, + BlobIdBinaryParseError, BlobIdTextParseError, CanonicalCatalog, CanonicalLayoutRecord, + CanonicalPublicationHead, CatalogAdmissionError, CatalogAllocationPhase, CatalogDecodeError, + CatalogEncodeError, CatalogEntryDecodeError, CatalogPublicationError, + CatalogPublicationExpectation, CatalogPublicationOutcome, CatalogPublicationPhase, + CatalogPublicationReadiness, CatalogPublicationReceipt, CatalogPublicationStorage, + CatalogRestartArtifact, CatalogRestartByteLimit, CatalogRestartByteLimitError, + CatalogRestartError, CatalogRestartPhase, CatalogRestartPolicy, CatalogSnapshot, + CatalogSnapshotError, CatalogSuccessor, CatalogTransitionError, ChecksummedCatalog, + ChecksummedPublicationHead, ChecksummedSegmentRecord, ClosedSegment, + FilesystemCatalogPublicationError, FilesystemCatalogPublisher, FilesystemCatalogSnapshot, + FilesystemPlatformAdmission, FilesystemRecoveryInventoryReader, FilesystemRecoveryStageError, + FilesystemSegmentStage, FilesystemWriterLock, LayoutDecodeError, LayoutDecodePolicy, + LayoutEncodeError, LayoutIdBinaryParseError, LayoutIdTextParseError, + PublicationHeadDecodeError, RecoveryCatalogStage, RecoveryCatalogStageError, RecoveryEntryName, + RecoveryEntryNameError, RecoveryEntryRole, RecoveryInventory, RecoveryInventoryEntry, + RecoveryInventoryError, RecoveryInventoryLimit, RecoveryInventoryLimitError, + RecoveryInventoryOperation, RecoveryInventoryStorage, RecoveryNameClassificationError, + RecoveryNameManifest, RecoveryNamedEntry, RecoveryNamespace, RecoveryNextHeadStage, + RecoveryNextHeadStageError, RecoveryPoolNameError, RecoveryRequiredEntry, RecoverySegmentStage, + RecoverySegmentStageError, RecoverySegmentTruncation, RecoveryStage, RecoveryStageAssessment, + RecoveryStageAssessmentError, RecoveryStageByteAdmissionError, RecoveryStageEvidence, + RecoveryStageFingerprint, RecoveryStageFingerprintAlgorithm, RecoveryStageFingerprintError, + RecoveryStageLength, RecoveryStageMetadata, RecoveryStageMetadataError, + RecoveryStageNamespacePhase, ReusableRecoverySegment, SealedSegment, SegmentDigest, + SegmentDurabilityPhase, SegmentHeader, SegmentHeaderError, SegmentPublication, + SegmentPublicationError, SegmentReadError, SegmentReadPolicy, SegmentRecordAdmissionError, + SegmentRecordChecksum, SegmentRecordDecodeError, SegmentRecordHeader, SegmentRecordHeaderError, + SegmentRecordIdentity, SegmentRecordLength, SegmentRecordLimit, SegmentRecordLimitError, + SegmentRecordPayloadLength, SegmentRecords, SegmentSeal, SegmentSealError, SegmentStage, + SegmentStageCreateError, SegmentWriteError, SegmentWritePhase, StagedSegment, + StorageProfileIdParseError, StoreInitializationError, StoreInitializationPhase, + StoreInitializationReceipt, StoreInitializationStorage, WriterLockAcquireError, + WriterLockAcquirePhase, admit_recovery_stage_bytes, assess_recovery_stage, classify_recovery_catalog_stage, classify_recovery_names, classify_recovery_next_head_stage, classify_recovery_segment_stage, fingerprint_recovery_stage, initialize_store, publish_catalog_generation, read_recovery_inventory, diff --git a/tests/recovery_stage_assessment.rs b/tests/recovery_stage_assessment.rs new file mode 100644 index 0000000..b62dd00 --- /dev/null +++ b/tests/recovery_stage_assessment.rs @@ -0,0 +1,39 @@ +//! Fingerprint-bound read-only recovery-stage assessment laws. + +#[path = "recovery_stage_assessment/admission_laws.rs"] +mod admission_laws; +#[path = "recovery_stage_assessment/assessment_laws.rs"] +mod assessment_laws; +mod support; + +use std::error::Error; + +use keep::{ + LayoutEntryLimit, RecoveryStage, RecoveryStageEvidence, RecoveryStageMetadata, + SegmentReadPolicy, SegmentRecordLimit, fingerprint_recovery_stage, +}; +use support::decode_hex; + +const SEGMENT_HEX: &str = include_str!("../conformance/segment-store/v1/one-zero-segment.hex"); +const CATALOG_HEX: &str = include_str!("../conformance/segment-store/v1/one-zero-catalog.hex"); +const HEAD_HEX: &str = include_str!("../conformance/segment-store/v1/one-zero-head.hex"); + +fn fixture(hex: &str) -> Result, Box> { + decode_hex( + hex.strip_suffix('\n') + .ok_or("recovery fixture must end in one LF")?, + ) + .map_err(Into::into) +} + +fn evidence(stage: RecoveryStage, encoded: &[u8]) -> Result> { + let length = u64::try_from(encoded.len())?; + Ok(fingerprint_recovery_stage( + RecoveryStageMetadata::new(stage, length)?, + encoded, + )?) +} + +const fn maximum_policy() -> SegmentReadPolicy { + SegmentReadPolicy::new(SegmentRecordLimit::MAXIMUM, LayoutEntryLimit::MAXIMUM) +} diff --git a/tests/recovery_stage_assessment/admission_laws.rs b/tests/recovery_stage_assessment/admission_laws.rs new file mode 100644 index 0000000..3d82cb4 --- /dev/null +++ b/tests/recovery_stage_assessment/admission_laws.rs @@ -0,0 +1,86 @@ +//! Exact stage, length, and fingerprint binding laws. + +use std::error::Error; + +use keep::{RecoveryStage, RecoveryStageByteAdmissionError, admit_recovery_stage_bytes}; + +use super::{SEGMENT_HEX, evidence, fixture}; + +#[test] +fn exact_stage_bytes_retain_their_prior_evidence() -> Result<(), Box> { + let encoded = fixture(SEGMENT_HEX)?; + let expected = evidence(RecoveryStage::Segment, &encoded)?; + + let admitted = admit_recovery_stage_bytes(RecoveryStage::Segment, expected, &encoded)?; + + assert_eq!(admitted.stage(), RecoveryStage::Segment); + assert_eq!(admitted.evidence(), expected); + assert_eq!(admitted.encoded(), encoded); + Ok(()) +} + +#[test] +fn canonical_name_stage_mismatch_refuses_before_byte_admission() -> Result<(), Box> { + let encoded = fixture(SEGMENT_HEX)?; + let observed = evidence(RecoveryStage::Segment, &encoded)?; + + let error = admit_recovery_stage_bytes(RecoveryStage::Catalog, observed, &encoded) + .err() + .ok_or("wrong canonical stage was admitted")?; + + assert!(matches!( + error, + RecoveryStageByteAdmissionError::StageMismatch { + expected: RecoveryStage::Catalog, + observed: RecoveryStage::Segment, + } + )); + Ok(()) +} + +#[test] +fn changed_length_refuses_before_fingerprint_comparison() -> Result<(), Box> { + let encoded = fixture(SEGMENT_HEX)?; + let expected = evidence(RecoveryStage::Segment, &encoded)?; + let changed_length = encoded.len().checked_sub(1).ok_or("segment underflow")?; + let changed = encoded + .get(..changed_length) + .ok_or("missing truncated segment")?; + + let error = admit_recovery_stage_bytes(RecoveryStage::Segment, expected, changed) + .err() + .ok_or("changed stage length was admitted")?; + + assert!(matches!( + error, + RecoveryStageByteAdmissionError::LengthMismatch { + stage: RecoveryStage::Segment, + expected: expected_length, + observed, + } if expected_length.get() == u64::try_from(encoded.len())? + && observed == u64::try_from(changed.len())? + )); + Ok(()) +} + +#[test] +fn same_length_mutation_refuses_by_fingerprint() -> Result<(), Box> { + let mut encoded = fixture(SEGMENT_HEX)?; + let expected = evidence(RecoveryStage::Segment, &encoded)?; + let byte = encoded.last_mut().ok_or("missing segment byte")?; + *byte ^= 1; + + let error = admit_recovery_stage_bytes(RecoveryStage::Segment, expected, &encoded) + .err() + .ok_or("changed stage fingerprint was admitted")?; + + assert!(matches!( + error, + RecoveryStageByteAdmissionError::FingerprintMismatch { + stage: RecoveryStage::Segment, + expected: expected_fingerprint, + observed, + } if expected_fingerprint == expected.fingerprint() && observed != expected_fingerprint + )); + Ok(()) +} diff --git a/tests/recovery_stage_assessment/assessment_laws.rs b/tests/recovery_stage_assessment/assessment_laws.rs new file mode 100644 index 0000000..ee4f663 --- /dev/null +++ b/tests/recovery_stage_assessment/assessment_laws.rs @@ -0,0 +1,84 @@ +//! Name-selected semantic dispatch laws. + +use std::error::Error; + +use keep::{ + CatalogDecodeError, RecoveryCatalogStage, RecoveryNextHeadStage, RecoverySegmentStage, + RecoveryStage, RecoveryStageAssessment, RecoveryStageAssessmentError, + admit_recovery_stage_bytes, assess_recovery_stage, +}; + +use super::{CATALOG_HEX, HEAD_HEX, SEGMENT_HEX, evidence, fixture, maximum_policy}; + +#[test] +fn every_fixed_stage_dispatches_to_its_only_semantic_classifier() -> Result<(), Box> { + let segment = fixture(SEGMENT_HEX)?; + let catalog = fixture(CATALOG_HEX)?; + let head = fixture(HEAD_HEX)?; + let segment_evidence = evidence(RecoveryStage::Segment, &segment)?; + let catalog_evidence = evidence(RecoveryStage::Catalog, &catalog)?; + let head_evidence = evidence(RecoveryStage::NextHead, &head)?; + + let segment = assess(RecoveryStage::Segment, segment_evidence, &segment)?; + let catalog = assess(RecoveryStage::Catalog, catalog_evidence, &catalog)?; + let head = assess(RecoveryStage::NextHead, head_evidence, &head)?; + + assert_eq!(segment.evidence(), segment_evidence); + assert_eq!(catalog.evidence(), catalog_evidence); + assert_eq!(head.evidence(), head_evidence); + assert!(matches!( + segment, + RecoveryStageAssessment::Segment { + state: RecoverySegmentStage::Complete(_), + .. + } + )); + assert!(matches!( + catalog, + RecoveryStageAssessment::Catalog { + state: RecoveryCatalogStage::Complete(_), + .. + } + )); + assert!(matches!( + head, + RecoveryStageAssessment::NextHead { + state: RecoveryNextHeadStage::Complete(_), + .. + } + )); + Ok(()) +} + +#[test] +fn matching_evidence_does_not_sanitize_corrupt_content() -> Result<(), Box> { + let mut catalog = fixture(CATALOG_HEX)?; + let byte = catalog.last_mut().ok_or("missing catalog checksum")?; + *byte ^= 1; + let observed = evidence(RecoveryStage::Catalog, &catalog)?; + let admitted = admit_recovery_stage_bytes(RecoveryStage::Catalog, observed, &catalog)?; + + let error = assess_recovery_stage(&admitted, maximum_policy()) + .err() + .ok_or("fingerprint-bound corrupt catalog was classified as lawful")?; + + assert!(matches!( + error, + RecoveryStageAssessmentError::Catalog { + source: keep::RecoveryCatalogStageError::Complete { + source: CatalogDecodeError::ChecksumMismatch { .. } + | CatalogDecodeError::DigestMismatch { .. }, + }, + } + )); + Ok(()) +} + +fn assess( + stage: RecoveryStage, + observed: keep::RecoveryStageEvidence, + encoded: &[u8], +) -> Result, Box> { + let admitted = admit_recovery_stage_bytes(stage, observed, encoded)?; + Ok(assess_recovery_stage(&admitted, maximum_policy())?) +} From 73c1f154376b2be0b11d45e9b232391df90b40ef Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 10:28:18 -0700 Subject: [PATCH 15/49] Add: Execute idempotent stage discard --- CHANGELOG.md | 5 + README.md | 11 +- docs/formats/segment-store-v1/recovery.md | 7 + docs/formats/segment-store-v1/requirements.md | 7 +- src/adapters/mod.rs | 22 +++ src/adapters/recovery_stage.rs | 10 +- src/adapters/recovery_stage_discard_error.rs | 44 ++++++ .../recovery_stage_discard_executor.rs | 29 ++++ .../recovery_stage_discard_outcome.rs | 11 ++ .../recovery_stage_discard_plan_error.rs | 28 ++++ .../recovery_stage_discard_planner.rs | 55 +++++++ src/adapters/recovery_stage_discard_reason.rs | 32 +++++ .../recovery_stage_discard_receipt.rs | 44 ++++++ .../recovery_stage_discard_request.rs | 36 +++++ .../recovery_stage_discard_storage.rs | 36 +++++ .../recovery_stage_discard_storage_error.rs | 50 +++++++ src/adapters/recovery_stage_parent.rs | 11 ++ src/lib.rs | 16 ++- tests/recovery_stage_discard.rs | 60 ++++++++ .../recovery_stage_discard/execution_laws.rs | 135 ++++++++++++++++++ tests/recovery_stage_discard/planning_laws.rs | 91 ++++++++++++ .../recovery_stage_discard/storage_double.rs | 83 +++++++++++ 22 files changed, 810 insertions(+), 13 deletions(-) create mode 100644 src/adapters/recovery_stage_discard_error.rs create mode 100644 src/adapters/recovery_stage_discard_executor.rs create mode 100644 src/adapters/recovery_stage_discard_outcome.rs create mode 100644 src/adapters/recovery_stage_discard_plan_error.rs create mode 100644 src/adapters/recovery_stage_discard_planner.rs create mode 100644 src/adapters/recovery_stage_discard_reason.rs create mode 100644 src/adapters/recovery_stage_discard_receipt.rs create mode 100644 src/adapters/recovery_stage_discard_request.rs create mode 100644 src/adapters/recovery_stage_discard_storage.rs create mode 100644 src/adapters/recovery_stage_discard_storage_error.rs create mode 100644 src/adapters/recovery_stage_parent.rs create mode 100644 tests/recovery_stage_discard.rs create mode 100644 tests/recovery_stage_discard/execution_laws.rs create mode 100644 tests/recovery_stage_discard/planning_laws.rs create mode 100644 tests/recovery_stage_discard/storage_double.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 615c478..627b75f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,6 +43,11 @@ after its public API and format compatibility policies are established. their canonical-name stage, exact length, and recomputed versioned fingerprint equal prior observation evidence, then dispatches through the stage-selected semantic classifier. +- Explicit truncated-stage recovery now plans only from an exact semantic + truncation, retains its evidence and reason, refuses changed evidence before + mutation, and returns a discard receipt only after the name-selected parent + directory is synchronized. An already absent stage remains an idempotent + input and still requires synchronization. - Store initialization now exposes one storage-port state machine that admits the platform before mutation, opens and locks `writer.lock`, admits the three protocol directories in order, synchronizes the root, and preserves the diff --git a/README.md b/README.md index d80a027..b2261ce 100644 --- a/README.md +++ b/README.md @@ -77,10 +77,13 @@ complete admitted segment, or exact truncation. Catalog and next-head stages likewise distinguish exact truncation from complete canonical bytes. Materialized bytes enter read-only semantic assessment only after their stage, length, and recomputed fingerprint match prior observation evidence. -Transitive publication-view admission and filesystem-streaming classification -remain planned. -Crash-injection execution, explicit recovery, retention, compaction, and -garbage collection remain planned. Presence in the reference CAS does not +Only an exact truncation assessment may form an explicit discard request; the +semantic executor refuses evidence drift and returns a receipt only after +exact removal or admitted prior absence is followed by parent synchronization. +The pinned-filesystem discard adapter, transitive publication-view admission, +and filesystem-streaming classification remain planned. +Crash-injection execution, remaining recovery actions, retention, compaction, +and garbage collection remain planned. Presence in the reference CAS does not claim retention, crash recovery, or durability. ```rust diff --git a/docs/formats/segment-store-v1/recovery.md b/docs/formats/segment-store-v1/recovery.md index 5c80ef9..e213a0e 100644 --- a/docs/formats/segment-store-v1/recovery.md +++ b/docs/formats/segment-store-v1/recovery.md @@ -50,6 +50,13 @@ length, and recomputed stage fingerprint to match prior observation evidence; only `assess_recovery_stage` may dispatch those admitted bytes to a semantic classifier. Matching evidence does not convert corrupt bytes into lawful content. +`plan_recovery_stage_discard` admits only an exact truncation assessment and +retains both its evidence and typed truncation reason. The semantic +`execute_recovery_stage_discard` port refuses evidence drift before mutation, +treats an absent canonical name as an idempotent input, synchronizes the +name-selected parent in either success case, and returns a receipt only after +that synchronization. The pinned-filesystem implementation remains +unimplemented. The sole admissible duplicate digest is one fixed staging name and its exact digest-derived pool name after a link transition. Recovery admits that pair diff --git a/docs/formats/segment-store-v1/requirements.md b/docs/formats/segment-store-v1/requirements.md index 09e185f..a06c367 100644 --- a/docs/formats/segment-store-v1/requirements.md +++ b/docs/formats/segment-store-v1/requirements.md @@ -92,8 +92,10 @@ machine to a fail-closed Linux ext4 adapter and canonical namespace. These slices now also classify canonical recovery names before opening artifact bytes, bind materialized bytes back to prior stage evidence, and dispatch complete caller-supplied fixed-stage bytes through their name-selected -classifiers. They do not yet claim transitive publication-view admission, -process-death injection, or recovery execution. +classifiers. An exact truncation assessment may now authorize one +evidence-bound, retry-safe discard through a semantic storage port. These +slices do not yet claim a pinned-filesystem discard implementation, transitive +publication-view admission, or process-death injection. @@ -111,6 +113,7 @@ process-death injection, or recovery execution. | `KEEP-RECOVERY-010` | Whole-byte segment-stage classification distinguishes a validated reusable prefix, a complete admitted immutable segment, and exact header, record, or seal truncation; complete-looking corruption, duplicates, and resource-limit excess remain typed refusals | Canonical prefix and corruption matrix | `tests/recovery_segment_classification.rs`, `tests/recovery_segment_classification/*.rs` | Implemented in #17 | | `KEEP-RECOVERY-011` | Whole-byte catalog and next-head stage classification distinguishes exact fixed-header, declared-body, and fixed-width truncation from complete canonical bytes; complete-looking corruption and oversize remain typed format or metadata refusals | Canonical publication-artifact truncation and corruption matrix | `tests/recovery_publication_stage_classification.rs`, `tests/recovery_publication_stage_classification/*.rs` | Implemented in #17 | | `KEEP-RECOVERY-012` | Read-only semantic assessment admits materialized stage bytes only when the canonical-name stage, exact observed length, and `KEEP:RECOVERY:STAGE\0` fingerprint equal prior evidence, then dispatches through the name-selected segment, catalog, or next-head classifier | Evidence-binding mutation matrix and canonical stage assessments | `tests/recovery_stage_assessment.rs`, `tests/recovery_stage_assessment/*.rs` | Implemented in #17 | +| `KEEP-RECOVERY-013` | Explicit discard plans only from an exact truncation assessment, retains the observation evidence and typed truncation reason, refuses changed evidence without mutation, synchronizes the name-selected parent after exact removal or admitted absence, and returns a receipt only after synchronization | Truncation-planning, evidence-drift, operation-order, and retry matrix | `tests/recovery_stage_discard.rs`, `tests/recovery_stage_discard/*.rs` | Implemented in #17 | diff --git a/src/adapters/mod.rs b/src/adapters/mod.rs index a1ded6a..9f7b651 100644 --- a/src/adapters/mod.rs +++ b/src/adapters/mod.rs @@ -150,6 +150,16 @@ mod recovery_stage_assessment_error; mod recovery_stage_assessor; mod recovery_stage_byte_admission; mod recovery_stage_byte_admission_error; +mod recovery_stage_discard_error; +mod recovery_stage_discard_executor; +mod recovery_stage_discard_outcome; +mod recovery_stage_discard_plan_error; +mod recovery_stage_discard_planner; +mod recovery_stage_discard_reason; +mod recovery_stage_discard_receipt; +mod recovery_stage_discard_request; +mod recovery_stage_discard_storage; +mod recovery_stage_discard_storage_error; mod recovery_stage_evidence; mod recovery_stage_fingerprint; mod recovery_stage_fingerprint_algorithm; @@ -158,6 +168,7 @@ mod recovery_stage_fingerprinter; mod recovery_stage_length; mod recovery_stage_metadata; mod recovery_stage_metadata_error; +mod recovery_stage_parent; mod sealed_segment; mod segment_digest; mod segment_digest_builder; @@ -310,6 +321,16 @@ pub use recovery_stage_assessment_error::RecoveryStageAssessmentError; pub use recovery_stage_assessor::assess_recovery_stage; pub use recovery_stage_byte_admission::admit_recovery_stage_bytes; pub use recovery_stage_byte_admission_error::RecoveryStageByteAdmissionError; +pub use recovery_stage_discard_error::RecoveryStageDiscardError; +pub use recovery_stage_discard_executor::execute_recovery_stage_discard; +pub use recovery_stage_discard_outcome::RecoveryStageDiscardOutcome; +pub use recovery_stage_discard_plan_error::RecoveryStageDiscardPlanError; +pub use recovery_stage_discard_planner::plan_recovery_stage_discard; +pub use recovery_stage_discard_reason::RecoveryStageDiscardReason; +pub use recovery_stage_discard_receipt::RecoveryStageDiscardReceipt; +pub use recovery_stage_discard_request::RecoveryStageDiscardRequest; +pub use recovery_stage_discard_storage::RecoveryStageDiscardStorage; +pub use recovery_stage_discard_storage_error::RecoveryStageDiscardStorageError; pub use recovery_stage_evidence::RecoveryStageEvidence; pub use recovery_stage_fingerprint::RecoveryStageFingerprint; pub use recovery_stage_fingerprint_algorithm::RecoveryStageFingerprintAlgorithm; @@ -318,6 +339,7 @@ pub use recovery_stage_fingerprinter::fingerprint_recovery_stage; pub use recovery_stage_length::RecoveryStageLength; pub use recovery_stage_metadata::RecoveryStageMetadata; pub use recovery_stage_metadata_error::RecoveryStageMetadataError; +pub use recovery_stage_parent::RecoveryStageParent; pub use sealed_segment::SealedSegment; pub use segment_digest::SegmentDigest; pub use segment_header::SegmentHeader; diff --git a/src/adapters/recovery_stage.rs b/src/adapters/recovery_stage.rs index dacd428..34156be 100644 --- a/src/adapters/recovery_stage.rs +++ b/src/adapters/recovery_stage.rs @@ -2,7 +2,7 @@ use std::fmt; -use super::segment_header; +use super::{RecoveryStageParent, segment_header}; use crate::CatalogLength; /// One fixed mutable artifact retained for explicit recovery. @@ -25,6 +25,14 @@ impl RecoveryStage { } } + /// Returns the only protocol parent containing this fixed stage. + pub const fn parent(self) -> RecoveryStageParent { + match self { + Self::Segment | Self::Catalog => RecoveryStageParent::Staging, + Self::NextHead => RecoveryStageParent::Root, + } + } + /// Returns the name-selected version-1 maximum byte length. #[must_use] pub const fn maximum_length(self) -> u64 { diff --git a/src/adapters/recovery_stage_discard_error.rs b/src/adapters/recovery_stage_discard_error.rs new file mode 100644 index 0000000..3c84022 --- /dev/null +++ b/src/adapters/recovery_stage_discard_error.rs @@ -0,0 +1,44 @@ +//! This module owns ordered truncated-stage discard execution failures. + +use std::error::Error; +use std::fmt; +use std::io; + +use super::{RecoveryStage, RecoveryStageDiscardStorageError}; + +/// Why explicit truncated-stage discard did not return a durable receipt. +#[derive(Debug)] +pub enum RecoveryStageDiscardError { + /// Exact-evidence removal or absence admission failed. + Remove { + /// Exact storage refusal. + source: RecoveryStageDiscardStorageError, + }, + /// Synchronizing the name-selected parent directory failed. + Synchronize { + /// Canonical stage selecting `staging` or the store root. + stage: RecoveryStage, + /// Exact parent-directory synchronization failure. + source: io::Error, + }, +} + +impl fmt::Display for RecoveryStageDiscardError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Remove { source } => write!(formatter, "stage discard was refused: {source}"), + Self::Synchronize { stage, source } => { + write!(formatter, "{stage} parent synchronization failed: {source}") + } + } + } +} + +impl Error for RecoveryStageDiscardError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Remove { source } => Some(source), + Self::Synchronize { source, .. } => Some(source), + } + } +} diff --git a/src/adapters/recovery_stage_discard_executor.rs b/src/adapters/recovery_stage_discard_executor.rs new file mode 100644 index 0000000..195b54d --- /dev/null +++ b/src/adapters/recovery_stage_discard_executor.rs @@ -0,0 +1,29 @@ +//! This module owns ordered, idempotent truncated-stage discard execution. + +use super::{ + RecoveryStageDiscardError, RecoveryStageDiscardReceipt, RecoveryStageDiscardRequest, + RecoveryStageDiscardStorage, +}; + +/// Removes one exact truncated stage and durably synchronizes its parent. +/// +/// An already absent canonical name still requires parent synchronization. +/// The call performs no allocation beyond work owned by the storage adapter. +/// +/// # Errors +/// +/// Returns [`RecoveryStageDiscardError`] without a receipt when exact-evidence +/// removal, absence admission, or parent synchronization fails. +pub fn execute_recovery_stage_discard( + storage: &mut impl RecoveryStageDiscardStorage, + request: RecoveryStageDiscardRequest, +) -> Result { + let stage = request.stage(); + let outcome = storage + .remove_if_matching(request.evidence()) + .map_err(|source| RecoveryStageDiscardError::Remove { source })?; + storage + .synchronize_parent(stage.parent()) + .map_err(|source| RecoveryStageDiscardError::Synchronize { stage, source })?; + Ok(RecoveryStageDiscardReceipt::new(request, outcome)) +} diff --git a/src/adapters/recovery_stage_discard_outcome.rs b/src/adapters/recovery_stage_discard_outcome.rs new file mode 100644 index 0000000..eb1a555 --- /dev/null +++ b/src/adapters/recovery_stage_discard_outcome.rs @@ -0,0 +1,11 @@ +//! This module owns idempotent fixed-stage removal outcomes. + +/// Namespace state observed by exact-evidence stage removal. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum RecoveryStageDiscardOutcome { + /// The exact fingerprint-bound stage was removed. + Removed, + /// The canonical stage name was already absent. + AlreadyAbsent, +} diff --git a/src/adapters/recovery_stage_discard_plan_error.rs b/src/adapters/recovery_stage_discard_plan_error.rs new file mode 100644 index 0000000..f0748c8 --- /dev/null +++ b/src/adapters/recovery_stage_discard_plan_error.rs @@ -0,0 +1,28 @@ +//! This module owns truncated-stage discard-planning refusals. + +use std::error::Error; +use std::fmt; + +use super::RecoveryStage; + +/// Why a semantic stage assessment cannot authorize truncated-stage discard. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum RecoveryStageDiscardPlanError { + /// The stage is reusable or complete rather than exactly truncated. + NotTruncated { + /// Canonical fixed stage whose lawful state forbids this discard plan. + stage: RecoveryStage, + }, +} + +impl fmt::Display for RecoveryStageDiscardPlanError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::NotTruncated { stage } => { + write!(formatter, "{stage} is not an exactly truncated stage") + } + } + } +} + +impl Error for RecoveryStageDiscardPlanError {} diff --git a/src/adapters/recovery_stage_discard_planner.rs b/src/adapters/recovery_stage_discard_planner.rs new file mode 100644 index 0000000..39fe9bc --- /dev/null +++ b/src/adapters/recovery_stage_discard_planner.rs @@ -0,0 +1,55 @@ +//! This module owns truncation-only recovery-stage discard planning. + +use super::{ + RecoveryCatalogStage, RecoveryNextHeadStage, RecoverySegmentStage, RecoveryStageAssessment, + RecoveryStageDiscardPlanError, RecoveryStageDiscardReason, RecoveryStageDiscardRequest, +}; + +/// Plans explicit discard only from an exact fingerprint-bound truncation. +/// +/// The call performs no I/O or allocation. Reusable and complete stages remain +/// ineligible because their lawful recovery action is not implicit deletion. +/// +/// # Errors +/// +/// Returns [`RecoveryStageDiscardPlanError`] when the stage is not truncated. +pub const fn plan_recovery_stage_discard( + assessment: &RecoveryStageAssessment<'_>, +) -> Result { + let evidence = assessment.evidence(); + let reason = match assessment { + RecoveryStageAssessment::Segment { + state: RecoverySegmentStage::Truncated(reason), + .. + } => RecoveryStageDiscardReason::Segment(*reason), + RecoveryStageAssessment::Catalog { + state: RecoveryCatalogStage::HeaderTruncated { required, observed }, + .. + } => RecoveryStageDiscardReason::CatalogHeader { + required: *required, + observed: *observed, + }, + RecoveryStageAssessment::Catalog { + state: RecoveryCatalogStage::BodyTruncated { expected, observed }, + .. + } => RecoveryStageDiscardReason::CatalogBody { + expected: *expected, + observed: *observed, + }, + RecoveryStageAssessment::NextHead { + state: RecoveryNextHeadStage::Truncated { required, observed }, + .. + } => RecoveryStageDiscardReason::NextHead { + required: *required, + observed: *observed, + }, + RecoveryStageAssessment::Segment { .. } + | RecoveryStageAssessment::Catalog { .. } + | RecoveryStageAssessment::NextHead { .. } => { + return Err(RecoveryStageDiscardPlanError::NotTruncated { + stage: evidence.stage(), + }); + } + }; + Ok(RecoveryStageDiscardRequest::new(evidence, reason)) +} diff --git a/src/adapters/recovery_stage_discard_reason.rs b/src/adapters/recovery_stage_discard_reason.rs new file mode 100644 index 0000000..8a32358 --- /dev/null +++ b/src/adapters/recovery_stage_discard_reason.rs @@ -0,0 +1,32 @@ +//! This module owns exact semantic reasons for truncated-stage discard. + +use super::RecoverySegmentTruncation; + +/// Exact truncation that makes a fixed stage eligible for explicit discard. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum RecoveryStageDiscardReason { + /// Exact incomplete segment boundary. + Segment(RecoverySegmentTruncation), + /// Incomplete fixed catalog header. + CatalogHeader { + /// Required fixed-header byte count. + required: usize, + /// Supplied byte count. + observed: usize, + }, + /// Admitted catalog header whose declared body is incomplete. + CatalogBody { + /// Declared canonical catalog byte count. + expected: u64, + /// Supplied byte count. + observed: usize, + }, + /// Incomplete fixed-width next publication head. + NextHead { + /// Required publication-head byte count. + required: usize, + /// Supplied byte count. + observed: usize, + }, +} diff --git a/src/adapters/recovery_stage_discard_receipt.rs b/src/adapters/recovery_stage_discard_receipt.rs new file mode 100644 index 0000000..371237b --- /dev/null +++ b/src/adapters/recovery_stage_discard_receipt.rs @@ -0,0 +1,44 @@ +//! This module owns durable truncated-stage discard receipts. + +use super::{ + RecoveryStage, RecoveryStageDiscardOutcome, RecoveryStageDiscardReason, + RecoveryStageDiscardRequest, RecoveryStageEvidence, +}; + +/// Proof that exact stage removal or absence was followed by parent sync. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct RecoveryStageDiscardReceipt { + request: RecoveryStageDiscardRequest, + outcome: RecoveryStageDiscardOutcome, +} + +impl RecoveryStageDiscardReceipt { + pub(super) const fn new( + request: RecoveryStageDiscardRequest, + outcome: RecoveryStageDiscardOutcome, + ) -> Self { + Self { request, outcome } + } + + /// Returns the canonical fixed stage whose absence is durable. + #[must_use] + pub const fn stage(self) -> RecoveryStage { + self.request.stage() + } + + /// Returns the exact evidence bound into the discard request. + pub const fn evidence(self) -> RecoveryStageEvidence { + self.request.evidence() + } + + /// Returns the exact truncation that authorized discard. + pub const fn reason(self) -> RecoveryStageDiscardReason { + self.request.reason() + } + + /// Returns whether execution removed the stage or admitted prior absence. + pub const fn outcome(self) -> RecoveryStageDiscardOutcome { + self.outcome + } +} diff --git a/src/adapters/recovery_stage_discard_request.rs b/src/adapters/recovery_stage_discard_request.rs new file mode 100644 index 0000000..69b5eaf --- /dev/null +++ b/src/adapters/recovery_stage_discard_request.rs @@ -0,0 +1,36 @@ +//! This module owns explicit fingerprint-bound stage-discard requests. + +use super::{RecoveryStage, RecoveryStageDiscardReason, RecoveryStageEvidence}; + +/// Immutable request to discard one exactly observed truncated fixed stage. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct RecoveryStageDiscardRequest { + evidence: RecoveryStageEvidence, + reason: RecoveryStageDiscardReason, +} + +impl RecoveryStageDiscardRequest { + pub(super) const fn new( + evidence: RecoveryStageEvidence, + reason: RecoveryStageDiscardReason, + ) -> Self { + Self { evidence, reason } + } + + /// Returns the canonical fixed stage selected by the request. + #[must_use] + pub const fn stage(self) -> RecoveryStage { + self.evidence.stage() + } + + /// Returns the exact length and fingerprint that mutation must reverify. + pub const fn evidence(self) -> RecoveryStageEvidence { + self.evidence + } + + /// Returns the exact truncation that authorized explicit discard. + pub const fn reason(self) -> RecoveryStageDiscardReason { + self.reason + } +} diff --git a/src/adapters/recovery_stage_discard_storage.rs b/src/adapters/recovery_stage_discard_storage.rs new file mode 100644 index 0000000..8a932d1 --- /dev/null +++ b/src/adapters/recovery_stage_discard_storage.rs @@ -0,0 +1,36 @@ +//! This module owns the storage port for exact truncated-stage discard. + +use std::io; + +use super::{ + RecoveryStageDiscardOutcome, RecoveryStageDiscardStorageError, RecoveryStageEvidence, + RecoveryStageParent, +}; + +/// Semantic storage operations required by explicit truncated-stage discard. +/// +/// The implementation must retain writer authority. Removal must select the +/// canonical name from `expected.stage()`, reopen without following links, +/// bound the complete read by that stage's protocol maximum, reverify exact +/// length, fingerprint, namespace identity, and regular-file type, and refuse +/// disagreement without mutation. An absent canonical name is an idempotent +/// input. The orchestration layer owns operation ordering and receipt timing. +pub trait RecoveryStageDiscardStorage { + /// Removes the exact stage or reports that its canonical name is absent. + /// + /// # Errors + /// + /// Returns a typed evidence mismatch without mutation or preserves the + /// exact storage error from reopen, verification, or removal. + fn remove_if_matching( + &mut self, + expected: RecoveryStageEvidence, + ) -> Result; + + /// Synchronizes the protocol-selected parent. + /// + /// # Errors + /// + /// Returns the exact parent-directory synchronization failure. + fn synchronize_parent(&mut self, parent: RecoveryStageParent) -> io::Result<()>; +} diff --git a/src/adapters/recovery_stage_discard_storage_error.rs b/src/adapters/recovery_stage_discard_storage_error.rs new file mode 100644 index 0000000..517ba42 --- /dev/null +++ b/src/adapters/recovery_stage_discard_storage_error.rs @@ -0,0 +1,50 @@ +//! This module owns semantic storage refusals during stage removal. + +use std::error::Error; +use std::fmt; +use std::io; + +use super::RecoveryStageEvidence; + +/// Why storage could not remove one exact fingerprint-bound stage. +#[derive(Debug)] +pub enum RecoveryStageDiscardStorageError { + /// The canonical stage name resolves to different evidence. + EvidenceMismatch { + /// Evidence bound into the explicit discard request. + expected: RecoveryStageEvidence, + /// Evidence observed immediately before the refused mutation. + observed: RecoveryStageEvidence, + }, + /// The storage boundary failed while reopening, verifying, or removing. + Storage { + /// Exact underlying storage failure. + source: io::Error, + }, +} + +impl fmt::Display for RecoveryStageDiscardStorageError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::EvidenceMismatch { expected, observed } => write!( + formatter, + "{} evidence changed from length {} to length {}", + expected.stage(), + expected.length().get(), + observed.length().get() + ), + Self::Storage { source } => { + write!(formatter, "recovery stage removal failed: {source}") + } + } + } +} + +impl Error for RecoveryStageDiscardStorageError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Storage { source } => Some(source), + Self::EvidenceMismatch { .. } => None, + } + } +} diff --git a/src/adapters/recovery_stage_parent.rs b/src/adapters/recovery_stage_parent.rs new file mode 100644 index 0000000..5fd6855 --- /dev/null +++ b/src/adapters/recovery_stage_parent.rs @@ -0,0 +1,11 @@ +//! This module owns semantic parent directories for fixed recovery stages. + +/// Protocol parent directory selected by one canonical fixed-stage name. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum RecoveryStageParent { + /// The `staging` protocol directory. + Staging, + /// The store root. + Root, +} diff --git a/src/lib.rs b/src/lib.rs index 5cba013..c82bf76 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -53,10 +53,13 @@ pub use adapters::{ RecoveryNameManifest, RecoveryNamedEntry, RecoveryNamespace, RecoveryNextHeadStage, RecoveryNextHeadStageError, RecoveryPoolNameError, RecoveryRequiredEntry, RecoverySegmentStage, RecoverySegmentStageError, RecoverySegmentTruncation, RecoveryStage, RecoveryStageAssessment, - RecoveryStageAssessmentError, RecoveryStageByteAdmissionError, RecoveryStageEvidence, - RecoveryStageFingerprint, RecoveryStageFingerprintAlgorithm, RecoveryStageFingerprintError, - RecoveryStageLength, RecoveryStageMetadata, RecoveryStageMetadataError, - RecoveryStageNamespacePhase, ReusableRecoverySegment, SealedSegment, SegmentDigest, + RecoveryStageAssessmentError, RecoveryStageByteAdmissionError, RecoveryStageDiscardError, + RecoveryStageDiscardOutcome, RecoveryStageDiscardPlanError, RecoveryStageDiscardReason, + RecoveryStageDiscardReceipt, RecoveryStageDiscardRequest, RecoveryStageDiscardStorage, + RecoveryStageDiscardStorageError, RecoveryStageEvidence, RecoveryStageFingerprint, + RecoveryStageFingerprintAlgorithm, RecoveryStageFingerprintError, RecoveryStageLength, + RecoveryStageMetadata, RecoveryStageMetadataError, RecoveryStageNamespacePhase, + RecoveryStageParent, ReusableRecoverySegment, SealedSegment, SegmentDigest, SegmentDurabilityPhase, SegmentHeader, SegmentHeaderError, SegmentPublication, SegmentPublicationError, SegmentReadError, SegmentReadPolicy, SegmentRecordAdmissionError, SegmentRecordChecksum, SegmentRecordDecodeError, SegmentRecordHeader, SegmentRecordHeaderError, @@ -67,8 +70,9 @@ pub use adapters::{ StoreInitializationReceipt, StoreInitializationStorage, WriterLockAcquireError, WriterLockAcquirePhase, admit_recovery_stage_bytes, assess_recovery_stage, classify_recovery_catalog_stage, classify_recovery_names, classify_recovery_next_head_stage, - classify_recovery_segment_stage, fingerprint_recovery_stage, initialize_store, - publish_catalog_generation, read_recovery_inventory, + classify_recovery_segment_stage, execute_recovery_stage_discard, fingerprint_recovery_stage, + initialize_store, plan_recovery_stage_discard, publish_catalog_generation, + read_recovery_inventory, }; pub use blob::{ BlobHashError, BlobHasher, BlobId, BlobLength, BlobReadError, ByteLength, ByteOffset, diff --git a/tests/recovery_stage_discard.rs b/tests/recovery_stage_discard.rs new file mode 100644 index 0000000..e8f4ff3 --- /dev/null +++ b/tests/recovery_stage_discard.rs @@ -0,0 +1,60 @@ +//! Explicit fingerprint-bound stage-discard laws. + +#[path = "recovery_stage_discard/execution_laws.rs"] +mod execution_laws; +#[path = "recovery_stage_discard/planning_laws.rs"] +mod planning_laws; +#[path = "recovery_stage_discard/storage_double.rs"] +pub mod storage_double; +mod support; + +use std::error::Error; + +use keep::{ + LayoutEntryLimit, RecoveryStage, RecoveryStageAssessment, RecoveryStageDiscardRequest, + RecoveryStageEvidence, RecoveryStageMetadata, SegmentReadPolicy, SegmentRecordLimit, + admit_recovery_stage_bytes, assess_recovery_stage, fingerprint_recovery_stage, + plan_recovery_stage_discard, +}; +use support::decode_hex; + +const SEGMENT_HEX: &str = include_str!("../conformance/segment-store/v1/one-zero-segment.hex"); +const CATALOG_HEX: &str = include_str!("../conformance/segment-store/v1/one-zero-catalog.hex"); +const HEAD_HEX: &str = include_str!("../conformance/segment-store/v1/one-zero-head.hex"); +const SEGMENT_SEAL_LENGTH: usize = 128; + +fn fixture(hex: &str) -> Result, Box> { + decode_hex( + hex.strip_suffix('\n') + .ok_or("recovery fixture must end in one LF")?, + ) + .map_err(Into::into) +} + +fn evidence(stage: RecoveryStage, encoded: &[u8]) -> Result> { + let length = u64::try_from(encoded.len())?; + Ok(fingerprint_recovery_stage( + RecoveryStageMetadata::new(stage, length)?, + encoded, + )?) +} + +fn assessment( + stage: RecoveryStage, + encoded: &[u8], +) -> Result, Box> { + let observed = evidence(stage, encoded)?; + let admitted = admit_recovery_stage_bytes(stage, observed, encoded)?; + Ok(assess_recovery_stage(&admitted, maximum_policy())?) +} + +fn discard_request( + stage: RecoveryStage, + encoded: &[u8], +) -> Result> { + Ok(plan_recovery_stage_discard(&assessment(stage, encoded)?)?) +} + +const fn maximum_policy() -> SegmentReadPolicy { + SegmentReadPolicy::new(SegmentRecordLimit::MAXIMUM, LayoutEntryLimit::MAXIMUM) +} diff --git a/tests/recovery_stage_discard/execution_laws.rs b/tests/recovery_stage_discard/execution_laws.rs new file mode 100644 index 0000000..ffd4522 --- /dev/null +++ b/tests/recovery_stage_discard/execution_laws.rs @@ -0,0 +1,135 @@ +//! Ordered, retry-safe stage-discard execution laws. + +use std::error::Error; +use std::io; + +use keep::{ + RecoveryStage, RecoveryStageDiscardError, RecoveryStageDiscardOutcome, + RecoveryStageDiscardStorageError, RecoveryStageParent, execute_recovery_stage_discard, +}; + +use super::storage_double::{Operation, StageDiscardDouble}; +use super::{discard_request, evidence}; + +#[test] +fn exact_evidence_is_removed_before_its_parent_is_synchronized() -> Result<(), Box> { + let bytes = [0_u8]; + let request = discard_request(RecoveryStage::Segment, &bytes)?; + let mut storage = StageDiscardDouble::new(Some(request.evidence())); + + let receipt = execute_recovery_stage_discard(&mut storage, request)?; + + assert_eq!(receipt.evidence(), request.evidence()); + assert_eq!(receipt.reason(), request.reason()); + assert_eq!(receipt.outcome(), RecoveryStageDiscardOutcome::Removed); + assert_eq!( + storage.operations(), + &[ + Operation::Remove(request.evidence()), + Operation::Synchronize(RecoveryStageParent::Staging), + ] + ); + assert_eq!(storage.present(), None); + Ok(()) +} + +#[test] +fn absent_exact_retry_still_synchronizes_the_selected_parent() -> Result<(), Box> { + for stage in [ + RecoveryStage::Segment, + RecoveryStage::Catalog, + RecoveryStage::NextHead, + ] { + let bytes = [0_u8]; + let request = discard_request(stage, &bytes)?; + let mut storage = StageDiscardDouble::new(None); + + let receipt = execute_recovery_stage_discard(&mut storage, request)?; + + assert_eq!( + receipt.outcome(), + RecoveryStageDiscardOutcome::AlreadyAbsent + ); + assert_eq!( + storage.operations(), + &[ + Operation::Remove(request.evidence()), + Operation::Synchronize(expected_parent(stage)), + ] + ); + } + Ok(()) +} + +#[test] +fn changed_evidence_refuses_without_removal_or_parent_sync() -> Result<(), Box> { + let bytes = [0_u8]; + let changed = [1_u8]; + let request = discard_request(RecoveryStage::Segment, &bytes)?; + let observed = evidence(RecoveryStage::Segment, &changed)?; + let mut storage = StageDiscardDouble::new(Some(observed)); + + let error = execute_recovery_stage_discard(&mut storage, request) + .err() + .ok_or("changed stage evidence was discarded")?; + + assert!(matches!( + error, + RecoveryStageDiscardError::Remove { + source: RecoveryStageDiscardStorageError::EvidenceMismatch { + expected, + observed: actual, + }, + } if expected == request.evidence() && actual == observed + )); + assert_eq!(storage.present(), Some(observed)); + assert_eq!( + storage.operations(), + &[Operation::Remove(request.evidence())] + ); + Ok(()) +} + +#[test] +fn retry_after_remove_before_parent_sync_is_idempotent() -> Result<(), Box> { + let bytes = [0_u8]; + let request = discard_request(RecoveryStage::NextHead, &bytes)?; + let mut storage = StageDiscardDouble::new(Some(request.evidence())).fail_next_synchronization(); + + let error = execute_recovery_stage_discard(&mut storage, request) + .err() + .ok_or("injected parent synchronization failure was ignored")?; + + assert!(matches!( + error, + RecoveryStageDiscardError::Synchronize { + stage: RecoveryStage::NextHead, + source, + } if source.kind() == io::ErrorKind::Other + )); + assert_eq!(storage.present(), None); + + let receipt = execute_recovery_stage_discard(&mut storage, request)?; + + assert_eq!( + receipt.outcome(), + RecoveryStageDiscardOutcome::AlreadyAbsent + ); + assert_eq!( + storage.operations(), + &[ + Operation::Remove(request.evidence()), + Operation::Synchronize(RecoveryStageParent::Root), + Operation::Remove(request.evidence()), + Operation::Synchronize(RecoveryStageParent::Root), + ] + ); + Ok(()) +} + +const fn expected_parent(stage: RecoveryStage) -> RecoveryStageParent { + match stage { + RecoveryStage::Segment | RecoveryStage::Catalog => RecoveryStageParent::Staging, + RecoveryStage::NextHead => RecoveryStageParent::Root, + } +} diff --git a/tests/recovery_stage_discard/planning_laws.rs b/tests/recovery_stage_discard/planning_laws.rs new file mode 100644 index 0000000..583d5b1 --- /dev/null +++ b/tests/recovery_stage_discard/planning_laws.rs @@ -0,0 +1,91 @@ +//! Truncation-only discard-planning laws. + +use std::error::Error; + +use keep::{ + RecoverySegmentTruncation, RecoveryStage, RecoveryStageDiscardPlanError, + RecoveryStageDiscardReason, plan_recovery_stage_discard, +}; + +use super::{ + CATALOG_HEX, HEAD_HEX, SEGMENT_HEX, SEGMENT_SEAL_LENGTH, assessment, evidence, fixture, +}; + +#[test] +fn every_exact_truncation_retains_its_reason_and_evidence() -> Result<(), Box> { + let segment = [0_u8]; + let catalog = [0_u8]; + let head = [0_u8]; + let segment_evidence = evidence(RecoveryStage::Segment, &segment)?; + let catalog_evidence = evidence(RecoveryStage::Catalog, &catalog)?; + let head_evidence = evidence(RecoveryStage::NextHead, &head)?; + + let segment_request = + plan_recovery_stage_discard(&assessment(RecoveryStage::Segment, &segment)?)?; + let catalog_request = + plan_recovery_stage_discard(&assessment(RecoveryStage::Catalog, &catalog)?)?; + let head_request = plan_recovery_stage_discard(&assessment(RecoveryStage::NextHead, &head)?)?; + + assert_eq!(segment_request.evidence(), segment_evidence); + assert_eq!(catalog_request.evidence(), catalog_evidence); + assert_eq!(head_request.evidence(), head_evidence); + assert!(matches!( + segment_request.reason(), + RecoveryStageDiscardReason::Segment(RecoverySegmentTruncation::Header { observed: 1, .. }) + )); + assert!(matches!( + catalog_request.reason(), + RecoveryStageDiscardReason::CatalogHeader { observed: 1, .. } + )); + assert!(matches!( + head_request.reason(), + RecoveryStageDiscardReason::NextHead { observed: 1, .. } + )); + Ok(()) +} + +#[test] +fn complete_stages_cannot_form_truncation_discard_requests() -> Result<(), Box> { + for (stage, encoded) in [ + (RecoveryStage::Segment, fixture(SEGMENT_HEX)?), + (RecoveryStage::Catalog, fixture(CATALOG_HEX)?), + (RecoveryStage::NextHead, fixture(HEAD_HEX)?), + ] { + let assessed = assessment(stage, &encoded)?; + let error = plan_recovery_stage_discard(&assessed) + .err() + .ok_or("complete stage formed a truncation-discard request")?; + + assert!(matches!( + error, + RecoveryStageDiscardPlanError::NotTruncated { stage: observed } + if observed == stage + )); + } + Ok(()) +} + +#[test] +fn reusable_segment_prefix_cannot_form_a_discard_request() -> Result<(), Box> { + let complete = fixture(SEGMENT_HEX)?; + let prefix_length = complete + .len() + .checked_sub(SEGMENT_SEAL_LENGTH) + .ok_or("canonical segment is shorter than its fixed seal")?; + let prefix = complete + .get(..prefix_length) + .ok_or("canonical reusable prefix is unavailable")?; + let assessed = assessment(RecoveryStage::Segment, prefix)?; + + let error = plan_recovery_stage_discard(&assessed) + .err() + .ok_or("reusable segment prefix formed a discard request")?; + + assert_eq!( + error, + RecoveryStageDiscardPlanError::NotTruncated { + stage: RecoveryStage::Segment, + } + ); + Ok(()) +} diff --git a/tests/recovery_stage_discard/storage_double.rs b/tests/recovery_stage_discard/storage_double.rs new file mode 100644 index 0000000..e4d7088 --- /dev/null +++ b/tests/recovery_stage_discard/storage_double.rs @@ -0,0 +1,83 @@ +//! Deterministic storage double for stage-discard orchestration. + +use std::io; + +use keep::{ + RecoveryStageDiscardOutcome, RecoveryStageDiscardStorage, RecoveryStageDiscardStorageError, + RecoveryStageEvidence, RecoveryStageParent, +}; + +/// One semantic operation observed by the deterministic storage double. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum Operation { + /// Exact-evidence removal attempt. + Remove(RecoveryStageEvidence), + /// Name-selected parent synchronization attempt. + Synchronize(RecoveryStageParent), +} + +/// In-memory stage-discard storage with deterministic failure injection. +pub struct StageDiscardDouble { + present: Option, + operations: Vec, + fail_synchronizations: usize, +} + +impl StageDiscardDouble { + /// Creates a double with the supplied canonical-stage observation. + pub const fn new(present: Option) -> Self { + Self { + present, + operations: Vec::new(), + fail_synchronizations: 0, + } + } + + /// Configures the next parent synchronization to fail exactly once. + #[must_use] + pub const fn fail_next_synchronization(mut self) -> Self { + self.fail_synchronizations = 1; + self + } + + /// Returns the stage evidence currently retained by the double. + pub const fn present(&self) -> Option { + self.present + } + + /// Returns every semantic storage operation in call order. + pub fn operations(&self) -> &[Operation] { + &self.operations + } +} + +impl RecoveryStageDiscardStorage for StageDiscardDouble { + fn remove_if_matching( + &mut self, + expected: RecoveryStageEvidence, + ) -> Result { + self.operations.push(Operation::Remove(expected)); + match self.present { + None => Ok(RecoveryStageDiscardOutcome::AlreadyAbsent), + Some(observed) if observed == expected => { + self.present = None; + Ok(RecoveryStageDiscardOutcome::Removed) + } + Some(observed) => { + Err(RecoveryStageDiscardStorageError::EvidenceMismatch { expected, observed }) + } + } + } + + fn synchronize_parent(&mut self, parent: RecoveryStageParent) -> io::Result<()> { + self.operations.push(Operation::Synchronize(parent)); + if self.fail_synchronizations == 0 { + return Ok(()); + } + self.fail_synchronizations = self + .fail_synchronizations + .checked_sub(1) + .ok_or_else(|| io::Error::other("synchronization counter underflow"))?; + Err(io::Error::other("injected parent synchronization failure")) + } +} From 9c24b77d50c199f7b6b756affcb5e69151f91a01 Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 10:46:43 -0700 Subject: [PATCH 16/49] Add: Bind recovery discard to filesystem --- CHANGELOG.md | 5 + README.md | 12 +- docs/formats/segment-store-v1/recovery.md | 7 +- docs/formats/segment-store-v1/requirements.md | 6 +- .../filesystem_recovery_inventory_reader.rs | 20 ++- ...ystem_recovery_stage_discard_open_error.rs | 67 +++++++ ...lesystem_recovery_stage_discard_storage.rs | 95 ++++++++++ ...filesystem_recovery_stage_discard_tests.rs | 170 ++++++++++++++++++ .../fixture.rs | 76 ++++++++ .../filesystem_recovery_stage_discarder.rs | 72 ++++++++ src/adapters/filesystem_writer_lock.rs | 4 + src/adapters/mod.rs | 7 + src/lib.rs | 7 +- 13 files changed, 527 insertions(+), 21 deletions(-) create mode 100644 src/adapters/filesystem_recovery_stage_discard_open_error.rs create mode 100644 src/adapters/filesystem_recovery_stage_discard_storage.rs create mode 100644 src/adapters/filesystem_recovery_stage_discard_tests.rs create mode 100644 src/adapters/filesystem_recovery_stage_discard_tests/fixture.rs create mode 100644 src/adapters/filesystem_recovery_stage_discarder.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 627b75f..ab07067 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,6 +48,11 @@ after its public API and format compatibility policies are established. mutation, and returns a discard receipt only after the name-selected parent directory is synchronized. An already absent stage remains an idempotent input and still requires synchronization. +- Filesystem truncated-stage discard now admits the platform, retains the root + and `writer.lock` locks, pins every protocol namespace, reopens stage bytes + without following links, refuses replacement or fingerprint drift before + unlink, and synchronizes the typed `staging` or root parent before returning + a receipt. - Store initialization now exposes one storage-port state machine that admits the platform before mutation, opens and locks `writer.lock`, admits the three protocol directories in order, synchronizes the root, and preserves the diff --git a/README.md b/README.md index b2261ce..c5178f9 100644 --- a/README.md +++ b/README.md @@ -80,11 +80,13 @@ length, and recomputed fingerprint match prior observation evidence. Only an exact truncation assessment may form an explicit discard request; the semantic executor refuses evidence drift and returns a receipt only after exact removal or admitted prior absence is followed by parent synchronization. -The pinned-filesystem discard adapter, transitive publication-view admission, -and filesystem-streaming classification remain planned. -Crash-injection execution, remaining recovery actions, retention, compaction, -and garbage collection remain planned. Presence in the reference CAS does not -claim retention, crash recovery, or durability. +The pinned-filesystem adapter retains writer authority, revalidates stage +evidence without following links, removes only an exact match, and +synchronizes the protocol-selected parent. Transitive publication-view +admission and filesystem-streaming classification remain planned. +Crash-injection execution, stage completion and next-head finalization, +retention, compaction, and garbage collection remain planned. Presence in the +reference CAS does not claim retention, crash recovery, or durability. ```rust use keep::BlobId; diff --git a/docs/formats/segment-store-v1/recovery.md b/docs/formats/segment-store-v1/recovery.md index e213a0e..db19766 100644 --- a/docs/formats/segment-store-v1/recovery.md +++ b/docs/formats/segment-store-v1/recovery.md @@ -55,8 +55,11 @@ retains both its evidence and typed truncation reason. The semantic `execute_recovery_stage_discard` port refuses evidence drift before mutation, treats an absent canonical name as an idempotent input, synchronizes the name-selected parent in either success case, and returns a receipt only after -that synchronization. The pinned-filesystem implementation remains -unimplemented. +that synchronization. `FilesystemRecoveryStageDiscarder` admits the supported +platform, retains the root and `writer.lock` locks, pins all protocol +directories, reopens the stage without following links, bounds and verifies +the complete fingerprint, refuses namespace or entry replacement, unlinks only +an exact evidence match, and synchronizes the typed `staging` or root parent. The sole admissible duplicate digest is one fixed staging name and its exact digest-derived pool name after a link transition. Recovery admits that pair diff --git a/docs/formats/segment-store-v1/requirements.md b/docs/formats/segment-store-v1/requirements.md index a06c367..51f855e 100644 --- a/docs/formats/segment-store-v1/requirements.md +++ b/docs/formats/segment-store-v1/requirements.md @@ -94,8 +94,9 @@ bytes, bind materialized bytes back to prior stage evidence, and dispatch complete caller-supplied fixed-stage bytes through their name-selected classifiers. An exact truncation assessment may now authorize one evidence-bound, retry-safe discard through a semantic storage port. These -slices do not yet claim a pinned-filesystem discard implementation, transitive -publication-view admission, or process-death injection. +slices now bind that discard to pinned writer-authorized filesystem storage. +They do not yet claim transitive publication-view admission or process-death +injection. @@ -114,6 +115,7 @@ publication-view admission, or process-death injection. | `KEEP-RECOVERY-011` | Whole-byte catalog and next-head stage classification distinguishes exact fixed-header, declared-body, and fixed-width truncation from complete canonical bytes; complete-looking corruption and oversize remain typed format or metadata refusals | Canonical publication-artifact truncation and corruption matrix | `tests/recovery_publication_stage_classification.rs`, `tests/recovery_publication_stage_classification/*.rs` | Implemented in #17 | | `KEEP-RECOVERY-012` | Read-only semantic assessment admits materialized stage bytes only when the canonical-name stage, exact observed length, and `KEEP:RECOVERY:STAGE\0` fingerprint equal prior evidence, then dispatches through the name-selected segment, catalog, or next-head classifier | Evidence-binding mutation matrix and canonical stage assessments | `tests/recovery_stage_assessment.rs`, `tests/recovery_stage_assessment/*.rs` | Implemented in #17 | | `KEEP-RECOVERY-013` | Explicit discard plans only from an exact truncation assessment, retains the observation evidence and typed truncation reason, refuses changed evidence without mutation, synchronizes the name-selected parent after exact removal or admitted absence, and returns a receipt only after synchronization | Truncation-planning, evidence-drift, operation-order, and retry matrix | `tests/recovery_stage_discard.rs`, `tests/recovery_stage_discard/*.rs` | Implemented in #17 | +| `KEEP-RECOVERY-014` | Filesystem discard retains root and `writer.lock` authority, pins every protocol directory, never follows a fixed-stage link, revalidates bounded fingerprint and entry identity before unlink, refuses drift without mutation, and synchronizes the typed parent after removal or admitted absence | Exact removal, absent retry, mismatch, symlink, replacement, and writer-exclusion matrix | `src/adapters/filesystem_recovery_stage_discard_tests.rs`, `src/adapters/filesystem_recovery_stage_discard_tests/fixture.rs` | Implemented in #17 | diff --git a/src/adapters/filesystem_recovery_inventory_reader.rs b/src/adapters/filesystem_recovery_inventory_reader.rs index cf296f1..d8466c5 100644 --- a/src/adapters/filesystem_recovery_inventory_reader.rs +++ b/src/adapters/filesystem_recovery_inventory_reader.rs @@ -11,7 +11,7 @@ use super::{ FilesystemRecoveryStageError, RecoveryEntryName, RecoveryInventory, RecoveryInventoryError, RecoveryInventoryLimit, RecoveryInventoryOperation, RecoveryInventoryStorage, RecoveryNamespace, RecoveryStage, RecoveryStageEvidence, RecoveryStageNamespacePhase, - filesystem_platform_profile, filesystem_recovery_inventory_scan, + RecoveryStageParent, filesystem_platform_profile, filesystem_recovery_inventory_scan, filesystem_recovery_namespace::PinnedRecoveryDirectory, filesystem_recovery_stage, read_recovery_inventory, }; @@ -69,7 +69,7 @@ impl FilesystemRecoveryInventoryReader { Self::from_root(root) } - fn from_root(root: Dir) -> Result { + pub(super) fn from_root(root: Dir) -> Result { let staging = PinnedRecoveryDirectory::open(&root, RecoveryNamespace::Staging, STAGING_NAME)?; let segments = @@ -153,7 +153,7 @@ impl FilesystemRecoveryInventoryReader { self.catalogs.verify(&self.root) } - fn verify_stage_namespaces( + pub(super) fn verify_stage_namespaces( &self, stage: RecoveryStage, phase: RecoveryStageNamespacePhase, @@ -175,14 +175,16 @@ impl FilesystemRecoveryInventoryReader { } } - const fn stage_directory(&self, stage: RecoveryStage) -> &Dir { - match stage { - RecoveryStage::Segment | RecoveryStage::Catalog => { - self.directory(RecoveryNamespace::Staging) - } - RecoveryStage::NextHead => self.directory(RecoveryNamespace::Root), + pub(super) const fn parent_directory(&self, parent: RecoveryStageParent) -> &Dir { + match parent { + RecoveryStageParent::Staging => self.directory(RecoveryNamespace::Staging), + RecoveryStageParent::Root => self.directory(RecoveryNamespace::Root), } } + + pub(super) const fn stage_directory(&self, stage: RecoveryStage) -> &Dir { + self.parent_directory(stage.parent()) + } } impl RecoveryInventoryStorage for FilesystemRecoveryInventoryReader { diff --git a/src/adapters/filesystem_recovery_stage_discard_open_error.rs b/src/adapters/filesystem_recovery_stage_discard_open_error.rs new file mode 100644 index 0000000..69307c4 --- /dev/null +++ b/src/adapters/filesystem_recovery_stage_discard_open_error.rs @@ -0,0 +1,67 @@ +//! This module owns filesystem stage-discard authority acquisition failures. + +use std::error::Error; +use std::fmt; +use std::io; + +use super::{RecoveryInventoryError, WriterLockAcquireError}; + +/// Why a pinned writer-authorized stage discarder could not be opened. +#[derive(Debug)] +pub enum FilesystemRecoveryStageDiscardOpenError { + /// The store root did not satisfy the supported platform profile. + Platform { + /// Exact platform-admission failure. + source: io::Error, + }, + /// Exclusive writer authority could not be acquired. + WriterLock { + /// Exact writer-lock acquisition refusal. + source: WriterLockAcquireError, + }, + /// The locked root capability could not be cloned for recovery inventory. + CloneRoot { + /// Exact root-capability clone failure. + source: io::Error, + }, + /// One pinned protocol namespace could not be admitted. + Namespace { + /// Exact recovery-namespace admission refusal. + source: RecoveryInventoryError, + }, +} + +impl fmt::Display for FilesystemRecoveryStageDiscardOpenError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Platform { source } => { + write!(formatter, "recovery discard platform was refused: {source}") + } + Self::WriterLock { source } => { + write!( + formatter, + "recovery discard writer lock was refused: {source}" + ) + } + Self::CloneRoot { source } => { + write!(formatter, "locked recovery root clone failed: {source}") + } + Self::Namespace { source } => { + write!( + formatter, + "recovery discard namespace was refused: {source}" + ) + } + } + } +} + +impl Error for FilesystemRecoveryStageDiscardOpenError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Platform { source } | Self::CloneRoot { source } => Some(source), + Self::WriterLock { source } => Some(source), + Self::Namespace { source } => Some(source), + } + } +} diff --git a/src/adapters/filesystem_recovery_stage_discard_storage.rs b/src/adapters/filesystem_recovery_stage_discard_storage.rs new file mode 100644 index 0000000..8bdb30d --- /dev/null +++ b/src/adapters/filesystem_recovery_stage_discard_storage.rs @@ -0,0 +1,95 @@ +//! This module owns filesystem execution of exact recovery-stage discard. + +use std::io; + +use cap_std::fs::Dir; + +use super::{ + FilesystemRecoveryStageDiscarder, FilesystemRecoveryStageError, RecoveryStage, + RecoveryStageDiscardOutcome, RecoveryStageDiscardStorage, RecoveryStageDiscardStorageError, + RecoveryStageEvidence, RecoveryStageNamespacePhase, RecoveryStageParent, + filesystem_catalog_artifact, filesystem_recovery_stage, +}; + +impl RecoveryStageDiscardStorage for FilesystemRecoveryStageDiscarder { + fn remove_if_matching( + &mut self, + expected: RecoveryStageEvidence, + ) -> Result { + remove_with(self, expected, filesystem_recovery_stage::fingerprint) + } + + fn synchronize_parent(&mut self, parent: RecoveryStageParent) -> io::Result<()> { + filesystem_catalog_artifact::synchronize_directory(self.inventory.parent_directory(parent)) + } +} + +impl FilesystemRecoveryStageDiscarder { + #[cfg(test)] + pub(super) fn remove_if_matching_with( + &self, + expected: RecoveryStageEvidence, + after_open: F, + ) -> Result + where + F: FnOnce(), + { + remove_with(self, expected, |directory, stage| { + filesystem_recovery_stage::fingerprint_with(directory, stage, after_open) + }) + } +} + +fn remove_with( + discarder: &FilesystemRecoveryStageDiscarder, + expected: RecoveryStageEvidence, + observe: F, +) -> Result +where + F: FnOnce(&Dir, RecoveryStage) -> Result, +{ + let stage = expected.stage(); + discarder + .inventory + .verify_stage_namespaces(stage, RecoveryStageNamespacePhase::BeforeObservation) + .map_err(stage_error)?; + let directory = discarder.inventory.stage_directory(stage); + if stage_is_absent(directory, stage)? { + discarder + .inventory + .verify_stage_namespaces(stage, RecoveryStageNamespacePhase::AfterObservation) + .map_err(stage_error)?; + return Ok(RecoveryStageDiscardOutcome::AlreadyAbsent); + } + let observed = observe(directory, stage).map_err(stage_error)?; + if observed != expected { + return Err(RecoveryStageDiscardStorageError::EvidenceMismatch { expected, observed }); + } + discarder + .inventory + .verify_stage_namespaces(stage, RecoveryStageNamespacePhase::AfterObservation) + .map_err(stage_error)?; + directory + .remove_file(stage.file_name()) + .map_err(storage_error)?; + Ok(RecoveryStageDiscardOutcome::Removed) +} + +fn stage_is_absent( + directory: &Dir, + stage: RecoveryStage, +) -> Result { + match directory.symlink_metadata(stage.file_name()) { + Ok(_) => Ok(false), + Err(source) if source.kind() == io::ErrorKind::NotFound => Ok(true), + Err(source) => Err(storage_error(source)), + } +} + +fn stage_error(source: FilesystemRecoveryStageError) -> RecoveryStageDiscardStorageError { + storage_error(io::Error::other(source)) +} + +const fn storage_error(source: io::Error) -> RecoveryStageDiscardStorageError { + RecoveryStageDiscardStorageError::Storage { source } +} diff --git a/src/adapters/filesystem_recovery_stage_discard_tests.rs b/src/adapters/filesystem_recovery_stage_discard_tests.rs new file mode 100644 index 0000000..e0bf5ca --- /dev/null +++ b/src/adapters/filesystem_recovery_stage_discard_tests.rs @@ -0,0 +1,170 @@ +//! Pinned-filesystem truncated-stage discard laws. + +use std::error::Error; +use std::fs; + +use super::{ + FilesystemRecoveryStageDiscardOpenError, FilesystemRecoveryStageError, RecoveryStage, + RecoveryStageDiscardError, RecoveryStageDiscardOutcome, RecoveryStageDiscardStorageError, + WriterLockAcquireError, execute_recovery_stage_discard, +}; + +mod fixture; + +use fixture::{DiscardFixture, evidence, request}; + +#[test] +fn exact_stage_discard_and_absent_retry_are_durable_for_every_parent() -> Result<(), Box> +{ + let fixture = DiscardFixture::new("filesystem-stage-discard")?; + let cases: [(RecoveryStage, &[u8]); 3] = [ + (RecoveryStage::Segment, b"s"), + (RecoveryStage::Catalog, b"c"), + (RecoveryStage::NextHead, b"h"), + ]; + for (stage, bytes) in cases { + fs::write(fixture.stage_path(stage), bytes)?; + } + let mut discarder = fixture.discarder()?; + + for (stage, bytes) in cases { + let request = request(stage, bytes)?; + let removed = execute_recovery_stage_discard(&mut discarder, request)?; + let retried = execute_recovery_stage_discard(&mut discarder, request)?; + + assert_eq!(removed.outcome(), RecoveryStageDiscardOutcome::Removed); + assert_eq!( + retried.outcome(), + RecoveryStageDiscardOutcome::AlreadyAbsent + ); + assert!(!fixture.stage_path(stage).exists()); + } + drop(discarder); + fixture.remove()?; + Ok(()) +} + +#[test] +fn changed_stage_evidence_is_preserved_and_refused_before_unlink() -> Result<(), Box> { + let fixture = DiscardFixture::new("filesystem-stage-discard-mismatch")?; + let expected = request(RecoveryStage::Segment, b"old")?; + fs::write(fixture.stage_path(RecoveryStage::Segment), b"new")?; + let observed = evidence(RecoveryStage::Segment, b"new")?; + let mut discarder = fixture.discarder()?; + + let error = execute_recovery_stage_discard(&mut discarder, expected) + .err() + .ok_or("changed recovery stage was removed")?; + + assert!(matches!( + error, + RecoveryStageDiscardError::Remove { + source: RecoveryStageDiscardStorageError::EvidenceMismatch { + expected: actual_expected, + observed: actual_observed, + }, + } if actual_expected == expected.evidence() && actual_observed == observed + )); + assert_eq!( + fs::read(fixture.stage_path(RecoveryStage::Segment))?, + b"new" + ); + drop(discarder); + fixture.remove()?; + Ok(()) +} + +#[test] +fn symbolic_stage_is_never_followed_or_removed() -> Result<(), Box> { + use std::os::unix::fs::symlink; + + let fixture = DiscardFixture::new("filesystem-stage-discard-symlink")?; + let request = request(RecoveryStage::Segment, b"outside")?; + let target = fixture.root().join("outside"); + fs::write(&target, b"outside")?; + symlink(&target, fixture.stage_path(RecoveryStage::Segment))?; + let mut discarder = fixture.discarder()?; + + let error = execute_recovery_stage_discard(&mut discarder, request) + .err() + .ok_or("symbolic recovery stage was followed")?; + + let RecoveryStageDiscardError::Remove { source } = error else { + return Err("symbolic-stage refusal lost its removal phase".into()); + }; + assert!(matches!( + filesystem_stage_source(&source)?, + FilesystemRecoveryStageError::Open { + stage: RecoveryStage::Segment, + .. + } + )); + assert_eq!(fs::read(&target)?, b"outside"); + assert!(fixture.stage_path(RecoveryStage::Segment).is_symlink()); + drop(discarder); + fixture.remove()?; + Ok(()) +} + +#[test] +fn replacement_after_open_refuses_without_removing_the_new_entry() -> Result<(), Box> { + let fixture = DiscardFixture::new("filesystem-stage-discard-replaced")?; + let stage_path = fixture.stage_path(RecoveryStage::Segment); + let retained_path = fixture.root().join("retained-stage"); + fs::write(&stage_path, b"old")?; + let request = request(RecoveryStage::Segment, b"old")?; + let discarder = fixture.discarder()?; + let mut hook_result = Ok(()); + + let result = discarder.remove_if_matching_with(request.evidence(), || { + hook_result = + fs::rename(&stage_path, &retained_path).and_then(|()| fs::write(&stage_path, b"new")); + }); + + hook_result?; + let error = result.err().ok_or("replaced recovery stage was removed")?; + assert!(matches!( + filesystem_stage_source(&error)?, + FilesystemRecoveryStageError::Replaced { + stage: RecoveryStage::Segment, + } + )); + assert_eq!(fs::read(&stage_path)?, b"new"); + assert_eq!(fs::read(&retained_path)?, b"old"); + drop(discarder); + fixture.remove()?; + Ok(()) +} + +fn filesystem_stage_source( + error: &RecoveryStageDiscardStorageError, +) -> Result<&FilesystemRecoveryStageError, &'static str> { + let RecoveryStageDiscardStorageError::Storage { source } = error else { + return Err("recovery-stage failure lost its storage boundary"); + }; + source + .get_ref() + .and_then(|source| source.downcast_ref::()) + .ok_or("recovery-stage storage failure lost its typed source") +} + +#[test] +fn retained_discarder_authority_excludes_a_second_writer() -> Result<(), Box> { + let fixture = DiscardFixture::new("filesystem-stage-discard-lock")?; + let first = fixture.discarder()?; + + let error = fixture + .discarder() + .err() + .ok_or("second recovery writer acquired authority")?; + + assert!(matches!( + error, + FilesystemRecoveryStageDiscardOpenError::WriterLock { + source: WriterLockAcquireError::Busy, + } + )); + drop(first); + fixture.remove()?; + Ok(()) +} diff --git a/src/adapters/filesystem_recovery_stage_discard_tests/fixture.rs b/src/adapters/filesystem_recovery_stage_discard_tests/fixture.rs new file mode 100644 index 0000000..5b6346a --- /dev/null +++ b/src/adapters/filesystem_recovery_stage_discard_tests/fixture.rs @@ -0,0 +1,76 @@ +//! Deterministic initialized-store fixture for filesystem stage discard. + +use std::error::Error; +use std::fs; + +use crate::LayoutEntryLimit; + +use super::super::{ + FilesystemRecoveryStageDiscardOpenError, FilesystemRecoveryStageDiscarder, RecoveryStage, + RecoveryStageDiscardRequest, RecoveryStageEvidence, RecoveryStageMetadata, SegmentReadPolicy, + SegmentRecordLimit, admit_recovery_stage_bytes, assess_recovery_stage, + filesystem_test_sandbox::TestDirectory, fingerprint_recovery_stage, + plan_recovery_stage_discard, +}; + +pub(super) fn request( + stage: RecoveryStage, + bytes: &[u8], +) -> Result> { + let observed = evidence(stage, bytes)?; + let admitted = admit_recovery_stage_bytes(stage, observed, bytes)?; + let assessed = assess_recovery_stage(&admitted, maximum_policy())?; + Ok(plan_recovery_stage_discard(&assessed)?) +} + +pub(super) fn evidence( + stage: RecoveryStage, + bytes: &[u8], +) -> Result> { + let length = u64::try_from(bytes.len())?; + Ok(fingerprint_recovery_stage( + RecoveryStageMetadata::new(stage, length)?, + bytes, + )?) +} + +const fn maximum_policy() -> SegmentReadPolicy { + SegmentReadPolicy::new(SegmentRecordLimit::MAXIMUM, LayoutEntryLimit::MAXIMUM) +} + +pub(super) struct DiscardFixture { + directory: TestDirectory, +} + +impl DiscardFixture { + pub(super) fn new(name: &str) -> Result> { + let directory = TestDirectory::create(name)?; + fs::write(directory.path().join("writer.lock"), [])?; + for name in ["staging", "segments", "catalogs"] { + fs::create_dir(directory.path().join(name))?; + } + Ok(Self { directory }) + } + + pub(super) fn root(&self) -> &std::path::Path { + self.directory.path() + } + + pub(super) fn stage_path(&self, stage: RecoveryStage) -> std::path::PathBuf { + match stage { + RecoveryStage::Segment => self.root().join("staging/current.seg"), + RecoveryStage::Catalog => self.root().join("staging/current.cat"), + RecoveryStage::NextHead => self.root().join("head.next"), + } + } + + pub(super) fn discarder( + &self, + ) -> Result { + FilesystemRecoveryStageDiscarder::open_unchecked_for_tests(self.root()) + } + + pub(super) fn remove(self) -> std::io::Result<()> { + self.directory.remove() + } +} diff --git a/src/adapters/filesystem_recovery_stage_discarder.rs b/src/adapters/filesystem_recovery_stage_discarder.rs new file mode 100644 index 0000000..9cc71bd --- /dev/null +++ b/src/adapters/filesystem_recovery_stage_discarder.rs @@ -0,0 +1,72 @@ +//! This module owns pinned writer authority for filesystem stage discard. + +use std::path::Path; + +#[cfg(test)] +use cap_std::ambient_authority; +use cap_std::fs::Dir; + +use super::{ + FilesystemRecoveryInventoryReader, FilesystemRecoveryStageDiscardOpenError, + FilesystemWriterLock, filesystem_platform_profile, +}; +#[cfg(test)] +use super::{RecoveryInventoryError, RecoveryInventoryOperation, RecoveryNamespace}; + +/// Writer-authorized pinned filesystem adapter for exact stage discard. +/// +/// Opening proves the supported platform, pins and exclusively locks the store +/// root and `writer.lock`, then pins all three protocol child directories +/// without following links. The synchronous adapter may block on filesystem +/// I/O and retains writer authority until dropped. +#[must_use] +pub struct FilesystemRecoveryStageDiscarder { + pub(super) inventory: FilesystemRecoveryInventoryReader, + _authority: FilesystemWriterLock, +} + +impl FilesystemRecoveryStageDiscarder { + /// Opens an initialized supported store for explicit stage discard. + /// + /// The call performs no protocol mutation. + /// + /// # Errors + /// + /// Returns [`FilesystemRecoveryStageDiscardOpenError`] on platform, + /// writer-authority, root-clone, or namespace admission failure. + pub fn open(store_root: &Path) -> Result { + let root = filesystem_platform_profile::open(store_root) + .map_err(|source| FilesystemRecoveryStageDiscardOpenError::Platform { source })?; + Self::from_root(root) + } + + #[cfg(test)] + pub(super) fn open_unchecked_for_tests( + store_root: &Path, + ) -> Result { + let root = Dir::open_ambient_dir(store_root, ambient_authority()).map_err(|source| { + FilesystemRecoveryStageDiscardOpenError::Namespace { + source: RecoveryInventoryError::io( + RecoveryNamespace::Root, + RecoveryInventoryOperation::OpenNamespace, + source, + ), + } + })?; + Self::from_root(root) + } + + fn from_root(root: Dir) -> Result { + let authority = FilesystemWriterLock::try_acquire_in(root) + .map_err(|source| FilesystemRecoveryStageDiscardOpenError::WriterLock { source })?; + let inventory_root = authority + .clone_directory() + .map_err(|source| FilesystemRecoveryStageDiscardOpenError::CloneRoot { source })?; + let inventory = FilesystemRecoveryInventoryReader::from_root(inventory_root) + .map_err(|source| FilesystemRecoveryStageDiscardOpenError::Namespace { source })?; + Ok(Self { + inventory, + _authority: authority, + }) + } +} diff --git a/src/adapters/filesystem_writer_lock.rs b/src/adapters/filesystem_writer_lock.rs index b0ee0de..4270198 100644 --- a/src/adapters/filesystem_writer_lock.rs +++ b/src/adapters/filesystem_writer_lock.rs @@ -72,6 +72,10 @@ impl FilesystemWriterLock { Dir::open_ambient_dir(store_root, ambient_authority()).map_err(|source| { WriterLockAcquireError::io(WriterLockAcquirePhase::OpenRoot, source) })?; + Self::try_acquire_in(directory) + } + + pub(super) fn try_acquire_in(directory: Dir) -> Result { let root_lock_file = acquire_root(&directory)?; let lock_file = open_existing(&directory)?; Self::acquire(directory, root_lock_file, lock_file) diff --git a/src/adapters/mod.rs b/src/adapters/mod.rs index 9f7b651..530bb81 100644 --- a/src/adapters/mod.rs +++ b/src/adapters/mod.rs @@ -86,6 +86,11 @@ mod filesystem_recovery_inventory_scan; mod filesystem_recovery_inventory_tests; mod filesystem_recovery_namespace; mod filesystem_recovery_stage; +mod filesystem_recovery_stage_discard_open_error; +mod filesystem_recovery_stage_discard_storage; +#[cfg(all(test, unix))] +mod filesystem_recovery_stage_discard_tests; +mod filesystem_recovery_stage_discarder; mod filesystem_recovery_stage_error; #[cfg(all(test, unix))] mod filesystem_recovery_stage_tests; @@ -279,6 +284,8 @@ pub use filesystem_catalog_publisher::FilesystemCatalogPublisher; pub use filesystem_catalog_snapshot::FilesystemCatalogSnapshot; pub use filesystem_platform_admission::FilesystemPlatformAdmission; pub use filesystem_recovery_inventory_reader::FilesystemRecoveryInventoryReader; +pub use filesystem_recovery_stage_discard_open_error::FilesystemRecoveryStageDiscardOpenError; +pub use filesystem_recovery_stage_discarder::FilesystemRecoveryStageDiscarder; pub use filesystem_recovery_stage_error::{ FilesystemRecoveryStageError, RecoveryStageNamespacePhase, }; diff --git a/src/lib.rs b/src/lib.rs index c82bf76..783daa7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -43,9 +43,10 @@ pub use adapters::{ CatalogSnapshotError, CatalogSuccessor, CatalogTransitionError, ChecksummedCatalog, ChecksummedPublicationHead, ChecksummedSegmentRecord, ClosedSegment, FilesystemCatalogPublicationError, FilesystemCatalogPublisher, FilesystemCatalogSnapshot, - FilesystemPlatformAdmission, FilesystemRecoveryInventoryReader, FilesystemRecoveryStageError, - FilesystemSegmentStage, FilesystemWriterLock, LayoutDecodeError, LayoutDecodePolicy, - LayoutEncodeError, LayoutIdBinaryParseError, LayoutIdTextParseError, + FilesystemPlatformAdmission, FilesystemRecoveryInventoryReader, + FilesystemRecoveryStageDiscardOpenError, FilesystemRecoveryStageDiscarder, + FilesystemRecoveryStageError, FilesystemSegmentStage, FilesystemWriterLock, LayoutDecodeError, + LayoutDecodePolicy, LayoutEncodeError, LayoutIdBinaryParseError, LayoutIdTextParseError, PublicationHeadDecodeError, RecoveryCatalogStage, RecoveryCatalogStageError, RecoveryEntryName, RecoveryEntryNameError, RecoveryEntryRole, RecoveryInventory, RecoveryInventoryEntry, RecoveryInventoryError, RecoveryInventoryLimit, RecoveryInventoryLimitError, From 65d41b25c7932866974fb475fbd337db6c504111 Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 11:08:42 -0700 Subject: [PATCH 17/49] Add: Execute complete stage recovery --- CHANGELOG.md | 6 + docs/formats/segment-store-v1/recovery.md | 21 +- docs/formats/segment-store-v1/requirements.md | 7 +- src/adapters/mod.rs | 22 +++ .../recovery_stage_completion_error.rs | 101 ++++++++++ .../recovery_stage_completion_executor.rs | 49 +++++ .../recovery_stage_completion_plan_error.rs | 36 ++++ .../recovery_stage_completion_planner.rs | 50 +++++ .../recovery_stage_completion_pool.rs | 21 ++ .../recovery_stage_completion_receipt.rs | 60 ++++++ .../recovery_stage_completion_request.rs | 35 ++++ .../recovery_stage_completion_storage.rs | 76 ++++++++ .../recovery_stage_completion_target.rs | 34 ++++ src/adapters/recovery_stage_pool_outcome.rs | 10 + .../recovery_stage_synchronization_outcome.rs | 10 + src/lib.rs | 30 +-- tests/recovery_stage_completion.rs | 64 ++++++ .../execution_laws.rs | 159 +++++++++++++++ .../planning_laws.rs | 97 ++++++++++ tests/recovery_stage_completion/retry_laws.rs | 104 ++++++++++ .../storage_double.rs | 182 ++++++++++++++++++ 21 files changed, 1154 insertions(+), 20 deletions(-) create mode 100644 src/adapters/recovery_stage_completion_error.rs create mode 100644 src/adapters/recovery_stage_completion_executor.rs create mode 100644 src/adapters/recovery_stage_completion_plan_error.rs create mode 100644 src/adapters/recovery_stage_completion_planner.rs create mode 100644 src/adapters/recovery_stage_completion_pool.rs create mode 100644 src/adapters/recovery_stage_completion_receipt.rs create mode 100644 src/adapters/recovery_stage_completion_request.rs create mode 100644 src/adapters/recovery_stage_completion_storage.rs create mode 100644 src/adapters/recovery_stage_completion_target.rs create mode 100644 src/adapters/recovery_stage_pool_outcome.rs create mode 100644 src/adapters/recovery_stage_synchronization_outcome.rs create mode 100644 tests/recovery_stage_completion.rs create mode 100644 tests/recovery_stage_completion/execution_laws.rs create mode 100644 tests/recovery_stage_completion/planning_laws.rs create mode 100644 tests/recovery_stage_completion/retry_laws.rs create mode 100644 tests/recovery_stage_completion/storage_double.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index ab07067..a66c7a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -53,6 +53,12 @@ after its public API and format compatibility policies are established. without following links, refuses replacement or fingerprint drift before unlink, and synchronizes the typed `staging` or root parent before returning a receipt. +- Explicit complete-stage recovery now plans only from exact complete segment + or catalog assessments, owns bounded stage evidence and immutable-pool + coordinates, re-synchronizes an exact present stage before linking, verifies + existing pool entries, synchronizes the selected pool before exact stage + removal, and returns a valid-orphan receipt only after staging + synchronization. It never creates or finalizes a publication head. - Store initialization now exposes one storage-port state machine that admits the platform before mutation, opens and locks `writer.lock`, admits the three protocol directories in order, synchronizes the root, and preserves the diff --git a/docs/formats/segment-store-v1/recovery.md b/docs/formats/segment-store-v1/recovery.md index db19766..ce4ba3b 100644 --- a/docs/formats/segment-store-v1/recovery.md +++ b/docs/formats/segment-store-v1/recovery.md @@ -104,11 +104,22 @@ digest-derived pool coordinate. Under the writer lock, the executor reopens without following links, reverifies and resynchronizes the complete staged artifact, and refuses any drift. -Segment completion reuses `KEEP-CRASH-009`–`012`; catalog completion reuses -`KEEP-CRASH-017`–`020`. The executor performs the same no-clobber link, -post-link pool verification, pool-directory synchronization, exact stage -unlink, and staging-directory synchronization as forward publication. An -existing exact pool entry is an idempotent input, not proof by name. +The public semantic boundary admits only +`RecoverySegmentStage::Complete` and `RecoveryCatalogStage::Complete`. +`plan_recovery_stage_completion` converts either borrowed assessment into a +bounded owned request containing exact stage evidence and the validated pool +coordinate. Reusable, truncated, and `head.next` states remain ineligible. +`execute_recovery_stage_completion` requires a storage port to perform the +ordered transition and returns `RecoveryStageCompletionReceipt` only after +pool and staging durability. The receipt proves a valid orphan; it does not +prove reachability or retention. + +Segment completion reuses `KEEP-CRASH-008`–`012`; catalog completion reuses +`KEEP-CRASH-016`–`020`. The executor performs the same staged-file +synchronization, no-clobber link, post-link pool verification, pool-directory +synchronization, exact stage unlink, and staging-directory synchronization as +forward publication. An existing exact pool entry is an idempotent input, not +proof by name. After a crash, retry accepts only the exact verified stage/pool pair, the reappeared exact stage, or the already completed pool entry with an absent diff --git a/docs/formats/segment-store-v1/requirements.md b/docs/formats/segment-store-v1/requirements.md index 51f855e..23ad4ac 100644 --- a/docs/formats/segment-store-v1/requirements.md +++ b/docs/formats/segment-store-v1/requirements.md @@ -95,8 +95,10 @@ complete caller-supplied fixed-stage bytes through their name-selected classifiers. An exact truncation assessment may now authorize one evidence-bound, retry-safe discard through a semantic storage port. These slices now bind that discard to pinned writer-authorized filesystem storage. -They do not yet claim transitive publication-view admission or process-death -injection. +A complete segment or catalog assessment may now authorize an owned, +evidence-bound valid-orphan transition through a semantic storage port. They +do not yet bind complete-stage recovery to the filesystem or claim transitive +publication-view admission or process-death injection. @@ -116,6 +118,7 @@ injection. | `KEEP-RECOVERY-012` | Read-only semantic assessment admits materialized stage bytes only when the canonical-name stage, exact observed length, and `KEEP:RECOVERY:STAGE\0` fingerprint equal prior evidence, then dispatches through the name-selected segment, catalog, or next-head classifier | Evidence-binding mutation matrix and canonical stage assessments | `tests/recovery_stage_assessment.rs`, `tests/recovery_stage_assessment/*.rs` | Implemented in #17 | | `KEEP-RECOVERY-013` | Explicit discard plans only from an exact truncation assessment, retains the observation evidence and typed truncation reason, refuses changed evidence without mutation, synchronizes the name-selected parent after exact removal or admitted absence, and returns a receipt only after synchronization | Truncation-planning, evidence-drift, operation-order, and retry matrix | `tests/recovery_stage_discard.rs`, `tests/recovery_stage_discard/*.rs` | Implemented in #17 | | `KEEP-RECOVERY-014` | Filesystem discard retains root and `writer.lock` authority, pins every protocol directory, never follows a fixed-stage link, revalidates bounded fingerprint and entry identity before unlink, refuses drift without mutation, and synchronizes the typed parent after removal or admitted absence | Exact removal, absent retry, mismatch, symlink, replacement, and writer-exclusion matrix | `src/adapters/filesystem_recovery_stage_discard_tests.rs`, `src/adapters/filesystem_recovery_stage_discard_tests/fixture.rs` | Implemented in #17 | +| `KEEP-RECOVERY-015` | Immutable-pool completion plans only from exact complete segment or catalog assessments, owns bounded evidence and validated coordinates, re-synchronizes an exact present stage before linking, verifies an existing pool entry before admission, synchronizes the pool before exact stage removal, synchronizes staging before receipt, accepts completed retries, and never finalizes a head | Complete-only planning, operation-order, staged-file-sync, pool-conflict, and retry matrix | `tests/recovery_stage_completion.rs`, `tests/recovery_stage_completion/*.rs` | Implemented in #17 | diff --git a/src/adapters/mod.rs b/src/adapters/mod.rs index 530bb81..6442abf 100644 --- a/src/adapters/mod.rs +++ b/src/adapters/mod.rs @@ -155,6 +155,15 @@ mod recovery_stage_assessment_error; mod recovery_stage_assessor; mod recovery_stage_byte_admission; mod recovery_stage_byte_admission_error; +mod recovery_stage_completion_error; +mod recovery_stage_completion_executor; +mod recovery_stage_completion_plan_error; +mod recovery_stage_completion_planner; +mod recovery_stage_completion_pool; +mod recovery_stage_completion_receipt; +mod recovery_stage_completion_request; +mod recovery_stage_completion_storage; +mod recovery_stage_completion_target; mod recovery_stage_discard_error; mod recovery_stage_discard_executor; mod recovery_stage_discard_outcome; @@ -174,6 +183,8 @@ mod recovery_stage_length; mod recovery_stage_metadata; mod recovery_stage_metadata_error; mod recovery_stage_parent; +mod recovery_stage_pool_outcome; +mod recovery_stage_synchronization_outcome; mod sealed_segment; mod segment_digest; mod segment_digest_builder; @@ -328,6 +339,15 @@ pub use recovery_stage_assessment_error::RecoveryStageAssessmentError; pub use recovery_stage_assessor::assess_recovery_stage; pub use recovery_stage_byte_admission::admit_recovery_stage_bytes; pub use recovery_stage_byte_admission_error::RecoveryStageByteAdmissionError; +pub use recovery_stage_completion_error::RecoveryStageCompletionError; +pub use recovery_stage_completion_executor::execute_recovery_stage_completion; +pub use recovery_stage_completion_plan_error::RecoveryStageCompletionPlanError; +pub use recovery_stage_completion_planner::plan_recovery_stage_completion; +pub use recovery_stage_completion_pool::RecoveryStageCompletionPool; +pub use recovery_stage_completion_receipt::RecoveryStageCompletionReceipt; +pub use recovery_stage_completion_request::RecoveryStageCompletionRequest; +pub use recovery_stage_completion_storage::RecoveryStageCompletionStorage; +pub use recovery_stage_completion_target::RecoveryStageCompletionTarget; pub use recovery_stage_discard_error::RecoveryStageDiscardError; pub use recovery_stage_discard_executor::execute_recovery_stage_discard; pub use recovery_stage_discard_outcome::RecoveryStageDiscardOutcome; @@ -347,6 +367,8 @@ pub use recovery_stage_length::RecoveryStageLength; pub use recovery_stage_metadata::RecoveryStageMetadata; pub use recovery_stage_metadata_error::RecoveryStageMetadataError; pub use recovery_stage_parent::RecoveryStageParent; +pub use recovery_stage_pool_outcome::RecoveryStagePoolOutcome; +pub use recovery_stage_synchronization_outcome::RecoveryStageSynchronizationOutcome; pub use sealed_segment::SealedSegment; pub use segment_digest::SegmentDigest; pub use segment_header::SegmentHeader; diff --git a/src/adapters/recovery_stage_completion_error.rs b/src/adapters/recovery_stage_completion_error.rs new file mode 100644 index 0000000..ede9e3b --- /dev/null +++ b/src/adapters/recovery_stage_completion_error.rs @@ -0,0 +1,101 @@ +//! This module owns ordered complete-stage recovery failures. + +use std::error::Error; +use std::fmt; +use std::io; + +use super::{ + RecoveryStage, RecoveryStageCompletionPool, RecoveryStageCompletionTarget, + RecoveryStageDiscardStorageError, +}; + +/// Exact failed phase of complete-stage recovery execution. +#[derive(Debug)] +pub enum RecoveryStageCompletionError { + /// The exact present stage could not be verified and synchronized. + SynchronizeStage { + /// Fixed stage that could not be made durable. + stage: RecoveryStage, + /// Exact underlying verification or synchronization failure. + source: io::Error, + }, + /// The exact stage could not be linked or an existing coordinate admitted. + LinkOrAdmit { + /// Validated immutable-pool target. + target: RecoveryStageCompletionTarget, + /// Exact underlying storage failure. + source: io::Error, + }, + /// The immutable-pool entry did not verify exactly. + VerifyPool { + /// Validated immutable-pool target. + target: RecoveryStageCompletionTarget, + /// Exact underlying verification failure. + source: io::Error, + }, + /// The immutable-pool directory could not be synchronized. + SynchronizePool { + /// Selected immutable pool. + pool: RecoveryStageCompletionPool, + /// Exact underlying synchronization failure. + source: io::Error, + }, + /// The exact stage could not be removed or admitted absent. + RemoveStage { + /// Exact semantic removal refusal. + source: RecoveryStageDiscardStorageError, + }, + /// The staging directory could not be synchronized after stage removal. + SynchronizeStaging { + /// Stage whose absence was not durably confirmed. + stage: RecoveryStage, + /// Exact underlying synchronization failure. + source: io::Error, + }, +} + +impl fmt::Display for RecoveryStageCompletionError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::SynchronizeStage { stage, source } => { + write!( + formatter, + "failed to synchronize complete {stage}: {source}" + ) + } + Self::LinkOrAdmit { target, source } => write!( + formatter, + "failed to link or admit the {} recovery target: {source}", + target.pool() + ), + Self::VerifyPool { target, source } => write!( + formatter, + "failed to verify the {} recovery target: {source}", + target.pool() + ), + Self::SynchronizePool { pool, source } => { + write!(formatter, "failed to synchronize the {pool} pool: {source}") + } + Self::RemoveStage { source } => { + write!(formatter, "failed to remove the completed stage: {source}") + } + Self::SynchronizeStaging { stage, source } => write!( + formatter, + "failed to synchronize staging after {stage} removal: {source}" + ), + } + } +} + +impl Error for RecoveryStageCompletionError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::SynchronizeStage { source, .. } + | Self::LinkOrAdmit { source, .. } + | Self::VerifyPool { source, .. } + | Self::SynchronizePool { source, .. } + | Self::SynchronizeStaging { source, .. } => Some(source), + Self::RemoveStage { source } => Some(source), + } + } +} diff --git a/src/adapters/recovery_stage_completion_executor.rs b/src/adapters/recovery_stage_completion_executor.rs new file mode 100644 index 0000000..44a3b67 --- /dev/null +++ b/src/adapters/recovery_stage_completion_executor.rs @@ -0,0 +1,49 @@ +//! This module owns ordered, idempotent complete-stage recovery. + +use super::{ + RecoveryStageCompletionError, RecoveryStageCompletionReceipt, RecoveryStageCompletionRequest, + RecoveryStageCompletionStorage, +}; + +/// Completes one exact stage into a durable, verified immutable orphan. +/// +/// The operation never creates, replaces, or finalizes a catalog head. A +/// receipt is returned only after the immutable pool is synchronized, the exact +/// fixed stage is absent, and staging is synchronized. +/// +/// # Errors +/// +/// Returns [`RecoveryStageCompletionError`] without a receipt at the exact +/// failed link, verification, pool-sync, removal, or staging-sync phase. +pub fn execute_recovery_stage_completion( + storage: &mut impl RecoveryStageCompletionStorage, + request: RecoveryStageCompletionRequest, +) -> Result { + let target = request.target(); + let pool = request.pool(); + let stage = request.evidence().stage(); + let synchronization_outcome = storage + .synchronize_stage_if_present(request) + .map_err(|source| RecoveryStageCompletionError::SynchronizeStage { stage, source })?; + let pool_outcome = storage + .link_stage_or_admit_pool(request) + .map_err(|source| RecoveryStageCompletionError::LinkOrAdmit { target, source })?; + storage + .verify_pool(request) + .map_err(|source| RecoveryStageCompletionError::VerifyPool { target, source })?; + storage + .synchronize_pool(pool) + .map_err(|source| RecoveryStageCompletionError::SynchronizePool { pool, source })?; + let stage_outcome = storage + .remove_stage_if_matching(request.evidence()) + .map_err(|source| RecoveryStageCompletionError::RemoveStage { source })?; + storage + .synchronize_staging() + .map_err(|source| RecoveryStageCompletionError::SynchronizeStaging { stage, source })?; + Ok(RecoveryStageCompletionReceipt::new( + request, + synchronization_outcome, + pool_outcome, + stage_outcome, + )) +} diff --git a/src/adapters/recovery_stage_completion_plan_error.rs b/src/adapters/recovery_stage_completion_plan_error.rs new file mode 100644 index 0000000..bfd8f30 --- /dev/null +++ b/src/adapters/recovery_stage_completion_plan_error.rs @@ -0,0 +1,36 @@ +//! This module owns complete-stage recovery planning refusals. + +use std::error::Error; +use std::fmt; + +use super::RecoveryStage; + +/// Why an assessed stage cannot enter immutable-pool completion. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum RecoveryStageCompletionPlanError { + /// A segment or catalog stage is not semantically complete. + NotComplete { + /// Fixed stage that requires a different recovery action. + stage: RecoveryStage, + }, + /// The stage belongs to a different recovery protocol. + NotPoolStage { + /// Fixed stage that cannot name an immutable-pool artifact. + stage: RecoveryStage, + }, +} + +impl fmt::Display for RecoveryStageCompletionPlanError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::NotComplete { stage } => { + write!(formatter, "{stage} is not a complete immutable artifact") + } + Self::NotPoolStage { stage } => { + write!(formatter, "{stage} does not publish into an immutable pool") + } + } + } +} + +impl Error for RecoveryStageCompletionPlanError {} diff --git a/src/adapters/recovery_stage_completion_planner.rs b/src/adapters/recovery_stage_completion_planner.rs new file mode 100644 index 0000000..99836fd --- /dev/null +++ b/src/adapters/recovery_stage_completion_planner.rs @@ -0,0 +1,50 @@ +//! This module owns exact complete-stage recovery planning. + +use super::{ + RecoveryCatalogStage, RecoverySegmentStage, RecoveryStageAssessment, + RecoveryStageCompletionPlanError, RecoveryStageCompletionRequest, + RecoveryStageCompletionTarget, +}; + +/// Plans immutable-pool completion from one exact complete stage assessment. +/// +/// The returned request owns only bounded evidence and validated coordinates; +/// it does not retain or allocate the assessed stage bytes. +/// +/// # Errors +/// +/// Returns [`RecoveryStageCompletionPlanError`] for incomplete segment or +/// catalog stages and for `head.next`, which has a dedicated finalization +/// protocol. +pub const fn plan_recovery_stage_completion( + assessment: &RecoveryStageAssessment<'_>, +) -> Result { + let evidence = assessment.evidence(); + let target = match assessment { + RecoveryStageAssessment::Segment { + state: RecoverySegmentStage::Complete(segment), + .. + } => RecoveryStageCompletionTarget::Segment { + digest: segment.digest(), + }, + RecoveryStageAssessment::Catalog { + state: RecoveryCatalogStage::Complete(catalog), + .. + } => RecoveryStageCompletionTarget::Catalog { + generation: catalog.generation(), + length: catalog.length(), + digest: catalog.digest(), + }, + RecoveryStageAssessment::Segment { .. } | RecoveryStageAssessment::Catalog { .. } => { + return Err(RecoveryStageCompletionPlanError::NotComplete { + stage: evidence.stage(), + }); + } + RecoveryStageAssessment::NextHead { .. } => { + return Err(RecoveryStageCompletionPlanError::NotPoolStage { + stage: evidence.stage(), + }); + } + }; + Ok(RecoveryStageCompletionRequest::new(evidence, target)) +} diff --git a/src/adapters/recovery_stage_completion_pool.rs b/src/adapters/recovery_stage_completion_pool.rs new file mode 100644 index 0000000..42af4a8 --- /dev/null +++ b/src/adapters/recovery_stage_completion_pool.rs @@ -0,0 +1,21 @@ +//! This module owns immutable pools selected by recovery completion. + +use std::fmt; + +/// Immutable artifact pool selected by a complete recovery stage. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum RecoveryStageCompletionPool { + /// Immutable sealed-segment pool. + Segments, + /// Immutable checksummed-catalog pool. + Catalogs, +} + +impl fmt::Display for RecoveryStageCompletionPool { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Segments => formatter.write_str("segments"), + Self::Catalogs => formatter.write_str("catalogs"), + } + } +} diff --git a/src/adapters/recovery_stage_completion_receipt.rs b/src/adapters/recovery_stage_completion_receipt.rs new file mode 100644 index 0000000..2cad1eb --- /dev/null +++ b/src/adapters/recovery_stage_completion_receipt.rs @@ -0,0 +1,60 @@ +//! This module owns durable valid-orphan recovery receipts. + +use super::{ + RecoveryStageCompletionRequest, RecoveryStageCompletionTarget, RecoveryStageDiscardOutcome, + RecoveryStageEvidence, RecoveryStagePoolOutcome, RecoveryStageSynchronizationOutcome, +}; + +/// Proof that one exact artifact is pooled and its fixed stage is durably absent. +/// +/// This receipt establishes a valid immutable orphan only. It makes no claim +/// that a catalog head names the artifact or that retention keeps it live. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct RecoveryStageCompletionReceipt { + request: RecoveryStageCompletionRequest, + synchronization_outcome: RecoveryStageSynchronizationOutcome, + pool_outcome: RecoveryStagePoolOutcome, + stage_outcome: RecoveryStageDiscardOutcome, +} + +impl RecoveryStageCompletionReceipt { + pub(super) const fn new( + request: RecoveryStageCompletionRequest, + synchronization_outcome: RecoveryStageSynchronizationOutcome, + pool_outcome: RecoveryStagePoolOutcome, + stage_outcome: RecoveryStageDiscardOutcome, + ) -> Self { + Self { + request, + synchronization_outcome, + pool_outcome, + stage_outcome, + } + } + + /// Returns whether the exact stage was synchronized or already absent. + pub const fn synchronization_outcome(self) -> RecoveryStageSynchronizationOutcome { + self.synchronization_outcome + } + + /// Returns the exact stage evidence bound into the request. + pub const fn evidence(self) -> RecoveryStageEvidence { + self.request.evidence() + } + + /// Returns the verified immutable-pool coordinate. + pub const fn target(self) -> RecoveryStageCompletionTarget { + self.request.target() + } + + /// Returns whether the pool entry was linked or already present. + pub const fn pool_outcome(self) -> RecoveryStagePoolOutcome { + self.pool_outcome + } + + /// Returns whether the exact stage was removed or already absent. + pub const fn stage_outcome(self) -> RecoveryStageDiscardOutcome { + self.stage_outcome + } +} diff --git a/src/adapters/recovery_stage_completion_request.rs b/src/adapters/recovery_stage_completion_request.rs new file mode 100644 index 0000000..8a5ca02 --- /dev/null +++ b/src/adapters/recovery_stage_completion_request.rs @@ -0,0 +1,35 @@ +//! This module owns explicit fingerprint-bound stage-completion requests. + +use super::{RecoveryStageCompletionPool, RecoveryStageCompletionTarget, RecoveryStageEvidence}; + +/// Authorized completion of one exact stage into one immutable pool. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct RecoveryStageCompletionRequest { + evidence: RecoveryStageEvidence, + target: RecoveryStageCompletionTarget, +} + +impl RecoveryStageCompletionRequest { + pub(super) const fn new( + evidence: RecoveryStageEvidence, + target: RecoveryStageCompletionTarget, + ) -> Self { + Self { evidence, target } + } + + /// Returns the exact stage evidence that authorized the request. + pub const fn evidence(self) -> RecoveryStageEvidence { + self.evidence + } + + /// Returns the verified immutable-pool coordinate. + pub const fn target(self) -> RecoveryStageCompletionTarget { + self.target + } + + /// Returns the immutable pool selected by the target. + pub const fn pool(self) -> RecoveryStageCompletionPool { + self.target.pool() + } +} diff --git a/src/adapters/recovery_stage_completion_storage.rs b/src/adapters/recovery_stage_completion_storage.rs new file mode 100644 index 0000000..57cf140 --- /dev/null +++ b/src/adapters/recovery_stage_completion_storage.rs @@ -0,0 +1,76 @@ +//! This module owns the storage port for exact complete-stage recovery. + +use std::io; + +use super::{ + RecoveryStageCompletionPool, RecoveryStageCompletionRequest, RecoveryStageDiscardOutcome, + RecoveryStageDiscardStorageError, RecoveryStageEvidence, RecoveryStagePoolOutcome, + RecoveryStageSynchronizationOutcome, +}; + +/// Semantic storage operations required by complete-stage recovery. +/// +/// Implementations must retain writer authority throughout execution. A link +/// must never replace a pool entry. An existing entry is input to exact +/// verification, not proof by name. Stage and pool reads must reopen without +/// following links, remain bounded by the selected protocol, and verify exact +/// evidence and semantic coordinates before mutation. The orchestration layer +/// owns operation order and receipt timing. +pub trait RecoveryStageCompletionStorage { + /// Verifies and synchronizes the exact stage when it remains present. + /// + /// An absent stage is an idempotent input only when the subsequent + /// link-or-admit operation can select an existing pool coordinate. + /// + /// # Errors + /// + /// Returns a source-preserving evidence, verification, or staged-file + /// synchronization failure. + fn synchronize_stage_if_present( + &mut self, + request: RecoveryStageCompletionRequest, + ) -> io::Result; + + /// Links an exact stage or admits an already-present pool coordinate. + /// + /// # Errors + /// + /// Returns a source-preserving storage error when neither an exact stage + /// nor an existing pool coordinate can continue the request. + fn link_stage_or_admit_pool( + &mut self, + request: RecoveryStageCompletionRequest, + ) -> io::Result; + + /// Verifies that the selected pool entry exactly satisfies the request. + /// + /// # Errors + /// + /// Returns a source-preserving verification or storage error. + fn verify_pool(&mut self, request: RecoveryStageCompletionRequest) -> io::Result<()>; + + /// Synchronizes the selected immutable-pool directory. + /// + /// # Errors + /// + /// Returns the exact directory synchronization failure. + fn synchronize_pool(&mut self, pool: RecoveryStageCompletionPool) -> io::Result<()>; + + /// Removes the exact stage or reports that its canonical name is absent. + /// + /// # Errors + /// + /// Returns a typed evidence mismatch without mutation or preserves the + /// exact storage error from reopen, verification, or removal. + fn remove_stage_if_matching( + &mut self, + expected: RecoveryStageEvidence, + ) -> Result; + + /// Synchronizes the staging directory after exact removal or absent retry. + /// + /// # Errors + /// + /// Returns the exact staging-directory synchronization failure. + fn synchronize_staging(&mut self) -> io::Result<()>; +} diff --git a/src/adapters/recovery_stage_completion_target.rs b/src/adapters/recovery_stage_completion_target.rs new file mode 100644 index 0000000..cfd0f0d --- /dev/null +++ b/src/adapters/recovery_stage_completion_target.rs @@ -0,0 +1,34 @@ +//! This module owns validated immutable-pool recovery coordinates. + +use super::{RecoveryStageCompletionPool, SegmentDigest}; +use crate::{CatalogDigest, CatalogGeneration, CatalogLength}; + +/// Exact immutable-pool coordinate derived from a complete stage. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum RecoveryStageCompletionTarget { + /// One fully admitted immutable segment. + Segment { + /// Verified physical segment digest. + digest: SegmentDigest, + }, + /// One framing-, checksum-, digest-, and entry-verified catalog. + Catalog { + /// Positive catalog generation. + generation: CatalogGeneration, + /// Exact canonical catalog byte length. + length: CatalogLength, + /// Verified physical catalog digest. + digest: CatalogDigest, + }, +} + +impl RecoveryStageCompletionTarget { + /// Returns the immutable pool selected by this coordinate. + pub const fn pool(self) -> RecoveryStageCompletionPool { + match self { + Self::Segment { .. } => RecoveryStageCompletionPool::Segments, + Self::Catalog { .. } => RecoveryStageCompletionPool::Catalogs, + } + } +} diff --git a/src/adapters/recovery_stage_pool_outcome.rs b/src/adapters/recovery_stage_pool_outcome.rs new file mode 100644 index 0000000..fc26179 --- /dev/null +++ b/src/adapters/recovery_stage_pool_outcome.rs @@ -0,0 +1,10 @@ +//! This module owns immutable-pool admission outcomes during recovery. + +/// Whether recovery linked an exact artifact or admitted an existing one. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum RecoveryStagePoolOutcome { + /// The exact stage was linked into its immutable pool. + Linked, + /// The immutable pool coordinate already existed and required verification. + AlreadyPresent, +} diff --git a/src/adapters/recovery_stage_synchronization_outcome.rs b/src/adapters/recovery_stage_synchronization_outcome.rs new file mode 100644 index 0000000..001dfa0 --- /dev/null +++ b/src/adapters/recovery_stage_synchronization_outcome.rs @@ -0,0 +1,10 @@ +//! This module owns staged-file synchronization outcomes during recovery. + +/// Whether complete-stage recovery synchronized a stage or found it absent. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum RecoveryStageSynchronizationOutcome { + /// The exact present stage was verified and synchronized. + Synchronized, + /// The fixed stage was absent and recovery continued from the pool. + AlreadyAbsent, +} diff --git a/src/lib.rs b/src/lib.rs index 783daa7..f945f99 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -15,10 +15,10 @@ //! generations, platform-gated filesystem publication mechanics, bounded //! immutable restart snapshots, typed store-initialization orchestration, and //! production initialization for the admitted Linux ext4 profile. Recovery -//! inventory, name classification, and bounded stage fingerprinting are -//! read-only; semantic recovery planning, execution, retention, and garbage -//! collection APIs remain intentionally absent until their contracts have -//! executable specifications. +//! inventory, name classification, bounded stage fingerprinting, exact +//! truncated-stage discard, and complete-stage valid-orphan recovery are +//! explicit. Head finalization, retention, and garbage collection APIs remain +//! intentionally absent until their contracts have executable specifications. #[cfg(test)] extern crate self as keep; @@ -54,13 +54,16 @@ pub use adapters::{ RecoveryNameManifest, RecoveryNamedEntry, RecoveryNamespace, RecoveryNextHeadStage, RecoveryNextHeadStageError, RecoveryPoolNameError, RecoveryRequiredEntry, RecoverySegmentStage, RecoverySegmentStageError, RecoverySegmentTruncation, RecoveryStage, RecoveryStageAssessment, - RecoveryStageAssessmentError, RecoveryStageByteAdmissionError, RecoveryStageDiscardError, - RecoveryStageDiscardOutcome, RecoveryStageDiscardPlanError, RecoveryStageDiscardReason, - RecoveryStageDiscardReceipt, RecoveryStageDiscardRequest, RecoveryStageDiscardStorage, - RecoveryStageDiscardStorageError, RecoveryStageEvidence, RecoveryStageFingerprint, - RecoveryStageFingerprintAlgorithm, RecoveryStageFingerprintError, RecoveryStageLength, - RecoveryStageMetadata, RecoveryStageMetadataError, RecoveryStageNamespacePhase, - RecoveryStageParent, ReusableRecoverySegment, SealedSegment, SegmentDigest, + RecoveryStageAssessmentError, RecoveryStageByteAdmissionError, RecoveryStageCompletionError, + RecoveryStageCompletionPlanError, RecoveryStageCompletionPool, RecoveryStageCompletionReceipt, + RecoveryStageCompletionRequest, RecoveryStageCompletionStorage, RecoveryStageCompletionTarget, + RecoveryStageDiscardError, RecoveryStageDiscardOutcome, RecoveryStageDiscardPlanError, + RecoveryStageDiscardReason, RecoveryStageDiscardReceipt, RecoveryStageDiscardRequest, + RecoveryStageDiscardStorage, RecoveryStageDiscardStorageError, RecoveryStageEvidence, + RecoveryStageFingerprint, RecoveryStageFingerprintAlgorithm, RecoveryStageFingerprintError, + RecoveryStageLength, RecoveryStageMetadata, RecoveryStageMetadataError, + RecoveryStageNamespacePhase, RecoveryStageParent, RecoveryStagePoolOutcome, + RecoveryStageSynchronizationOutcome, ReusableRecoverySegment, SealedSegment, SegmentDigest, SegmentDurabilityPhase, SegmentHeader, SegmentHeaderError, SegmentPublication, SegmentPublicationError, SegmentReadError, SegmentReadPolicy, SegmentRecordAdmissionError, SegmentRecordChecksum, SegmentRecordDecodeError, SegmentRecordHeader, SegmentRecordHeaderError, @@ -71,8 +74,9 @@ pub use adapters::{ StoreInitializationReceipt, StoreInitializationStorage, WriterLockAcquireError, WriterLockAcquirePhase, admit_recovery_stage_bytes, assess_recovery_stage, classify_recovery_catalog_stage, classify_recovery_names, classify_recovery_next_head_stage, - classify_recovery_segment_stage, execute_recovery_stage_discard, fingerprint_recovery_stage, - initialize_store, plan_recovery_stage_discard, publish_catalog_generation, + classify_recovery_segment_stage, execute_recovery_stage_completion, + execute_recovery_stage_discard, fingerprint_recovery_stage, initialize_store, + plan_recovery_stage_completion, plan_recovery_stage_discard, publish_catalog_generation, read_recovery_inventory, }; pub use blob::{ diff --git a/tests/recovery_stage_completion.rs b/tests/recovery_stage_completion.rs new file mode 100644 index 0000000..3823700 --- /dev/null +++ b/tests/recovery_stage_completion.rs @@ -0,0 +1,64 @@ +//! Exact complete-stage recovery laws. + +#[path = "recovery_stage_completion/execution_laws.rs"] +mod execution_laws; +#[path = "recovery_stage_completion/planning_laws.rs"] +mod planning_laws; +#[path = "recovery_stage_completion/retry_laws.rs"] +mod retry_laws; +#[path = "recovery_stage_completion/storage_double.rs"] +pub mod storage_double; +mod support; + +use std::error::Error; + +use keep::{ + LayoutEntryLimit, RecoveryStage, RecoveryStageAssessment, RecoveryStageCompletionRequest, + RecoveryStageEvidence, RecoveryStageMetadata, SegmentReadPolicy, SegmentRecordLimit, + admit_recovery_stage_bytes, assess_recovery_stage, fingerprint_recovery_stage, + plan_recovery_stage_completion, +}; +use support::decode_hex; + +const SEGMENT_HEX: &str = include_str!("../conformance/segment-store/v1/one-zero-segment.hex"); +const CATALOG_HEX: &str = include_str!("../conformance/segment-store/v1/one-zero-catalog.hex"); +const HEAD_HEX: &str = include_str!("../conformance/segment-store/v1/one-zero-head.hex"); +const SEGMENT_SEAL_LENGTH: usize = 128; + +fn fixture(hex: &str) -> Result, Box> { + decode_hex( + hex.strip_suffix('\n') + .ok_or("recovery fixture must end in one LF")?, + ) + .map_err(Into::into) +} + +fn evidence(stage: RecoveryStage, encoded: &[u8]) -> Result> { + let length = u64::try_from(encoded.len())?; + Ok(fingerprint_recovery_stage( + RecoveryStageMetadata::new(stage, length)?, + encoded, + )?) +} + +fn assessment( + stage: RecoveryStage, + encoded: &[u8], +) -> Result, Box> { + let observed = evidence(stage, encoded)?; + let admitted = admit_recovery_stage_bytes(stage, observed, encoded)?; + Ok(assess_recovery_stage(&admitted, maximum_policy())?) +} + +fn completion_request( + stage: RecoveryStage, + encoded: &[u8], +) -> Result> { + Ok(plan_recovery_stage_completion(&assessment( + stage, encoded, + )?)?) +} + +const fn maximum_policy() -> SegmentReadPolicy { + SegmentReadPolicy::new(SegmentRecordLimit::MAXIMUM, LayoutEntryLimit::MAXIMUM) +} diff --git a/tests/recovery_stage_completion/execution_laws.rs b/tests/recovery_stage_completion/execution_laws.rs new file mode 100644 index 0000000..a64a03d --- /dev/null +++ b/tests/recovery_stage_completion/execution_laws.rs @@ -0,0 +1,159 @@ +//! Ordered complete-stage recovery execution laws. + +use std::error::Error; + +use keep::{ + RecoveryStage, RecoveryStageCompletionPool, RecoveryStageDiscardOutcome, + RecoveryStagePoolOutcome, RecoveryStageSynchronizationOutcome, + execute_recovery_stage_completion, +}; + +use super::storage_double::{Operation, StageCompletionDouble}; +use super::{CATALOG_HEX, SEGMENT_HEX, completion_request, fixture}; + +#[test] +fn complete_segment_becomes_a_durable_orphan_before_stage_removal() -> Result<(), Box> { + let bytes = fixture(SEGMENT_HEX)?; + let request = completion_request(RecoveryStage::Segment, &bytes)?; + let mut storage = StageCompletionDouble::new(Some(request.evidence()), None); + + let receipt = execute_recovery_stage_completion(&mut storage, request)?; + + assert_eq!(receipt.evidence(), request.evidence()); + assert_eq!(receipt.target(), request.target()); + assert_eq!( + receipt.synchronization_outcome(), + RecoveryStageSynchronizationOutcome::Synchronized + ); + assert_eq!(receipt.pool_outcome(), RecoveryStagePoolOutcome::Linked); + assert_eq!( + receipt.stage_outcome(), + RecoveryStageDiscardOutcome::Removed + ); + assert_eq!( + storage.operations(), + &[ + Operation::SynchronizeStage(request), + Operation::LinkOrAdmit(request), + Operation::VerifyPool(request), + Operation::SynchronizePool(RecoveryStageCompletionPool::Segments), + Operation::RemoveStage(request.evidence()), + Operation::SynchronizeStaging, + ] + ); + assert_eq!(storage.pool(), Some(request)); + assert_eq!(storage.stage(), None); + Ok(()) +} + +#[test] +fn complete_catalog_selects_the_catalog_pool() -> Result<(), Box> { + let bytes = fixture(CATALOG_HEX)?; + let request = completion_request(RecoveryStage::Catalog, &bytes)?; + let mut storage = StageCompletionDouble::new(Some(request.evidence()), None); + + let receipt = execute_recovery_stage_completion(&mut storage, request)?; + + assert_eq!(receipt.pool_outcome(), RecoveryStagePoolOutcome::Linked); + assert_eq!( + storage.operations(), + &[ + Operation::SynchronizeStage(request), + Operation::LinkOrAdmit(request), + Operation::VerifyPool(request), + Operation::SynchronizePool(RecoveryStageCompletionPool::Catalogs), + Operation::RemoveStage(request.evidence()), + Operation::SynchronizeStaging, + ] + ); + Ok(()) +} + +#[test] +fn existing_exact_pool_is_verified_before_stage_removal() -> Result<(), Box> { + let bytes = fixture(SEGMENT_HEX)?; + let request = completion_request(RecoveryStage::Segment, &bytes)?; + let mut storage = StageCompletionDouble::new(Some(request.evidence()), Some(request)); + + let receipt = execute_recovery_stage_completion(&mut storage, request)?; + + assert_eq!( + receipt.pool_outcome(), + RecoveryStagePoolOutcome::AlreadyPresent + ); + assert_eq!( + receipt.synchronization_outcome(), + RecoveryStageSynchronizationOutcome::Synchronized + ); + assert_eq!( + receipt.stage_outcome(), + RecoveryStageDiscardOutcome::Removed + ); + assert_eq!( + storage.operations(), + &[ + Operation::SynchronizeStage(request), + Operation::LinkOrAdmit(request), + Operation::VerifyPool(request), + Operation::SynchronizePool(RecoveryStageCompletionPool::Segments), + Operation::RemoveStage(request.evidence()), + Operation::SynchronizeStaging, + ] + ); + Ok(()) +} + +#[test] +fn absent_stage_with_exact_pool_is_an_idempotent_completed_retry() -> Result<(), Box> { + let bytes = fixture(CATALOG_HEX)?; + let request = completion_request(RecoveryStage::Catalog, &bytes)?; + let mut storage = StageCompletionDouble::new(None, Some(request)); + + let receipt = execute_recovery_stage_completion(&mut storage, request)?; + + assert_eq!( + receipt.pool_outcome(), + RecoveryStagePoolOutcome::AlreadyPresent + ); + assert_eq!( + receipt.synchronization_outcome(), + RecoveryStageSynchronizationOutcome::AlreadyAbsent + ); + assert_eq!( + receipt.stage_outcome(), + RecoveryStageDiscardOutcome::AlreadyAbsent + ); + assert_eq!(storage.pool(), Some(request)); + assert_eq!(storage.stage(), None); + Ok(()) +} + +#[test] +fn conflicting_existing_pool_refuses_before_sync_or_stage_removal() -> Result<(), Box> { + let segment = fixture(SEGMENT_HEX)?; + let catalog = fixture(CATALOG_HEX)?; + let request = completion_request(RecoveryStage::Segment, &segment)?; + let conflict = completion_request(RecoveryStage::Catalog, &catalog)?; + let mut storage = StageCompletionDouble::new(Some(request.evidence()), Some(conflict)); + + let error = execute_recovery_stage_completion(&mut storage, request) + .err() + .ok_or("conflicting pool artifact was accepted")?; + + assert!(matches!( + error, + keep::RecoveryStageCompletionError::VerifyPool { target, .. } + if target == request.target() + )); + assert_eq!( + storage.operations(), + &[ + Operation::SynchronizeStage(request), + Operation::LinkOrAdmit(request), + Operation::VerifyPool(request), + ] + ); + assert_eq!(storage.stage(), Some(request.evidence())); + assert_eq!(storage.pool(), Some(conflict)); + Ok(()) +} diff --git a/tests/recovery_stage_completion/planning_laws.rs b/tests/recovery_stage_completion/planning_laws.rs new file mode 100644 index 0000000..7b5a0d8 --- /dev/null +++ b/tests/recovery_stage_completion/planning_laws.rs @@ -0,0 +1,97 @@ +//! Complete-stage recovery planning laws. + +use std::error::Error; + +use keep::{ + AdmittedSegment, ChecksummedCatalog, RecoveryStage, RecoveryStageCompletionPlanError, + RecoveryStageCompletionTarget, plan_recovery_stage_completion, +}; + +use super::{ + CATALOG_HEX, HEAD_HEX, SEGMENT_HEX, SEGMENT_SEAL_LENGTH, assessment, fixture, maximum_policy, +}; + +#[test] +fn complete_segment_plan_retains_exact_evidence_and_pool_coordinate() -> Result<(), Box> +{ + let bytes = fixture(SEGMENT_HEX)?; + let assessed = assessment(RecoveryStage::Segment, &bytes)?; + let expected = AdmittedSegment::decode(&bytes, maximum_policy())?; + + let request = plan_recovery_stage_completion(&assessed)?; + + assert_eq!(request.evidence(), assessed.evidence()); + assert_eq!( + request.target(), + RecoveryStageCompletionTarget::Segment { + digest: expected.digest(), + } + ); + Ok(()) +} + +#[test] +fn complete_catalog_plan_retains_exact_evidence_and_pool_coordinate() -> Result<(), Box> +{ + let bytes = fixture(CATALOG_HEX)?; + let assessed = assessment(RecoveryStage::Catalog, &bytes)?; + let expected = ChecksummedCatalog::decode(&bytes)?; + + let request = plan_recovery_stage_completion(&assessed)?; + + assert_eq!(request.evidence(), assessed.evidence()); + assert_eq!( + request.target(), + RecoveryStageCompletionTarget::Catalog { + generation: expected.generation(), + length: expected.length(), + digest: expected.digest(), + } + ); + Ok(()) +} + +#[test] +fn reusable_and_truncated_pool_stages_are_not_completion_requests() -> Result<(), Box> { + let segment = fixture(SEGMENT_HEX)?; + let reusable_length = segment + .len() + .checked_sub(SEGMENT_SEAL_LENGTH) + .ok_or("segment fixture is shorter than its seal")?; + let reusable_bytes = segment + .get(..reusable_length) + .ok_or("reusable segment prefix is outside the fixture")?; + let reusable = assessment(RecoveryStage::Segment, reusable_bytes)?; + let truncated_catalog = assessment(RecoveryStage::Catalog, &[0_u8])?; + + for assessed in [&reusable, &truncated_catalog] { + let error = plan_recovery_stage_completion(assessed) + .err() + .ok_or("incomplete stage produced a completion request")?; + assert_eq!( + error, + RecoveryStageCompletionPlanError::NotComplete { + stage: assessed.evidence().stage(), + } + ); + } + Ok(()) +} + +#[test] +fn next_head_requires_its_dedicated_finalization_protocol() -> Result<(), Box> { + let bytes = fixture(HEAD_HEX)?; + let assessed = assessment(RecoveryStage::NextHead, &bytes)?; + + let error = plan_recovery_stage_completion(&assessed) + .err() + .ok_or("next-head stage entered immutable-pool completion")?; + + assert_eq!( + error, + RecoveryStageCompletionPlanError::NotPoolStage { + stage: RecoveryStage::NextHead, + } + ); + Ok(()) +} diff --git a/tests/recovery_stage_completion/retry_laws.rs b/tests/recovery_stage_completion/retry_laws.rs new file mode 100644 index 0000000..4cfaa0a --- /dev/null +++ b/tests/recovery_stage_completion/retry_laws.rs @@ -0,0 +1,104 @@ +//! Process-death retry laws for complete-stage recovery. + +use std::error::Error; + +use keep::{ + RecoveryStage, RecoveryStageCompletionError, RecoveryStageDiscardOutcome, + RecoveryStagePoolOutcome, execute_recovery_stage_completion, +}; + +use super::storage_double::StageCompletionDouble; +use super::{SEGMENT_HEX, completion_request, fixture}; + +#[test] +fn stage_sync_failure_stops_before_link_or_pool_admission() -> Result<(), Box> { + let bytes = fixture(SEGMENT_HEX)?; + let request = completion_request(RecoveryStage::Segment, &bytes)?; + let mut storage = StageCompletionDouble::new(Some(request.evidence()), None) + .fail_next_stage_synchronization(); + + let error = execute_recovery_stage_completion(&mut storage, request) + .err() + .ok_or("injected stage synchronization failure was ignored")?; + + assert!(matches!( + error, + RecoveryStageCompletionError::SynchronizeStage { + stage: RecoveryStage::Segment, + .. + } + )); + assert_eq!(storage.pool(), None); + assert_eq!(storage.stage(), Some(request.evidence())); + assert_eq!( + storage.operations(), + &[super::storage_double::Operation::SynchronizeStage(request)] + ); + Ok(()) +} + +#[test] +fn retry_after_pool_sync_failure_reverifies_before_stage_removal() -> Result<(), Box> { + let bytes = fixture(SEGMENT_HEX)?; + let request = completion_request(RecoveryStage::Segment, &bytes)?; + let mut storage = + StageCompletionDouble::new(Some(request.evidence()), None).fail_next_pool_synchronization(); + + let error = execute_recovery_stage_completion(&mut storage, request) + .err() + .ok_or("injected pool synchronization failure was ignored")?; + + assert!(matches!( + error, + RecoveryStageCompletionError::SynchronizePool { pool, .. } + if pool == request.pool() + )); + assert_eq!(storage.pool(), Some(request)); + assert_eq!(storage.stage(), Some(request.evidence())); + + let receipt = execute_recovery_stage_completion(&mut storage, request)?; + + assert_eq!( + receipt.pool_outcome(), + RecoveryStagePoolOutcome::AlreadyPresent + ); + assert_eq!( + receipt.stage_outcome(), + RecoveryStageDiscardOutcome::Removed + ); + Ok(()) +} + +#[test] +fn retry_after_stage_removal_reestablishes_staging_durability() -> Result<(), Box> { + let bytes = fixture(SEGMENT_HEX)?; + let request = completion_request(RecoveryStage::Segment, &bytes)?; + let mut storage = StageCompletionDouble::new(Some(request.evidence()), None) + .fail_next_staging_synchronization(); + + let error = execute_recovery_stage_completion(&mut storage, request) + .err() + .ok_or("injected staging synchronization failure was ignored")?; + + assert!(matches!( + error, + RecoveryStageCompletionError::SynchronizeStaging { + stage: RecoveryStage::Segment, + .. + } + )); + assert_eq!(storage.pool(), Some(request)); + assert_eq!(storage.stage(), None); + + let receipt = execute_recovery_stage_completion(&mut storage, request)?; + + assert_eq!( + receipt.pool_outcome(), + RecoveryStagePoolOutcome::AlreadyPresent + ); + assert_eq!( + receipt.stage_outcome(), + RecoveryStageDiscardOutcome::AlreadyAbsent + ); + Ok(()) +} diff --git a/tests/recovery_stage_completion/storage_double.rs b/tests/recovery_stage_completion/storage_double.rs new file mode 100644 index 0000000..fa2c658 --- /dev/null +++ b/tests/recovery_stage_completion/storage_double.rs @@ -0,0 +1,182 @@ +//! Deterministic storage double for complete-stage recovery. + +use std::io; + +use keep::{ + RecoveryStageCompletionPool, RecoveryStageCompletionRequest, RecoveryStageCompletionStorage, + RecoveryStageDiscardOutcome, RecoveryStageDiscardStorageError, RecoveryStageEvidence, + RecoveryStagePoolOutcome, RecoveryStageSynchronizationOutcome, +}; + +/// One semantic operation observed by the deterministic storage double. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum Operation { + /// Exact stage verification and file synchronization. + SynchronizeStage(RecoveryStageCompletionRequest), + /// Stage-to-pool link or completed-pool admission. + LinkOrAdmit(RecoveryStageCompletionRequest), + /// Exact completed-pool verification. + VerifyPool(RecoveryStageCompletionRequest), + /// Selected immutable-pool synchronization. + SynchronizePool(RecoveryStageCompletionPool), + /// Exact-evidence stage removal. + RemoveStage(RecoveryStageEvidence), + /// Staging-directory synchronization. + SynchronizeStaging, +} + +/// In-memory complete-stage storage with deterministic failure injection. +pub struct StageCompletionDouble { + stage: Option, + pool: Option, + operations: Vec, + fail_stage_synchronizations: usize, + fail_pool_synchronizations: usize, + fail_staging_synchronizations: usize, +} + +impl StageCompletionDouble { + /// Creates a double with the supplied stage and pool observations. + pub const fn new( + stage: Option, + pool: Option, + ) -> Self { + Self { + stage, + pool, + operations: Vec::new(), + fail_stage_synchronizations: 0, + fail_pool_synchronizations: 0, + fail_staging_synchronizations: 0, + } + } + + /// Configures the next staged-file synchronization to fail once. + #[must_use] + pub const fn fail_next_stage_synchronization(mut self) -> Self { + self.fail_stage_synchronizations = 1; + self + } + + /// Configures the next immutable-pool synchronization to fail once. + #[must_use] + pub const fn fail_next_pool_synchronization(mut self) -> Self { + self.fail_pool_synchronizations = 1; + self + } + + /// Configures the next staging synchronization to fail once. + #[must_use] + pub const fn fail_next_staging_synchronization(mut self) -> Self { + self.fail_staging_synchronizations = 1; + self + } + + /// Returns the current canonical stage evidence. + pub const fn stage(&self) -> Option { + self.stage + } + + /// Returns the current pooled request coordinate. + pub const fn pool(&self) -> Option { + self.pool + } + + /// Returns every semantic operation in call order. + pub fn operations(&self) -> &[Operation] { + &self.operations + } +} + +impl RecoveryStageCompletionStorage for StageCompletionDouble { + fn synchronize_stage_if_present( + &mut self, + request: RecoveryStageCompletionRequest, + ) -> io::Result { + self.operations.push(Operation::SynchronizeStage(request)); + match self.stage { + Some(observed) if observed == request.evidence() => { + fail_once( + &mut self.fail_stage_synchronizations, + "injected stage synchronization failure", + )?; + Ok(RecoveryStageSynchronizationOutcome::Synchronized) + } + Some(_) => Err(io::Error::other("stage evidence changed")), + None => Ok(RecoveryStageSynchronizationOutcome::AlreadyAbsent), + } + } + + fn link_stage_or_admit_pool( + &mut self, + request: RecoveryStageCompletionRequest, + ) -> io::Result { + self.operations.push(Operation::LinkOrAdmit(request)); + if self.pool.is_some() { + return Ok(RecoveryStagePoolOutcome::AlreadyPresent); + } + match self.stage { + Some(observed) if observed == request.evidence() => { + self.pool = Some(request); + Ok(RecoveryStagePoolOutcome::Linked) + } + Some(_) => Err(io::Error::other("stage evidence changed")), + None => Err(io::Error::new( + io::ErrorKind::NotFound, + "stage and pool are absent", + )), + } + } + + fn verify_pool(&mut self, request: RecoveryStageCompletionRequest) -> io::Result<()> { + self.operations.push(Operation::VerifyPool(request)); + if self.pool == Some(request) { + Ok(()) + } else { + Err(io::Error::other("pool bytes conflict with request")) + } + } + + fn synchronize_pool(&mut self, pool: RecoveryStageCompletionPool) -> io::Result<()> { + self.operations.push(Operation::SynchronizePool(pool)); + fail_once( + &mut self.fail_pool_synchronizations, + "injected pool synchronization failure", + ) + } + + fn remove_stage_if_matching( + &mut self, + expected: RecoveryStageEvidence, + ) -> Result { + self.operations.push(Operation::RemoveStage(expected)); + match self.stage { + None => Ok(RecoveryStageDiscardOutcome::AlreadyAbsent), + Some(observed) if observed == expected => { + self.stage = None; + Ok(RecoveryStageDiscardOutcome::Removed) + } + Some(observed) => { + Err(RecoveryStageDiscardStorageError::EvidenceMismatch { expected, observed }) + } + } + } + + fn synchronize_staging(&mut self) -> io::Result<()> { + self.operations.push(Operation::SynchronizeStaging); + fail_once( + &mut self.fail_staging_synchronizations, + "injected staging synchronization failure", + ) + } +} + +fn fail_once(remaining: &mut usize, message: &'static str) -> io::Result<()> { + if *remaining == 0 { + return Ok(()); + } + *remaining = remaining + .checked_sub(1) + .ok_or_else(|| io::Error::other("failure counter underflow"))?; + Err(io::Error::other(message)) +} From 61ff6c66747587933018936da0bef6c3e2ce00f7 Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 11:30:54 -0700 Subject: [PATCH 18/49] Add: Bind complete stage recovery to filesystem --- CHANGELOG.md | 5 + docs/formats/segment-store-v1/recovery.md | 9 + docs/formats/segment-store-v1/requirements.md | 8 +- .../filesystem_recovery_inventory_reader.rs | 18 +- src/adapters/filesystem_recovery_stage.rs | 71 ++++++- .../filesystem_recovery_stage_completer.rs | 41 ++++ ...em_recovery_stage_completion_open_error.rs | 87 +++++++++ ...lesystem_recovery_stage_completion_pool.rs | 184 ++++++++++++++++++ ...ystem_recovery_stage_completion_storage.rs | 135 +++++++++++++ ...esystem_recovery_stage_completion_tests.rs | 96 +++++++++ .../fixture.rs | 103 ++++++++++ .../refusal_laws.rs | 179 +++++++++++++++++ .../replacement_laws.rs | 70 +++++++ ...lesystem_recovery_stage_discard_storage.rs | 37 ++-- .../filesystem_recovery_stage_error.rs | 14 ++ .../filesystem_recovery_stage_sync.rs | 78 ++++++++ src/adapters/mod.rs | 11 ++ .../recovery_stage_completion_error.rs | 17 +- .../recovery_stage_completion_storage.rs | 12 +- ...recovery_stage_completion_storage_error.rs | 67 +++++++ src/lib.rs | 45 ++--- .../storage_double.rs | 41 ++-- 22 files changed, 1250 insertions(+), 78 deletions(-) create mode 100644 src/adapters/filesystem_recovery_stage_completer.rs create mode 100644 src/adapters/filesystem_recovery_stage_completion_open_error.rs create mode 100644 src/adapters/filesystem_recovery_stage_completion_pool.rs create mode 100644 src/adapters/filesystem_recovery_stage_completion_storage.rs create mode 100644 src/adapters/filesystem_recovery_stage_completion_tests.rs create mode 100644 src/adapters/filesystem_recovery_stage_completion_tests/fixture.rs create mode 100644 src/adapters/filesystem_recovery_stage_completion_tests/refusal_laws.rs create mode 100644 src/adapters/filesystem_recovery_stage_completion_tests/replacement_laws.rs create mode 100644 src/adapters/filesystem_recovery_stage_sync.rs create mode 100644 src/adapters/recovery_stage_completion_storage_error.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index a66c7a4..b2e5084 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -59,6 +59,11 @@ after its public API and format compatibility policies are established. existing pool entries, synchronizes the selected pool before exact stage removal, and returns a valid-orphan receipt only after staging synchronization. It never creates or finalizes a publication head. +- Filesystem complete-stage recovery now retains pinned root and writer + authority, revalidates exact stage evidence at synchronization and link + boundaries, uses no-clobber immutable-pool links, never follows stage or pool + links, preserves conflicting or replaced entries, verifies exact pool bytes, + and accepts stage/pool, reappeared-stage, and completed pool-only retries. - Store initialization now exposes one storage-port state machine that admits the platform before mutation, opens and locks `writer.lock`, admits the three protocol directories in order, synchronizes the root, and preserves the diff --git a/docs/formats/segment-store-v1/recovery.md b/docs/formats/segment-store-v1/recovery.md index ce4ba3b..e53992f 100644 --- a/docs/formats/segment-store-v1/recovery.md +++ b/docs/formats/segment-store-v1/recovery.md @@ -114,6 +114,15 @@ ordered transition and returns `RecoveryStageCompletionReceipt` only after pool and staging durability. The receipt proves a valid orphan; it does not prove reachability or retention. +`FilesystemRecoveryStageCompleter` binds that port to the admitted filesystem +profile. It retains the pinned root and `writer.lock` authority, pins all +protocol directories, reopens stages and pool entries without following +links, bounds every complete read by the stage grammar, rechecks stage evidence +at the link boundary, and refuses entry replacement or fingerprint drift +without removing the fixed stage. An absent fixed stage can continue only when +the canonical pool coordinate exists and verifies to the exact request +evidence. + Segment completion reuses `KEEP-CRASH-008`–`012`; catalog completion reuses `KEEP-CRASH-016`–`020`. The executor performs the same staged-file synchronization, no-clobber link, post-link pool verification, pool-directory diff --git a/docs/formats/segment-store-v1/requirements.md b/docs/formats/segment-store-v1/requirements.md index 23ad4ac..ba97ebd 100644 --- a/docs/formats/segment-store-v1/requirements.md +++ b/docs/formats/segment-store-v1/requirements.md @@ -96,9 +96,10 @@ classifiers. An exact truncation assessment may now authorize one evidence-bound, retry-safe discard through a semantic storage port. These slices now bind that discard to pinned writer-authorized filesystem storage. A complete segment or catalog assessment may now authorize an owned, -evidence-bound valid-orphan transition through a semantic storage port. They -do not yet bind complete-stage recovery to the filesystem or claim transitive -publication-view admission or process-death injection. +evidence-bound valid-orphan transition through a semantic storage port, and +the filesystem completer now binds that transition to pinned writer-authorized +storage. They do not yet claim next-head finalization, transitive +publication-view admission, or process-death injection. @@ -119,6 +120,7 @@ publication-view admission or process-death injection. | `KEEP-RECOVERY-013` | Explicit discard plans only from an exact truncation assessment, retains the observation evidence and typed truncation reason, refuses changed evidence without mutation, synchronizes the name-selected parent after exact removal or admitted absence, and returns a receipt only after synchronization | Truncation-planning, evidence-drift, operation-order, and retry matrix | `tests/recovery_stage_discard.rs`, `tests/recovery_stage_discard/*.rs` | Implemented in #17 | | `KEEP-RECOVERY-014` | Filesystem discard retains root and `writer.lock` authority, pins every protocol directory, never follows a fixed-stage link, revalidates bounded fingerprint and entry identity before unlink, refuses drift without mutation, and synchronizes the typed parent after removal or admitted absence | Exact removal, absent retry, mismatch, symlink, replacement, and writer-exclusion matrix | `src/adapters/filesystem_recovery_stage_discard_tests.rs`, `src/adapters/filesystem_recovery_stage_discard_tests/fixture.rs` | Implemented in #17 | | `KEEP-RECOVERY-015` | Immutable-pool completion plans only from exact complete segment or catalog assessments, owns bounded evidence and validated coordinates, re-synchronizes an exact present stage before linking, verifies an existing pool entry before admission, synchronizes the pool before exact stage removal, synchronizes staging before receipt, accepts completed retries, and never finalizes a head | Complete-only planning, operation-order, staged-file-sync, pool-conflict, and retry matrix | `tests/recovery_stage_completion.rs`, `tests/recovery_stage_completion/*.rs` | Implemented in #17 | +| `KEEP-RECOVERY-016` | Filesystem completion retains root and `writer.lock` authority, pins every protocol directory, re-synchronizes and re-fingerprints exact stage evidence before no-clobber link, never follows stage or pool links, verifies exact pool evidence before removal, preserves conflicting and replaced entries, accepts exact stage/pool, reappeared-stage, and pool-only retries, and returns only after pool and staging synchronization | Segment/catalog completion, three retry states, conflict, link, replacement, stale-evidence, missing-artifact, and writer-exclusion matrix | `src/adapters/filesystem_recovery_stage_completion_tests.rs`, `src/adapters/filesystem_recovery_stage_completion_tests/*.rs` | Implemented in #17 | diff --git a/src/adapters/filesystem_recovery_inventory_reader.rs b/src/adapters/filesystem_recovery_inventory_reader.rs index d8466c5..f269125 100644 --- a/src/adapters/filesystem_recovery_inventory_reader.rs +++ b/src/adapters/filesystem_recovery_inventory_reader.rs @@ -10,10 +10,10 @@ use cap_std::fs::Dir; use super::{ FilesystemRecoveryStageError, RecoveryEntryName, RecoveryInventory, RecoveryInventoryError, RecoveryInventoryLimit, RecoveryInventoryOperation, RecoveryInventoryStorage, - RecoveryNamespace, RecoveryStage, RecoveryStageEvidence, RecoveryStageNamespacePhase, - RecoveryStageParent, filesystem_platform_profile, filesystem_recovery_inventory_scan, - filesystem_recovery_namespace::PinnedRecoveryDirectory, filesystem_recovery_stage, - read_recovery_inventory, + RecoveryNamespace, RecoveryStage, RecoveryStageCompletionPool, RecoveryStageEvidence, + RecoveryStageNamespacePhase, RecoveryStageParent, filesystem_platform_profile, + filesystem_recovery_inventory_scan, filesystem_recovery_namespace::PinnedRecoveryDirectory, + filesystem_recovery_stage, read_recovery_inventory, }; const STAGING_NAME: &str = "staging"; @@ -185,6 +185,16 @@ impl FilesystemRecoveryInventoryReader { pub(super) const fn stage_directory(&self, stage: RecoveryStage) -> &Dir { self.parent_directory(stage.parent()) } + + pub(super) const fn completion_pool_directory( + &self, + pool: RecoveryStageCompletionPool, + ) -> &Dir { + match pool { + RecoveryStageCompletionPool::Segments => self.directory(RecoveryNamespace::Segments), + RecoveryStageCompletionPool::Catalogs => self.directory(RecoveryNamespace::Catalogs), + } + } } impl RecoveryInventoryStorage for FilesystemRecoveryInventoryReader { diff --git a/src/adapters/filesystem_recovery_stage.rs b/src/adapters/filesystem_recovery_stage.rs index f977a1d..8cf45b0 100644 --- a/src/adapters/filesystem_recovery_stage.rs +++ b/src/adapters/filesystem_recovery_stage.rs @@ -28,11 +28,44 @@ struct AdmittedStage { metadata: RecoveryStageMetadata, } +pub(super) struct ObservedRecoveryStage { + file: File, + admitted: AdmittedStage, + evidence: RecoveryStageEvidence, +} + +impl ObservedRecoveryStage { + pub(super) const fn evidence(&self) -> RecoveryStageEvidence { + self.evidence + } + + pub(super) fn synchronize( + &self, + directory: &Dir, + name: &str, + stage: RecoveryStage, + ) -> Result<(), FilesystemRecoveryStageError> { + self.file + .sync_all() + .map_err(|source| FilesystemRecoveryStageError::Synchronize { stage, source })?; + verify_opened_handle(&self.file, stage, &self.admitted)?; + verify_current_entry(directory, name, stage, &self.admitted) + } +} + pub(super) fn fingerprint( directory: &Dir, stage: RecoveryStage, ) -> Result { - observe(directory, stage, || {}) + fingerprint_named(directory, stage.file_name(), stage) +} + +pub(super) fn fingerprint_named( + directory: &Dir, + name: &str, + stage: RecoveryStage, +) -> Result { + Ok(observe_named(directory, name, stage)?.evidence()) } #[cfg(test)] @@ -44,31 +77,48 @@ pub(super) fn fingerprint_with( where F: FnOnce(), { - observe(directory, stage, after_open) + Ok(observe_named_with(directory, stage.file_name(), stage, after_open)?.evidence()) } -fn observe( +pub(super) fn observe_named( directory: &Dir, + name: &str, + stage: RecoveryStage, +) -> Result { + observe_named_with(directory, name, stage, || {}) +} + +pub(super) fn observe_named_with( + directory: &Dir, + name: &str, stage: RecoveryStage, after_open: F, -) -> Result +) -> Result where F: FnOnce(), { - let mut file = open_stage(directory, stage)?; + let mut file = open_stage(directory, name, stage)?; let admitted = admit_stage(&file, stage)?; after_open(); let evidence = fingerprint_recovery_stage(admitted.metadata, &mut file) .map_err(|source| FilesystemRecoveryStageError::Fingerprint { stage, source })?; verify_length(stage, admitted.metadata.length(), evidence.length().get())?; verify_opened_handle(&file, stage, &admitted)?; - verify_current_entry(directory, stage, &admitted)?; - Ok(evidence) + verify_current_entry(directory, name, stage, &admitted)?; + Ok(ObservedRecoveryStage { + file, + admitted, + evidence, + }) } -fn open_stage(directory: &Dir, stage: RecoveryStage) -> Result { +fn open_stage( + directory: &Dir, + name: &str, + stage: RecoveryStage, +) -> Result { directory - .open_with(stage.file_name(), &read_options()) + .open_with(name, &read_options()) .map_err(|source| FilesystemRecoveryStageError::Open { stage, source }) } @@ -104,11 +154,12 @@ fn verify_opened_handle( fn verify_current_entry( directory: &Dir, + name: &str, stage: RecoveryStage, admitted: &AdmittedStage, ) -> Result<(), FilesystemRecoveryStageError> { let file = directory - .open_with(stage.file_name(), &read_options()) + .open_with(name, &read_options()) .map_err(|source| FilesystemRecoveryStageError::VerifyEntry { stage, source })?; let metadata = file .metadata() diff --git a/src/adapters/filesystem_recovery_stage_completer.rs b/src/adapters/filesystem_recovery_stage_completer.rs new file mode 100644 index 0000000..90debae --- /dev/null +++ b/src/adapters/filesystem_recovery_stage_completer.rs @@ -0,0 +1,41 @@ +//! This module owns pinned writer authority for filesystem stage completion. + +use std::path::Path; + +use super::{FilesystemRecoveryStageCompletionOpenError, FilesystemRecoveryStageDiscarder}; + +/// Writer-authorized pinned filesystem adapter for exact stage completion. +/// +/// Opening proves the supported platform, pins and exclusively locks the store +/// root and `writer.lock`, then pins all three protocol child directories +/// without following links. The synchronous adapter may block on filesystem +/// I/O and retains writer authority until dropped. +#[must_use] +pub struct FilesystemRecoveryStageCompleter { + pub(super) discarder: FilesystemRecoveryStageDiscarder, +} + +impl FilesystemRecoveryStageCompleter { + /// Opens an initialized supported store for explicit stage completion. + /// + /// The call performs no protocol mutation. + /// + /// # Errors + /// + /// Returns [`FilesystemRecoveryStageCompletionOpenError`] on platform, + /// writer-authority, root-clone, or namespace admission failure. + pub fn open(store_root: &Path) -> Result { + FilesystemRecoveryStageDiscarder::open(store_root) + .map(|discarder| Self { discarder }) + .map_err(Into::into) + } + + #[cfg(test)] + pub(super) fn open_unchecked_for_tests( + store_root: &Path, + ) -> Result { + FilesystemRecoveryStageDiscarder::open_unchecked_for_tests(store_root) + .map(|discarder| Self { discarder }) + .map_err(Into::into) + } +} diff --git a/src/adapters/filesystem_recovery_stage_completion_open_error.rs b/src/adapters/filesystem_recovery_stage_completion_open_error.rs new file mode 100644 index 0000000..f3e3638 --- /dev/null +++ b/src/adapters/filesystem_recovery_stage_completion_open_error.rs @@ -0,0 +1,87 @@ +//! This module owns filesystem stage-completion authority acquisition failures. + +use std::error::Error; +use std::fmt; +use std::io; + +use super::{ + FilesystemRecoveryStageDiscardOpenError, RecoveryInventoryError, WriterLockAcquireError, +}; + +/// Why a pinned writer-authorized stage completer could not be opened. +#[derive(Debug)] +pub enum FilesystemRecoveryStageCompletionOpenError { + /// The store root did not satisfy the supported platform profile. + Platform { + /// Exact platform-admission failure. + source: io::Error, + }, + /// Exclusive writer authority could not be acquired. + WriterLock { + /// Exact writer-lock acquisition refusal. + source: WriterLockAcquireError, + }, + /// The locked root capability could not be cloned for recovery inventory. + CloneRoot { + /// Exact root-capability clone failure. + source: io::Error, + }, + /// One pinned protocol namespace could not be admitted. + Namespace { + /// Exact recovery-namespace admission refusal. + source: RecoveryInventoryError, + }, +} + +impl From for FilesystemRecoveryStageCompletionOpenError { + fn from(source: FilesystemRecoveryStageDiscardOpenError) -> Self { + match source { + FilesystemRecoveryStageDiscardOpenError::Platform { source } => { + Self::Platform { source } + } + FilesystemRecoveryStageDiscardOpenError::WriterLock { source } => { + Self::WriterLock { source } + } + FilesystemRecoveryStageDiscardOpenError::CloneRoot { source } => { + Self::CloneRoot { source } + } + FilesystemRecoveryStageDiscardOpenError::Namespace { source } => { + Self::Namespace { source } + } + } + } +} + +impl fmt::Display for FilesystemRecoveryStageCompletionOpenError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Platform { source } => { + write!( + formatter, + "recovery completion platform was refused: {source}" + ) + } + Self::WriterLock { source } => write!( + formatter, + "recovery completion writer lock was refused: {source}" + ), + Self::CloneRoot { source } => { + write!(formatter, "locked recovery root clone failed: {source}") + } + Self::Namespace { source } => write!( + formatter, + "recovery completion namespace was refused: {source}" + ), + } + } +} + +impl Error for FilesystemRecoveryStageCompletionOpenError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Platform { source } | Self::CloneRoot { source } => Some(source), + Self::WriterLock { source } => Some(source), + Self::Namespace { source } => Some(source), + } + } +} diff --git a/src/adapters/filesystem_recovery_stage_completion_pool.rs b/src/adapters/filesystem_recovery_stage_completion_pool.rs new file mode 100644 index 0000000..a6884d9 --- /dev/null +++ b/src/adapters/filesystem_recovery_stage_completion_pool.rs @@ -0,0 +1,184 @@ +//! This module owns filesystem immutable-pool recovery transitions. + +use std::io; + +use cap_std::fs::Dir; + +use super::{ + FilesystemRecoveryInventoryReader, FilesystemRecoveryStageCompleter, + FilesystemRecoveryStageError, RecoveryStageCompletionPool, RecoveryStageCompletionRequest, + RecoveryStageCompletionStorageError, RecoveryStageCompletionTarget, RecoveryStageEvidence, + RecoveryStageNamespacePhase, RecoveryStagePoolOutcome, filesystem_catalog_artifact, + filesystem_recovery_stage, physical_pool_name, +}; + +pub(super) fn link_or_admit( + inventory: &FilesystemRecoveryInventoryReader, + request: RecoveryStageCompletionRequest, +) -> Result { + verify_namespaces( + inventory, + request, + RecoveryStageNamespacePhase::BeforeObservation, + )?; + let stage = request.evidence().stage(); + let stage_directory = inventory.stage_directory(stage); + let pool_directory = inventory.completion_pool_directory(request.pool()); + let pool_name = pool_name(request.target()); + let outcome = if entry_is_absent(stage_directory, stage.file_name())? { + if entry_is_absent(pool_directory, &pool_name)? { + return Err(RecoveryStageCompletionStorageError::Missing { request }); + } + RecoveryStagePoolOutcome::AlreadyPresent + } else { + let observed = + filesystem_recovery_stage::fingerprint(stage_directory, stage).map_err(stage_error)?; + require_evidence(request.evidence(), observed)?; + link( + stage_directory, + stage.file_name(), + pool_directory, + &pool_name, + )? + }; + verify_namespaces( + inventory, + request, + RecoveryStageNamespacePhase::AfterObservation, + )?; + Ok(outcome) +} + +pub(super) fn verify( + inventory: &FilesystemRecoveryInventoryReader, + request: RecoveryStageCompletionRequest, +) -> Result<(), RecoveryStageCompletionStorageError> { + verify_namespaces( + inventory, + request, + RecoveryStageNamespacePhase::BeforeObservation, + )?; + let observed = filesystem_recovery_stage::fingerprint_named( + inventory.completion_pool_directory(request.pool()), + &pool_name(request.target()), + request.evidence().stage(), + ) + .map_err(stage_error)?; + require_evidence(request.evidence(), observed)?; + verify_namespaces( + inventory, + request, + RecoveryStageNamespacePhase::AfterObservation, + ) +} + +pub(super) fn synchronize( + inventory: &FilesystemRecoveryInventoryReader, + pool: RecoveryStageCompletionPool, +) -> io::Result<()> { + filesystem_catalog_artifact::synchronize_directory(inventory.completion_pool_directory(pool)) +} + +impl FilesystemRecoveryStageCompleter { + #[cfg(test)] + pub(super) fn verify_pool_with( + &self, + request: RecoveryStageCompletionRequest, + after_open: F, + ) -> Result<(), RecoveryStageCompletionStorageError> + where + F: FnOnce(), + { + verify_with(&self.discarder.inventory, request, after_open) + } +} + +#[cfg(test)] +fn verify_with( + inventory: &FilesystemRecoveryInventoryReader, + request: RecoveryStageCompletionRequest, + after_open: F, +) -> Result<(), RecoveryStageCompletionStorageError> +where + F: FnOnce(), +{ + verify_namespaces( + inventory, + request, + RecoveryStageNamespacePhase::BeforeObservation, + )?; + let observed = filesystem_recovery_stage::observe_named_with( + inventory.completion_pool_directory(request.pool()), + &pool_name(request.target()), + request.evidence().stage(), + after_open, + ) + .map_err(stage_error)? + .evidence(); + require_evidence(request.evidence(), observed)?; + verify_namespaces( + inventory, + request, + RecoveryStageNamespacePhase::AfterObservation, + ) +} + +fn require_evidence( + expected: RecoveryStageEvidence, + observed: RecoveryStageEvidence, +) -> Result<(), RecoveryStageCompletionStorageError> { + if observed == expected { + Ok(()) + } else { + Err(RecoveryStageCompletionStorageError::EvidenceMismatch { expected, observed }) + } +} + +fn link( + stage_directory: &Dir, + stage_name: &str, + pool_directory: &Dir, + pool_name: &str, +) -> Result { + match stage_directory.hard_link(stage_name, pool_directory, pool_name) { + Ok(()) => Ok(RecoveryStagePoolOutcome::Linked), + Err(source) if source.kind() == io::ErrorKind::AlreadyExists => { + Ok(RecoveryStagePoolOutcome::AlreadyPresent) + } + Err(source) => Err(RecoveryStageCompletionStorageError::storage(source)), + } +} + +fn entry_is_absent( + directory: &Dir, + name: &str, +) -> Result { + match directory.symlink_metadata(name) { + Ok(_) => Ok(false), + Err(source) if source.kind() == io::ErrorKind::NotFound => Ok(true), + Err(source) => Err(RecoveryStageCompletionStorageError::storage(source)), + } +} + +fn verify_namespaces( + inventory: &FilesystemRecoveryInventoryReader, + request: RecoveryStageCompletionRequest, + phase: RecoveryStageNamespacePhase, +) -> Result<(), RecoveryStageCompletionStorageError> { + inventory + .verify_stage_namespaces(request.evidence().stage(), phase) + .map_err(stage_error) +} + +fn pool_name(target: RecoveryStageCompletionTarget) -> String { + match target { + RecoveryStageCompletionTarget::Segment { digest } => physical_pool_name::segment(digest), + RecoveryStageCompletionTarget::Catalog { + generation, digest, .. + } => physical_pool_name::catalog(generation, digest), + } +} + +fn stage_error(source: FilesystemRecoveryStageError) -> RecoveryStageCompletionStorageError { + RecoveryStageCompletionStorageError::storage(io::Error::other(source)) +} diff --git a/src/adapters/filesystem_recovery_stage_completion_storage.rs b/src/adapters/filesystem_recovery_stage_completion_storage.rs new file mode 100644 index 0000000..fa97f79 --- /dev/null +++ b/src/adapters/filesystem_recovery_stage_completion_storage.rs @@ -0,0 +1,135 @@ +//! This module owns filesystem execution of exact stage completion. + +use std::io; + +use super::{ + FilesystemRecoveryInventoryReader, FilesystemRecoveryStageCompleter, + FilesystemRecoveryStageError, RecoveryStageCompletionPool, RecoveryStageCompletionRequest, + RecoveryStageCompletionStorage, RecoveryStageCompletionStorageError, + RecoveryStageDiscardOutcome, RecoveryStageDiscardStorageError, RecoveryStageEvidence, + RecoveryStageNamespacePhase, RecoveryStageParent, RecoveryStagePoolOutcome, + RecoveryStageSynchronizationOutcome, filesystem_catalog_artifact, + filesystem_recovery_stage_completion_pool, filesystem_recovery_stage_discard_storage, + filesystem_recovery_stage_sync, +}; + +impl RecoveryStageCompletionStorage for FilesystemRecoveryStageCompleter { + fn synchronize_stage_if_present( + &mut self, + request: RecoveryStageCompletionRequest, + ) -> Result { + let inventory = &self.discarder.inventory; + verify_before(inventory, request)?; + let outcome = filesystem_recovery_stage_sync::synchronize_if_matching( + inventory.stage_directory(request.evidence().stage()), + request.evidence(), + )?; + verify_after(inventory, request)?; + Ok(outcome) + } + + fn link_stage_or_admit_pool( + &mut self, + request: RecoveryStageCompletionRequest, + ) -> Result { + filesystem_recovery_stage_completion_pool::link_or_admit(&self.discarder.inventory, request) + } + + fn verify_pool( + &mut self, + request: RecoveryStageCompletionRequest, + ) -> Result<(), RecoveryStageCompletionStorageError> { + filesystem_recovery_stage_completion_pool::verify(&self.discarder.inventory, request) + } + + fn synchronize_pool(&mut self, pool: RecoveryStageCompletionPool) -> io::Result<()> { + filesystem_recovery_stage_completion_pool::synchronize(&self.discarder.inventory, pool) + } + + fn remove_stage_if_matching( + &mut self, + expected: RecoveryStageEvidence, + ) -> Result { + filesystem_recovery_stage_discard_storage::remove_if_matching( + &self.discarder.inventory, + expected, + ) + } + + fn synchronize_staging(&mut self) -> io::Result<()> { + filesystem_catalog_artifact::synchronize_directory( + self.discarder + .inventory + .parent_directory(RecoveryStageParent::Staging), + ) + } +} + +impl FilesystemRecoveryStageCompleter { + #[cfg(test)] + pub(super) fn synchronize_stage_if_present_with( + &self, + request: RecoveryStageCompletionRequest, + after_open: F, + ) -> Result + where + F: FnOnce(), + { + synchronize_stage_with(&self.discarder.inventory, request, after_open) + } +} + +#[cfg(test)] +fn synchronize_stage_with( + inventory: &FilesystemRecoveryInventoryReader, + request: RecoveryStageCompletionRequest, + after_open: F, +) -> Result +where + F: FnOnce(), +{ + verify_before(inventory, request)?; + let outcome = filesystem_recovery_stage_sync::synchronize_if_matching_with( + inventory.stage_directory(request.evidence().stage()), + request.evidence(), + after_open, + )?; + verify_after(inventory, request)?; + Ok(outcome) +} + +fn verify_before( + inventory: &FilesystemRecoveryInventoryReader, + request: RecoveryStageCompletionRequest, +) -> Result<(), RecoveryStageCompletionStorageError> { + verify_namespaces( + inventory, + request, + RecoveryStageNamespacePhase::BeforeObservation, + ) +} + +fn verify_after( + inventory: &FilesystemRecoveryInventoryReader, + request: RecoveryStageCompletionRequest, +) -> Result<(), RecoveryStageCompletionStorageError> { + verify_namespaces( + inventory, + request, + RecoveryStageNamespacePhase::AfterObservation, + ) +} + +fn verify_namespaces( + inventory: &FilesystemRecoveryInventoryReader, + request: RecoveryStageCompletionRequest, + phase: RecoveryStageNamespacePhase, +) -> Result<(), RecoveryStageCompletionStorageError> { + inventory + .verify_stage_namespaces(request.evidence().stage(), phase) + .map_err(stage_error) +} + +fn stage_error(source: FilesystemRecoveryStageError) -> RecoveryStageCompletionStorageError { + RecoveryStageCompletionStorageError::storage(io::Error::other(source)) +} diff --git a/src/adapters/filesystem_recovery_stage_completion_tests.rs b/src/adapters/filesystem_recovery_stage_completion_tests.rs new file mode 100644 index 0000000..0f01720 --- /dev/null +++ b/src/adapters/filesystem_recovery_stage_completion_tests.rs @@ -0,0 +1,96 @@ +//! Pinned-filesystem complete-stage recovery laws. + +use std::error::Error; +use std::fs; + +use super::{ + FilesystemRecoveryStageCompletionOpenError, RecoveryStage, RecoveryStageDiscardOutcome, + RecoveryStagePoolOutcome, RecoveryStageSynchronizationOutcome, WriterLockAcquireError, + execute_recovery_stage_completion, +}; + +mod fixture; +mod refusal_laws; +mod replacement_laws; + +use fixture::{CompletionFixture, catalog_bytes, request, segment_bytes}; + +#[test] +fn exact_segment_and_catalog_complete_to_durable_orphans() -> Result<(), Box> { + let fixture = CompletionFixture::new("filesystem-stage-completion")?; + let cases = [ + (RecoveryStage::Segment, segment_bytes()?), + (RecoveryStage::Catalog, catalog_bytes()?), + ]; + let mut completer = fixture.completer()?; + + for (stage, bytes) in cases { + let request = request(stage, &bytes)?; + fs::write(fixture.stage_path(stage), &bytes)?; + + let completed = execute_recovery_stage_completion(&mut completer, request)?; + let retried = execute_recovery_stage_completion(&mut completer, request)?; + fs::write(fixture.stage_path(stage), &bytes)?; + let reappeared = execute_recovery_stage_completion(&mut completer, request)?; + + assert_eq!( + completed.synchronization_outcome(), + RecoveryStageSynchronizationOutcome::Synchronized + ); + assert_eq!(completed.pool_outcome(), RecoveryStagePoolOutcome::Linked); + assert_eq!( + completed.stage_outcome(), + RecoveryStageDiscardOutcome::Removed + ); + assert_eq!( + retried.synchronization_outcome(), + RecoveryStageSynchronizationOutcome::AlreadyAbsent + ); + assert_eq!( + retried.pool_outcome(), + RecoveryStagePoolOutcome::AlreadyPresent + ); + assert_eq!( + retried.stage_outcome(), + RecoveryStageDiscardOutcome::AlreadyAbsent + ); + assert_eq!( + reappeared.synchronization_outcome(), + RecoveryStageSynchronizationOutcome::Synchronized + ); + assert_eq!( + reappeared.pool_outcome(), + RecoveryStagePoolOutcome::AlreadyPresent + ); + assert_eq!( + reappeared.stage_outcome(), + RecoveryStageDiscardOutcome::Removed + ); + assert_eq!(fs::read(fixture.pool_path(request))?, bytes); + assert!(!fixture.stage_path(stage).exists()); + } + drop(completer); + fixture.remove()?; + Ok(()) +} + +#[test] +fn retained_completer_authority_excludes_a_second_writer() -> Result<(), Box> { + let fixture = CompletionFixture::new("filesystem-stage-completion-lock")?; + let first = fixture.completer()?; + + let error = fixture + .completer() + .err() + .ok_or("second recovery writer acquired authority")?; + + assert!(matches!( + error, + FilesystemRecoveryStageCompletionOpenError::WriterLock { + source: WriterLockAcquireError::Busy, + } + )); + drop(first); + fixture.remove()?; + Ok(()) +} diff --git a/src/adapters/filesystem_recovery_stage_completion_tests/fixture.rs b/src/adapters/filesystem_recovery_stage_completion_tests/fixture.rs new file mode 100644 index 0000000..ee90a64 --- /dev/null +++ b/src/adapters/filesystem_recovery_stage_completion_tests/fixture.rs @@ -0,0 +1,103 @@ +//! Deterministic initialized-store fixture for filesystem stage completion. + +use std::error::Error; +use std::fs; +use std::path::{Path, PathBuf}; + +use crate::LayoutEntryLimit; + +use super::super::{ + FilesystemRecoveryStageCompleter, FilesystemRecoveryStageCompletionOpenError, RecoveryStage, + RecoveryStageCompletionRequest, RecoveryStageCompletionTarget, RecoveryStageMetadata, + SegmentReadPolicy, SegmentRecordLimit, admit_recovery_stage_bytes, assess_recovery_stage, + filesystem_test_sandbox::TestDirectory, fingerprint_recovery_stage, physical_pool_name, + plan_recovery_stage_completion, test_support::decode_hex, +}; + +const SEGMENT_HEX: &str = + include_str!("../../../conformance/segment-store/v1/one-zero-segment.hex"); +const CATALOG_HEX: &str = + include_str!("../../../conformance/segment-store/v1/one-zero-catalog.hex"); + +pub(super) fn segment_bytes() -> Result, Box> { + fixture(SEGMENT_HEX) +} + +pub(super) fn catalog_bytes() -> Result, Box> { + fixture(CATALOG_HEX) +} + +pub(super) fn request( + stage: RecoveryStage, + bytes: &[u8], +) -> Result> { + let length = u64::try_from(bytes.len())?; + let observed = fingerprint_recovery_stage(RecoveryStageMetadata::new(stage, length)?, bytes)?; + let admitted = admit_recovery_stage_bytes(stage, observed, bytes)?; + let assessed = assess_recovery_stage(&admitted, maximum_policy())?; + Ok(plan_recovery_stage_completion(&assessed)?) +} + +fn fixture(hex: &str) -> Result, Box> { + decode_hex( + hex.strip_suffix('\n') + .ok_or("recovery fixture must end in one LF")?, + ) + .map_err(Into::into) +} + +const fn maximum_policy() -> SegmentReadPolicy { + SegmentReadPolicy::new(SegmentRecordLimit::MAXIMUM, LayoutEntryLimit::MAXIMUM) +} + +pub(super) struct CompletionFixture { + directory: TestDirectory, +} + +impl CompletionFixture { + pub(super) fn new(name: &str) -> Result> { + let directory = TestDirectory::create(name)?; + fs::write(directory.path().join("writer.lock"), [])?; + for name in ["staging", "segments", "catalogs"] { + fs::create_dir(directory.path().join(name))?; + } + Ok(Self { directory }) + } + + pub(super) fn root(&self) -> &Path { + self.directory.path() + } + + pub(super) fn stage_path(&self, stage: RecoveryStage) -> PathBuf { + match stage { + RecoveryStage::Segment => self.root().join("staging/current.seg"), + RecoveryStage::Catalog => self.root().join("staging/current.cat"), + RecoveryStage::NextHead => self.root().join("head.next"), + } + } + + pub(super) fn pool_path(&self, request: RecoveryStageCompletionRequest) -> PathBuf { + match request.target() { + RecoveryStageCompletionTarget::Segment { digest } => self + .root() + .join("segments") + .join(physical_pool_name::segment(digest)), + RecoveryStageCompletionTarget::Catalog { + generation, digest, .. + } => self + .root() + .join("catalogs") + .join(physical_pool_name::catalog(generation, digest)), + } + } + + pub(super) fn completer( + &self, + ) -> Result { + FilesystemRecoveryStageCompleter::open_unchecked_for_tests(self.root()) + } + + pub(super) fn remove(self) -> std::io::Result<()> { + self.directory.remove() + } +} diff --git a/src/adapters/filesystem_recovery_stage_completion_tests/refusal_laws.rs b/src/adapters/filesystem_recovery_stage_completion_tests/refusal_laws.rs new file mode 100644 index 0000000..756d960 --- /dev/null +++ b/src/adapters/filesystem_recovery_stage_completion_tests/refusal_laws.rs @@ -0,0 +1,179 @@ +//! Filesystem complete-stage refusal and replacement laws. + +use std::error::Error; +use std::fs; + +use super::super::{ + FilesystemRecoveryStageError, RecoveryStage, RecoveryStageCompletionError, + RecoveryStageCompletionStorage, RecoveryStageCompletionStorageError, + execute_recovery_stage_completion, +}; +use super::fixture::{CompletionFixture, catalog_bytes, request, segment_bytes}; + +#[test] +fn conflicting_pool_is_refused_without_removing_the_exact_stage() -> Result<(), Box> { + let fixture = CompletionFixture::new("filesystem-stage-completion-conflict")?; + let bytes = segment_bytes()?; + let request = request(RecoveryStage::Segment, &bytes)?; + fs::write(fixture.stage_path(RecoveryStage::Segment), &bytes)?; + fs::write(fixture.pool_path(request), b"conflict")?; + let mut completer = fixture.completer()?; + + let error = execute_recovery_stage_completion(&mut completer, request) + .err() + .ok_or("conflicting pool artifact was accepted")?; + + assert!(matches!( + error, + RecoveryStageCompletionError::VerifyPool { target, .. } + if target == request.target() + )); + assert_eq!(fs::read(fixture.stage_path(RecoveryStage::Segment))?, bytes); + assert_eq!(fs::read(fixture.pool_path(request))?, b"conflict"); + drop(completer); + fixture.remove()?; + Ok(()) +} + +#[test] +fn symbolic_stage_is_never_followed_or_linked() -> Result<(), Box> { + use std::os::unix::fs::symlink; + + let fixture = CompletionFixture::new("filesystem-stage-completion-stage-link")?; + let bytes = segment_bytes()?; + let request = request(RecoveryStage::Segment, &bytes)?; + let target = fixture.root().join("outside-stage"); + fs::write(&target, &bytes)?; + symlink(&target, fixture.stage_path(RecoveryStage::Segment))?; + let mut completer = fixture.completer()?; + + let error = execute_recovery_stage_completion(&mut completer, request) + .err() + .ok_or("symbolic recovery stage was followed")?; + + assert!(matches!( + filesystem_stage_source(&error)?, + FilesystemRecoveryStageError::Open { + stage: RecoveryStage::Segment, + .. + } + )); + assert_eq!(fs::read(&target)?, bytes); + assert!(fixture.stage_path(RecoveryStage::Segment).is_symlink()); + assert!(!fixture.pool_path(request).exists()); + drop(completer); + fixture.remove()?; + Ok(()) +} + +#[test] +fn symbolic_pool_is_never_followed_or_admitted() -> Result<(), Box> { + use std::os::unix::fs::symlink; + + let fixture = CompletionFixture::new("filesystem-stage-completion-pool-link")?; + let bytes = catalog_bytes()?; + let request = request(RecoveryStage::Catalog, &bytes)?; + let target = fixture.root().join("outside-pool"); + fs::write(fixture.stage_path(RecoveryStage::Catalog), &bytes)?; + fs::write(&target, &bytes)?; + symlink(&target, fixture.pool_path(request))?; + let mut completer = fixture.completer()?; + + let error = execute_recovery_stage_completion(&mut completer, request) + .err() + .ok_or("symbolic pool artifact was followed")?; + + assert!(matches!( + filesystem_stage_source(&error)?, + FilesystemRecoveryStageError::Open { + stage: RecoveryStage::Catalog, + .. + } + )); + assert_eq!(fs::read(&target)?, bytes); + assert!(fixture.pool_path(request).is_symlink()); + assert_eq!(fs::read(fixture.stage_path(RecoveryStage::Catalog))?, bytes); + drop(completer); + fixture.remove()?; + Ok(()) +} + +#[test] +fn absent_stage_and_pool_refuse_without_creating_state() -> Result<(), Box> { + let fixture = CompletionFixture::new("filesystem-stage-completion-absent")?; + let bytes = segment_bytes()?; + let request = request(RecoveryStage::Segment, &bytes)?; + let mut completer = fixture.completer()?; + + let error = execute_recovery_stage_completion(&mut completer, request) + .err() + .ok_or("absent stage and pool produced a receipt")?; + + assert!(matches!( + error, + RecoveryStageCompletionError::LinkOrAdmit { + source: RecoveryStageCompletionStorageError::Missing { + request: missing, + }, + .. + } if missing == request + )); + assert!(!fixture.stage_path(RecoveryStage::Segment).exists()); + assert!(!fixture.pool_path(request).exists()); + drop(completer); + fixture.remove()?; + Ok(()) +} + +#[test] +fn changed_stage_is_refused_before_a_pool_link_is_created() -> Result<(), Box> { + let fixture = CompletionFixture::new("filesystem-stage-completion-link-mismatch")?; + let bytes = segment_bytes()?; + let request = request(RecoveryStage::Segment, &bytes)?; + fs::write(fixture.stage_path(RecoveryStage::Segment), b"different")?; + let mut completer = fixture.completer()?; + + let error = completer + .link_stage_or_admit_pool(request) + .err() + .ok_or("changed stage was linked into the pool")?; + + assert!(matches!( + error, + RecoveryStageCompletionStorageError::EvidenceMismatch { + expected, + observed, + } if expected == request.evidence() && observed != expected + )); + assert!(!fixture.pool_path(request).exists()); + assert_eq!( + fs::read(fixture.stage_path(RecoveryStage::Segment))?, + b"different" + ); + drop(completer); + fixture.remove()?; + Ok(()) +} + +fn filesystem_stage_source( + error: &RecoveryStageCompletionError, +) -> Result<&FilesystemRecoveryStageError, &'static str> { + let completion_source = match error { + RecoveryStageCompletionError::SynchronizeStage { source, .. } + | RecoveryStageCompletionError::VerifyPool { source, .. } => source, + _ => return Err("filesystem stage refusal lost its completion phase"), + }; + filesystem_storage_source(completion_source) +} + +pub(super) fn filesystem_storage_source( + error: &RecoveryStageCompletionStorageError, +) -> Result<&FilesystemRecoveryStageError, &'static str> { + let RecoveryStageCompletionStorageError::Storage { source } = error else { + return Err("filesystem stage refusal lost its storage boundary"); + }; + source + .get_ref() + .and_then(|source| source.downcast_ref::()) + .ok_or("completion failure lost its typed filesystem stage source") +} diff --git a/src/adapters/filesystem_recovery_stage_completion_tests/replacement_laws.rs b/src/adapters/filesystem_recovery_stage_completion_tests/replacement_laws.rs new file mode 100644 index 0000000..a155d52 --- /dev/null +++ b/src/adapters/filesystem_recovery_stage_completion_tests/replacement_laws.rs @@ -0,0 +1,70 @@ +//! Filesystem complete-stage replacement refusal laws. + +use std::error::Error; +use std::fs; + +use super::super::{FilesystemRecoveryStageError, RecoveryStage}; +use super::fixture::{CompletionFixture, catalog_bytes, request, segment_bytes}; +use super::refusal_laws::filesystem_storage_source; + +#[test] +fn replacement_after_stage_open_is_preserved_and_refused() -> Result<(), Box> { + let fixture = CompletionFixture::new("filesystem-stage-completion-stage-replaced")?; + let bytes = segment_bytes()?; + let request = request(RecoveryStage::Segment, &bytes)?; + let stage_path = fixture.stage_path(RecoveryStage::Segment); + let retained_path = fixture.root().join("retained-stage"); + fs::write(&stage_path, &bytes)?; + let completer = fixture.completer()?; + let mut hook_result = Ok(()); + + let result = completer.synchronize_stage_if_present_with(request, || { + hook_result = + fs::rename(&stage_path, &retained_path).and_then(|()| fs::write(&stage_path, b"new")); + }); + + hook_result?; + let error = result.err().ok_or("replaced stage was synchronized")?; + assert!(matches!( + filesystem_storage_source(&error)?, + FilesystemRecoveryStageError::Replaced { + stage: RecoveryStage::Segment, + } + )); + assert_eq!(fs::read(&stage_path)?, b"new"); + assert_eq!(fs::read(&retained_path)?, bytes); + drop(completer); + fixture.remove()?; + Ok(()) +} + +#[test] +fn replacement_after_pool_open_is_preserved_and_refused() -> Result<(), Box> { + let fixture = CompletionFixture::new("filesystem-stage-completion-pool-replaced")?; + let bytes = catalog_bytes()?; + let request = request(RecoveryStage::Catalog, &bytes)?; + let pool_path = fixture.pool_path(request); + let retained_path = fixture.root().join("retained-pool"); + fs::write(&pool_path, &bytes)?; + let completer = fixture.completer()?; + let mut hook_result = Ok(()); + + let result = completer.verify_pool_with(request, || { + hook_result = + fs::rename(&pool_path, &retained_path).and_then(|()| fs::write(&pool_path, b"new")); + }); + + hook_result?; + let error = result.err().ok_or("replaced pool was admitted")?; + assert!(matches!( + filesystem_storage_source(&error)?, + FilesystemRecoveryStageError::Replaced { + stage: RecoveryStage::Catalog, + } + )); + assert_eq!(fs::read(&pool_path)?, b"new"); + assert_eq!(fs::read(&retained_path)?, bytes); + drop(completer); + fixture.remove()?; + Ok(()) +} diff --git a/src/adapters/filesystem_recovery_stage_discard_storage.rs b/src/adapters/filesystem_recovery_stage_discard_storage.rs index 8bdb30d..5f9b03f 100644 --- a/src/adapters/filesystem_recovery_stage_discard_storage.rs +++ b/src/adapters/filesystem_recovery_stage_discard_storage.rs @@ -5,10 +5,11 @@ use std::io; use cap_std::fs::Dir; use super::{ - FilesystemRecoveryStageDiscarder, FilesystemRecoveryStageError, RecoveryStage, - RecoveryStageDiscardOutcome, RecoveryStageDiscardStorage, RecoveryStageDiscardStorageError, - RecoveryStageEvidence, RecoveryStageNamespacePhase, RecoveryStageParent, - filesystem_catalog_artifact, filesystem_recovery_stage, + FilesystemRecoveryInventoryReader, FilesystemRecoveryStageDiscarder, + FilesystemRecoveryStageError, RecoveryStage, RecoveryStageDiscardOutcome, + RecoveryStageDiscardStorage, RecoveryStageDiscardStorageError, RecoveryStageEvidence, + RecoveryStageNamespacePhase, RecoveryStageParent, filesystem_catalog_artifact, + filesystem_recovery_stage, }; impl RecoveryStageDiscardStorage for FilesystemRecoveryStageDiscarder { @@ -16,7 +17,11 @@ impl RecoveryStageDiscardStorage for FilesystemRecoveryStageDiscarder { &mut self, expected: RecoveryStageEvidence, ) -> Result { - remove_with(self, expected, filesystem_recovery_stage::fingerprint) + remove_with( + &self.inventory, + expected, + filesystem_recovery_stage::fingerprint, + ) } fn synchronize_parent(&mut self, parent: RecoveryStageParent) -> io::Result<()> { @@ -34,14 +39,21 @@ impl FilesystemRecoveryStageDiscarder { where F: FnOnce(), { - remove_with(self, expected, |directory, stage| { + remove_with(&self.inventory, expected, |directory, stage| { filesystem_recovery_stage::fingerprint_with(directory, stage, after_open) }) } } +pub(super) fn remove_if_matching( + inventory: &FilesystemRecoveryInventoryReader, + expected: RecoveryStageEvidence, +) -> Result { + remove_with(inventory, expected, filesystem_recovery_stage::fingerprint) +} + fn remove_with( - discarder: &FilesystemRecoveryStageDiscarder, + inventory: &FilesystemRecoveryInventoryReader, expected: RecoveryStageEvidence, observe: F, ) -> Result @@ -49,14 +61,12 @@ where F: FnOnce(&Dir, RecoveryStage) -> Result, { let stage = expected.stage(); - discarder - .inventory + inventory .verify_stage_namespaces(stage, RecoveryStageNamespacePhase::BeforeObservation) .map_err(stage_error)?; - let directory = discarder.inventory.stage_directory(stage); + let directory = inventory.stage_directory(stage); if stage_is_absent(directory, stage)? { - discarder - .inventory + inventory .verify_stage_namespaces(stage, RecoveryStageNamespacePhase::AfterObservation) .map_err(stage_error)?; return Ok(RecoveryStageDiscardOutcome::AlreadyAbsent); @@ -65,8 +75,7 @@ where if observed != expected { return Err(RecoveryStageDiscardStorageError::EvidenceMismatch { expected, observed }); } - discarder - .inventory + inventory .verify_stage_namespaces(stage, RecoveryStageNamespacePhase::AfterObservation) .map_err(stage_error)?; directory diff --git a/src/adapters/filesystem_recovery_stage_error.rs b/src/adapters/filesystem_recovery_stage_error.rs index 1305e4a..2b5915f 100644 --- a/src/adapters/filesystem_recovery_stage_error.rs +++ b/src/adapters/filesystem_recovery_stage_error.rs @@ -54,6 +54,13 @@ pub enum FilesystemRecoveryStageError { /// Exact streaming refusal. source: RecoveryStageFingerprintError, }, + /// The exact complete stage could not be synchronized. + Synchronize { + /// Fixed stage being synchronized. + stage: RecoveryStage, + /// Exact staged-file synchronization failure. + source: io::Error, + }, /// The fixed stage entry could not be reopened for identity verification. VerifyEntry { /// Fixed stage being observed. @@ -117,6 +124,12 @@ impl fmt::Display for FilesystemRecoveryStageError { formatter, "recovery stage {stage} fingerprint failed: {source}" ), + Self::Synchronize { stage, source } => { + write!( + formatter, + "failed to synchronize recovery stage {stage}: {source}" + ) + } Self::VerifyEntry { stage, source } => write!( formatter, "failed to verify recovery stage entry {stage}: {source}" @@ -152,6 +165,7 @@ impl Error for FilesystemRecoveryStageError { Self::Namespace { source, .. } => Some(source), Self::Open { source, .. } | Self::Inspect { source, .. } + | Self::Synchronize { source, .. } | Self::VerifyEntry { source, .. } => Some(source), Self::MetadataAdmission { source, .. } => Some(source), Self::Fingerprint { source, .. } => Some(source), diff --git a/src/adapters/filesystem_recovery_stage_sync.rs b/src/adapters/filesystem_recovery_stage_sync.rs new file mode 100644 index 0000000..f5bd57b --- /dev/null +++ b/src/adapters/filesystem_recovery_stage_sync.rs @@ -0,0 +1,78 @@ +//! This module owns exact recovery-stage file synchronization. + +use std::io; + +use cap_std::fs::Dir; + +use super::{ + FilesystemRecoveryStageError, RecoveryStageCompletionStorageError, RecoveryStageEvidence, + RecoveryStageSynchronizationOutcome, filesystem_recovery_stage, +}; + +pub(super) fn synchronize_if_matching( + directory: &Dir, + expected: RecoveryStageEvidence, +) -> Result { + synchronize_with(directory, expected, || {}) +} + +#[cfg(test)] +pub(super) fn synchronize_if_matching_with( + directory: &Dir, + expected: RecoveryStageEvidence, + after_open: F, +) -> Result +where + F: FnOnce(), +{ + synchronize_with(directory, expected, after_open) +} + +fn synchronize_with( + directory: &Dir, + expected: RecoveryStageEvidence, + after_open: F, +) -> Result +where + F: FnOnce(), +{ + let stage = expected.stage(); + match directory.symlink_metadata(stage.file_name()) { + Ok(_) => {} + Err(source) if source.kind() == io::ErrorKind::NotFound => { + return Ok(RecoveryStageSynchronizationOutcome::AlreadyAbsent); + } + Err(source) => { + return Err(RecoveryStageCompletionStorageError::storage(source)); + } + } + let observed = filesystem_recovery_stage::observe_named_with( + directory, + stage.file_name(), + stage, + after_open, + ) + .map_err(stage_error)?; + require_evidence(expected, observed.evidence())?; + observed + .synchronize(directory, stage.file_name(), stage) + .map_err(stage_error)?; + let verified = filesystem_recovery_stage::fingerprint(directory, stage).map_err(stage_error)?; + require_evidence(expected, verified)?; + Ok(RecoveryStageSynchronizationOutcome::Synchronized) +} + +fn require_evidence( + expected: RecoveryStageEvidence, + observed: RecoveryStageEvidence, +) -> Result<(), RecoveryStageCompletionStorageError> { + if observed == expected { + Ok(()) + } else { + Err(RecoveryStageCompletionStorageError::EvidenceMismatch { expected, observed }) + } +} + +fn stage_error(source: FilesystemRecoveryStageError) -> RecoveryStageCompletionStorageError { + RecoveryStageCompletionStorageError::storage(io::Error::other(source)) +} diff --git a/src/adapters/mod.rs b/src/adapters/mod.rs index 6442abf..f4164ea 100644 --- a/src/adapters/mod.rs +++ b/src/adapters/mod.rs @@ -86,12 +86,19 @@ mod filesystem_recovery_inventory_scan; mod filesystem_recovery_inventory_tests; mod filesystem_recovery_namespace; mod filesystem_recovery_stage; +mod filesystem_recovery_stage_completer; +mod filesystem_recovery_stage_completion_open_error; +mod filesystem_recovery_stage_completion_pool; +mod filesystem_recovery_stage_completion_storage; +#[cfg(all(test, unix))] +mod filesystem_recovery_stage_completion_tests; mod filesystem_recovery_stage_discard_open_error; mod filesystem_recovery_stage_discard_storage; #[cfg(all(test, unix))] mod filesystem_recovery_stage_discard_tests; mod filesystem_recovery_stage_discarder; mod filesystem_recovery_stage_error; +mod filesystem_recovery_stage_sync; #[cfg(all(test, unix))] mod filesystem_recovery_stage_tests; mod filesystem_segment_stage; @@ -163,6 +170,7 @@ mod recovery_stage_completion_pool; mod recovery_stage_completion_receipt; mod recovery_stage_completion_request; mod recovery_stage_completion_storage; +mod recovery_stage_completion_storage_error; mod recovery_stage_completion_target; mod recovery_stage_discard_error; mod recovery_stage_discard_executor; @@ -295,6 +303,8 @@ pub use filesystem_catalog_publisher::FilesystemCatalogPublisher; pub use filesystem_catalog_snapshot::FilesystemCatalogSnapshot; pub use filesystem_platform_admission::FilesystemPlatformAdmission; pub use filesystem_recovery_inventory_reader::FilesystemRecoveryInventoryReader; +pub use filesystem_recovery_stage_completer::FilesystemRecoveryStageCompleter; +pub use filesystem_recovery_stage_completion_open_error::FilesystemRecoveryStageCompletionOpenError; pub use filesystem_recovery_stage_discard_open_error::FilesystemRecoveryStageDiscardOpenError; pub use filesystem_recovery_stage_discarder::FilesystemRecoveryStageDiscarder; pub use filesystem_recovery_stage_error::{ @@ -347,6 +357,7 @@ pub use recovery_stage_completion_pool::RecoveryStageCompletionPool; pub use recovery_stage_completion_receipt::RecoveryStageCompletionReceipt; pub use recovery_stage_completion_request::RecoveryStageCompletionRequest; pub use recovery_stage_completion_storage::RecoveryStageCompletionStorage; +pub use recovery_stage_completion_storage_error::RecoveryStageCompletionStorageError; pub use recovery_stage_completion_target::RecoveryStageCompletionTarget; pub use recovery_stage_discard_error::RecoveryStageDiscardError; pub use recovery_stage_discard_executor::execute_recovery_stage_discard; diff --git a/src/adapters/recovery_stage_completion_error.rs b/src/adapters/recovery_stage_completion_error.rs index ede9e3b..6275664 100644 --- a/src/adapters/recovery_stage_completion_error.rs +++ b/src/adapters/recovery_stage_completion_error.rs @@ -5,8 +5,8 @@ use std::fmt; use std::io; use super::{ - RecoveryStage, RecoveryStageCompletionPool, RecoveryStageCompletionTarget, - RecoveryStageDiscardStorageError, + RecoveryStage, RecoveryStageCompletionPool, RecoveryStageCompletionStorageError, + RecoveryStageCompletionTarget, RecoveryStageDiscardStorageError, }; /// Exact failed phase of complete-stage recovery execution. @@ -17,21 +17,21 @@ pub enum RecoveryStageCompletionError { /// Fixed stage that could not be made durable. stage: RecoveryStage, /// Exact underlying verification or synchronization failure. - source: io::Error, + source: RecoveryStageCompletionStorageError, }, /// The exact stage could not be linked or an existing coordinate admitted. LinkOrAdmit { /// Validated immutable-pool target. target: RecoveryStageCompletionTarget, /// Exact underlying storage failure. - source: io::Error, + source: RecoveryStageCompletionStorageError, }, /// The immutable-pool entry did not verify exactly. VerifyPool { /// Validated immutable-pool target. target: RecoveryStageCompletionTarget, /// Exact underlying verification failure. - source: io::Error, + source: RecoveryStageCompletionStorageError, }, /// The immutable-pool directory could not be synchronized. SynchronizePool { @@ -92,9 +92,10 @@ impl Error for RecoveryStageCompletionError { match self { Self::SynchronizeStage { source, .. } | Self::LinkOrAdmit { source, .. } - | Self::VerifyPool { source, .. } - | Self::SynchronizePool { source, .. } - | Self::SynchronizeStaging { source, .. } => Some(source), + | Self::VerifyPool { source, .. } => Some(source), + Self::SynchronizePool { source, .. } | Self::SynchronizeStaging { source, .. } => { + Some(source) + } Self::RemoveStage { source } => Some(source), } } diff --git a/src/adapters/recovery_stage_completion_storage.rs b/src/adapters/recovery_stage_completion_storage.rs index 57cf140..46fbb07 100644 --- a/src/adapters/recovery_stage_completion_storage.rs +++ b/src/adapters/recovery_stage_completion_storage.rs @@ -3,7 +3,8 @@ use std::io; use super::{ - RecoveryStageCompletionPool, RecoveryStageCompletionRequest, RecoveryStageDiscardOutcome, + RecoveryStageCompletionPool, RecoveryStageCompletionRequest, + RecoveryStageCompletionStorageError, RecoveryStageDiscardOutcome, RecoveryStageDiscardStorageError, RecoveryStageEvidence, RecoveryStagePoolOutcome, RecoveryStageSynchronizationOutcome, }; @@ -29,7 +30,7 @@ pub trait RecoveryStageCompletionStorage { fn synchronize_stage_if_present( &mut self, request: RecoveryStageCompletionRequest, - ) -> io::Result; + ) -> Result; /// Links an exact stage or admits an already-present pool coordinate. /// @@ -40,14 +41,17 @@ pub trait RecoveryStageCompletionStorage { fn link_stage_or_admit_pool( &mut self, request: RecoveryStageCompletionRequest, - ) -> io::Result; + ) -> Result; /// Verifies that the selected pool entry exactly satisfies the request. /// /// # Errors /// /// Returns a source-preserving verification or storage error. - fn verify_pool(&mut self, request: RecoveryStageCompletionRequest) -> io::Result<()>; + fn verify_pool( + &mut self, + request: RecoveryStageCompletionRequest, + ) -> Result<(), RecoveryStageCompletionStorageError>; /// Synchronizes the selected immutable-pool directory. /// diff --git a/src/adapters/recovery_stage_completion_storage_error.rs b/src/adapters/recovery_stage_completion_storage_error.rs new file mode 100644 index 0000000..c4fda75 --- /dev/null +++ b/src/adapters/recovery_stage_completion_storage_error.rs @@ -0,0 +1,67 @@ +//! This module owns semantic storage refusals during stage completion. + +use std::error::Error; +use std::fmt; +use std::io; + +use super::{RecoveryStageCompletionRequest, RecoveryStageEvidence}; + +/// Why storage could not continue one exact complete-stage request. +#[derive(Debug)] +pub enum RecoveryStageCompletionStorageError { + /// A canonical stage or pool entry resolves to different evidence. + EvidenceMismatch { + /// Evidence bound into the explicit completion request. + expected: RecoveryStageEvidence, + /// Evidence observed immediately before the refused transition. + observed: RecoveryStageEvidence, + }, + /// Neither the fixed stage nor its immutable-pool coordinate exists. + Missing { + /// Exact completion request that has no recoverable artifact. + request: RecoveryStageCompletionRequest, + }, + /// The storage boundary failed while observing or mutating. + Storage { + /// Exact underlying storage failure. + source: io::Error, + }, +} + +impl RecoveryStageCompletionStorageError { + pub(super) const fn storage(source: io::Error) -> Self { + Self::Storage { source } + } +} + +impl fmt::Display for RecoveryStageCompletionStorageError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::EvidenceMismatch { expected, observed } => write!( + formatter, + "{} completion evidence changed from length {} to length {}", + expected.stage(), + expected.length().get(), + observed.length().get() + ), + Self::Missing { request } => write!( + formatter, + "{} and its {} pool coordinate are both absent", + request.evidence().stage(), + request.pool() + ), + Self::Storage { source } => { + write!(formatter, "recovery stage completion failed: {source}") + } + } + } +} + +impl Error for RecoveryStageCompletionStorageError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Storage { source } => Some(source), + Self::EvidenceMismatch { .. } | Self::Missing { .. } => None, + } + } +} diff --git a/src/lib.rs b/src/lib.rs index f945f99..f059e0b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -44,6 +44,7 @@ pub use adapters::{ ChecksummedPublicationHead, ChecksummedSegmentRecord, ClosedSegment, FilesystemCatalogPublicationError, FilesystemCatalogPublisher, FilesystemCatalogSnapshot, FilesystemPlatformAdmission, FilesystemRecoveryInventoryReader, + FilesystemRecoveryStageCompleter, FilesystemRecoveryStageCompletionOpenError, FilesystemRecoveryStageDiscardOpenError, FilesystemRecoveryStageDiscarder, FilesystemRecoveryStageError, FilesystemSegmentStage, FilesystemWriterLock, LayoutDecodeError, LayoutDecodePolicy, LayoutEncodeError, LayoutIdBinaryParseError, LayoutIdTextParseError, @@ -56,28 +57,28 @@ pub use adapters::{ RecoverySegmentStageError, RecoverySegmentTruncation, RecoveryStage, RecoveryStageAssessment, RecoveryStageAssessmentError, RecoveryStageByteAdmissionError, RecoveryStageCompletionError, RecoveryStageCompletionPlanError, RecoveryStageCompletionPool, RecoveryStageCompletionReceipt, - RecoveryStageCompletionRequest, RecoveryStageCompletionStorage, RecoveryStageCompletionTarget, - RecoveryStageDiscardError, RecoveryStageDiscardOutcome, RecoveryStageDiscardPlanError, - RecoveryStageDiscardReason, RecoveryStageDiscardReceipt, RecoveryStageDiscardRequest, - RecoveryStageDiscardStorage, RecoveryStageDiscardStorageError, RecoveryStageEvidence, - RecoveryStageFingerprint, RecoveryStageFingerprintAlgorithm, RecoveryStageFingerprintError, - RecoveryStageLength, RecoveryStageMetadata, RecoveryStageMetadataError, - RecoveryStageNamespacePhase, RecoveryStageParent, RecoveryStagePoolOutcome, - RecoveryStageSynchronizationOutcome, ReusableRecoverySegment, SealedSegment, SegmentDigest, - SegmentDurabilityPhase, SegmentHeader, SegmentHeaderError, SegmentPublication, - SegmentPublicationError, SegmentReadError, SegmentReadPolicy, SegmentRecordAdmissionError, - SegmentRecordChecksum, SegmentRecordDecodeError, SegmentRecordHeader, SegmentRecordHeaderError, - SegmentRecordIdentity, SegmentRecordLength, SegmentRecordLimit, SegmentRecordLimitError, - SegmentRecordPayloadLength, SegmentRecords, SegmentSeal, SegmentSealError, SegmentStage, - SegmentStageCreateError, SegmentWriteError, SegmentWritePhase, StagedSegment, - StorageProfileIdParseError, StoreInitializationError, StoreInitializationPhase, - StoreInitializationReceipt, StoreInitializationStorage, WriterLockAcquireError, - WriterLockAcquirePhase, admit_recovery_stage_bytes, assess_recovery_stage, - classify_recovery_catalog_stage, classify_recovery_names, classify_recovery_next_head_stage, - classify_recovery_segment_stage, execute_recovery_stage_completion, - execute_recovery_stage_discard, fingerprint_recovery_stage, initialize_store, - plan_recovery_stage_completion, plan_recovery_stage_discard, publish_catalog_generation, - read_recovery_inventory, + RecoveryStageCompletionRequest, RecoveryStageCompletionStorage, + RecoveryStageCompletionStorageError, RecoveryStageCompletionTarget, RecoveryStageDiscardError, + RecoveryStageDiscardOutcome, RecoveryStageDiscardPlanError, RecoveryStageDiscardReason, + RecoveryStageDiscardReceipt, RecoveryStageDiscardRequest, RecoveryStageDiscardStorage, + RecoveryStageDiscardStorageError, RecoveryStageEvidence, RecoveryStageFingerprint, + RecoveryStageFingerprintAlgorithm, RecoveryStageFingerprintError, RecoveryStageLength, + RecoveryStageMetadata, RecoveryStageMetadataError, RecoveryStageNamespacePhase, + RecoveryStageParent, RecoveryStagePoolOutcome, RecoveryStageSynchronizationOutcome, + ReusableRecoverySegment, SealedSegment, SegmentDigest, SegmentDurabilityPhase, SegmentHeader, + SegmentHeaderError, SegmentPublication, SegmentPublicationError, SegmentReadError, + SegmentReadPolicy, SegmentRecordAdmissionError, SegmentRecordChecksum, + SegmentRecordDecodeError, SegmentRecordHeader, SegmentRecordHeaderError, SegmentRecordIdentity, + SegmentRecordLength, SegmentRecordLimit, SegmentRecordLimitError, SegmentRecordPayloadLength, + SegmentRecords, SegmentSeal, SegmentSealError, SegmentStage, SegmentStageCreateError, + SegmentWriteError, SegmentWritePhase, StagedSegment, StorageProfileIdParseError, + StoreInitializationError, StoreInitializationPhase, StoreInitializationReceipt, + StoreInitializationStorage, WriterLockAcquireError, WriterLockAcquirePhase, + admit_recovery_stage_bytes, assess_recovery_stage, classify_recovery_catalog_stage, + classify_recovery_names, classify_recovery_next_head_stage, classify_recovery_segment_stage, + execute_recovery_stage_completion, execute_recovery_stage_discard, fingerprint_recovery_stage, + initialize_store, plan_recovery_stage_completion, plan_recovery_stage_discard, + publish_catalog_generation, read_recovery_inventory, }; pub use blob::{ BlobHashError, BlobHasher, BlobId, BlobLength, BlobReadError, ByteLength, ByteOffset, diff --git a/tests/recovery_stage_completion/storage_double.rs b/tests/recovery_stage_completion/storage_double.rs index fa2c658..7200c16 100644 --- a/tests/recovery_stage_completion/storage_double.rs +++ b/tests/recovery_stage_completion/storage_double.rs @@ -4,8 +4,9 @@ use std::io; use keep::{ RecoveryStageCompletionPool, RecoveryStageCompletionRequest, RecoveryStageCompletionStorage, - RecoveryStageDiscardOutcome, RecoveryStageDiscardStorageError, RecoveryStageEvidence, - RecoveryStagePoolOutcome, RecoveryStageSynchronizationOutcome, + RecoveryStageCompletionStorageError, RecoveryStageDiscardOutcome, + RecoveryStageDiscardStorageError, RecoveryStageEvidence, RecoveryStagePoolOutcome, + RecoveryStageSynchronizationOutcome, }; /// One semantic operation observed by the deterministic storage double. @@ -92,17 +93,21 @@ impl RecoveryStageCompletionStorage for StageCompletionDouble { fn synchronize_stage_if_present( &mut self, request: RecoveryStageCompletionRequest, - ) -> io::Result { + ) -> Result { self.operations.push(Operation::SynchronizeStage(request)); match self.stage { Some(observed) if observed == request.evidence() => { fail_once( &mut self.fail_stage_synchronizations, "injected stage synchronization failure", - )?; + ) + .map_err(|source| RecoveryStageCompletionStorageError::Storage { source })?; Ok(RecoveryStageSynchronizationOutcome::Synchronized) } - Some(_) => Err(io::Error::other("stage evidence changed")), + Some(observed) => Err(RecoveryStageCompletionStorageError::EvidenceMismatch { + expected: request.evidence(), + observed, + }), None => Ok(RecoveryStageSynchronizationOutcome::AlreadyAbsent), } } @@ -110,7 +115,7 @@ impl RecoveryStageCompletionStorage for StageCompletionDouble { fn link_stage_or_admit_pool( &mut self, request: RecoveryStageCompletionRequest, - ) -> io::Result { + ) -> Result { self.operations.push(Operation::LinkOrAdmit(request)); if self.pool.is_some() { return Ok(RecoveryStagePoolOutcome::AlreadyPresent); @@ -120,20 +125,30 @@ impl RecoveryStageCompletionStorage for StageCompletionDouble { self.pool = Some(request); Ok(RecoveryStagePoolOutcome::Linked) } - Some(_) => Err(io::Error::other("stage evidence changed")), - None => Err(io::Error::new( - io::ErrorKind::NotFound, - "stage and pool are absent", - )), + Some(observed) => Err(RecoveryStageCompletionStorageError::EvidenceMismatch { + expected: request.evidence(), + observed, + }), + None => Err(RecoveryStageCompletionStorageError::Missing { request }), } } - fn verify_pool(&mut self, request: RecoveryStageCompletionRequest) -> io::Result<()> { + fn verify_pool( + &mut self, + request: RecoveryStageCompletionRequest, + ) -> Result<(), RecoveryStageCompletionStorageError> { self.operations.push(Operation::VerifyPool(request)); if self.pool == Some(request) { Ok(()) } else { - Err(io::Error::other("pool bytes conflict with request")) + let observed = self + .pool + .map(RecoveryStageCompletionRequest::evidence) + .unwrap_or_else(|| request.evidence()); + Err(RecoveryStageCompletionStorageError::EvidenceMismatch { + expected: request.evidence(), + observed, + }) } } From a021b3095861fa2950d3a1979ab9877554bab287 Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 12:03:50 -0700 Subject: [PATCH 19/49] Add: Define exact next-head finalization --- CHANGELOG.md | 5 + README.md | 41 +++-- docs/formats/segment-store-v1/recovery.md | 15 ++ docs/formats/segment-store-v1/requirements.md | 7 +- src/adapters/catalog_publication.rs | 35 ++-- src/adapters/catalog_snapshot.rs | 12 +- src/adapters/catalog_transition.rs | 27 ++- ...esystem_recovery_stage_completion_tests.rs | 11 +- .../refusal_laws.rs | 24 +-- src/adapters/mod.rs | 22 +++ .../recovery_next_head_finalization_error.rs | 67 +++++++ ...ecovery_next_head_finalization_executor.rs | 51 ++++++ ...recovery_next_head_finalization_outcome.rs | 10 ++ ...overy_next_head_finalization_plan_error.rs | 95 ++++++++++ ...recovery_next_head_finalization_planner.rs | 95 ++++++++++ ...covery_next_head_finalization_readiness.rs | 10 ++ ...recovery_next_head_finalization_receipt.rs | 38 ++++ ...recovery_next_head_finalization_request.rs | 43 +++++ ...recovery_next_head_finalization_storage.rs | 45 +++++ ...ry_next_head_finalization_storage_error.rs | 99 ++++++++++ .../recovery_next_head_finalization_target.rs | 51 ++++++ .../recovery_stage_completion_error.rs | 8 +- .../recovery_stage_completion_executor.rs | 22 ++- src/lib.rs | 24 ++- tests/recovery_next_head_finalization.rs | 81 +++++++++ .../execution_laws.rs | 130 ++++++++++++++ .../planning_laws.rs | 169 ++++++++++++++++++ .../storage_double.rs | 124 +++++++++++++ .../storage_double.rs | 8 +- .../recovery_laws.rs | 24 ++- 30 files changed, 1301 insertions(+), 92 deletions(-) create mode 100644 src/adapters/recovery_next_head_finalization_error.rs create mode 100644 src/adapters/recovery_next_head_finalization_executor.rs create mode 100644 src/adapters/recovery_next_head_finalization_outcome.rs create mode 100644 src/adapters/recovery_next_head_finalization_plan_error.rs create mode 100644 src/adapters/recovery_next_head_finalization_planner.rs create mode 100644 src/adapters/recovery_next_head_finalization_readiness.rs create mode 100644 src/adapters/recovery_next_head_finalization_receipt.rs create mode 100644 src/adapters/recovery_next_head_finalization_request.rs create mode 100644 src/adapters/recovery_next_head_finalization_storage.rs create mode 100644 src/adapters/recovery_next_head_finalization_storage_error.rs create mode 100644 src/adapters/recovery_next_head_finalization_target.rs create mode 100644 tests/recovery_next_head_finalization.rs create mode 100644 tests/recovery_next_head_finalization/execution_laws.rs create mode 100644 tests/recovery_next_head_finalization/planning_laws.rs create mode 100644 tests/recovery_next_head_finalization/storage_double.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index b2e5084..947bc9c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -64,6 +64,11 @@ after its public API and format compatibility policies are established. boundaries, uses no-clobber immutable-pool links, never follows stage or pool links, preserves conflicting or replaced entries, verifies exact pool bytes, and accepts stage/pool, reappeared-stage, and completed pool-only retries. +- Storage-independent next-head recovery now binds a complete `head.next` + assessment to its exact transitive catalog snapshot, admits only generation + one over an uninitialized root or the exact successor of an expected current + snapshot, distinguishes ready from already-finalized retries, and returns a + receipt only after root synchronization. - Store initialization now exposes one storage-port state machine that admits the platform before mutation, opens and locks `writer.lock`, admits the three protocol directories in order, synchronizes the root, and preserves the diff --git a/README.md b/README.md index c5178f9..d0f3667 100644 --- a/README.md +++ b/README.md @@ -61,15 +61,15 @@ The reference CAS is executable evidence for M2 storage laws, not a durable backend. Its committed state is process memory; process death loses it all. The durable boundary can initialize and platform-admit a store only under the documented Linux ext4 contract. Acquiring `FilesystemWriterLock` alone cannot -construct a filesystem publisher. Leftover `head.next`, staged recovery -evidence, and ambiguous crash states remain explicit recovery work. An absent -`HEAD` is admitted for first publication only when both immutable pools are -empty. The public storage-independent recovery inventory counts all four -protocol namespaces before retaining names, applies a configurable ceiling no -greater than 2,097,152 entries, and returns duplicate-free deterministic raw -name order. `FilesystemRecoveryInventoryReader` implements that contract with -pinned, no-follow namespace capabilities and pre/post identity verification on -the admitted Linux ext4 profile. Its bounded stage-fingerprint operation opens +construct a filesystem publisher. Ambiguous crash states remain explicit +recovery work. An absent `HEAD` is admitted for first publication only when +both immutable pools are empty. The public storage-independent recovery +inventory counts all four protocol namespaces before retaining names, applies +a configurable ceiling no greater than 2,097,152 entries, and returns +duplicate-free deterministic raw name order. +`FilesystemRecoveryInventoryReader` implements that contract with pinned, +no-follow namespace capabilities and pre/post identity verification on the +admitted Linux ext4 profile. Its bounded stage-fingerprint operation opens fixed stages relative to those capabilities, refuses links and nonregular files, and verifies entry identity and length after reading. Complete caller-supplied segment-stage bytes can be classified as a reusable prefix, @@ -77,16 +77,19 @@ complete admitted segment, or exact truncation. Catalog and next-head stages likewise distinguish exact truncation from complete canonical bytes. Materialized bytes enter read-only semantic assessment only after their stage, length, and recomputed fingerprint match prior observation evidence. -Only an exact truncation assessment may form an explicit discard request; the -semantic executor refuses evidence drift and returns a receipt only after -exact removal or admitted prior absence is followed by parent synchronization. -The pinned-filesystem adapter retains writer authority, revalidates stage -evidence without following links, removes only an exact match, and -synchronizes the protocol-selected parent. Transitive publication-view -admission and filesystem-streaming classification remain planned. -Crash-injection execution, stage completion and next-head finalization, -retention, compaction, and garbage collection remain planned. Presence in the -reference CAS does not claim retention, crash recovery, or durability. + +Exact truncation assessments can authorize durable, evidence-bound discard. +Complete segment and catalog assessments can authorize verified immutable-pool +completion through `FilesystemRecoveryStageCompleter`; its receipt proves a +valid orphan, not reachability. A complete `head.next` and its transitive +`CatalogSnapshot` can authorize storage-independent finalization only when the +candidate is generation one over an uninitialized root or the exact successor +of the expected current snapshot. The executor distinguishes first +finalization from an already-finalized retry and returns only after root +synchronization. The filesystem adapter for that semantic finalization, +process-death injection, reusable-stage continuation, retention, compaction, +and garbage collection remain planned. Presence in the reference CAS does not +claim retention, crash recovery, or durability. ```rust use keep::BlobId; diff --git a/docs/formats/segment-store-v1/recovery.md b/docs/formats/segment-store-v1/recovery.md index e53992f..11679c6 100644 --- a/docs/formats/segment-store-v1/recovery.md +++ b/docs/formats/segment-store-v1/recovery.md @@ -186,6 +186,21 @@ exactly extends the verified current head. A lawful generation-1 candidate may instead extend a verified uninitialized root. Finalization reuses `KEEP-CRASH-025` and `KEEP-CRASH-026` without rewriting the candidate. +The public semantic boundary requires both a complete +`RecoveryNextHeadStage` assessment and the exact complete `CatalogSnapshot` +named by that head. `plan_recovery_next_head_finalization` refuses a mismatched +snapshot, a noninitial generation over an uninitialized root, and any +generation or predecessor other than the expected exact successor. Its owned +request retains the prior stage evidence, current-state expectation, and +candidate generation, length, and digest. + +`execute_recovery_next_head_finalization` revalidates durable current state and +the complete candidate view through its storage port. A ready candidate +atomically replaces `HEAD`; an already-finalized retry skips replacement. Both +paths synchronize the root before returning +`RecoveryNextHeadFinalizationReceipt`. Filesystem binding of this semantic +port remains planned. + A truncated, corrupt, stale, or otherwise unpublishable candidate remains invisible and blocks new publication. Recovery never rewrites a retained `head.next`. diff --git a/docs/formats/segment-store-v1/requirements.md b/docs/formats/segment-store-v1/requirements.md index ba97ebd..9a2d4a6 100644 --- a/docs/formats/segment-store-v1/requirements.md +++ b/docs/formats/segment-store-v1/requirements.md @@ -98,8 +98,10 @@ slices now bind that discard to pinned writer-authorized filesystem storage. A complete segment or catalog assessment may now authorize an owned, evidence-bound valid-orphan transition through a semantic storage port, and the filesystem completer now binds that transition to pinned writer-authorized -storage. They do not yet claim next-head finalization, transitive -publication-view admission, or process-death injection. +storage. A complete next-head assessment and exact transitive catalog snapshot +may now authorize a transition-checked finalization through a semantic storage +port. These slices do not yet claim filesystem next-head finalization or +process-death injection. @@ -121,6 +123,7 @@ publication-view admission, or process-death injection. | `KEEP-RECOVERY-014` | Filesystem discard retains root and `writer.lock` authority, pins every protocol directory, never follows a fixed-stage link, revalidates bounded fingerprint and entry identity before unlink, refuses drift without mutation, and synchronizes the typed parent after removal or admitted absence | Exact removal, absent retry, mismatch, symlink, replacement, and writer-exclusion matrix | `src/adapters/filesystem_recovery_stage_discard_tests.rs`, `src/adapters/filesystem_recovery_stage_discard_tests/fixture.rs` | Implemented in #17 | | `KEEP-RECOVERY-015` | Immutable-pool completion plans only from exact complete segment or catalog assessments, owns bounded evidence and validated coordinates, re-synchronizes an exact present stage before linking, verifies an existing pool entry before admission, synchronizes the pool before exact stage removal, synchronizes staging before receipt, accepts completed retries, and never finalizes a head | Complete-only planning, operation-order, staged-file-sync, pool-conflict, and retry matrix | `tests/recovery_stage_completion.rs`, `tests/recovery_stage_completion/*.rs` | Implemented in #17 | | `KEEP-RECOVERY-016` | Filesystem completion retains root and `writer.lock` authority, pins every protocol directory, re-synchronizes and re-fingerprints exact stage evidence before no-clobber link, never follows stage or pool links, verifies exact pool evidence before removal, preserves conflicting and replaced entries, accepts exact stage/pool, reappeared-stage, and pool-only retries, and returns only after pool and staging synchronization | Segment/catalog completion, three retry states, conflict, link, replacement, stale-evidence, missing-artifact, and writer-exclusion matrix | `src/adapters/filesystem_recovery_stage_completion_tests.rs`, `src/adapters/filesystem_recovery_stage_completion_tests/*.rs` | Implemented in #17 | +| `KEEP-RECOVERY-017` | Next-head finalization plans only from an exact complete `head.next` assessment and its matching complete transitive catalog snapshot, admits only generation one over an uninitialized root or the exact successor of an expected current snapshot, atomically replaces only a ready candidate, accepts an already-finalized retry, and returns a receipt only after root synchronization | Snapshot-coordinate, transition, operation-order, fault-stop, and post-replacement retry matrix | `tests/recovery_next_head_finalization.rs`, `tests/recovery_next_head_finalization/*.rs` | Implemented in #17 | diff --git a/src/adapters/catalog_publication.rs b/src/adapters/catalog_publication.rs index d3a9b10..4dcfb71 100644 --- a/src/adapters/catalog_publication.rs +++ b/src/adapters/catalog_publication.rs @@ -1,11 +1,12 @@ //! Preflighted catalog-generation publication orchestration. use super::catalog_publication_expectation::ExpectedCurrentCatalog; +use super::catalog_transition; use super::{ AdmittedCatalog, AdmittedSegment, CanonicalCatalog, CanonicalPublicationHead, CatalogPublicationError, CatalogPublicationExpectation, CatalogPublicationReadiness, - CatalogPublicationReceipt, CatalogPublicationStorage, CatalogTransitionError, - ChecksummedPublicationHead, SegmentPublication, catalog_publication_execution, + CatalogPublicationReceipt, CatalogPublicationStorage, ChecksummedPublicationHead, + SegmentPublication, catalog_publication_execution, }; /// Publishes one fully admitted canonical catalog generation. @@ -68,29 +69,13 @@ fn validate_transition( match expectation.current() { ExpectedCurrentCatalog::Uninitialized => validate_initial(candidate), ExpectedCurrentCatalog::Published { generation, digest } => { - let expected = - generation - .successor() - .map_err(|source| CatalogPublicationError::Transition { - source: CatalogTransitionError::GenerationExhausted { source }, - })?; - if candidate.generation() != expected { - return Err(CatalogPublicationError::Transition { - source: CatalogTransitionError::Generation { - expected, - observed: candidate.generation(), - }, - }); - } - if candidate.previous_catalog_digest() != Some(digest) { - return Err(CatalogPublicationError::Transition { - source: CatalogTransitionError::Predecessor { - expected: digest, - observed: candidate.previous_catalog_digest(), - }, - }); - } - Ok(()) + catalog_transition::validate_coordinates( + generation, + digest, + candidate.generation(), + candidate.previous_catalog_digest(), + ) + .map_err(|source| CatalogPublicationError::Transition { source }) } } } diff --git a/src/adapters/catalog_snapshot.rs b/src/adapters/catalog_snapshot.rs index a7dfa0e..98991fe 100644 --- a/src/adapters/catalog_snapshot.rs +++ b/src/adapters/catalog_snapshot.rs @@ -3,7 +3,7 @@ use super::{ AdmittedCatalog, AdmittedSegmentRecord, ChecksummedPublicationHead, SegmentRecordIdentity, }; -use crate::{CatalogDigest, CatalogGeneration}; +use crate::{CatalogDigest, CatalogGeneration, CatalogLength}; /// One complete immutable catalog generation pinned by a checksummed head. /// @@ -27,6 +27,16 @@ impl<'head, 'catalog, 'records> CatalogSnapshot<'head, 'catalog, 'records> { self.head.catalog_digest() } + /// Returns the verified catalog byte length pinned by the head. + pub const fn catalog_length(&self) -> CatalogLength { + self.catalog.length() + } + + /// Returns the verified predecessor coordinate from the admitted catalog. + pub const fn previous_catalog_digest(&self) -> Option { + self.catalog.previous_catalog_digest() + } + /// Returns the exact number of logical record bindings. #[must_use] pub const fn record_count(&self) -> u64 { diff --git a/src/adapters/catalog_transition.rs b/src/adapters/catalog_transition.rs index 064178b..883a4eb 100644 --- a/src/adapters/catalog_transition.rs +++ b/src/adapters/catalog_transition.rs @@ -1,23 +1,38 @@ //! Exact generation and predecessor transition admission. use super::{AdmittedCatalog, CatalogSuccessor, CatalogTransitionError}; +use crate::{CatalogDigest, CatalogGeneration}; pub(super) fn validate<'catalog, 'records>( current: &AdmittedCatalog<'_, '_>, candidate: AdmittedCatalog<'catalog, 'records>, ) -> Result, CatalogTransitionError> { - let expected = current - .generation() + validate_coordinates( + current.generation(), + current.digest(), + candidate.generation(), + candidate.previous_catalog_digest(), + )?; + Ok(CatalogSuccessor::new(candidate)) +} + +pub(super) fn validate_coordinates( + current_generation: CatalogGeneration, + current_digest: CatalogDigest, + candidate_generation: CatalogGeneration, + candidate_previous_digest: Option, +) -> Result<(), CatalogTransitionError> { + let expected = current_generation .successor() .map_err(|source| CatalogTransitionError::GenerationExhausted { source })?; - let observed = candidate.generation(); + let observed = candidate_generation; if observed != expected { return Err(CatalogTransitionError::Generation { expected, observed }); } - let expected = current.digest(); - let observed = candidate.previous_catalog_digest(); + let expected = current_digest; + let observed = candidate_previous_digest; if observed != Some(expected) { return Err(CatalogTransitionError::Predecessor { expected, observed }); } - Ok(CatalogSuccessor::new(candidate)) + Ok(()) } diff --git a/src/adapters/filesystem_recovery_stage_completion_tests.rs b/src/adapters/filesystem_recovery_stage_completion_tests.rs index 0f01720..5d43ddc 100644 --- a/src/adapters/filesystem_recovery_stage_completion_tests.rs +++ b/src/adapters/filesystem_recovery_stage_completion_tests.rs @@ -28,18 +28,21 @@ fn exact_segment_and_catalog_complete_to_durable_orphans() -> Result<(), Box Result<(), Box Result<(), Box Result<&FilesystemRecoveryStageError, &'static str> { - let completion_source = match error { - RecoveryStageCompletionError::SynchronizeStage { source, .. } - | RecoveryStageCompletionError::VerifyPool { source, .. } => source, - _ => return Err("filesystem stage refusal lost its completion phase"), + let (RecoveryStageCompletionError::SynchronizeStage { source, .. } + | RecoveryStageCompletionError::VerifyPool { source, .. }) = error + else { + return Err("filesystem stage refusal lost its completion phase"); }; - filesystem_storage_source(completion_source) + filesystem_storage_source(source.as_ref()) } pub(super) fn filesystem_storage_source( diff --git a/src/adapters/mod.rs b/src/adapters/mod.rs index f4164ea..a5a4abf 100644 --- a/src/adapters/mod.rs +++ b/src/adapters/mod.rs @@ -146,6 +146,17 @@ mod recovery_name_classification; mod recovery_name_classification_error; mod recovery_name_manifest; mod recovery_namespace; +mod recovery_next_head_finalization_error; +mod recovery_next_head_finalization_executor; +mod recovery_next_head_finalization_outcome; +mod recovery_next_head_finalization_plan_error; +mod recovery_next_head_finalization_planner; +mod recovery_next_head_finalization_readiness; +mod recovery_next_head_finalization_receipt; +mod recovery_next_head_finalization_request; +mod recovery_next_head_finalization_storage; +mod recovery_next_head_finalization_storage_error; +mod recovery_next_head_finalization_target; mod recovery_next_head_stage; mod recovery_next_head_stage_error; mod recovery_pool_name; @@ -332,6 +343,17 @@ pub use recovery_name_classification::classify_recovery_names; pub use recovery_name_classification_error::RecoveryNameClassificationError; pub use recovery_name_manifest::{RecoveryNameManifest, RecoveryNamedEntry}; pub use recovery_namespace::RecoveryNamespace; +pub use recovery_next_head_finalization_error::RecoveryNextHeadFinalizationError; +pub use recovery_next_head_finalization_executor::execute_recovery_next_head_finalization; +pub use recovery_next_head_finalization_outcome::RecoveryNextHeadFinalizationOutcome; +pub use recovery_next_head_finalization_plan_error::RecoveryNextHeadFinalizationPlanError; +pub use recovery_next_head_finalization_planner::plan_recovery_next_head_finalization; +pub use recovery_next_head_finalization_readiness::RecoveryNextHeadFinalizationReadiness; +pub use recovery_next_head_finalization_receipt::RecoveryNextHeadFinalizationReceipt; +pub use recovery_next_head_finalization_request::RecoveryNextHeadFinalizationRequest; +pub use recovery_next_head_finalization_storage::RecoveryNextHeadFinalizationStorage; +pub use recovery_next_head_finalization_storage_error::RecoveryNextHeadFinalizationStorageError; +pub use recovery_next_head_finalization_target::RecoveryNextHeadFinalizationTarget; pub use recovery_next_head_stage::RecoveryNextHeadStage; pub use recovery_next_head_stage_error::RecoveryNextHeadStageError; pub use recovery_pool_name_error::RecoveryPoolNameError; diff --git a/src/adapters/recovery_next_head_finalization_error.rs b/src/adapters/recovery_next_head_finalization_error.rs new file mode 100644 index 0000000..598b338 --- /dev/null +++ b/src/adapters/recovery_next_head_finalization_error.rs @@ -0,0 +1,67 @@ +//! This module owns ordered next-head finalization failures. + +use std::error::Error; +use std::fmt; +use std::io; + +use super::{ + RecoveryNextHeadFinalizationStorageError, RecoveryNextHeadFinalizationTarget, + RecoveryStageEvidence, +}; + +/// Exact failed phase of recovery next-head finalization. +#[derive(Debug)] +pub enum RecoveryNextHeadFinalizationError { + /// Durable current state or the candidate view did not verify. + Verify { + /// Exact candidate target that was refused. + target: RecoveryNextHeadFinalizationTarget, + /// Exact underlying verification failure. + source: Box, + }, + /// The exact candidate could not atomically replace durable `HEAD`. + Replace { + /// Exact `head.next` evidence that could not be finalized. + evidence: RecoveryStageEvidence, + /// Exact underlying replacement failure. + source: Box, + }, + /// The root directory could not be synchronized after replacement or retry. + SynchronizeRoot { + /// Candidate whose current state was not durably confirmed. + target: RecoveryNextHeadFinalizationTarget, + /// Exact underlying synchronization failure. + source: io::Error, + }, +} + +impl fmt::Display for RecoveryNextHeadFinalizationError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Verify { target, source } => write!( + formatter, + "failed to verify recovery head generation {}: {source}", + target.generation().get() + ), + Self::Replace { evidence, source } => write!( + formatter, + "failed to finalize {} evidence: {source}", + evidence.stage() + ), + Self::SynchronizeRoot { target, source } => write!( + formatter, + "failed to synchronize recovery head generation {}: {source}", + target.generation().get() + ), + } + } +} + +impl Error for RecoveryNextHeadFinalizationError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Verify { source, .. } | Self::Replace { source, .. } => Some(source.as_ref()), + Self::SynchronizeRoot { source, .. } => Some(source), + } + } +} diff --git a/src/adapters/recovery_next_head_finalization_executor.rs b/src/adapters/recovery_next_head_finalization_executor.rs new file mode 100644 index 0000000..a549815 --- /dev/null +++ b/src/adapters/recovery_next_head_finalization_executor.rs @@ -0,0 +1,51 @@ +//! This module owns ordered, idempotent recovery head finalization. + +use super::{ + RecoveryNextHeadFinalizationError, RecoveryNextHeadFinalizationOutcome, + RecoveryNextHeadFinalizationReadiness, RecoveryNextHeadFinalizationReceipt, + RecoveryNextHeadFinalizationRequest, RecoveryNextHeadFinalizationStorage, +}; + +/// Finalizes one exact complete recovery next head. +/// +/// A receipt is returned only after durable `HEAD` names the exact candidate +/// and the root directory has been synchronized. Retrying after replacement +/// but before directory synchronization re-admits the candidate and repeats the +/// root synchronization without replacing the head again. +/// Typed storage failures are boxed on the error path to keep the phase error +/// bounded without discarding expected or observed state. +/// +/// # Errors +/// +/// Returns [`RecoveryNextHeadFinalizationError`] without a receipt at the exact +/// verification, atomic replacement, or root synchronization phase. +pub fn execute_recovery_next_head_finalization( + storage: &mut impl RecoveryNextHeadFinalizationStorage, + request: RecoveryNextHeadFinalizationRequest, +) -> Result { + let target = request.target(); + let readiness = storage.verify_current(request).map_err(|source| { + RecoveryNextHeadFinalizationError::Verify { + target, + source: Box::new(source), + } + })?; + let outcome = match readiness { + RecoveryNextHeadFinalizationReadiness::Ready => { + storage.replace_head(request).map_err(|source| { + RecoveryNextHeadFinalizationError::Replace { + evidence: request.evidence(), + source: Box::new(source), + } + })?; + RecoveryNextHeadFinalizationOutcome::Finalized + } + RecoveryNextHeadFinalizationReadiness::AlreadyFinalized => { + RecoveryNextHeadFinalizationOutcome::AlreadyFinalized + } + }; + storage + .synchronize_root() + .map_err(|source| RecoveryNextHeadFinalizationError::SynchronizeRoot { target, source })?; + Ok(RecoveryNextHeadFinalizationReceipt::new(request, outcome)) +} diff --git a/src/adapters/recovery_next_head_finalization_outcome.rs b/src/adapters/recovery_next_head_finalization_outcome.rs new file mode 100644 index 0000000..61b0ffc --- /dev/null +++ b/src/adapters/recovery_next_head_finalization_outcome.rs @@ -0,0 +1,10 @@ +//! This module owns observable next-head finalization outcomes. + +/// How one exact recovery next head reached the durable current coordinate. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum RecoveryNextHeadFinalizationOutcome { + /// The exact candidate replaced the prior durable head. + Finalized, + /// The exact candidate was already current during retry. + AlreadyFinalized, +} diff --git a/src/adapters/recovery_next_head_finalization_plan_error.rs b/src/adapters/recovery_next_head_finalization_plan_error.rs new file mode 100644 index 0000000..3221e0f --- /dev/null +++ b/src/adapters/recovery_next_head_finalization_plan_error.rs @@ -0,0 +1,95 @@ +//! This module owns next-head finalization planning refusals. + +use std::error::Error; +use std::fmt; + +use super::{CatalogTransitionError, RecoveryStage}; +use crate::{CatalogDigest, CatalogGeneration, CatalogLength}; + +/// Why an assessed `head.next` cannot enter finalization. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum RecoveryNextHeadFinalizationPlanError { + /// The assessment belongs to a fixed stage other than `head.next`. + NotNextHead { + /// Fixed stage that requires a different recovery protocol. + stage: RecoveryStage, + }, + /// The `head.next` bytes are incomplete. + NotComplete, + /// The complete transitive snapshot does not match the assessed head. + SnapshotCoordinate { + /// Generation named by the assessed head. + expected_generation: CatalogGeneration, + /// Catalog length named by the assessed head. + expected_length: CatalogLength, + /// Catalog digest named by the assessed head. + expected_digest: CatalogDigest, + /// Generation pinned by the complete snapshot. + observed_generation: CatalogGeneration, + /// Catalog length pinned by the complete snapshot. + observed_length: CatalogLength, + /// Catalog digest pinned by the complete snapshot. + observed_digest: CatalogDigest, + }, + /// An uninitialized root can only admit generation one. + InitialGeneration { + /// Candidate generation observed in the complete snapshot. + observed: CatalogGeneration, + }, + /// The candidate is not the exact successor of the expected current head. + Transition { + /// Exact generation or predecessor refusal. + source: CatalogTransitionError, + }, +} + +impl fmt::Display for RecoveryNextHeadFinalizationPlanError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::NotNextHead { stage } => { + write!(formatter, "{stage} is not the recovery next head") + } + Self::NotComplete => formatter.write_str("recovery next head is incomplete"), + Self::SnapshotCoordinate { + expected_generation, + expected_length, + expected_digest, + observed_generation, + observed_length, + observed_digest, + } => write!( + formatter, + "recovery next-head coordinate generation {} length {} digest {:?} does not match snapshot generation {} length {} digest {:?}", + expected_generation.get(), + expected_length.get(), + expected_digest, + observed_generation.get(), + observed_length.get(), + observed_digest + ), + Self::InitialGeneration { observed } => write!( + formatter, + "uninitialized recovery requires generation 1, observed {}", + observed.get() + ), + Self::Transition { source } => { + write!( + formatter, + "recovery next head is not an exact successor: {source}" + ) + } + } + } +} + +impl Error for RecoveryNextHeadFinalizationPlanError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Transition { source } => Some(source), + Self::NotNextHead { .. } + | Self::NotComplete + | Self::SnapshotCoordinate { .. } + | Self::InitialGeneration { .. } => None, + } + } +} diff --git a/src/adapters/recovery_next_head_finalization_planner.rs b/src/adapters/recovery_next_head_finalization_planner.rs new file mode 100644 index 0000000..893579a --- /dev/null +++ b/src/adapters/recovery_next_head_finalization_planner.rs @@ -0,0 +1,95 @@ +//! This module owns complete next-head recovery planning. + +use super::catalog_publication_expectation::ExpectedCurrentCatalog; +use super::catalog_transition; +use super::{ + CatalogPublicationExpectation, CatalogSnapshot, RecoveryNextHeadFinalizationPlanError, + RecoveryNextHeadFinalizationRequest, RecoveryNextHeadFinalizationTarget, RecoveryNextHeadStage, + RecoveryStageAssessment, +}; + +/// Plans exact, transition-checked finalization of one complete `head.next`. +/// +/// The candidate snapshot must prove the complete transitive catalog view +/// named by the assessed head. The returned request owns only bounded evidence, +/// the expected current coordinate, and the validated candidate coordinate. +/// +/// # Errors +/// +/// Returns [`RecoveryNextHeadFinalizationPlanError`] unless the assessment is a +/// complete `head.next` whose snapshot is generation one over an uninitialized +/// root or the exact successor of the expected current snapshot. +pub fn plan_recovery_next_head_finalization( + assessment: &RecoveryStageAssessment<'_>, + candidate: &CatalogSnapshot<'_, '_, '_>, + expectation: CatalogPublicationExpectation, +) -> Result { + let head = match assessment { + RecoveryStageAssessment::NextHead { + state: RecoveryNextHeadStage::Complete(head), + .. + } => head, + RecoveryStageAssessment::NextHead { .. } => { + return Err(RecoveryNextHeadFinalizationPlanError::NotComplete); + } + RecoveryStageAssessment::Segment { .. } | RecoveryStageAssessment::Catalog { .. } => { + return Err(RecoveryNextHeadFinalizationPlanError::NotNextHead { + stage: assessment.evidence().stage(), + }); + } + }; + verify_snapshot(head, candidate)?; + verify_transition(expectation, candidate)?; + let target = RecoveryNextHeadFinalizationTarget::from_snapshot(candidate); + Ok(RecoveryNextHeadFinalizationRequest::new( + assessment.evidence(), + expectation, + target, + )) +} + +fn verify_snapshot( + head: &super::ChecksummedPublicationHead<'_>, + candidate: &CatalogSnapshot<'_, '_, '_>, +) -> Result<(), RecoveryNextHeadFinalizationPlanError> { + let coordinate_matches = head.generation() == candidate.generation() + && head.catalog_length() == candidate.catalog_length() + && head.catalog_digest() == candidate.catalog_digest(); + if coordinate_matches { + return Ok(()); + } + Err(RecoveryNextHeadFinalizationPlanError::SnapshotCoordinate { + expected_generation: head.generation(), + expected_length: head.catalog_length(), + expected_digest: head.catalog_digest(), + observed_generation: candidate.generation(), + observed_length: candidate.catalog_length(), + observed_digest: candidate.catalog_digest(), + }) +} + +fn verify_transition( + expectation: CatalogPublicationExpectation, + candidate: &CatalogSnapshot<'_, '_, '_>, +) -> Result<(), RecoveryNextHeadFinalizationPlanError> { + match expectation.current() { + ExpectedCurrentCatalog::Uninitialized => { + if candidate.generation().get() == 1 { + Ok(()) + } else { + Err(RecoveryNextHeadFinalizationPlanError::InitialGeneration { + observed: candidate.generation(), + }) + } + } + ExpectedCurrentCatalog::Published { generation, digest } => { + catalog_transition::validate_coordinates( + generation, + digest, + candidate.generation(), + candidate.previous_catalog_digest(), + ) + .map_err(|source| RecoveryNextHeadFinalizationPlanError::Transition { source }) + } + } +} diff --git a/src/adapters/recovery_next_head_finalization_readiness.rs b/src/adapters/recovery_next_head_finalization_readiness.rs new file mode 100644 index 0000000..5ae9232 --- /dev/null +++ b/src/adapters/recovery_next_head_finalization_readiness.rs @@ -0,0 +1,10 @@ +//! This module owns revalidated next-head finalization readiness. + +/// Durable current-state relationship to one finalization request. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum RecoveryNextHeadFinalizationReadiness { + /// Durable `HEAD` matches the request expectation and the candidate is ready. + Ready, + /// Durable `HEAD` already names the exact candidate coordinate. + AlreadyFinalized, +} diff --git a/src/adapters/recovery_next_head_finalization_receipt.rs b/src/adapters/recovery_next_head_finalization_receipt.rs new file mode 100644 index 0000000..5fd8133 --- /dev/null +++ b/src/adapters/recovery_next_head_finalization_receipt.rs @@ -0,0 +1,38 @@ +//! This module owns durable recovery head-finalization receipts. + +use super::{ + RecoveryNextHeadFinalizationOutcome, RecoveryNextHeadFinalizationRequest, + RecoveryNextHeadFinalizationTarget, RecoveryStageEvidence, +}; + +/// Proof that durable `HEAD` names one exact candidate after root synchronization. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct RecoveryNextHeadFinalizationReceipt { + request: RecoveryNextHeadFinalizationRequest, + outcome: RecoveryNextHeadFinalizationOutcome, +} + +impl RecoveryNextHeadFinalizationReceipt { + pub(super) const fn new( + request: RecoveryNextHeadFinalizationRequest, + outcome: RecoveryNextHeadFinalizationOutcome, + ) -> Self { + Self { request, outcome } + } + + /// Returns the exact `head.next` evidence bound into the request. + pub const fn evidence(self) -> RecoveryStageEvidence { + self.request.evidence() + } + + /// Returns the durable candidate catalog coordinate. + pub const fn target(self) -> RecoveryNextHeadFinalizationTarget { + self.request.target() + } + + /// Returns whether this execution replaced or re-admitted durable `HEAD`. + pub const fn outcome(self) -> RecoveryNextHeadFinalizationOutcome { + self.outcome + } +} diff --git a/src/adapters/recovery_next_head_finalization_request.rs b/src/adapters/recovery_next_head_finalization_request.rs new file mode 100644 index 0000000..73a433b --- /dev/null +++ b/src/adapters/recovery_next_head_finalization_request.rs @@ -0,0 +1,43 @@ +//! This module owns explicit requests to finalize one exact recovery next head. + +use super::{ + CatalogPublicationExpectation, RecoveryNextHeadFinalizationTarget, RecoveryStageEvidence, +}; + +/// Authorized finalization of one complete `head.next` candidate. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct RecoveryNextHeadFinalizationRequest { + evidence: RecoveryStageEvidence, + expectation: CatalogPublicationExpectation, + target: RecoveryNextHeadFinalizationTarget, +} + +impl RecoveryNextHeadFinalizationRequest { + pub(super) const fn new( + evidence: RecoveryStageEvidence, + expectation: CatalogPublicationExpectation, + target: RecoveryNextHeadFinalizationTarget, + ) -> Self { + Self { + evidence, + expectation, + target, + } + } + + /// Returns the exact `head.next` evidence that authorized the request. + pub const fn evidence(self) -> RecoveryStageEvidence { + self.evidence + } + + /// Returns the durable current-state expectation to revalidate. + pub const fn expectation(self) -> CatalogPublicationExpectation { + self.expectation + } + + /// Returns the complete candidate catalog coordinate. + pub const fn target(self) -> RecoveryNextHeadFinalizationTarget { + self.target + } +} diff --git a/src/adapters/recovery_next_head_finalization_storage.rs b/src/adapters/recovery_next_head_finalization_storage.rs new file mode 100644 index 0000000..74200e7 --- /dev/null +++ b/src/adapters/recovery_next_head_finalization_storage.rs @@ -0,0 +1,45 @@ +//! This module owns the storage port for exact next-head finalization. + +use std::io; + +use super::{ + RecoveryNextHeadFinalizationReadiness, RecoveryNextHeadFinalizationRequest, + RecoveryNextHeadFinalizationStorageError, +}; + +/// Semantic storage operations required by recovery head finalization. +/// +/// Implementations must retain writer authority throughout execution. +/// Verification must re-open named paths without following links and establish +/// either the exact expected-current/candidate relationship or an exact +/// already-finalized candidate. Replacement must be atomic. The orchestration +/// layer owns operation order and receipt timing. +pub trait RecoveryNextHeadFinalizationStorage { + /// Revalidates durable current state and the complete candidate view. + /// + /// # Errors + /// + /// Returns a source-preserving evidence, transition, verification, or + /// storage failure. + fn verify_current( + &mut self, + request: RecoveryNextHeadFinalizationRequest, + ) -> Result; + + /// Atomically replaces durable `HEAD` with the exact candidate. + /// + /// # Errors + /// + /// Returns a source-preserving evidence or storage failure. + fn replace_head( + &mut self, + request: RecoveryNextHeadFinalizationRequest, + ) -> Result<(), RecoveryNextHeadFinalizationStorageError>; + + /// Synchronizes the root directory after replacement or retry admission. + /// + /// # Errors + /// + /// Returns the exact root-directory synchronization failure. + fn synchronize_root(&mut self) -> io::Result<()>; +} diff --git a/src/adapters/recovery_next_head_finalization_storage_error.rs b/src/adapters/recovery_next_head_finalization_storage_error.rs new file mode 100644 index 0000000..2d958a1 --- /dev/null +++ b/src/adapters/recovery_next_head_finalization_storage_error.rs @@ -0,0 +1,99 @@ +//! This module owns semantic storage failures during next-head finalization. + +use std::error::Error; +use std::fmt; +use std::io; + +use super::{ + CatalogPublicationExpectation, RecoveryNextHeadFinalizationTarget, RecoveryStageEvidence, +}; + +/// Why storage could not continue one exact next-head finalization request. +#[derive(Debug)] +pub enum RecoveryNextHeadFinalizationStorageError { + /// The canonical `head.next` resolves to different evidence. + EvidenceMismatch { + /// Evidence bound into the explicit finalization request. + expected: RecoveryStageEvidence, + /// Evidence observed immediately before the refused transition. + observed: RecoveryStageEvidence, + }, + /// Durable `HEAD` is neither the expected current head nor the candidate. + CurrentMismatch { + /// Current-state coordinate bound into the request. + expected: CatalogPublicationExpectation, + /// Different valid durable head, or absence. + observed: Option, + }, + /// The complete candidate view resolves to different coordinates. + CandidateMismatch { + /// Candidate coordinate bound into the request. + expected: RecoveryNextHeadFinalizationTarget, + /// Different complete candidate coordinate. + observed: RecoveryNextHeadFinalizationTarget, + }, + /// The candidate is absent while durable `HEAD` still matches the expectation. + MissingCandidate { + /// Exact missing `head.next` evidence bound into the request. + expected: RecoveryStageEvidence, + }, + /// The storage boundary failed while observing or mutating. + Storage { + /// Exact underlying storage failure. + source: io::Error, + }, +} + +impl fmt::Display for RecoveryNextHeadFinalizationStorageError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::EvidenceMismatch { expected, observed } => write!( + formatter, + "{} finalization evidence changed from length {} to length {}", + expected.stage(), + expected.length().get(), + observed.length().get() + ), + Self::CurrentMismatch { expected, observed } => write!( + formatter, + "durable head does not match expected generation {:?} digest {:?}; observed {observed:?}", + expected.current_generation(), + expected.current_catalog_digest() + ), + Self::CandidateMismatch { expected, observed } => write!( + formatter, + "candidate generation {} length {} digest {:?} does not match generation {} length {} digest {:?}", + expected.generation().get(), + expected.length().get(), + expected.digest(), + observed.generation().get(), + observed.length().get(), + observed.digest() + ), + Self::MissingCandidate { expected } => write!( + formatter, + "{} candidate is absent for evidence length {}", + expected.stage(), + expected.length().get() + ), + Self::Storage { source } => { + write!( + formatter, + "recovery next-head finalization failed: {source}" + ) + } + } + } +} + +impl Error for RecoveryNextHeadFinalizationStorageError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Storage { source } => Some(source), + Self::EvidenceMismatch { .. } + | Self::CurrentMismatch { .. } + | Self::CandidateMismatch { .. } + | Self::MissingCandidate { .. } => None, + } + } +} diff --git a/src/adapters/recovery_next_head_finalization_target.rs b/src/adapters/recovery_next_head_finalization_target.rs new file mode 100644 index 0000000..b3e9277 --- /dev/null +++ b/src/adapters/recovery_next_head_finalization_target.rs @@ -0,0 +1,51 @@ +//! This module owns validated catalog coordinates for next-head finalization. + +use super::CatalogSnapshot; +use crate::{CatalogDigest, CatalogGeneration, CatalogLength}; + +/// Exact catalog coordinate named by a complete recovery `head.next`. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct RecoveryNextHeadFinalizationTarget { + generation: CatalogGeneration, + length: CatalogLength, + digest: CatalogDigest, +} + +impl RecoveryNextHeadFinalizationTarget { + /// Derives exact coordinates from a complete pinned catalog snapshot. + pub const fn from_snapshot(snapshot: &CatalogSnapshot<'_, '_, '_>) -> Self { + Self::new( + snapshot.generation(), + snapshot.catalog_length(), + snapshot.catalog_digest(), + ) + } + + pub(super) const fn new( + generation: CatalogGeneration, + length: CatalogLength, + digest: CatalogDigest, + ) -> Self { + Self { + generation, + length, + digest, + } + } + + /// Returns the exact candidate generation. + pub const fn generation(self) -> CatalogGeneration { + self.generation + } + + /// Returns the exact candidate catalog byte length. + pub const fn length(self) -> CatalogLength { + self.length + } + + /// Returns the exact candidate catalog digest. + pub const fn digest(self) -> CatalogDigest { + self.digest + } +} diff --git a/src/adapters/recovery_stage_completion_error.rs b/src/adapters/recovery_stage_completion_error.rs index 6275664..fc1bb1a 100644 --- a/src/adapters/recovery_stage_completion_error.rs +++ b/src/adapters/recovery_stage_completion_error.rs @@ -17,21 +17,21 @@ pub enum RecoveryStageCompletionError { /// Fixed stage that could not be made durable. stage: RecoveryStage, /// Exact underlying verification or synchronization failure. - source: RecoveryStageCompletionStorageError, + source: Box, }, /// The exact stage could not be linked or an existing coordinate admitted. LinkOrAdmit { /// Validated immutable-pool target. target: RecoveryStageCompletionTarget, /// Exact underlying storage failure. - source: RecoveryStageCompletionStorageError, + source: Box, }, /// The immutable-pool entry did not verify exactly. VerifyPool { /// Validated immutable-pool target. target: RecoveryStageCompletionTarget, /// Exact underlying verification failure. - source: RecoveryStageCompletionStorageError, + source: Box, }, /// The immutable-pool directory could not be synchronized. SynchronizePool { @@ -92,7 +92,7 @@ impl Error for RecoveryStageCompletionError { match self { Self::SynchronizeStage { source, .. } | Self::LinkOrAdmit { source, .. } - | Self::VerifyPool { source, .. } => Some(source), + | Self::VerifyPool { source, .. } => Some(source.as_ref()), Self::SynchronizePool { source, .. } | Self::SynchronizeStaging { source, .. } => { Some(source) } diff --git a/src/adapters/recovery_stage_completion_executor.rs b/src/adapters/recovery_stage_completion_executor.rs index 44a3b67..08acdfa 100644 --- a/src/adapters/recovery_stage_completion_executor.rs +++ b/src/adapters/recovery_stage_completion_executor.rs @@ -10,6 +10,8 @@ use super::{ /// The operation never creates, replaces, or finalizes a catalog head. A /// receipt is returned only after the immutable pool is synchronized, the exact /// fixed stage is absent, and staging is synchronized. +/// Typed storage failures are boxed on the error path to keep the phase error +/// bounded without discarding expected or observed state. /// /// # Errors /// @@ -22,15 +24,25 @@ pub fn execute_recovery_stage_completion( let target = request.target(); let pool = request.pool(); let stage = request.evidence().stage(); - let synchronization_outcome = storage - .synchronize_stage_if_present(request) - .map_err(|source| RecoveryStageCompletionError::SynchronizeStage { stage, source })?; + let synchronization_outcome = + storage + .synchronize_stage_if_present(request) + .map_err(|source| RecoveryStageCompletionError::SynchronizeStage { + stage, + source: Box::new(source), + })?; let pool_outcome = storage .link_stage_or_admit_pool(request) - .map_err(|source| RecoveryStageCompletionError::LinkOrAdmit { target, source })?; + .map_err(|source| RecoveryStageCompletionError::LinkOrAdmit { + target, + source: Box::new(source), + })?; storage .verify_pool(request) - .map_err(|source| RecoveryStageCompletionError::VerifyPool { target, source })?; + .map_err(|source| RecoveryStageCompletionError::VerifyPool { + target, + source: Box::new(source), + })?; storage .synchronize_pool(pool) .map_err(|source| RecoveryStageCompletionError::SynchronizePool { pool, source })?; diff --git a/src/lib.rs b/src/lib.rs index f059e0b..d62242e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -17,8 +17,10 @@ //! production initialization for the admitted Linux ext4 profile. Recovery //! inventory, name classification, bounded stage fingerprinting, exact //! truncated-stage discard, and complete-stage valid-orphan recovery are -//! explicit. Head finalization, retention, and garbage collection APIs remain -//! intentionally absent until their contracts have executable specifications. +//! explicit. Exact next-head finalization now has a storage-independent +//! contract; filesystem finalization, retention, and garbage collection APIs +//! remain intentionally absent until their contracts have executable +//! specifications. #[cfg(test)] extern crate self as keep; @@ -52,9 +54,14 @@ pub use adapters::{ RecoveryEntryNameError, RecoveryEntryRole, RecoveryInventory, RecoveryInventoryEntry, RecoveryInventoryError, RecoveryInventoryLimit, RecoveryInventoryLimitError, RecoveryInventoryOperation, RecoveryInventoryStorage, RecoveryNameClassificationError, - RecoveryNameManifest, RecoveryNamedEntry, RecoveryNamespace, RecoveryNextHeadStage, - RecoveryNextHeadStageError, RecoveryPoolNameError, RecoveryRequiredEntry, RecoverySegmentStage, - RecoverySegmentStageError, RecoverySegmentTruncation, RecoveryStage, RecoveryStageAssessment, + RecoveryNameManifest, RecoveryNamedEntry, RecoveryNamespace, RecoveryNextHeadFinalizationError, + RecoveryNextHeadFinalizationOutcome, RecoveryNextHeadFinalizationPlanError, + RecoveryNextHeadFinalizationReadiness, RecoveryNextHeadFinalizationReceipt, + RecoveryNextHeadFinalizationRequest, RecoveryNextHeadFinalizationStorage, + RecoveryNextHeadFinalizationStorageError, RecoveryNextHeadFinalizationTarget, + RecoveryNextHeadStage, RecoveryNextHeadStageError, RecoveryPoolNameError, + RecoveryRequiredEntry, RecoverySegmentStage, RecoverySegmentStageError, + RecoverySegmentTruncation, RecoveryStage, RecoveryStageAssessment, RecoveryStageAssessmentError, RecoveryStageByteAdmissionError, RecoveryStageCompletionError, RecoveryStageCompletionPlanError, RecoveryStageCompletionPool, RecoveryStageCompletionReceipt, RecoveryStageCompletionRequest, RecoveryStageCompletionStorage, @@ -76,9 +83,10 @@ pub use adapters::{ StoreInitializationStorage, WriterLockAcquireError, WriterLockAcquirePhase, admit_recovery_stage_bytes, assess_recovery_stage, classify_recovery_catalog_stage, classify_recovery_names, classify_recovery_next_head_stage, classify_recovery_segment_stage, - execute_recovery_stage_completion, execute_recovery_stage_discard, fingerprint_recovery_stage, - initialize_store, plan_recovery_stage_completion, plan_recovery_stage_discard, - publish_catalog_generation, read_recovery_inventory, + execute_recovery_next_head_finalization, execute_recovery_stage_completion, + execute_recovery_stage_discard, fingerprint_recovery_stage, initialize_store, + plan_recovery_next_head_finalization, plan_recovery_stage_completion, + plan_recovery_stage_discard, publish_catalog_generation, read_recovery_inventory, }; pub use blob::{ BlobHashError, BlobHasher, BlobId, BlobLength, BlobReadError, ByteLength, ByteOffset, diff --git a/tests/recovery_next_head_finalization.rs b/tests/recovery_next_head_finalization.rs new file mode 100644 index 0000000..22cece7 --- /dev/null +++ b/tests/recovery_next_head_finalization.rs @@ -0,0 +1,81 @@ +//! Exact next-head finalization laws. + +#[path = "recovery_next_head_finalization/execution_laws.rs"] +mod execution_laws; +#[path = "recovery_next_head_finalization/planning_laws.rs"] +mod planning_laws; +#[path = "recovery_next_head_finalization/storage_double.rs"] +pub mod storage_double; +mod support; + +use std::error::Error; + +use keep::{ + AdmittedSegment, CatalogPublicationExpectation, CatalogSnapshot, ChecksummedCatalog, + ChecksummedPublicationHead, LayoutEntryLimit, RecoveryNextHeadFinalizationRequest, + RecoveryStage, RecoveryStageAssessment, RecoveryStageEvidence, RecoveryStageMetadata, + SegmentReadPolicy, SegmentRecordLimit, admit_recovery_stage_bytes, assess_recovery_stage, + fingerprint_recovery_stage, plan_recovery_next_head_finalization, +}; +use support::decode_hex; + +const SEGMENT_HEX: &str = include_str!("../conformance/segment-store/v1/one-zero-segment.hex"); +const CATALOG_ONE_HEX: &str = include_str!("../conformance/segment-store/v1/one-zero-catalog.hex"); +const HEAD_ONE_HEX: &str = include_str!("../conformance/segment-store/v1/one-zero-head.hex"); +const CATALOG_TWO_HEX: &str = + include_str!("../conformance/segment-store/v1/one-zero-catalog-generation-two.hex"); +const HEAD_TWO_HEX: &str = + include_str!("../conformance/segment-store/v1/one-zero-head-generation-two.hex"); + +fn fixture(hex: &str) -> Result, Box> { + decode_hex( + hex.strip_suffix('\n') + .ok_or("recovery fixture must end in one LF")?, + ) + .map_err(Into::into) +} + +fn evidence(stage: RecoveryStage, encoded: &[u8]) -> Result> { + let length = u64::try_from(encoded.len())?; + Ok(fingerprint_recovery_stage( + RecoveryStageMetadata::new(stage, length)?, + encoded, + )?) +} + +fn assessment( + stage: RecoveryStage, + encoded: &[u8], +) -> Result, Box> { + let observed = evidence(stage, encoded)?; + let admitted = admit_recovery_stage_bytes(stage, observed, encoded)?; + Ok(assess_recovery_stage(&admitted, maximum_policy())?) +} + +fn snapshot<'bytes>( + head_bytes: &'bytes [u8], + catalog_bytes: &'bytes [u8], + segment_bytes: &'bytes [u8], +) -> Result, Box> { + let segments = [AdmittedSegment::decode(segment_bytes, maximum_policy())?]; + let catalog = ChecksummedCatalog::decode(catalog_bytes)?.admit(&segments)?; + Ok(ChecksummedPublicationHead::decode(head_bytes)?.admit(catalog)?) +} + +fn initial_request( + head_bytes: &[u8], + catalog_bytes: &[u8], + segment_bytes: &[u8], +) -> Result> { + let assessed = assessment(RecoveryStage::NextHead, head_bytes)?; + let candidate = snapshot(head_bytes, catalog_bytes, segment_bytes)?; + Ok(plan_recovery_next_head_finalization( + &assessed, + &candidate, + CatalogPublicationExpectation::uninitialized(), + )?) +} + +const fn maximum_policy() -> SegmentReadPolicy { + SegmentReadPolicy::new(SegmentRecordLimit::MAXIMUM, LayoutEntryLimit::MAXIMUM) +} diff --git a/tests/recovery_next_head_finalization/execution_laws.rs b/tests/recovery_next_head_finalization/execution_laws.rs new file mode 100644 index 0000000..db35380 --- /dev/null +++ b/tests/recovery_next_head_finalization/execution_laws.rs @@ -0,0 +1,130 @@ +//! Ordered and retry-safe next-head finalization laws. + +use std::error::Error; + +use keep::{ + RecoveryNextHeadFinalizationError, RecoveryNextHeadFinalizationOutcome, + RecoveryNextHeadFinalizationReadiness, execute_recovery_next_head_finalization, +}; + +use super::storage_double::{NextHeadDouble, Operation}; +use super::{CATALOG_ONE_HEX, HEAD_ONE_HEX, SEGMENT_HEX, fixture, initial_request}; + +#[test] +fn ready_candidate_replaces_head_before_root_synchronization() -> Result<(), Box> { + let segment = fixture(SEGMENT_HEX)?; + let catalog = fixture(CATALOG_ONE_HEX)?; + let head = fixture(HEAD_ONE_HEX)?; + let request = initial_request(&head, &catalog, &segment)?; + let mut storage = NextHeadDouble::new(RecoveryNextHeadFinalizationReadiness::Ready); + + let receipt = execute_recovery_next_head_finalization(&mut storage, request)?; + + assert_eq!(receipt.evidence(), request.evidence()); + assert_eq!(receipt.target(), request.target()); + assert_eq!( + receipt.outcome(), + RecoveryNextHeadFinalizationOutcome::Finalized + ); + assert_eq!( + storage.operations(), + &[ + Operation::Verify(request), + Operation::Replace(request), + Operation::SynchronizeRoot, + ] + ); + Ok(()) +} + +#[test] +fn already_current_retry_only_resynchronizes_root() -> Result<(), Box> { + let segment = fixture(SEGMENT_HEX)?; + let catalog = fixture(CATALOG_ONE_HEX)?; + let head = fixture(HEAD_ONE_HEX)?; + let request = initial_request(&head, &catalog, &segment)?; + let mut storage = NextHeadDouble::new(RecoveryNextHeadFinalizationReadiness::AlreadyFinalized); + + let receipt = execute_recovery_next_head_finalization(&mut storage, request)?; + + assert_eq!( + receipt.outcome(), + RecoveryNextHeadFinalizationOutcome::AlreadyFinalized + ); + assert_eq!( + storage.operations(), + &[Operation::Verify(request), Operation::SynchronizeRoot] + ); + Ok(()) +} + +#[test] +fn verification_and_replacement_failures_stop_later_mutation() -> Result<(), Box> { + let segment = fixture(SEGMENT_HEX)?; + let catalog = fixture(CATALOG_ONE_HEX)?; + let head = fixture(HEAD_ONE_HEX)?; + let request = initial_request(&head, &catalog, &segment)?; + let mut verify_failure = + NextHeadDouble::new(RecoveryNextHeadFinalizationReadiness::Ready).fail_next_verification(); + let mut replace_failure = + NextHeadDouble::new(RecoveryNextHeadFinalizationReadiness::Ready).fail_next_replacement(); + + let verify_error = execute_recovery_next_head_finalization(&mut verify_failure, request) + .err() + .ok_or("verification failure was ignored")?; + let replace_error = execute_recovery_next_head_finalization(&mut replace_failure, request) + .err() + .ok_or("replacement failure was ignored")?; + + assert!(matches!( + verify_error, + RecoveryNextHeadFinalizationError::Verify { .. } + )); + assert!(matches!( + replace_error, + RecoveryNextHeadFinalizationError::Replace { .. } + )); + assert_eq!(verify_failure.operations(), &[Operation::Verify(request)]); + assert_eq!( + replace_failure.operations(), + &[Operation::Verify(request), Operation::Replace(request)] + ); + Ok(()) +} + +#[test] +fn retry_after_replace_before_root_sync_is_already_finalized() -> Result<(), Box> { + let segment = fixture(SEGMENT_HEX)?; + let catalog = fixture(CATALOG_ONE_HEX)?; + let head = fixture(HEAD_ONE_HEX)?; + let request = initial_request(&head, &catalog, &segment)?; + let mut storage = NextHeadDouble::new(RecoveryNextHeadFinalizationReadiness::Ready) + .fail_next_synchronization(); + + let error = execute_recovery_next_head_finalization(&mut storage, request) + .err() + .ok_or("root synchronization failure was ignored")?; + + assert!(matches!( + error, + RecoveryNextHeadFinalizationError::SynchronizeRoot { .. } + )); + + let receipt = execute_recovery_next_head_finalization(&mut storage, request)?; + + assert_eq!( + receipt.outcome(), + RecoveryNextHeadFinalizationOutcome::AlreadyFinalized + ); + assert_eq!( + storage.operations(), + &[ + Operation::Verify(request), + Operation::Replace(request), + Operation::SynchronizeRoot, + Operation::Verify(request), + Operation::SynchronizeRoot, + ] + ); + Ok(()) +} diff --git a/tests/recovery_next_head_finalization/planning_laws.rs b/tests/recovery_next_head_finalization/planning_laws.rs new file mode 100644 index 0000000..d86c217 --- /dev/null +++ b/tests/recovery_next_head_finalization/planning_laws.rs @@ -0,0 +1,169 @@ +//! Next-head finalization planning laws. + +use std::error::Error; + +use keep::{ + CatalogPublicationExpectation, CatalogTransitionError, RecoveryNextHeadFinalizationPlanError, + RecoveryNextHeadFinalizationTarget, RecoveryStage, plan_recovery_next_head_finalization, +}; + +use super::{ + CATALOG_ONE_HEX, CATALOG_TWO_HEX, HEAD_ONE_HEX, HEAD_TWO_HEX, SEGMENT_HEX, assessment, fixture, + snapshot, +}; + +#[test] +fn generation_one_candidate_extends_an_uninitialized_root() -> Result<(), Box> { + let segment = fixture(SEGMENT_HEX)?; + let catalog = fixture(CATALOG_ONE_HEX)?; + let head = fixture(HEAD_ONE_HEX)?; + let assessed = assessment(RecoveryStage::NextHead, &head)?; + let candidate = snapshot(&head, &catalog, &segment)?; + + let request = plan_recovery_next_head_finalization( + &assessed, + &candidate, + CatalogPublicationExpectation::uninitialized(), + )?; + + assert_eq!(request.evidence(), assessed.evidence()); + assert_eq!( + request.target(), + RecoveryNextHeadFinalizationTarget::from_snapshot(&candidate) + ); + Ok(()) +} + +#[test] +fn generation_two_candidate_extends_the_exact_current_snapshot() -> Result<(), Box> { + let segment = fixture(SEGMENT_HEX)?; + let catalog_one = fixture(CATALOG_ONE_HEX)?; + let head_one = fixture(HEAD_ONE_HEX)?; + let catalog_two = fixture(CATALOG_TWO_HEX)?; + let head_two = fixture(HEAD_TWO_HEX)?; + let current = snapshot(&head_one, &catalog_one, &segment)?; + let candidate = snapshot(&head_two, &catalog_two, &segment)?; + let assessed = assessment(RecoveryStage::NextHead, &head_two)?; + let expectation = CatalogPublicationExpectation::successor_of(¤t); + + let request = plan_recovery_next_head_finalization(&assessed, &candidate, expectation)?; + + assert_eq!(request.expectation(), expectation); + assert_eq!(request.target().generation(), candidate.generation()); + assert_eq!( + candidate.previous_catalog_digest(), + expectation.current_catalog_digest() + ); + Ok(()) +} + +#[test] +fn candidate_snapshot_must_match_the_assessed_head() -> Result<(), Box> { + let segment = fixture(SEGMENT_HEX)?; + let head_one = fixture(HEAD_ONE_HEX)?; + let catalog_two = fixture(CATALOG_TWO_HEX)?; + let head_two = fixture(HEAD_TWO_HEX)?; + let assessed = assessment(RecoveryStage::NextHead, &head_one)?; + let wrong = snapshot(&head_two, &catalog_two, &segment)?; + + let error = plan_recovery_next_head_finalization( + &assessed, + &wrong, + CatalogPublicationExpectation::uninitialized(), + ) + .err() + .ok_or("mismatched candidate snapshot was accepted")?; + + assert!(matches!( + error, + RecoveryNextHeadFinalizationPlanError::SnapshotCoordinate { + expected_generation, + observed_generation, + .. + } if expected_generation.get() == 1 && observed_generation.get() == 2 + )); + Ok(()) +} + +#[test] +fn noninitial_candidate_cannot_extend_an_uninitialized_root() -> Result<(), Box> { + let segment = fixture(SEGMENT_HEX)?; + let catalog = fixture(CATALOG_TWO_HEX)?; + let head = fixture(HEAD_TWO_HEX)?; + let assessed = assessment(RecoveryStage::NextHead, &head)?; + let candidate = snapshot(&head, &catalog, &segment)?; + + let error = plan_recovery_next_head_finalization( + &assessed, + &candidate, + CatalogPublicationExpectation::uninitialized(), + ) + .err() + .ok_or("generation-two candidate extended an uninitialized root")?; + + assert!(matches!( + error, + RecoveryNextHeadFinalizationPlanError::InitialGeneration { observed } + if observed.get() == 2 + )); + Ok(()) +} + +#[test] +fn candidate_must_be_the_exact_expected_successor() -> Result<(), Box> { + let segment = fixture(SEGMENT_HEX)?; + let catalog = fixture(CATALOG_ONE_HEX)?; + let head = fixture(HEAD_ONE_HEX)?; + let current = snapshot(&head, &catalog, &segment)?; + let assessed = assessment(RecoveryStage::NextHead, &head)?; + let expectation = CatalogPublicationExpectation::successor_of(¤t); + + let error = plan_recovery_next_head_finalization(&assessed, ¤t, expectation) + .err() + .ok_or("current generation was accepted as its own successor")?; + + assert!(matches!( + error, + RecoveryNextHeadFinalizationPlanError::Transition { + source: CatalogTransitionError::Generation { expected, observed }, + } if expected.get() == 2 && observed.get() == 1 + )); + Ok(()) +} + +#[test] +fn only_a_complete_next_head_can_enter_finalization() -> Result<(), Box> { + let segment = fixture(SEGMENT_HEX)?; + let catalog = fixture(CATALOG_ONE_HEX)?; + let head = fixture(HEAD_ONE_HEX)?; + let candidate = snapshot(&head, &catalog, &segment)?; + let truncated = assessment(RecoveryStage::NextHead, &[0_u8])?; + let wrong_stage = assessment(RecoveryStage::Catalog, &[0_u8])?; + + let truncated_error = plan_recovery_next_head_finalization( + &truncated, + &candidate, + CatalogPublicationExpectation::uninitialized(), + ) + .err() + .ok_or("truncated next head entered finalization")?; + let wrong_error = plan_recovery_next_head_finalization( + &wrong_stage, + &candidate, + CatalogPublicationExpectation::uninitialized(), + ) + .err() + .ok_or("non-head stage entered finalization")?; + + assert_eq!( + truncated_error, + RecoveryNextHeadFinalizationPlanError::NotComplete + ); + assert_eq!( + wrong_error, + RecoveryNextHeadFinalizationPlanError::NotNextHead { + stage: RecoveryStage::Catalog, + } + ); + Ok(()) +} diff --git a/tests/recovery_next_head_finalization/storage_double.rs b/tests/recovery_next_head_finalization/storage_double.rs new file mode 100644 index 0000000..9fe2a49 --- /dev/null +++ b/tests/recovery_next_head_finalization/storage_double.rs @@ -0,0 +1,124 @@ +//! Deterministic storage double for next-head finalization. + +use std::io; + +use keep::{ + RecoveryNextHeadFinalizationReadiness, RecoveryNextHeadFinalizationRequest, + RecoveryNextHeadFinalizationStorage, RecoveryNextHeadFinalizationStorageError, +}; + +/// One semantic operation observed by the deterministic storage double. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum Operation { + /// Current-state and candidate-view verification. + Verify(RecoveryNextHeadFinalizationRequest), + /// Atomic candidate-head replacement. + Replace(RecoveryNextHeadFinalizationRequest), + /// Root-directory synchronization. + SynchronizeRoot, +} + +/// In-memory next-head storage with deterministic failure injection. +pub struct NextHeadDouble { + readiness: RecoveryNextHeadFinalizationReadiness, + operations: Vec, + fail_verifications: usize, + fail_replacements: usize, + fail_synchronizations: usize, +} + +impl NextHeadDouble { + /// Creates a double with the supplied current-state readiness. + pub const fn new(readiness: RecoveryNextHeadFinalizationReadiness) -> Self { + Self { + readiness, + operations: Vec::new(), + fail_verifications: 0, + fail_replacements: 0, + fail_synchronizations: 0, + } + } + + /// Configures the next verification to fail once. + #[must_use] + pub const fn fail_next_verification(mut self) -> Self { + self.fail_verifications = 1; + self + } + + /// Configures the next atomic replacement to fail once. + #[must_use] + pub const fn fail_next_replacement(mut self) -> Self { + self.fail_replacements = 1; + self + } + + /// Configures the next root synchronization to fail once. + #[must_use] + pub const fn fail_next_synchronization(mut self) -> Self { + self.fail_synchronizations = 1; + self + } + + /// Returns every semantic storage operation in call order. + pub fn operations(&self) -> &[Operation] { + &self.operations + } +} + +impl RecoveryNextHeadFinalizationStorage for NextHeadDouble { + fn verify_current( + &mut self, + request: RecoveryNextHeadFinalizationRequest, + ) -> Result + { + self.operations.push(Operation::Verify(request)); + fail_once( + &mut self.fail_verifications, + "injected current-state verification failure", + )?; + Ok(self.readiness) + } + + fn replace_head( + &mut self, + request: RecoveryNextHeadFinalizationRequest, + ) -> Result<(), RecoveryNextHeadFinalizationStorageError> { + self.operations.push(Operation::Replace(request)); + fail_once( + &mut self.fail_replacements, + "injected atomic replacement failure", + )?; + self.readiness = RecoveryNextHeadFinalizationReadiness::AlreadyFinalized; + Ok(()) + } + + fn synchronize_root(&mut self) -> io::Result<()> { + self.operations.push(Operation::SynchronizeRoot); + if self.fail_synchronizations == 0 { + return Ok(()); + } + self.fail_synchronizations = self + .fail_synchronizations + .checked_sub(1) + .ok_or_else(|| io::Error::other("synchronization counter underflow"))?; + Err(io::Error::other("injected root synchronization failure")) + } +} + +fn fail_once( + remaining: &mut usize, + message: &'static str, +) -> Result<(), RecoveryNextHeadFinalizationStorageError> { + if *remaining == 0 { + return Ok(()); + } + *remaining = remaining.checked_sub(1).ok_or_else(|| { + RecoveryNextHeadFinalizationStorageError::Storage { + source: io::Error::other("failure counter underflow"), + } + })?; + Err(RecoveryNextHeadFinalizationStorageError::Storage { + source: io::Error::other(message), + }) +} diff --git a/tests/recovery_stage_completion/storage_double.rs b/tests/recovery_stage_completion/storage_double.rs index 7200c16..e0592ec 100644 --- a/tests/recovery_stage_completion/storage_double.rs +++ b/tests/recovery_stage_completion/storage_double.rs @@ -141,10 +141,10 @@ impl RecoveryStageCompletionStorage for StageCompletionDouble { if self.pool == Some(request) { Ok(()) } else { - let observed = self - .pool - .map(RecoveryStageCompletionRequest::evidence) - .unwrap_or_else(|| request.evidence()); + let observed = self.pool.map_or_else( + || request.evidence(), + RecoveryStageCompletionRequest::evidence, + ); Err(RecoveryStageCompletionStorageError::EvidenceMismatch { expected: request.evidence(), observed, diff --git a/xtask/tests/segment_store_protocol_contract/recovery_laws.rs b/xtask/tests/segment_store_protocol_contract/recovery_laws.rs index d33915b..10a2e5f 100644 --- a/xtask/tests/segment_store_protocol_contract/recovery_laws.rs +++ b/xtask/tests/segment_store_protocol_contract/recovery_laws.rs @@ -40,9 +40,9 @@ fn durable_stage_recovery_can_complete_only_immutable_pool_publication() { "## Complete a durable stage", "reverifies and resynchronizes the complete staged\n\ artifact", - "reuses `KEEP-CRASH-009`–`012`", + "reuses `KEEP-CRASH-008`–`012`", "catalog completion reuses\n\ - `KEEP-CRASH-017`–`020`", + `KEEP-CRASH-016`–`020`", "returns a\n\ valid-orphan receipt", "never creates or finalizes a publication head", @@ -54,6 +54,26 @@ fn durable_stage_recovery_can_complete_only_immutable_pool_publication() { } } +#[test] +fn next_head_finalization_requires_one_exact_transition_and_durable_receipt() { + for required in [ + "## Leftover next head", + "complete transitive catalog view", + "`plan_recovery_next_head_finalization` refuses a mismatched\n\ + snapshot", + "generation one over an uninitialized root", + "expected exact successor", + "`execute_recovery_next_head_finalization` revalidates durable current state", + "an already-finalized retry skips replacement", + "`RecoveryNextHeadFinalizationReceipt`", + ] { + assert!( + SPECIFICATION.contains(required), + "missing next-head finalization law: {required}" + ); + } +} + #[test] fn truncated_stage_discard_fingerprint_has_one_preimage() { assert!( From d6dbd1bc18ef062beca1441dddfcc0d1d74e7118 Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 12:31:18 -0700 Subject: [PATCH 20/49] Add: Finalize recovery heads on filesystem --- CHANGELOG.md | 10 +- README.md | 8 +- docs/formats/segment-store-v1/recovery.md | 20 +- docs/formats/segment-store-v1/requirements.md | 8 +- ...filesystem_recovery_next_head_candidate.rs | 112 ++++++++++ ...overy_next_head_finalization_open_error.rs | 89 ++++++++ ...recovery_next_head_finalization_storage.rs | 141 +++++++++++++ ...m_recovery_next_head_finalization_tests.rs | 65 ++++++ .../fixture.rs | 185 +++++++++++++++++ .../namespace_laws.rs | 44 ++++ .../refusal_laws.rs | 196 ++++++++++++++++++ .../replacement_laws.rs | 55 +++++ ...filesystem_recovery_next_head_finalizer.rs | 58 ++++++ src/adapters/mod.rs | 8 + .../recovery_next_head_finalization_error.rs | 16 +- ...ecovery_next_head_finalization_executor.rs | 9 +- ...recovery_next_head_finalization_storage.rs | 14 ++ ...ry_next_head_finalization_storage_error.rs | 43 +++- src/lib.rs | 7 +- .../execution_laws.rs | 35 +++- .../storage_double.rs | 23 ++ .../recovery_laws.rs | 4 +- 22 files changed, 1128 insertions(+), 22 deletions(-) create mode 100644 src/adapters/filesystem_recovery_next_head_candidate.rs create mode 100644 src/adapters/filesystem_recovery_next_head_finalization_open_error.rs create mode 100644 src/adapters/filesystem_recovery_next_head_finalization_storage.rs create mode 100644 src/adapters/filesystem_recovery_next_head_finalization_tests.rs create mode 100644 src/adapters/filesystem_recovery_next_head_finalization_tests/fixture.rs create mode 100644 src/adapters/filesystem_recovery_next_head_finalization_tests/namespace_laws.rs create mode 100644 src/adapters/filesystem_recovery_next_head_finalization_tests/refusal_laws.rs create mode 100644 src/adapters/filesystem_recovery_next_head_finalization_tests/replacement_laws.rs create mode 100644 src/adapters/filesystem_recovery_next_head_finalizer.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 947bc9c..55e2d96 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -67,8 +67,14 @@ after its public API and format compatibility policies are established. - Storage-independent next-head recovery now binds a complete `head.next` assessment to its exact transitive catalog snapshot, admits only generation one over an uninitialized root or the exact successor of an expected current - snapshot, distinguishes ready from already-finalized retries, and returns a - receipt only after root synchronization. + snapshot, synchronizes a ready candidate before replacement, distinguishes + ready from already-finalized retries, and returns a receipt only after root + synchronization. +- Filesystem next-head recovery now retains pinned root and writer authority, + reconstructs complete current and candidate views under exact namespace and + stage evidence, synchronizes and reverifies the candidate before atomic + replacement, refuses reappeared candidates on retry, and returns only after + root synchronization. - Store initialization now exposes one storage-port state machine that admits the platform before mutation, opens and locks `writer.lock`, admits the three protocol directories in order, synchronizes the root, and preserves the diff --git a/README.md b/README.md index d0f3667..5afce36 100644 --- a/README.md +++ b/README.md @@ -86,8 +86,12 @@ valid orphan, not reachability. A complete `head.next` and its transitive candidate is generation one over an uninitialized root or the exact successor of the expected current snapshot. The executor distinguishes first finalization from an already-finalized retry and returns only after root -synchronization. The filesystem adapter for that semantic finalization, -process-death injection, reusable-stage continuation, retention, compaction, +synchronization. `FilesystemRecoveryNextHeadFinalizer` retains pinned writer +authority, reconstructs the complete current and candidate views without +following links, verifies namespace and stage identity, synchronizes and +reverifies the exact candidate, atomically replaces `HEAD`, and synchronizes +the root. An already-finalized retry requires `head.next` to be absent. +Process-death injection, reusable-stage continuation, retention, compaction, and garbage collection remain planned. Presence in the reference CAS does not claim retention, crash recovery, or durability. diff --git a/docs/formats/segment-store-v1/recovery.md b/docs/formats/segment-store-v1/recovery.md index 11679c6..5bde39c 100644 --- a/docs/formats/segment-store-v1/recovery.md +++ b/docs/formats/segment-store-v1/recovery.md @@ -184,7 +184,7 @@ its exact 128-byte grammar, checksum, generation, catalog length and digest, and complete transitive catalog view. Recovery finalizes it only when it exactly extends the verified current head. A lawful generation-1 candidate may instead extend a verified uninitialized root. Finalization reuses -`KEEP-CRASH-025` and `KEEP-CRASH-026` without rewriting the candidate. +`KEEP-CRASH-024`–`KEEP-CRASH-026` without rewriting the candidate. The public semantic boundary requires both a complete `RecoveryNextHeadStage` assessment and the exact complete `CatalogSnapshot` @@ -195,11 +195,19 @@ request retains the prior stage evidence, current-state expectation, and candidate generation, length, and digest. `execute_recovery_next_head_finalization` revalidates durable current state and -the complete candidate view through its storage port. A ready candidate -atomically replaces `HEAD`; an already-finalized retry skips replacement. Both -paths synchronize the root before returning -`RecoveryNextHeadFinalizationReceipt`. Filesystem binding of this semantic -port remains planned. +the complete candidate view through its storage port. A ready candidate is +synchronized and reverified before it atomically replaces `HEAD`; an +already-finalized retry requires `head.next` to be absent and skips replacement. +Both paths synchronize the root before returning +`RecoveryNextHeadFinalizationReceipt`. + +`FilesystemRecoveryNextHeadFinalizer` binds this port to pinned root, writer, +staging, segment-pool, and catalog-pool capabilities. It revalidates namespace +identity around bounded no-follow loads, binds `head.next` to the request +fingerprint before and after candidate synchronization, reconstructs the +complete transitive current and candidate views, refuses a reappeared candidate +after finalization, and revalidates the complete transition at the replacement +boundary. A truncated, corrupt, stale, or otherwise unpublishable candidate remains invisible and blocks new publication. diff --git a/docs/formats/segment-store-v1/requirements.md b/docs/formats/segment-store-v1/requirements.md index 9a2d4a6..fe52c11 100644 --- a/docs/formats/segment-store-v1/requirements.md +++ b/docs/formats/segment-store-v1/requirements.md @@ -100,8 +100,9 @@ evidence-bound valid-orphan transition through a semantic storage port, and the filesystem completer now binds that transition to pinned writer-authorized storage. A complete next-head assessment and exact transitive catalog snapshot may now authorize a transition-checked finalization through a semantic storage -port. These slices do not yet claim filesystem next-head finalization or -process-death injection. +port, and the filesystem finalizer binds that transition to pinned +writer-authorized storage. These slices do not yet claim reusable-stage +continuation or process-death injection. @@ -123,7 +124,8 @@ process-death injection. | `KEEP-RECOVERY-014` | Filesystem discard retains root and `writer.lock` authority, pins every protocol directory, never follows a fixed-stage link, revalidates bounded fingerprint and entry identity before unlink, refuses drift without mutation, and synchronizes the typed parent after removal or admitted absence | Exact removal, absent retry, mismatch, symlink, replacement, and writer-exclusion matrix | `src/adapters/filesystem_recovery_stage_discard_tests.rs`, `src/adapters/filesystem_recovery_stage_discard_tests/fixture.rs` | Implemented in #17 | | `KEEP-RECOVERY-015` | Immutable-pool completion plans only from exact complete segment or catalog assessments, owns bounded evidence and validated coordinates, re-synchronizes an exact present stage before linking, verifies an existing pool entry before admission, synchronizes the pool before exact stage removal, synchronizes staging before receipt, accepts completed retries, and never finalizes a head | Complete-only planning, operation-order, staged-file-sync, pool-conflict, and retry matrix | `tests/recovery_stage_completion.rs`, `tests/recovery_stage_completion/*.rs` | Implemented in #17 | | `KEEP-RECOVERY-016` | Filesystem completion retains root and `writer.lock` authority, pins every protocol directory, re-synchronizes and re-fingerprints exact stage evidence before no-clobber link, never follows stage or pool links, verifies exact pool evidence before removal, preserves conflicting and replaced entries, accepts exact stage/pool, reappeared-stage, and pool-only retries, and returns only after pool and staging synchronization | Segment/catalog completion, three retry states, conflict, link, replacement, stale-evidence, missing-artifact, and writer-exclusion matrix | `src/adapters/filesystem_recovery_stage_completion_tests.rs`, `src/adapters/filesystem_recovery_stage_completion_tests/*.rs` | Implemented in #17 | -| `KEEP-RECOVERY-017` | Next-head finalization plans only from an exact complete `head.next` assessment and its matching complete transitive catalog snapshot, admits only generation one over an uninitialized root or the exact successor of an expected current snapshot, atomically replaces only a ready candidate, accepts an already-finalized retry, and returns a receipt only after root synchronization | Snapshot-coordinate, transition, operation-order, fault-stop, and post-replacement retry matrix | `tests/recovery_next_head_finalization.rs`, `tests/recovery_next_head_finalization/*.rs` | Implemented in #17 | +| `KEEP-RECOVERY-017` | Next-head finalization plans only from an exact complete `head.next` assessment and its matching complete transitive catalog snapshot, admits only generation one over an uninitialized root or the exact successor of an expected current snapshot, synchronizes a ready candidate before atomic replacement, accepts an already-finalized retry, and returns a receipt only after root synchronization | Snapshot-coordinate, transition, candidate-sync, operation-order, fault-stop, and post-replacement retry matrix | `tests/recovery_next_head_finalization.rs`, `tests/recovery_next_head_finalization/*.rs` | Implemented in #17 | +| `KEEP-RECOVERY-018` | Filesystem next-head finalization retains root and `writer.lock` authority, pins every protocol directory, revalidates namespace identity and exact stage evidence around bounded complete current and candidate loads, synchronizes and reverifies the exact candidate before atomic replacement, refuses current drift, missing or reappeared candidates, links, corrupt transitive views, and namespace replacement, and returns only after root synchronization | Initial and successor finalization, exact retry, candidate-sync, evidence-drift, missing, link, corruption, namespace-replacement, current-drift, and writer-exclusion matrix | `src/adapters/filesystem_recovery_next_head_finalization_tests.rs`, `src/adapters/filesystem_recovery_next_head_finalization_tests/*.rs` | Implemented in #17 | diff --git a/src/adapters/filesystem_recovery_next_head_candidate.rs b/src/adapters/filesystem_recovery_next_head_candidate.rs new file mode 100644 index 0000000..14641c4 --- /dev/null +++ b/src/adapters/filesystem_recovery_next_head_candidate.rs @@ -0,0 +1,112 @@ +//! This module owns exact filesystem recovery-candidate revalidation. + +use std::io; + +use super::{ + FilesystemRecoveryNextHeadFinalizer, FilesystemRecoveryStageError, + RecoveryNextHeadFinalizationRequest, RecoveryNextHeadFinalizationStorageError, + RecoveryNextHeadFinalizationTarget, RecoveryStage, RecoveryStageEvidence, + RecoveryStageNamespacePhase, catalog_restart_loader, filesystem_recovery_stage, +}; + +const NEXT_HEAD: &str = "head.next"; + +pub(super) fn verify( + finalizer: &FilesystemRecoveryNextHeadFinalizer, + request: RecoveryNextHeadFinalizationRequest, +) -> Result<(), RecoveryNextHeadFinalizationStorageError> { + verify_evidence(finalizer, request)?; + let loaded = + catalog_restart_loader::load_from_directory(finalizer.root(), NEXT_HEAD, finalizer.policy) + .map_err( + |source| RecoveryNextHeadFinalizationStorageError::CandidateView { + source: Box::new(source), + }, + )?; + let snapshot = loaded.snapshot().map_err(|source| { + RecoveryNextHeadFinalizationStorageError::CandidateView { + source: Box::new(source), + } + })?; + let observed = RecoveryNextHeadFinalizationTarget::from_snapshot(&snapshot); + if observed != request.target() { + return Err( + RecoveryNextHeadFinalizationStorageError::CandidateMismatch { + expected: request.target(), + observed, + }, + ); + } + verify_evidence(finalizer, request) +} + +pub(super) fn synchronize( + finalizer: &FilesystemRecoveryNextHeadFinalizer, + request: RecoveryNextHeadFinalizationRequest, +) -> Result<(), RecoveryNextHeadFinalizationStorageError> { + verify_namespaces(finalizer, RecoveryStageNamespacePhase::BeforeObservation)?; + let stage = RecoveryStage::NextHead; + let observed = filesystem_recovery_stage::observe_named(finalizer.root(), NEXT_HEAD, stage) + .map_err(|source| map_stage(source, request.evidence()))?; + require_evidence(request.evidence(), observed.evidence())?; + observed + .synchronize(finalizer.root(), NEXT_HEAD, stage) + .map_err(|source| map_stage(source, request.evidence()))?; + verify_namespaces(finalizer, RecoveryStageNamespacePhase::AfterObservation)?; + verify(finalizer, request) +} + +pub(super) fn verify_namespaces( + finalizer: &FilesystemRecoveryNextHeadFinalizer, + phase: RecoveryStageNamespacePhase, +) -> Result<(), RecoveryNextHeadFinalizationStorageError> { + finalizer + .discarder + .inventory + .verify_stage_namespaces(RecoveryStage::NextHead, phase) + .map_err(stage) +} + +fn verify_evidence( + finalizer: &FilesystemRecoveryNextHeadFinalizer, + request: RecoveryNextHeadFinalizationRequest, +) -> Result<(), RecoveryNextHeadFinalizationStorageError> { + let observed = finalizer + .discarder + .inventory + .fingerprint_stage(RecoveryStage::NextHead) + .map_err(|source| map_stage(source, request.evidence()))?; + let expected = request.evidence(); + require_evidence(expected, observed) +} + +fn require_evidence( + expected: RecoveryStageEvidence, + observed: RecoveryStageEvidence, +) -> Result<(), RecoveryNextHeadFinalizationStorageError> { + if observed == expected { + Ok(()) + } else { + Err(RecoveryNextHeadFinalizationStorageError::EvidenceMismatch { expected, observed }) + } +} + +fn map_stage( + source: FilesystemRecoveryStageError, + expected: RecoveryStageEvidence, +) -> RecoveryNextHeadFinalizationStorageError { + match source { + FilesystemRecoveryStageError::Open { source, .. } + if source.kind() == io::ErrorKind::NotFound => + { + RecoveryNextHeadFinalizationStorageError::MissingCandidate { expected } + } + source => stage(source), + } +} + +fn stage(source: FilesystemRecoveryStageError) -> RecoveryNextHeadFinalizationStorageError { + RecoveryNextHeadFinalizationStorageError::Stage { + source: Box::new(source), + } +} diff --git a/src/adapters/filesystem_recovery_next_head_finalization_open_error.rs b/src/adapters/filesystem_recovery_next_head_finalization_open_error.rs new file mode 100644 index 0000000..a663480 --- /dev/null +++ b/src/adapters/filesystem_recovery_next_head_finalization_open_error.rs @@ -0,0 +1,89 @@ +//! This module owns filesystem next-head authority acquisition failures. + +use std::error::Error; +use std::fmt; +use std::io; + +use super::{ + FilesystemRecoveryStageDiscardOpenError, RecoveryInventoryError, WriterLockAcquireError, +}; + +/// Why a pinned writer-authorized next-head finalizer could not be opened. +#[derive(Debug)] +pub enum FilesystemRecoveryNextHeadFinalizationOpenError { + /// The store root did not satisfy the supported platform profile. + Platform { + /// Exact platform-admission failure. + source: io::Error, + }, + /// Exclusive writer authority could not be acquired. + WriterLock { + /// Exact writer-lock acquisition refusal. + source: WriterLockAcquireError, + }, + /// The locked root capability could not be cloned for recovery inventory. + CloneRoot { + /// Exact root-capability clone failure. + source: io::Error, + }, + /// One pinned protocol namespace could not be admitted. + Namespace { + /// Exact recovery-namespace admission refusal. + source: RecoveryInventoryError, + }, +} + +impl From + for FilesystemRecoveryNextHeadFinalizationOpenError +{ + fn from(source: FilesystemRecoveryStageDiscardOpenError) -> Self { + match source { + FilesystemRecoveryStageDiscardOpenError::Platform { source } => { + Self::Platform { source } + } + FilesystemRecoveryStageDiscardOpenError::WriterLock { source } => { + Self::WriterLock { source } + } + FilesystemRecoveryStageDiscardOpenError::CloneRoot { source } => { + Self::CloneRoot { source } + } + FilesystemRecoveryStageDiscardOpenError::Namespace { source } => { + Self::Namespace { source } + } + } + } +} + +impl fmt::Display for FilesystemRecoveryNextHeadFinalizationOpenError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Platform { source } => { + write!( + formatter, + "recovery next-head platform was refused: {source}" + ) + } + Self::WriterLock { source } => write!( + formatter, + "recovery next-head writer lock was refused: {source}" + ), + Self::CloneRoot { source } => { + write!(formatter, "locked recovery root clone failed: {source}") + } + Self::Namespace { source } => write!( + formatter, + "recovery next-head namespace was refused: {source}" + ), + } + } +} + +impl Error for FilesystemRecoveryNextHeadFinalizationOpenError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Platform { source } | Self::CloneRoot { source } => Some(source), + Self::WriterLock { source } => Some(source), + Self::Namespace { source } => Some(source), + } + } +} diff --git a/src/adapters/filesystem_recovery_next_head_finalization_storage.rs b/src/adapters/filesystem_recovery_next_head_finalization_storage.rs new file mode 100644 index 0000000..6c80a93 --- /dev/null +++ b/src/adapters/filesystem_recovery_next_head_finalization_storage.rs @@ -0,0 +1,141 @@ +//! This module binds next-head finalization to pinned filesystem storage. + +use std::io; + +use super::{ + CatalogRestartError, CatalogRestartPhase, FilesystemRecoveryNextHeadFinalizer, + RecoveryNextHeadFinalizationReadiness, RecoveryNextHeadFinalizationRequest, + RecoveryNextHeadFinalizationStorage, RecoveryNextHeadFinalizationStorageError, + RecoveryNextHeadFinalizationTarget, RecoveryStageNamespacePhase, catalog_restart_loader, + filesystem_catalog_artifact, filesystem_recovery_next_head_candidate, +}; + +const HEAD: &str = "HEAD"; +const NEXT_HEAD: &str = "head.next"; + +impl RecoveryNextHeadFinalizationStorage for FilesystemRecoveryNextHeadFinalizer { + fn verify_current( + &mut self, + request: RecoveryNextHeadFinalizationRequest, + ) -> Result + { + filesystem_recovery_next_head_candidate::verify_namespaces( + self, + RecoveryStageNamespacePhase::BeforeObservation, + )?; + let readiness = current_readiness(self, request)?; + match readiness { + RecoveryNextHeadFinalizationReadiness::Ready => { + filesystem_recovery_next_head_candidate::verify(self, request)?; + } + RecoveryNextHeadFinalizationReadiness::AlreadyFinalized => { + require_candidate_absent(self, request)?; + filesystem_recovery_next_head_candidate::verify_namespaces( + self, + RecoveryStageNamespacePhase::AfterObservation, + )?; + } + } + Ok(readiness) + } + + fn synchronize_candidate( + &mut self, + request: RecoveryNextHeadFinalizationRequest, + ) -> Result<(), RecoveryNextHeadFinalizationStorageError> { + filesystem_recovery_next_head_candidate::synchronize(self, request) + } + + fn replace_head( + &mut self, + request: RecoveryNextHeadFinalizationRequest, + ) -> Result<(), RecoveryNextHeadFinalizationStorageError> { + match self.verify_current(request)? { + RecoveryNextHeadFinalizationReadiness::Ready => self + .root() + .rename(NEXT_HEAD, self.root(), HEAD) + .map_err(storage), + RecoveryNextHeadFinalizationReadiness::AlreadyFinalized => { + Err(RecoveryNextHeadFinalizationStorageError::CurrentMismatch { + expected: request.expectation(), + observed: Some(request.target()), + }) + } + } + } + + fn synchronize_root(&mut self) -> io::Result<()> { + filesystem_catalog_artifact::synchronize_directory(self.root()) + } +} + +fn current_readiness( + finalizer: &FilesystemRecoveryNextHeadFinalizer, + request: RecoveryNextHeadFinalizationRequest, +) -> Result { + match catalog_restart_loader::load_from_directory(finalizer.root(), HEAD, finalizer.policy) { + Ok(loaded) => { + let snapshot = loaded.snapshot().map_err(current_view)?; + let observed = RecoveryNextHeadFinalizationTarget::from_snapshot(&snapshot); + let expected = request.expectation(); + if expected.current_generation() == Some(observed.generation()) + && expected.current_catalog_digest() == Some(observed.digest()) + { + Ok(RecoveryNextHeadFinalizationReadiness::Ready) + } else if observed == request.target() { + Ok(RecoveryNextHeadFinalizationReadiness::AlreadyFinalized) + } else { + Err(RecoveryNextHeadFinalizationStorageError::CurrentMismatch { + expected, + observed: Some(observed), + }) + } + } + Err(source) if missing_head(&source) => { + if request.expectation().current_generation().is_none() { + Ok(RecoveryNextHeadFinalizationReadiness::Ready) + } else { + Err(RecoveryNextHeadFinalizationStorageError::CurrentMismatch { + expected: request.expectation(), + observed: None, + }) + } + } + Err(source) => Err(current_view(source)), + } +} + +fn missing_head(error: &CatalogRestartError) -> bool { + matches!( + error, + CatalogRestartError::Io { + phase: CatalogRestartPhase::OpenHead, + source, + } if source.kind() == io::ErrorKind::NotFound + ) +} + +fn current_view(source: CatalogRestartError) -> RecoveryNextHeadFinalizationStorageError { + RecoveryNextHeadFinalizationStorageError::CurrentView { + source: Box::new(source), + } +} + +fn require_candidate_absent( + finalizer: &FilesystemRecoveryNextHeadFinalizer, + request: RecoveryNextHeadFinalizationRequest, +) -> Result<(), RecoveryNextHeadFinalizationStorageError> { + match finalizer.root().symlink_metadata(NEXT_HEAD) { + Err(source) if source.kind() == io::ErrorKind::NotFound => Ok(()), + Err(source) => Err(storage(source)), + Ok(_metadata) => Err( + RecoveryNextHeadFinalizationStorageError::UnexpectedCandidate { + expected: request.evidence(), + }, + ), + } +} + +const fn storage(source: io::Error) -> RecoveryNextHeadFinalizationStorageError { + RecoveryNextHeadFinalizationStorageError::Storage { source } +} diff --git a/src/adapters/filesystem_recovery_next_head_finalization_tests.rs b/src/adapters/filesystem_recovery_next_head_finalization_tests.rs new file mode 100644 index 0000000..00dfe2b --- /dev/null +++ b/src/adapters/filesystem_recovery_next_head_finalization_tests.rs @@ -0,0 +1,65 @@ +//! Filesystem next-head finalization laws. + +mod fixture; +mod namespace_laws; +mod refusal_laws; +mod replacement_laws; + +use std::error::Error; +use std::fs; + +use super::{ + RecoveryNextHeadFinalizationOutcome, RecoveryNextHeadFinalizationReadiness, + RecoveryNextHeadFinalizationStorage, execute_recovery_next_head_finalization, +}; +use fixture::FinalizationFixture; + +#[test] +fn generation_one_finalizes_and_retries_without_replacement() -> Result<(), Box> { + let fixture = FinalizationFixture::new("filesystem-next-head-initial")?; + let request = fixture.install_generation_one_candidate()?; + let mut finalizer = fixture.finalizer()?; + + assert_eq!( + finalizer.verify_current(request)?, + RecoveryNextHeadFinalizationReadiness::Ready + ); + let receipt = execute_recovery_next_head_finalization(&mut finalizer, request)?; + let retry = execute_recovery_next_head_finalization(&mut finalizer, request)?; + + assert_eq!( + receipt.outcome(), + RecoveryNextHeadFinalizationOutcome::Finalized + ); + assert_eq!( + retry.outcome(), + RecoveryNextHeadFinalizationOutcome::AlreadyFinalized + ); + assert_eq!( + fs::read(fixture.head_path())?, + FinalizationFixture::head_one()? + ); + assert!(!fixture.next_head_path().exists()); + drop(finalizer); + fixture.remove()?; + Ok(()) +} + +#[test] +fn one_writer_excludes_a_second_finalizer() -> Result<(), Box> { + let fixture = FinalizationFixture::new("filesystem-next-head-writer-exclusion")?; + let first = fixture.finalizer()?; + + let error = fixture + .finalizer() + .err() + .ok_or("second next-head finalizer acquired writer authority")?; + + assert!(matches!( + error.downcast_ref::(), + Some(super::FilesystemRecoveryNextHeadFinalizationOpenError::WriterLock { .. }) + )); + drop(first); + fixture.remove()?; + Ok(()) +} diff --git a/src/adapters/filesystem_recovery_next_head_finalization_tests/fixture.rs b/src/adapters/filesystem_recovery_next_head_finalization_tests/fixture.rs new file mode 100644 index 0000000..328d7b4 --- /dev/null +++ b/src/adapters/filesystem_recovery_next_head_finalization_tests/fixture.rs @@ -0,0 +1,185 @@ +//! Deterministic initialized-store fixture for filesystem head finalization. + +use std::error::Error; +use std::fs; +use std::path::{Path, PathBuf}; + +use crate::LayoutEntryLimit; + +use super::super::{ + AdmittedSegment, CatalogPublicationExpectation, CatalogRestartByteLimit, CatalogRestartPolicy, + CatalogSnapshot, ChecksummedCatalog, ChecksummedPublicationHead, + FilesystemRecoveryNextHeadFinalizer, RecoveryNextHeadFinalizationRequest, RecoveryStage, + RecoveryStageMetadata, SegmentReadPolicy, SegmentRecordLimit, admit_recovery_stage_bytes, + assess_recovery_stage, filesystem_test_sandbox::TestDirectory, fingerprint_recovery_stage, + physical_pool_name, plan_recovery_next_head_finalization, test_support::decode_hex, +}; + +const SEGMENT_HEX: &str = + include_str!("../../../conformance/segment-store/v1/one-zero-segment.hex"); +const CATALOG_ONE_HEX: &str = + include_str!("../../../conformance/segment-store/v1/one-zero-catalog.hex"); +const HEAD_ONE_HEX: &str = include_str!("../../../conformance/segment-store/v1/one-zero-head.hex"); +const CATALOG_TWO_HEX: &str = + include_str!("../../../conformance/segment-store/v1/one-zero-catalog-generation-two.hex"); +const HEAD_TWO_HEX: &str = + include_str!("../../../conformance/segment-store/v1/one-zero-head-generation-two.hex"); +const RETAINED_SEGMENT_LIMIT: u64 = 1_048_576; + +pub(super) struct FinalizationFixture { + directory: TestDirectory, +} + +impl FinalizationFixture { + pub(super) fn new(name: &str) -> Result> { + let directory = TestDirectory::create(name)?; + fs::write(directory.path().join("writer.lock"), [])?; + for name in ["staging", "segments", "catalogs"] { + fs::create_dir(directory.path().join(name))?; + } + Ok(Self { directory }) + } + + pub(super) fn root(&self) -> &Path { + self.directory.path() + } + + pub(super) fn head_path(&self) -> PathBuf { + self.root().join("HEAD") + } + + pub(super) fn next_head_path(&self) -> PathBuf { + self.root().join("head.next") + } + + pub(super) fn head_one() -> Result, Box> { + fixture(HEAD_ONE_HEX) + } + + pub(super) fn head_two() -> Result, Box> { + fixture(HEAD_TWO_HEX) + } + + pub(super) fn catalog_one_path(&self) -> Result> { + self.catalog_path(&fixture(CATALOG_ONE_HEX)?) + } + + pub(super) fn install_generation_one_candidate( + &self, + ) -> Result> { + let segment = fixture(SEGMENT_HEX)?; + let catalog = fixture(CATALOG_ONE_HEX)?; + let head = Self::head_one()?; + self.install_pool(&catalog, &segment)?; + fs::write(self.next_head_path(), &head)?; + request( + &head, + &catalog, + &segment, + CatalogPublicationExpectation::uninitialized(), + ) + } + + pub(super) fn install_generation_two_candidate( + &self, + ) -> Result> { + let segment = fixture(SEGMENT_HEX)?; + let catalog_one = fixture(CATALOG_ONE_HEX)?; + let head_one = Self::head_one()?; + let catalog_two = fixture(CATALOG_TWO_HEX)?; + let head_two = Self::head_two()?; + self.install_pool(&catalog_one, &segment)?; + self.install_pool(&catalog_two, &segment)?; + fs::write(self.head_path(), &head_one)?; + fs::write(self.next_head_path(), &head_two)?; + let current = snapshot(&head_one, &catalog_one, &segment)?; + request( + &head_two, + &catalog_two, + &segment, + CatalogPublicationExpectation::successor_of(¤t), + ) + } + + pub(super) fn finalizer(&self) -> Result> { + Ok(FilesystemRecoveryNextHeadFinalizer::open_unchecked_for_tests(self.root(), policy()?)?) + } + + pub(super) fn remove(self) -> std::io::Result<()> { + self.directory.remove() + } + + fn install_pool(&self, catalog: &[u8], segment: &[u8]) -> Result<(), Box> { + let admitted_segment = AdmittedSegment::decode(segment, maximum_segment_policy())?; + let checksummed_catalog = ChecksummedCatalog::decode(catalog)?; + let segment_name = physical_pool_name::segment(admitted_segment.digest()); + let catalog_name = physical_pool_name::catalog( + checksummed_catalog.generation(), + checksummed_catalog.digest(), + ); + fs::write(self.root().join("segments").join(segment_name), segment)?; + fs::write(self.root().join("catalogs").join(catalog_name), catalog)?; + Ok(()) + } + + fn catalog_path(&self, bytes: &[u8]) -> Result> { + let catalog = ChecksummedCatalog::decode(bytes)?; + Ok(self + .root() + .join("catalogs") + .join(physical_pool_name::catalog( + catalog.generation(), + catalog.digest(), + ))) + } +} + +fn request( + head: &[u8], + catalog: &[u8], + segment: &[u8], + expectation: CatalogPublicationExpectation, +) -> Result> { + let length = u64::try_from(head.len())?; + let evidence = fingerprint_recovery_stage( + RecoveryStageMetadata::new(RecoveryStage::NextHead, length)?, + head, + )?; + let admitted = admit_recovery_stage_bytes(RecoveryStage::NextHead, evidence, head)?; + let assessment = assess_recovery_stage(&admitted, maximum_segment_policy())?; + let candidate = snapshot(head, catalog, segment)?; + Ok(plan_recovery_next_head_finalization( + &assessment, + &candidate, + expectation, + )?) +} + +fn snapshot<'bytes>( + head: &'bytes [u8], + catalog: &'bytes [u8], + segment: &'bytes [u8], +) -> Result, Box> { + let segments = [AdmittedSegment::decode(segment, maximum_segment_policy())?]; + let admitted_catalog = ChecksummedCatalog::decode(catalog)?.admit(&segments)?; + Ok(ChecksummedPublicationHead::decode(head)?.admit(admitted_catalog)?) +} + +fn policy() -> Result> { + Ok(CatalogRestartPolicy::new( + maximum_segment_policy(), + CatalogRestartByteLimit::new(RETAINED_SEGMENT_LIMIT)?, + )) +} + +const fn maximum_segment_policy() -> SegmentReadPolicy { + SegmentReadPolicy::new(SegmentRecordLimit::MAXIMUM, LayoutEntryLimit::MAXIMUM) +} + +fn fixture(hex: &str) -> Result, Box> { + decode_hex( + hex.strip_suffix('\n') + .ok_or("recovery fixture must end in one LF")?, + ) + .map_err(Into::into) +} diff --git a/src/adapters/filesystem_recovery_next_head_finalization_tests/namespace_laws.rs b/src/adapters/filesystem_recovery_next_head_finalization_tests/namespace_laws.rs new file mode 100644 index 0000000..36c99c9 --- /dev/null +++ b/src/adapters/filesystem_recovery_next_head_finalization_tests/namespace_laws.rs @@ -0,0 +1,44 @@ +//! Pinned namespace laws for filesystem next-head finalization. + +use std::error::Error; +use std::fs; + +use super::super::{ + FilesystemRecoveryStageError, RecoveryNextHeadFinalizationError, + RecoveryNextHeadFinalizationStorageError, execute_recovery_next_head_finalization, +}; +use super::fixture::FinalizationFixture; + +#[test] +fn replaced_catalog_namespace_refuses_candidate_loading() -> Result<(), Box> { + let fixture = FinalizationFixture::new("filesystem-next-head-catalog-replaced")?; + let request = fixture.install_generation_one_candidate()?; + let mut finalizer = fixture.finalizer()?; + let catalog_directory = fixture.root().join("catalogs"); + let pinned_directory = fixture.root().join("catalogs.pinned"); + fs::rename(&catalog_directory, &pinned_directory)?; + fs::create_dir(&catalog_directory)?; + for entry in fs::read_dir(&pinned_directory)? { + let entry = entry?; + fs::copy(entry.path(), catalog_directory.join(entry.file_name()))?; + } + + let error = execute_recovery_next_head_finalization(&mut finalizer, request) + .err() + .ok_or("candidate used a replaced catalog namespace")?; + + let RecoveryNextHeadFinalizationError::Verify { source, .. } = error else { + return Err("namespace refusal lost the verification phase".into()); + }; + let RecoveryNextHeadFinalizationStorageError::Stage { source } = source.as_ref() else { + return Err("namespace refusal lost the stage boundary".into()); + }; + assert!(matches!( + source.as_ref(), + FilesystemRecoveryStageError::Namespace { .. } + )); + assert!(!fixture.head_path().exists()); + drop(finalizer); + fixture.remove()?; + Ok(()) +} diff --git a/src/adapters/filesystem_recovery_next_head_finalization_tests/refusal_laws.rs b/src/adapters/filesystem_recovery_next_head_finalization_tests/refusal_laws.rs new file mode 100644 index 0000000..194c8e2 --- /dev/null +++ b/src/adapters/filesystem_recovery_next_head_finalization_tests/refusal_laws.rs @@ -0,0 +1,196 @@ +//! Filesystem next-head evidence and namespace refusal laws. + +use std::error::Error; +use std::fs; + +use super::super::{ + FilesystemRecoveryStageError, RecoveryNextHeadFinalizationError, + RecoveryNextHeadFinalizationReadiness, RecoveryNextHeadFinalizationStorage, + RecoveryNextHeadFinalizationStorageError, execute_recovery_next_head_finalization, +}; +use super::fixture::FinalizationFixture; + +#[test] +fn symbolic_candidate_is_never_followed_or_published() -> Result<(), Box> { + use std::os::unix::fs::symlink; + + let fixture = FinalizationFixture::new("filesystem-next-head-link")?; + let request = fixture.install_generation_one_candidate()?; + let target = fixture.root().join("outside-head"); + fs::rename(fixture.next_head_path(), &target)?; + symlink(&target, fixture.next_head_path())?; + let mut finalizer = fixture.finalizer()?; + + let error = execute_recovery_next_head_finalization(&mut finalizer, request) + .err() + .ok_or("symbolic next head was followed")?; + + let RecoveryNextHeadFinalizationError::Verify { source, .. } = error else { + return Err("symbolic candidate refusal lost the verification phase".into()); + }; + let RecoveryNextHeadFinalizationStorageError::Stage { source } = source.as_ref() else { + return Err("symbolic candidate refusal lost the stage boundary".into()); + }; + assert!(matches!( + source.as_ref(), + FilesystemRecoveryStageError::Open { .. } + )); + assert!(!fixture.head_path().exists()); + assert!(fixture.next_head_path().is_symlink()); + drop(finalizer); + fixture.remove()?; + Ok(()) +} + +#[test] +fn changed_candidate_is_refused_at_the_replacement_boundary() -> Result<(), Box> { + let fixture = FinalizationFixture::new("filesystem-next-head-replacement-drift")?; + let request = fixture.install_generation_one_candidate()?; + let mut finalizer = fixture.finalizer()?; + assert_eq!( + finalizer.verify_current(request)?, + RecoveryNextHeadFinalizationReadiness::Ready + ); + fs::write(fixture.next_head_path(), [0_u8])?; + + let error = finalizer + .replace_head(request) + .err() + .ok_or("changed next head replaced durable HEAD")?; + + assert!(matches!( + error, + RecoveryNextHeadFinalizationStorageError::EvidenceMismatch { .. } + )); + assert!(!fixture.head_path().exists()); + assert_eq!(fs::read(fixture.next_head_path())?, [0_u8]); + drop(finalizer); + fixture.remove()?; + Ok(()) +} + +#[test] +fn disappeared_current_head_refuses_successor_without_mutation() -> Result<(), Box> { + let fixture = FinalizationFixture::new("filesystem-next-head-current-missing")?; + let request = fixture.install_generation_two_candidate()?; + fs::remove_file(fixture.head_path())?; + let mut finalizer = fixture.finalizer()?; + + let error = execute_recovery_next_head_finalization(&mut finalizer, request) + .err() + .ok_or("successor finalized after current HEAD disappeared")?; + + assert!(matches!( + error, + RecoveryNextHeadFinalizationError::Verify { source, .. } + if matches!( + source.as_ref(), + RecoveryNextHeadFinalizationStorageError::CurrentMismatch { + observed: None, + .. + } + ) + )); + assert_eq!( + fs::read(fixture.next_head_path())?, + FinalizationFixture::head_two()? + ); + assert!(!fixture.head_path().exists()); + drop(finalizer); + fixture.remove()?; + Ok(()) +} + +#[test] +fn corrupt_candidate_catalog_refuses_before_head_replacement() -> Result<(), Box> { + let fixture = FinalizationFixture::new("filesystem-next-head-catalog-corrupt")?; + let request = fixture.install_generation_one_candidate()?; + fs::write(fixture.catalog_one_path()?, b"corrupt")?; + let mut finalizer = fixture.finalizer()?; + + let error = execute_recovery_next_head_finalization(&mut finalizer, request) + .err() + .ok_or("candidate with corrupt catalog was finalized")?; + + assert!(matches!( + error, + RecoveryNextHeadFinalizationError::Verify { source, .. } + if matches!( + source.as_ref(), + RecoveryNextHeadFinalizationStorageError::CandidateView { .. } + ) + )); + assert!(!fixture.head_path().exists()); + assert_eq!( + fs::read(fixture.next_head_path())?, + FinalizationFixture::head_one()? + ); + drop(finalizer); + fixture.remove()?; + Ok(()) +} + +#[test] +fn already_finalized_retry_refuses_a_reappeared_candidate() -> Result<(), Box> { + let fixture = FinalizationFixture::new("filesystem-next-head-reappeared")?; + let request = fixture.install_generation_one_candidate()?; + let mut finalizer = fixture.finalizer()?; + let first_receipt = execute_recovery_next_head_finalization(&mut finalizer, request)?; + fs::write(fixture.next_head_path(), FinalizationFixture::head_one()?)?; + + let error = execute_recovery_next_head_finalization(&mut finalizer, request) + .err() + .ok_or("already-finalized retry ignored a reappeared candidate")?; + + assert_eq!( + first_receipt.outcome(), + super::super::RecoveryNextHeadFinalizationOutcome::Finalized + ); + assert!(matches!( + error, + RecoveryNextHeadFinalizationError::Verify { source, .. } + if matches!( + source.as_ref(), + RecoveryNextHeadFinalizationStorageError::UnexpectedCandidate { .. } + ) + )); + assert_eq!( + fs::read(fixture.head_path())?, + FinalizationFixture::head_one()? + ); + assert_eq!( + fs::read(fixture.next_head_path())?, + FinalizationFixture::head_one()? + ); + drop(finalizer); + fixture.remove()?; + Ok(()) +} + +#[test] +fn absent_candidate_reports_typed_missing_without_creating_head() -> Result<(), Box> { + let fixture = FinalizationFixture::new("filesystem-next-head-missing")?; + let request = fixture.install_generation_one_candidate()?; + fs::remove_file(fixture.next_head_path())?; + let mut finalizer = fixture.finalizer()?; + + let error = execute_recovery_next_head_finalization(&mut finalizer, request) + .err() + .ok_or("absent candidate produced a finalization receipt")?; + + assert!(matches!( + error, + RecoveryNextHeadFinalizationError::Verify { source, .. } + if matches!( + source.as_ref(), + RecoveryNextHeadFinalizationStorageError::MissingCandidate { + expected, + } if *expected == request.evidence() + ) + )); + assert!(!fixture.head_path().exists()); + assert!(!fixture.next_head_path().exists()); + drop(finalizer); + fixture.remove()?; + Ok(()) +} diff --git a/src/adapters/filesystem_recovery_next_head_finalization_tests/replacement_laws.rs b/src/adapters/filesystem_recovery_next_head_finalization_tests/replacement_laws.rs new file mode 100644 index 0000000..e450cc6 --- /dev/null +++ b/src/adapters/filesystem_recovery_next_head_finalization_tests/replacement_laws.rs @@ -0,0 +1,55 @@ +//! Atomic replacement and retry laws for filesystem head finalization. + +use std::error::Error; +use std::fs; + +use super::super::{RecoveryNextHeadFinalizationOutcome, execute_recovery_next_head_finalization}; +use super::fixture::FinalizationFixture; + +#[test] +fn exact_successor_atomically_replaces_the_current_head() -> Result<(), Box> { + let fixture = FinalizationFixture::new("filesystem-next-head-successor")?; + let request = fixture.install_generation_two_candidate()?; + let mut finalizer = fixture.finalizer()?; + + let receipt = execute_recovery_next_head_finalization(&mut finalizer, request)?; + + assert_eq!( + receipt.outcome(), + RecoveryNextHeadFinalizationOutcome::Finalized + ); + assert_eq!( + fs::read(fixture.head_path())?, + FinalizationFixture::head_two()? + ); + assert!(!fixture.next_head_path().exists()); + drop(finalizer); + fixture.remove()?; + Ok(()) +} + +#[test] +fn successor_retry_admits_the_exact_published_view() -> Result<(), Box> { + let fixture = FinalizationFixture::new("filesystem-next-head-successor-retry")?; + let request = fixture.install_generation_two_candidate()?; + let mut finalizer = fixture.finalizer()?; + let first_receipt = execute_recovery_next_head_finalization(&mut finalizer, request)?; + + let retry = execute_recovery_next_head_finalization(&mut finalizer, request)?; + + assert_eq!( + first_receipt.outcome(), + RecoveryNextHeadFinalizationOutcome::Finalized + ); + assert_eq!( + retry.outcome(), + RecoveryNextHeadFinalizationOutcome::AlreadyFinalized + ); + assert_eq!( + fs::read(fixture.head_path())?, + FinalizationFixture::head_two()? + ); + drop(finalizer); + fixture.remove()?; + Ok(()) +} diff --git a/src/adapters/filesystem_recovery_next_head_finalizer.rs b/src/adapters/filesystem_recovery_next_head_finalizer.rs new file mode 100644 index 0000000..1f1daeb --- /dev/null +++ b/src/adapters/filesystem_recovery_next_head_finalizer.rs @@ -0,0 +1,58 @@ +//! This module owns pinned writer authority for filesystem head finalization. + +use std::path::Path; + +use cap_std::fs::Dir; + +use super::{ + CatalogRestartPolicy, FilesystemRecoveryNextHeadFinalizationOpenError, + FilesystemRecoveryStageDiscarder, RecoveryStageParent, +}; + +/// Writer-authorized pinned filesystem adapter for exact next-head finalization. +/// +/// Opening proves the supported platform, pins and exclusively locks the store +/// root and `writer.lock`, then pins all three protocol child directories +/// without following links. The synchronous adapter may block on bounded +/// filesystem I/O and retains writer authority until dropped. +#[must_use] +pub struct FilesystemRecoveryNextHeadFinalizer { + pub(super) discarder: FilesystemRecoveryStageDiscarder, + pub(super) policy: CatalogRestartPolicy, +} + +impl FilesystemRecoveryNextHeadFinalizer { + /// Opens an initialized supported store for explicit head finalization. + /// + /// The call performs no protocol mutation and does not read `HEAD` or + /// `head.next`. + /// + /// # Errors + /// + /// Returns [`FilesystemRecoveryNextHeadFinalizationOpenError`] on platform, + /// writer-authority, root-clone, or namespace admission failure. + pub fn open( + store_root: &Path, + policy: CatalogRestartPolicy, + ) -> Result { + FilesystemRecoveryStageDiscarder::open(store_root) + .map(|discarder| Self { discarder, policy }) + .map_err(Into::into) + } + + #[cfg(test)] + pub(super) fn open_unchecked_for_tests( + store_root: &Path, + policy: CatalogRestartPolicy, + ) -> Result { + FilesystemRecoveryStageDiscarder::open_unchecked_for_tests(store_root) + .map(|discarder| Self { discarder, policy }) + .map_err(Into::into) + } + + pub(super) const fn root(&self) -> &Dir { + self.discarder + .inventory + .parent_directory(RecoveryStageParent::Root) + } +} diff --git a/src/adapters/mod.rs b/src/adapters/mod.rs index a5a4abf..088822d 100644 --- a/src/adapters/mod.rs +++ b/src/adapters/mod.rs @@ -85,6 +85,12 @@ mod filesystem_recovery_inventory_scan; #[cfg(all(test, unix))] mod filesystem_recovery_inventory_tests; mod filesystem_recovery_namespace; +mod filesystem_recovery_next_head_candidate; +mod filesystem_recovery_next_head_finalization_open_error; +mod filesystem_recovery_next_head_finalization_storage; +#[cfg(all(test, unix))] +mod filesystem_recovery_next_head_finalization_tests; +mod filesystem_recovery_next_head_finalizer; mod filesystem_recovery_stage; mod filesystem_recovery_stage_completer; mod filesystem_recovery_stage_completion_open_error; @@ -314,6 +320,8 @@ pub use filesystem_catalog_publisher::FilesystemCatalogPublisher; pub use filesystem_catalog_snapshot::FilesystemCatalogSnapshot; pub use filesystem_platform_admission::FilesystemPlatformAdmission; pub use filesystem_recovery_inventory_reader::FilesystemRecoveryInventoryReader; +pub use filesystem_recovery_next_head_finalization_open_error::FilesystemRecoveryNextHeadFinalizationOpenError; +pub use filesystem_recovery_next_head_finalizer::FilesystemRecoveryNextHeadFinalizer; pub use filesystem_recovery_stage_completer::FilesystemRecoveryStageCompleter; pub use filesystem_recovery_stage_completion_open_error::FilesystemRecoveryStageCompletionOpenError; pub use filesystem_recovery_stage_discard_open_error::FilesystemRecoveryStageDiscardOpenError; diff --git a/src/adapters/recovery_next_head_finalization_error.rs b/src/adapters/recovery_next_head_finalization_error.rs index 598b338..295c18b 100644 --- a/src/adapters/recovery_next_head_finalization_error.rs +++ b/src/adapters/recovery_next_head_finalization_error.rs @@ -19,6 +19,13 @@ pub enum RecoveryNextHeadFinalizationError { /// Exact underlying verification failure. source: Box, }, + /// The exact candidate could not be made durable before replacement. + SynchronizeCandidate { + /// Exact `head.next` evidence that was not made durable. + evidence: RecoveryStageEvidence, + /// Exact underlying synchronization or verification failure. + source: Box, + }, /// The exact candidate could not atomically replace durable `HEAD`. Replace { /// Exact `head.next` evidence that could not be finalized. @@ -43,6 +50,11 @@ impl fmt::Display for RecoveryNextHeadFinalizationError { "failed to verify recovery head generation {}: {source}", target.generation().get() ), + Self::SynchronizeCandidate { evidence, source } => write!( + formatter, + "failed to synchronize {} evidence before finalization: {source}", + evidence.stage() + ), Self::Replace { evidence, source } => write!( formatter, "failed to finalize {} evidence: {source}", @@ -60,7 +72,9 @@ impl fmt::Display for RecoveryNextHeadFinalizationError { impl Error for RecoveryNextHeadFinalizationError { fn source(&self) -> Option<&(dyn Error + 'static)> { match self { - Self::Verify { source, .. } | Self::Replace { source, .. } => Some(source.as_ref()), + Self::Verify { source, .. } + | Self::SynchronizeCandidate { source, .. } + | Self::Replace { source, .. } => Some(source.as_ref()), Self::SynchronizeRoot { source, .. } => Some(source), } } diff --git a/src/adapters/recovery_next_head_finalization_executor.rs b/src/adapters/recovery_next_head_finalization_executor.rs index a549815..75d990c 100644 --- a/src/adapters/recovery_next_head_finalization_executor.rs +++ b/src/adapters/recovery_next_head_finalization_executor.rs @@ -11,7 +11,8 @@ use super::{ /// A receipt is returned only after durable `HEAD` names the exact candidate /// and the root directory has been synchronized. Retrying after replacement /// but before directory synchronization re-admits the candidate and repeats the -/// root synchronization without replacing the head again. +/// root synchronization without replacing the head again. A ready candidate is +/// synchronized and reverified before atomic replacement. /// Typed storage failures are boxed on the error path to keep the phase error /// bounded without discarding expected or observed state. /// @@ -32,6 +33,12 @@ pub fn execute_recovery_next_head_finalization( })?; let outcome = match readiness { RecoveryNextHeadFinalizationReadiness::Ready => { + storage.synchronize_candidate(request).map_err(|source| { + RecoveryNextHeadFinalizationError::SynchronizeCandidate { + evidence: request.evidence(), + source: Box::new(source), + } + })?; storage.replace_head(request).map_err(|source| { RecoveryNextHeadFinalizationError::Replace { evidence: request.evidence(), diff --git a/src/adapters/recovery_next_head_finalization_storage.rs b/src/adapters/recovery_next_head_finalization_storage.rs index 74200e7..0b86310 100644 --- a/src/adapters/recovery_next_head_finalization_storage.rs +++ b/src/adapters/recovery_next_head_finalization_storage.rs @@ -26,6 +26,20 @@ pub trait RecoveryNextHeadFinalizationStorage { request: RecoveryNextHeadFinalizationRequest, ) -> Result; + /// Synchronizes the exact complete candidate before head replacement. + /// + /// Implementations must bind the synchronized file to the request evidence + /// and revalidate its complete transitive view after synchronization. + /// + /// # Errors + /// + /// Returns a source-preserving evidence, synchronization, verification, or + /// storage failure. + fn synchronize_candidate( + &mut self, + request: RecoveryNextHeadFinalizationRequest, + ) -> Result<(), RecoveryNextHeadFinalizationStorageError>; + /// Atomically replaces durable `HEAD` with the exact candidate. /// /// # Errors diff --git a/src/adapters/recovery_next_head_finalization_storage_error.rs b/src/adapters/recovery_next_head_finalization_storage_error.rs index 2d958a1..7f83d78 100644 --- a/src/adapters/recovery_next_head_finalization_storage_error.rs +++ b/src/adapters/recovery_next_head_finalization_storage_error.rs @@ -5,12 +5,18 @@ use std::fmt; use std::io; use super::{ - CatalogPublicationExpectation, RecoveryNextHeadFinalizationTarget, RecoveryStageEvidence, + CatalogPublicationExpectation, CatalogRestartError, FilesystemRecoveryStageError, + RecoveryNextHeadFinalizationTarget, RecoveryStageEvidence, }; /// Why storage could not continue one exact next-head finalization request. #[derive(Debug)] pub enum RecoveryNextHeadFinalizationStorageError { + /// The canonical `head.next` could not be observed exactly. + Stage { + /// Exact no-follow stage observation failure. + source: Box, + }, /// The canonical `head.next` resolves to different evidence. EvidenceMismatch { /// Evidence bound into the explicit finalization request. @@ -37,6 +43,21 @@ pub enum RecoveryNextHeadFinalizationStorageError { /// Exact missing `head.next` evidence bound into the request. expected: RecoveryStageEvidence, }, + /// Durable `HEAD` is final but the fixed candidate name reappeared. + UnexpectedCandidate { + /// Evidence from the request whose completed retry requires absence. + expected: RecoveryStageEvidence, + }, + /// The complete candidate snapshot could not be reconstructed. + CandidateView { + /// Exact bounded restart-loading failure. + source: Box, + }, + /// Durable current `HEAD` could not be reconstructed. + CurrentView { + /// Exact bounded restart-loading failure. + source: Box, + }, /// The storage boundary failed while observing or mutating. Storage { /// Exact underlying storage failure. @@ -47,6 +68,9 @@ pub enum RecoveryNextHeadFinalizationStorageError { impl fmt::Display for RecoveryNextHeadFinalizationStorageError { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { match self { + Self::Stage { source } => { + write!(formatter, "recovery next-head observation failed: {source}") + } Self::EvidenceMismatch { expected, observed } => write!( formatter, "{} finalization evidence changed from length {} to length {}", @@ -76,6 +100,18 @@ impl fmt::Display for RecoveryNextHeadFinalizationStorageError { expected.stage(), expected.length().get() ), + Self::UnexpectedCandidate { expected } => write!( + formatter, + "{} candidate reappeared after finalization for evidence length {}", + expected.stage(), + expected.length().get() + ), + Self::CandidateView { source } => { + write!(formatter, "recovery candidate view is invalid: {source}") + } + Self::CurrentView { source } => { + write!(formatter, "durable current view is invalid: {source}") + } Self::Storage { source } => { write!( formatter, @@ -90,10 +126,13 @@ impl Error for RecoveryNextHeadFinalizationStorageError { fn source(&self) -> Option<&(dyn Error + 'static)> { match self { Self::Storage { source } => Some(source), + Self::Stage { source } => Some(source.as_ref()), + Self::CandidateView { source } | Self::CurrentView { source } => Some(source.as_ref()), Self::EvidenceMismatch { .. } | Self::CurrentMismatch { .. } | Self::CandidateMismatch { .. } - | Self::MissingCandidate { .. } => None, + | Self::MissingCandidate { .. } + | Self::UnexpectedCandidate { .. } => None, } } } diff --git a/src/lib.rs b/src/lib.rs index d62242e..2bdbd7f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -18,9 +18,9 @@ //! inventory, name classification, bounded stage fingerprinting, exact //! truncated-stage discard, and complete-stage valid-orphan recovery are //! explicit. Exact next-head finalization now has a storage-independent -//! contract; filesystem finalization, retention, and garbage collection APIs -//! remain intentionally absent until their contracts have executable -//! specifications. +//! contract and a pinned writer-authorized filesystem adapter. Reusable-stage +//! continuation, retention, and garbage collection APIs remain intentionally +//! absent until their contracts have executable specifications. #[cfg(test)] extern crate self as keep; @@ -46,6 +46,7 @@ pub use adapters::{ ChecksummedPublicationHead, ChecksummedSegmentRecord, ClosedSegment, FilesystemCatalogPublicationError, FilesystemCatalogPublisher, FilesystemCatalogSnapshot, FilesystemPlatformAdmission, FilesystemRecoveryInventoryReader, + FilesystemRecoveryNextHeadFinalizationOpenError, FilesystemRecoveryNextHeadFinalizer, FilesystemRecoveryStageCompleter, FilesystemRecoveryStageCompletionOpenError, FilesystemRecoveryStageDiscardOpenError, FilesystemRecoveryStageDiscarder, FilesystemRecoveryStageError, FilesystemSegmentStage, FilesystemWriterLock, LayoutDecodeError, diff --git a/tests/recovery_next_head_finalization/execution_laws.rs b/tests/recovery_next_head_finalization/execution_laws.rs index db35380..891bb7f 100644 --- a/tests/recovery_next_head_finalization/execution_laws.rs +++ b/tests/recovery_next_head_finalization/execution_laws.rs @@ -30,6 +30,7 @@ fn ready_candidate_replaces_head_before_root_synchronization() -> Result<(), Box storage.operations(), &[ Operation::Verify(request), + Operation::SynchronizeCandidate(request), Operation::Replace(request), Operation::SynchronizeRoot, ] @@ -87,7 +88,38 @@ fn verification_and_replacement_failures_stop_later_mutation() -> Result<(), Box assert_eq!(verify_failure.operations(), &[Operation::Verify(request)]); assert_eq!( replace_failure.operations(), - &[Operation::Verify(request), Operation::Replace(request)] + &[ + Operation::Verify(request), + Operation::SynchronizeCandidate(request), + Operation::Replace(request), + ] + ); + Ok(()) +} + +#[test] +fn candidate_sync_failure_stops_before_head_replacement() -> Result<(), Box> { + let segment = fixture(SEGMENT_HEX)?; + let catalog = fixture(CATALOG_ONE_HEX)?; + let head = fixture(HEAD_ONE_HEX)?; + let request = initial_request(&head, &catalog, &segment)?; + let mut storage = NextHeadDouble::new(RecoveryNextHeadFinalizationReadiness::Ready) + .fail_next_candidate_synchronization(); + + let error = execute_recovery_next_head_finalization(&mut storage, request) + .err() + .ok_or("candidate synchronization failure was ignored")?; + + assert!(matches!( + error, + RecoveryNextHeadFinalizationError::SynchronizeCandidate { .. } + )); + assert_eq!( + storage.operations(), + &[ + Operation::Verify(request), + Operation::SynchronizeCandidate(request), + ] ); Ok(()) } @@ -120,6 +152,7 @@ fn retry_after_replace_before_root_sync_is_already_finalized() -> Result<(), Box storage.operations(), &[ Operation::Verify(request), + Operation::SynchronizeCandidate(request), Operation::Replace(request), Operation::SynchronizeRoot, Operation::Verify(request), diff --git a/tests/recovery_next_head_finalization/storage_double.rs b/tests/recovery_next_head_finalization/storage_double.rs index 9fe2a49..f4e49c2 100644 --- a/tests/recovery_next_head_finalization/storage_double.rs +++ b/tests/recovery_next_head_finalization/storage_double.rs @@ -12,6 +12,8 @@ use keep::{ pub enum Operation { /// Current-state and candidate-view verification. Verify(RecoveryNextHeadFinalizationRequest), + /// Exact candidate-file synchronization. + SynchronizeCandidate(RecoveryNextHeadFinalizationRequest), /// Atomic candidate-head replacement. Replace(RecoveryNextHeadFinalizationRequest), /// Root-directory synchronization. @@ -23,6 +25,7 @@ pub struct NextHeadDouble { readiness: RecoveryNextHeadFinalizationReadiness, operations: Vec, fail_verifications: usize, + fail_candidate_synchronizations: usize, fail_replacements: usize, fail_synchronizations: usize, } @@ -34,11 +37,19 @@ impl NextHeadDouble { readiness, operations: Vec::new(), fail_verifications: 0, + fail_candidate_synchronizations: 0, fail_replacements: 0, fail_synchronizations: 0, } } + /// Configures the next candidate synchronization to fail once. + #[must_use] + pub const fn fail_next_candidate_synchronization(mut self) -> Self { + self.fail_candidate_synchronizations = 1; + self + } + /// Configures the next verification to fail once. #[must_use] pub const fn fail_next_verification(mut self) -> Self { @@ -80,6 +91,18 @@ impl RecoveryNextHeadFinalizationStorage for NextHeadDouble { Ok(self.readiness) } + fn synchronize_candidate( + &mut self, + request: RecoveryNextHeadFinalizationRequest, + ) -> Result<(), RecoveryNextHeadFinalizationStorageError> { + self.operations + .push(Operation::SynchronizeCandidate(request)); + fail_once( + &mut self.fail_candidate_synchronizations, + "injected candidate synchronization failure", + ) + } + fn replace_head( &mut self, request: RecoveryNextHeadFinalizationRequest, diff --git a/xtask/tests/segment_store_protocol_contract/recovery_laws.rs b/xtask/tests/segment_store_protocol_contract/recovery_laws.rs index 10a2e5f..3383661 100644 --- a/xtask/tests/segment_store_protocol_contract/recovery_laws.rs +++ b/xtask/tests/segment_store_protocol_contract/recovery_laws.rs @@ -64,7 +64,9 @@ fn next_head_finalization_requires_one_exact_transition_and_durable_receipt() { "generation one over an uninitialized root", "expected exact successor", "`execute_recovery_next_head_finalization` revalidates durable current state", - "an already-finalized retry skips replacement", + "synchronized and reverified before it atomically replaces `HEAD`", + "requires `head.next` to be absent and skips replacement", + "`FilesystemRecoveryNextHeadFinalizer` binds this port", "`RecoveryNextHeadFinalizationReceipt`", ] { assert!( From 6e00a0327d41bf05f92996372b2a6f58c54830e3 Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 12:54:49 -0700 Subject: [PATCH 21/49] Add: Resume reusable recovery segments --- CHANGELOG.md | 5 + README.md | 11 +- docs/formats/segment-store-v1/recovery.md | 25 ++++ docs/formats/segment-store-v1/requirements.md | 1 + src/adapters/mod.rs | 17 +++ src/adapters/opened_reusable_segment.rs | 32 +++++ src/adapters/recovery_segment_resume_error.rs | 66 +++++++++++ .../recovery_segment_resume_executor.rs | 48 ++++++++ .../recovery_segment_resume_plan_error.rs | 48 ++++++++ .../recovery_segment_resume_planner.rs | 44 +++++++ .../recovery_segment_resume_request.rs | 54 +++++++++ src/adapters/recovery_segment_resume_state.rs | 67 +++++++++++ .../recovery_segment_resume_storage.rs | 31 +++++ .../recovery_segment_resume_storage_error.rs | 65 ++++++++++ src/adapters/staged_segment.rs | 23 +++- src/lib.rs | 38 +++--- tests/recovery_segment_resume.rs | 71 +++++++++++ .../recovery_segment_resume/execution_laws.rs | 88 ++++++++++++++ .../recovery_segment_resume/planning_laws.rs | 111 ++++++++++++++++++ .../recovery_segment_resume/storage_double.rs | 81 +++++++++++++ 20 files changed, 903 insertions(+), 23 deletions(-) create mode 100644 src/adapters/opened_reusable_segment.rs create mode 100644 src/adapters/recovery_segment_resume_error.rs create mode 100644 src/adapters/recovery_segment_resume_executor.rs create mode 100644 src/adapters/recovery_segment_resume_plan_error.rs create mode 100644 src/adapters/recovery_segment_resume_planner.rs create mode 100644 src/adapters/recovery_segment_resume_request.rs create mode 100644 src/adapters/recovery_segment_resume_state.rs create mode 100644 src/adapters/recovery_segment_resume_storage.rs create mode 100644 src/adapters/recovery_segment_resume_storage_error.rs create mode 100644 tests/recovery_segment_resume.rs create mode 100644 tests/recovery_segment_resume/execution_laws.rs create mode 100644 tests/recovery_segment_resume/planning_laws.rs create mode 100644 tests/recovery_segment_resume/storage_double.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 55e2d96..d5e4319 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,11 @@ after its public API and format compatibility policies are established. reusable prefix, a complete admitted immutable segment, or an exact truncation. Complete-looking corruption, duplicate identities, and caller-policy excess remain typed refusals. +- Storage-independent reusable-segment recovery now plans only from an exact + reusable assessment within the selected resource policy, consumes reopening + authority, re-admits the materialized prefix against saved evidence, rebuilds + digest and duplicate-identity state, and returns the ordinary append-only + stage without rewriting admitted bytes. - Complete caller-supplied catalog and candidate-head stages now distinguish exact fixed-header, declared-body, or fixed-width truncation from canonical bytes. Complete-looking corruption and oversized stages remain typed diff --git a/README.md b/README.md index 5afce36..772bca5 100644 --- a/README.md +++ b/README.md @@ -77,6 +77,10 @@ complete admitted segment, or exact truncation. Catalog and next-head stages likewise distinguish exact truncation from complete canonical bytes. Materialized bytes enter read-only semantic assessment only after their stage, length, and recomputed fingerprint match prior observation evidence. +An exact reusable segment assessment can authorize storage-independent +continuation: the executor consumes writer authority, re-admits the complete +bounded prefix, rebuilds digest and duplicate-identity state, and returns the +ordinary append-only stage without rewriting admitted bytes. Exact truncation assessments can authorize durable, evidence-bound discard. Complete segment and catalog assessments can authorize verified immutable-pool @@ -91,9 +95,10 @@ authority, reconstructs the complete current and candidate views without following links, verifies namespace and stage identity, synchronizes and reverifies the exact candidate, atomically replaces `HEAD`, and synchronizes the root. An already-finalized retry requires `head.next` to be absent. -Process-death injection, reusable-stage continuation, retention, compaction, -and garbage collection remain planned. Presence in the reference CAS does not -claim retention, crash recovery, or durability. +Process-death injection, the filesystem binding for reusable-stage +continuation, retention, compaction, and garbage collection remain planned. +Presence in the reference CAS does not claim retention, crash recovery, or +durability. ```rust use keep::BlobId; diff --git a/docs/formats/segment-store-v1/recovery.md b/docs/formats/segment-store-v1/recovery.md index 5bde39c..2365a3d 100644 --- a/docs/formats/segment-store-v1/recovery.md +++ b/docs/formats/segment-store-v1/recovery.md @@ -96,6 +96,31 @@ same crash points and must be idempotent. It may not silently promote the newest artifact, truncate to the last plausible boundary, rewrite a checksum, delete a valid orphan, or select by timestamp. +## Resume a reusable segment + +The public semantic boundary admits only +`RecoverySegmentStage::Reusable`. `plan_recovery_segment_resume` binds the +exact prior stage evidence, complete-record count, append boundary, and caller +resource policy into an owned request. Complete, truncated, catalog, and +candidate-head states remain ineligible. A selected policy below the already +admitted record count is refused before storage access. + +`execute_recovery_segment_resume` consumes a +`RecoverySegmentResumeStorage` capability so exclusive writer authority can +remain owned by the returned stage. The storage port returns one +protocol-bounded materialization and a writable stage positioned immediately +after those exact bytes. Before returning that stage, the executor recomputes +the saved fingerprint, repeats semantic classification under the selected +policy, and rebuilds the incremental segment digest and duplicate-identity +index from the complete prefix. + +Continuation returns the ordinary `StagedSegment` state machine. Subsequent +append and seal operations therefore retain the forward protocol's checked +record count, length bounds, duplicate refusal, flushes, synchronization, and +canonical seal. The admitted prefix is never rewritten. A storage adapter that +cannot prove the writable object, canonical entry, exact materialized bytes, +and end position agree must refuse before returning the stage. + ## Complete a durable stage The recovery plan may bind one fully verified `current.seg` or `current.cat` diff --git a/docs/formats/segment-store-v1/requirements.md b/docs/formats/segment-store-v1/requirements.md index fe52c11..c63c629 100644 --- a/docs/formats/segment-store-v1/requirements.md +++ b/docs/formats/segment-store-v1/requirements.md @@ -126,6 +126,7 @@ continuation or process-death injection. | `KEEP-RECOVERY-016` | Filesystem completion retains root and `writer.lock` authority, pins every protocol directory, re-synchronizes and re-fingerprints exact stage evidence before no-clobber link, never follows stage or pool links, verifies exact pool evidence before removal, preserves conflicting and replaced entries, accepts exact stage/pool, reappeared-stage, and pool-only retries, and returns only after pool and staging synchronization | Segment/catalog completion, three retry states, conflict, link, replacement, stale-evidence, missing-artifact, and writer-exclusion matrix | `src/adapters/filesystem_recovery_stage_completion_tests.rs`, `src/adapters/filesystem_recovery_stage_completion_tests/*.rs` | Implemented in #17 | | `KEEP-RECOVERY-017` | Next-head finalization plans only from an exact complete `head.next` assessment and its matching complete transitive catalog snapshot, admits only generation one over an uninitialized root or the exact successor of an expected current snapshot, synchronizes a ready candidate before atomic replacement, accepts an already-finalized retry, and returns a receipt only after root synchronization | Snapshot-coordinate, transition, candidate-sync, operation-order, fault-stop, and post-replacement retry matrix | `tests/recovery_next_head_finalization.rs`, `tests/recovery_next_head_finalization/*.rs` | Implemented in #17 | | `KEEP-RECOVERY-018` | Filesystem next-head finalization retains root and `writer.lock` authority, pins every protocol directory, revalidates namespace identity and exact stage evidence around bounded complete current and candidate loads, synchronizes and reverifies the exact candidate before atomic replacement, refuses current drift, missing or reappeared candidates, links, corrupt transitive views, and namespace replacement, and returns only after root synchronization | Initial and successor finalization, exact retry, candidate-sync, evidence-drift, missing, link, corruption, namespace-replacement, current-drift, and writer-exclusion matrix | `src/adapters/filesystem_recovery_next_head_finalization_tests.rs`, `src/adapters/filesystem_recovery_next_head_finalization_tests/*.rs` | Implemented in #17 | +| `KEEP-RECOVERY-019` | Reusable-segment continuation plans only from an exact reusable `current.seg` assessment within the selected record policy, consumes the storage authority that reopens the stage, re-admits the complete materialized prefix against prior evidence, rebuilds digest and duplicate-identity state, and returns the ordinary append-only stage without rewriting admitted bytes | Reusable-only planning, policy refusal, changed-evidence, storage-failure, duplicate-identity, append, seal, and independent decode matrix | `tests/recovery_segment_resume.rs`, `tests/recovery_segment_resume/*.rs` | Implemented in #17 | diff --git a/src/adapters/mod.rs b/src/adapters/mod.rs index 088822d..829bd0a 100644 --- a/src/adapters/mod.rs +++ b/src/adapters/mod.rs @@ -134,6 +134,7 @@ mod layout_record_format; mod layout_record_framing; mod loaded_segment; mod lower_hex; +mod opened_reusable_segment; mod physical_pool_name; mod publication_head_decode_error; mod publication_head_decode_error_display; @@ -170,6 +171,14 @@ mod recovery_pool_name_error; mod recovery_publication_stage_classifier; mod recovery_required_entry; mod recovery_segment_classifier; +mod recovery_segment_resume_error; +mod recovery_segment_resume_executor; +mod recovery_segment_resume_plan_error; +mod recovery_segment_resume_planner; +mod recovery_segment_resume_request; +mod recovery_segment_resume_state; +mod recovery_segment_resume_storage; +mod recovery_segment_resume_storage_error; mod recovery_segment_stage; mod recovery_segment_stage_error; mod recovery_segment_truncation; @@ -337,6 +346,7 @@ pub use layout_encode_error::LayoutEncodeError; pub use layout_id_binary_error::LayoutIdBinaryParseError; pub use layout_id_text_error::LayoutIdTextParseError; pub use layout_record::CanonicalLayoutRecord; +pub use opened_reusable_segment::OpenedReusableSegment; pub use publication_head_decode_error::PublicationHeadDecodeError; pub use recovery_catalog_stage::RecoveryCatalogStage; pub use recovery_catalog_stage_error::RecoveryCatalogStageError; @@ -370,6 +380,13 @@ pub use recovery_publication_stage_classifier::{ }; pub use recovery_required_entry::RecoveryRequiredEntry; pub use recovery_segment_classifier::classify_recovery_segment_stage; +pub use recovery_segment_resume_error::RecoverySegmentResumeError; +pub use recovery_segment_resume_executor::execute_recovery_segment_resume; +pub use recovery_segment_resume_plan_error::RecoverySegmentResumePlanError; +pub use recovery_segment_resume_planner::plan_recovery_segment_resume; +pub use recovery_segment_resume_request::RecoverySegmentResumeRequest; +pub use recovery_segment_resume_storage::RecoverySegmentResumeStorage; +pub use recovery_segment_resume_storage_error::RecoverySegmentResumeStorageError; pub use recovery_segment_stage::{RecoverySegmentStage, ReusableRecoverySegment}; pub use recovery_segment_stage_error::RecoverySegmentStageError; pub use recovery_segment_truncation::RecoverySegmentTruncation; diff --git a/src/adapters/opened_reusable_segment.rs b/src/adapters/opened_reusable_segment.rs new file mode 100644 index 0000000..6617bf9 --- /dev/null +++ b/src/adapters/opened_reusable_segment.rs @@ -0,0 +1,32 @@ +//! This module owns a storage-opened reusable segment materialization. + +use super::SegmentStage; + +/// Writable stage and exact bounded prefix returned by continuation storage. +/// +/// The storage adapter constructing this value must prove that `stage` +/// contains exactly `encoded`, is positioned immediately after those bytes, +/// and retains exclusive writer authority for its lifetime. The recovery +/// executor independently re-admits `encoded` before returning the stage. +#[must_use] +pub struct OpenedReusableSegment +where + S: SegmentStage, +{ + stage: S, + encoded: Box<[u8]>, +} + +impl OpenedReusableSegment +where + S: SegmentStage, +{ + /// Binds one storage-proven writable stage to its materialized prefix. + pub const fn new(stage: S, encoded: Box<[u8]>) -> Self { + Self { stage, encoded } + } + + pub(super) fn into_parts(self) -> (S, Box<[u8]>) { + (self.stage, self.encoded) + } +} diff --git a/src/adapters/recovery_segment_resume_error.rs b/src/adapters/recovery_segment_resume_error.rs new file mode 100644 index 0000000..d9aed65 --- /dev/null +++ b/src/adapters/recovery_segment_resume_error.rs @@ -0,0 +1,66 @@ +//! This module owns reusable-segment continuation execution failures. + +use std::error::Error; +use std::fmt; + +use super::{ + RecoverySegmentResumeStorageError, RecoveryStageAssessmentError, + RecoveryStageByteAdmissionError, SegmentReadError, +}; + +/// Why one reusable segment prefix could not become writable again. +#[derive(Debug)] +pub enum RecoverySegmentResumeError { + /// Storage could not reopen the exact stage. + Open { + /// Exact storage refusal. + source: RecoverySegmentResumeStorageError, + }, + /// Materialized bytes no longer match prior evidence. + Admission { + /// Exact evidence-admission refusal. + source: RecoveryStageByteAdmissionError, + }, + /// Reopened bytes no longer admit the segment grammar. + Assessment { + /// Exact semantic assessment refusal. + source: RecoveryStageAssessmentError, + }, + /// Reopened bytes no longer classify as a reusable prefix. + NotReusable, + /// Rebuilding append state from re-admitted records failed. + Rebuild { + /// Exact record-cursor or allocation refusal. + source: SegmentReadError, + }, +} + +impl fmt::Display for RecoverySegmentResumeError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Open { source } => write!(formatter, "cannot reopen reusable segment: {source}"), + Self::Admission { source } => { + write!(formatter, "reopened segment evidence disagrees: {source}") + } + Self::Assessment { source } => { + write!(formatter, "reopened segment is invalid: {source}") + } + Self::NotReusable => write!(formatter, "reopened segment is no longer reusable"), + Self::Rebuild { source } => { + write!(formatter, "cannot rebuild reusable segment state: {source}") + } + } + } +} + +impl Error for RecoverySegmentResumeError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Open { source } => Some(source), + Self::Admission { source } => Some(source), + Self::Assessment { source } => Some(source), + Self::Rebuild { source } => Some(source), + Self::NotReusable => None, + } + } +} diff --git a/src/adapters/recovery_segment_resume_executor.rs b/src/adapters/recovery_segment_resume_executor.rs new file mode 100644 index 0000000..ed92951 --- /dev/null +++ b/src/adapters/recovery_segment_resume_executor.rs @@ -0,0 +1,48 @@ +//! This module owns exact reusable-segment continuation execution. + +use super::{ + RecoverySegmentResumeError, RecoverySegmentResumeRequest, RecoverySegmentResumeStorage, + RecoverySegmentStage, RecoveryStage, RecoveryStageAssessment, StagedSegment, + admit_recovery_stage_bytes, assess_recovery_stage, +}; + +/// Reopens one exact reusable prefix as the ordinary append-only stage state. +/// +/// Storage is consumed so its writer authority remains owned by the returned +/// stage. The complete prefix is materialized once, re-admitted against prior +/// evidence, reclassified under the continuation policy, and used to rebuild +/// digest and duplicate-identity state before the first new write. +/// +/// # Errors +/// +/// Returns [`RecoverySegmentResumeError`] before returning a writable stage +/// when storage, evidence admission, semantic classification, or state +/// reconstruction fails. +pub fn execute_recovery_segment_resume( + storage: S, + request: RecoverySegmentResumeRequest, +) -> Result, RecoverySegmentResumeError> +where + S: RecoverySegmentResumeStorage, +{ + let opened = storage + .open_reusable(request) + .map_err(|source| RecoverySegmentResumeError::Open { source })?; + let (stage, encoded) = opened.into_parts(); + let admitted = admit_recovery_stage_bytes(RecoveryStage::Segment, request.evidence(), &encoded) + .map_err(|source| RecoverySegmentResumeError::Admission { source })?; + let assessment = assess_recovery_stage(&admitted, request.policy()) + .map_err(|source| RecoverySegmentResumeError::Assessment { source })?; + let RecoveryStageAssessment::Segment { + state: RecoverySegmentStage::Reusable(reusable), + .. + } = assessment + else { + return Err(RecoverySegmentResumeError::NotReusable); + }; + if reusable.record_count() != request.record_count() || reusable.length() != request.length() { + return Err(RecoverySegmentResumeError::NotReusable); + } + StagedSegment::resume_admitted(stage, &encoded, request) + .map_err(|source| RecoverySegmentResumeError::Rebuild { source }) +} diff --git a/src/adapters/recovery_segment_resume_plan_error.rs b/src/adapters/recovery_segment_resume_plan_error.rs new file mode 100644 index 0000000..7f6d88e --- /dev/null +++ b/src/adapters/recovery_segment_resume_plan_error.rs @@ -0,0 +1,48 @@ +//! This module owns reusable-segment continuation planning refusals. + +use std::error::Error; +use std::fmt; + +use super::{RecoveryStage, SegmentRecordLimit}; + +/// Why an assessed stage cannot enter reusable-segment continuation. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum RecoverySegmentResumePlanError { + /// A segment stage is not a reusable complete-record prefix. + NotReusable { + /// Fixed stage that requires a different recovery action. + stage: RecoveryStage, + }, + /// The stage belongs to a different recovery protocol. + NotSegment { + /// Fixed stage that cannot be resumed as a segment. + stage: RecoveryStage, + }, + /// The selected continuation policy is below the admitted record count. + RecordLimit { + /// Maximum complete-record count allowed by the policy. + maximum: SegmentRecordLimit, + /// Complete records already present in the reusable prefix. + observed: u32, + }, +} + +impl fmt::Display for RecoverySegmentResumePlanError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::NotReusable { stage } => { + write!(formatter, "{stage} is not a reusable segment prefix") + } + Self::NotSegment { stage } => { + write!(formatter, "{stage} cannot be resumed as a segment") + } + Self::RecordLimit { maximum, observed } => write!( + formatter, + "reusable segment has {observed} records, above continuation limit {}", + maximum.get() + ), + } + } +} + +impl Error for RecoverySegmentResumePlanError {} diff --git a/src/adapters/recovery_segment_resume_planner.rs b/src/adapters/recovery_segment_resume_planner.rs new file mode 100644 index 0000000..50e9cc3 --- /dev/null +++ b/src/adapters/recovery_segment_resume_planner.rs @@ -0,0 +1,44 @@ +//! This module owns pure planning for reusable-segment continuation. + +use super::{ + RecoverySegmentResumePlanError, RecoverySegmentResumeRequest, RecoverySegmentStage, + RecoveryStageAssessment, SegmentReadPolicy, +}; + +/// Plans continuation only from an exact reusable `current.seg` assessment. +/// +/// The returned request owns the observation evidence, complete-record count, +/// exact append boundary, and resource policy needed to re-admit the stage at +/// execution time. +/// +/// # Errors +/// +/// Returns [`RecoverySegmentResumePlanError`] when the assessment is not a +/// reusable segment prefix or the selected policy is already exceeded. +pub const fn plan_recovery_segment_resume( + assessment: &RecoveryStageAssessment<'_>, + policy: SegmentReadPolicy, +) -> Result { + let RecoveryStageAssessment::Segment { evidence, state } = assessment else { + return Err(RecoverySegmentResumePlanError::NotSegment { + stage: assessment.evidence().stage(), + }); + }; + let RecoverySegmentStage::Reusable(reusable) = state else { + return Err(RecoverySegmentResumePlanError::NotReusable { + stage: evidence.stage(), + }); + }; + if reusable.record_count() > policy.record_limit().get() { + return Err(RecoverySegmentResumePlanError::RecordLimit { + maximum: policy.record_limit(), + observed: reusable.record_count(), + }); + } + Ok(RecoverySegmentResumeRequest::new( + *evidence, + reusable.record_count(), + reusable.length(), + policy, + )) +} diff --git a/src/adapters/recovery_segment_resume_request.rs b/src/adapters/recovery_segment_resume_request.rs new file mode 100644 index 0000000..dcafb6b --- /dev/null +++ b/src/adapters/recovery_segment_resume_request.rs @@ -0,0 +1,54 @@ +//! This module owns one exact reusable-segment continuation request. + +use super::{RecoveryStageEvidence, RecoveryStageLength, SegmentReadPolicy, SegmentRecordLimit}; + +/// Owned authority to continue one exact validated reusable segment prefix. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[must_use] +pub struct RecoverySegmentResumeRequest { + evidence: RecoveryStageEvidence, + record_count: u32, + length: RecoveryStageLength, + policy: SegmentReadPolicy, +} + +impl RecoverySegmentResumeRequest { + pub(super) const fn new( + evidence: RecoveryStageEvidence, + record_count: u32, + length: RecoveryStageLength, + policy: SegmentReadPolicy, + ) -> Self { + Self { + evidence, + record_count, + length, + policy, + } + } + + /// Returns the exact prior observation that must still match. + pub const fn evidence(self) -> RecoveryStageEvidence { + self.evidence + } + + /// Returns the number of complete records already admitted. + pub const fn record_count(self) -> u32 { + self.record_count + } + + /// Returns the exact byte boundary after the admitted record prefix. + pub const fn length(self) -> RecoveryStageLength { + self.length + } + + /// Returns the resource policy that governs re-admission and continuation. + pub const fn policy(self) -> SegmentReadPolicy { + self.policy + } + + /// Returns the maximum complete-record count after continuation. + pub const fn record_limit(self) -> SegmentRecordLimit { + self.policy.record_limit() + } +} diff --git a/src/adapters/recovery_segment_resume_state.rs b/src/adapters/recovery_segment_resume_state.rs new file mode 100644 index 0000000..9632291 --- /dev/null +++ b/src/adapters/recovery_segment_resume_state.rs @@ -0,0 +1,67 @@ +//! This module owns reconstruction of append state from a reusable prefix. + +use std::collections::HashSet; + +use super::segment_digest_builder::SegmentDigestBuilder; +use super::segment_record_cursor::SegmentRecordCursor; +use super::{ + RecoverySegmentResumeRequest, SegmentHeader, SegmentReadError, SegmentRecordIdentity, + SegmentRecordLimit, +}; + +pub(super) struct RecoverySegmentResumeState { + pub(super) digest: SegmentDigestBuilder, + pub(super) identities: HashSet, + pub(super) record_limit: SegmentRecordLimit, + pub(super) record_count: u32, + pub(super) bytes_written: u64, +} + +impl RecoverySegmentResumeState { + pub(super) fn rebuild( + encoded: &[u8], + request: RecoverySegmentResumeRequest, + ) -> Result { + let records = + encoded + .get(SegmentHeader::ENCODED_LENGTH..) + .ok_or(SegmentReadError::WrongLength { + minimum: SegmentHeader::ENCODED_LENGTH, + observed: encoded.len(), + })?; + let mut identities = reserve_identities(request.record_count())?; + let mut cursor = + SegmentRecordCursor::new(records, request.record_count(), request.policy()); + while let Some(located) = cursor.next_record()? { + let _inserted = identities.insert(located.record.identity()); + } + cursor.finish()?; + let mut digest = SegmentDigestBuilder::new(); + digest.update(encoded); + Ok(Self { + digest, + identities, + record_limit: request.record_limit(), + record_count: request.record_count(), + bytes_written: request.length().get(), + }) + } +} + +fn reserve_identities( + record_count: u32, +) -> Result, SegmentReadError> { + let capacity = usize::try_from(record_count).map_err(|_source| { + SegmentReadError::RecordCountHostWidth { + observed: record_count, + } + })?; + let mut identities = HashSet::new(); + identities.try_reserve(capacity).map_err(|source| { + SegmentReadError::IdentityIndexAllocation { + record_count, + source, + } + })?; + Ok(identities) +} diff --git a/src/adapters/recovery_segment_resume_storage.rs b/src/adapters/recovery_segment_resume_storage.rs new file mode 100644 index 0000000..9196318 --- /dev/null +++ b/src/adapters/recovery_segment_resume_storage.rs @@ -0,0 +1,31 @@ +//! This module owns the writable storage port for segment continuation. + +use super::{ + OpenedReusableSegment, RecoverySegmentResumeRequest, RecoverySegmentResumeStorageError, + SegmentStage, +}; + +/// Reopens one exact reusable segment prefix under exclusive writer authority. +/// +/// The operation consumes the storage capability so the returned stage can +/// retain writer authority for its full writable lifetime. On success, the +/// stage must contain exactly the returned bytes and be positioned immediately +/// after them. The returned materialization must be bounded by the segment +/// protocol maximum and must have been revalidated against `request` after the +/// writable handle and canonical directory entry were both admitted. +pub trait RecoverySegmentResumeStorage: Sized { + /// Exclusively owned writable stage that retains writer authority. + type Stage: SegmentStage; + + /// Reopens and materializes the exact reusable prefix. + /// + /// # Errors + /// + /// Returns [`RecoverySegmentResumeStorageError`] when the stage is absent, + /// differs from the request, cannot be opened safely, or cannot be + /// materialized and positioned exactly. + fn open_reusable( + self, + request: RecoverySegmentResumeRequest, + ) -> Result, RecoverySegmentResumeStorageError>; +} diff --git a/src/adapters/recovery_segment_resume_storage_error.rs b/src/adapters/recovery_segment_resume_storage_error.rs new file mode 100644 index 0000000..256a9c8 --- /dev/null +++ b/src/adapters/recovery_segment_resume_storage_error.rs @@ -0,0 +1,65 @@ +//! This module owns semantic storage refusals during segment continuation. + +use std::error::Error; +use std::fmt; +use std::io; + +use super::{RecoverySegmentResumeRequest, RecoveryStageEvidence}; + +/// Why storage could not reopen one exact reusable segment prefix. +#[derive(Debug)] +pub enum RecoverySegmentResumeStorageError { + /// The fixed stage resolves to different evidence. + EvidenceMismatch { + /// Evidence bound into the explicit continuation request. + expected: RecoveryStageEvidence, + /// Evidence observed while reopening the writable stage. + observed: RecoveryStageEvidence, + }, + /// The fixed segment stage is absent. + Missing { + /// Exact continuation request whose stage is absent. + request: RecoverySegmentResumeRequest, + }, + /// The storage boundary failed while reopening or materializing. + Storage { + /// Exact underlying storage failure. + source: io::Error, + }, +} + +impl RecoverySegmentResumeStorageError { + /// Wraps an underlying storage failure without discarding its source. + pub const fn storage(source: io::Error) -> Self { + Self::Storage { source } + } +} + +impl fmt::Display for RecoverySegmentResumeStorageError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::EvidenceMismatch { expected, observed } => write!( + formatter, + "{} continuation evidence changed from length {} to length {}", + expected.stage(), + expected.length().get(), + observed.length().get() + ), + Self::Missing { request } => { + write!(formatter, "{} is absent", request.evidence().stage()) + } + Self::Storage { source } => { + write!(formatter, "recovery segment continuation failed: {source}") + } + } + } +} + +impl Error for RecoverySegmentResumeStorageError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Storage { source } => Some(source), + Self::EvidenceMismatch { .. } | Self::Missing { .. } => None, + } + } +} diff --git a/src/adapters/staged_segment.rs b/src/adapters/staged_segment.rs index d21f86e..f1ab83e 100644 --- a/src/adapters/staged_segment.rs +++ b/src/adapters/staged_segment.rs @@ -2,11 +2,12 @@ use std::collections::HashSet; +use super::recovery_segment_resume_state::RecoverySegmentResumeState; use super::segment_digest_builder::SegmentDigestBuilder; use super::{ - AdmittedSegmentRecord, SealedSegment, SegmentDurabilityPhase, SegmentHeader, - SegmentRecordIdentity, SegmentRecordLimit, SegmentSeal, SegmentStage, SegmentWriteError, - SegmentWritePhase, segment_seal_builder, segment_stage_write, + AdmittedSegmentRecord, RecoverySegmentResumeRequest, SealedSegment, SegmentDurabilityPhase, + SegmentHeader, SegmentReadError, SegmentRecordIdentity, SegmentRecordLimit, SegmentSeal, + SegmentStage, SegmentWriteError, SegmentWritePhase, segment_seal_builder, segment_stage_write, }; /// An exclusively owned append-only segment stage. @@ -63,6 +64,22 @@ where }) } + pub(super) fn resume_admitted( + stage: S, + encoded: &[u8], + request: RecoverySegmentResumeRequest, + ) -> Result { + let recovered = RecoverySegmentResumeState::rebuild(encoded, request)?; + Ok(Self { + stage, + digest: recovered.digest, + identities: recovered.identities, + record_limit: recovered.record_limit, + record_count: recovered.record_count, + bytes_written: recovered.bytes_written, + }) + } + /// Appends one complete content-admitted record. /// /// All count, duplicate, allocation, and complete-segment length checks diff --git a/src/lib.rs b/src/lib.rs index 2bdbd7f..7f305b5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -19,8 +19,9 @@ //! truncated-stage discard, and complete-stage valid-orphan recovery are //! explicit. Exact next-head finalization now has a storage-independent //! contract and a pinned writer-authorized filesystem adapter. Reusable-stage -//! continuation, retention, and garbage collection APIs remain intentionally -//! absent until their contracts have executable specifications. +//! continuation has a storage-independent planning and execution boundary. +//! Its filesystem binding, retention, and garbage collection remain +//! intentionally absent until their contracts have executable specifications. #[cfg(test)] extern crate self as keep; @@ -51,17 +52,19 @@ pub use adapters::{ FilesystemRecoveryStageDiscardOpenError, FilesystemRecoveryStageDiscarder, FilesystemRecoveryStageError, FilesystemSegmentStage, FilesystemWriterLock, LayoutDecodeError, LayoutDecodePolicy, LayoutEncodeError, LayoutIdBinaryParseError, LayoutIdTextParseError, - PublicationHeadDecodeError, RecoveryCatalogStage, RecoveryCatalogStageError, RecoveryEntryName, - RecoveryEntryNameError, RecoveryEntryRole, RecoveryInventory, RecoveryInventoryEntry, - RecoveryInventoryError, RecoveryInventoryLimit, RecoveryInventoryLimitError, - RecoveryInventoryOperation, RecoveryInventoryStorage, RecoveryNameClassificationError, - RecoveryNameManifest, RecoveryNamedEntry, RecoveryNamespace, RecoveryNextHeadFinalizationError, - RecoveryNextHeadFinalizationOutcome, RecoveryNextHeadFinalizationPlanError, - RecoveryNextHeadFinalizationReadiness, RecoveryNextHeadFinalizationReceipt, - RecoveryNextHeadFinalizationRequest, RecoveryNextHeadFinalizationStorage, - RecoveryNextHeadFinalizationStorageError, RecoveryNextHeadFinalizationTarget, - RecoveryNextHeadStage, RecoveryNextHeadStageError, RecoveryPoolNameError, - RecoveryRequiredEntry, RecoverySegmentStage, RecoverySegmentStageError, + OpenedReusableSegment, PublicationHeadDecodeError, RecoveryCatalogStage, + RecoveryCatalogStageError, RecoveryEntryName, RecoveryEntryNameError, RecoveryEntryRole, + RecoveryInventory, RecoveryInventoryEntry, RecoveryInventoryError, RecoveryInventoryLimit, + RecoveryInventoryLimitError, RecoveryInventoryOperation, RecoveryInventoryStorage, + RecoveryNameClassificationError, RecoveryNameManifest, RecoveryNamedEntry, RecoveryNamespace, + RecoveryNextHeadFinalizationError, RecoveryNextHeadFinalizationOutcome, + RecoveryNextHeadFinalizationPlanError, RecoveryNextHeadFinalizationReadiness, + RecoveryNextHeadFinalizationReceipt, RecoveryNextHeadFinalizationRequest, + RecoveryNextHeadFinalizationStorage, RecoveryNextHeadFinalizationStorageError, + RecoveryNextHeadFinalizationTarget, RecoveryNextHeadStage, RecoveryNextHeadStageError, + RecoveryPoolNameError, RecoveryRequiredEntry, RecoverySegmentResumeError, + RecoverySegmentResumePlanError, RecoverySegmentResumeRequest, RecoverySegmentResumeStorage, + RecoverySegmentResumeStorageError, RecoverySegmentStage, RecoverySegmentStageError, RecoverySegmentTruncation, RecoveryStage, RecoveryStageAssessment, RecoveryStageAssessmentError, RecoveryStageByteAdmissionError, RecoveryStageCompletionError, RecoveryStageCompletionPlanError, RecoveryStageCompletionPool, RecoveryStageCompletionReceipt, @@ -84,10 +87,11 @@ pub use adapters::{ StoreInitializationStorage, WriterLockAcquireError, WriterLockAcquirePhase, admit_recovery_stage_bytes, assess_recovery_stage, classify_recovery_catalog_stage, classify_recovery_names, classify_recovery_next_head_stage, classify_recovery_segment_stage, - execute_recovery_next_head_finalization, execute_recovery_stage_completion, - execute_recovery_stage_discard, fingerprint_recovery_stage, initialize_store, - plan_recovery_next_head_finalization, plan_recovery_stage_completion, - plan_recovery_stage_discard, publish_catalog_generation, read_recovery_inventory, + execute_recovery_next_head_finalization, execute_recovery_segment_resume, + execute_recovery_stage_completion, execute_recovery_stage_discard, fingerprint_recovery_stage, + initialize_store, plan_recovery_next_head_finalization, plan_recovery_segment_resume, + plan_recovery_stage_completion, plan_recovery_stage_discard, publish_catalog_generation, + read_recovery_inventory, }; pub use blob::{ BlobHashError, BlobHasher, BlobId, BlobLength, BlobReadError, ByteLength, ByteOffset, diff --git a/tests/recovery_segment_resume.rs b/tests/recovery_segment_resume.rs new file mode 100644 index 0000000..9f07605 --- /dev/null +++ b/tests/recovery_segment_resume.rs @@ -0,0 +1,71 @@ +//! Exact reusable-segment recovery laws. + +#[path = "recovery_segment_resume/execution_laws.rs"] +mod execution_laws; +#[path = "recovery_segment_resume/planning_laws.rs"] +mod planning_laws; +#[path = "recovery_segment_resume/storage_double.rs"] +pub mod storage_double; +#[path = "support/mod.rs"] +mod support; + +use std::error::Error; + +use keep::{ + LayoutEntryLimit, RecoverySegmentResumeRequest, RecoveryStage, RecoveryStageAssessment, + RecoveryStageEvidence, RecoveryStageMetadata, SegmentReadPolicy, SegmentRecordLimit, + admit_recovery_stage_bytes, assess_recovery_stage, fingerprint_recovery_stage, + plan_recovery_segment_resume, +}; +use support::decode_hex; + +const SEGMENT_HEX: &str = include_str!("../conformance/segment-store/v1/one-zero-segment.hex"); +const HEAD_HEX: &str = include_str!("../conformance/segment-store/v1/one-zero-head.hex"); +const SEGMENT_HEADER_LENGTH: usize = 64; +const SEGMENT_SEAL_LENGTH: usize = 128; + +fn fixture(hex: &str) -> Result, Box> { + decode_hex( + hex.strip_suffix('\n') + .ok_or("recovery fixture must end in one LF")?, + ) + .map_err(Into::into) +} + +fn reusable_prefix() -> Result, Box> { + let mut encoded = fixture(SEGMENT_HEX)?; + let prefix_length = encoded + .len() + .checked_sub(SEGMENT_SEAL_LENGTH) + .ok_or("segment fixture is shorter than its seal")?; + encoded.truncate(prefix_length); + Ok(encoded) +} + +fn evidence(stage: RecoveryStage, encoded: &[u8]) -> Result> { + let length = u64::try_from(encoded.len())?; + Ok(fingerprint_recovery_stage( + RecoveryStageMetadata::new(stage, length)?, + encoded, + )?) +} + +fn assessment( + stage: RecoveryStage, + encoded: &[u8], +) -> Result, Box> { + let observed = evidence(stage, encoded)?; + let admitted = admit_recovery_stage_bytes(stage, observed, encoded)?; + Ok(assess_recovery_stage(&admitted, maximum_policy())?) +} + +fn resume_request(encoded: &[u8]) -> Result> { + Ok(plan_recovery_segment_resume( + &assessment(RecoveryStage::Segment, encoded)?, + maximum_policy(), + )?) +} + +const fn maximum_policy() -> SegmentReadPolicy { + SegmentReadPolicy::new(SegmentRecordLimit::MAXIMUM, LayoutEntryLimit::MAXIMUM) +} diff --git a/tests/recovery_segment_resume/execution_laws.rs b/tests/recovery_segment_resume/execution_laws.rs new file mode 100644 index 0000000..00b0e24 --- /dev/null +++ b/tests/recovery_segment_resume/execution_laws.rs @@ -0,0 +1,88 @@ +//! Reusable-segment recovery execution laws. + +use std::error::Error; + +use keep::{ + AdmittedSegment, AdmittedSegmentRecord, RecoverySegmentResumeError, SegmentWriteError, + execute_recovery_segment_resume, +}; + +use super::storage_double::MemoryResumeStorage; +use super::support::require_error; +use super::{maximum_policy, resume_request, reusable_prefix}; + +#[test] +fn resumed_prefix_appends_and_seals_without_rewriting_admitted_bytes() -> Result<(), Box> +{ + let prefix = reusable_prefix()?; + let storage = MemoryResumeStorage::available(&prefix); + let probe = storage.probe(); + + let resumed = execute_recovery_segment_resume(storage, resume_request(&prefix)?)?; + let sealed = resumed + .append(AdmittedSegmentRecord::for_chunk(&[1])?)? + .seal()?; + let observed = probe.borrow().clone(); + let admitted = AdmittedSegment::decode(&observed, maximum_policy())?; + + assert!(observed.starts_with(&prefix)); + assert_eq!(sealed.record_count(), 2); + assert_eq!(admitted.record_count(), 2); + Ok(()) +} + +#[test] +fn resumed_prefix_retains_duplicate_identity_refusal() -> Result<(), Box> { + let prefix = reusable_prefix()?; + let storage = MemoryResumeStorage::available(&prefix); + let resumed = execute_recovery_segment_resume(storage, resume_request(&prefix)?)?; + let duplicate = AdmittedSegmentRecord::for_chunk(&[0])?; + + let error = require_error( + resumed.append(duplicate), + "resumed prefix must retain prior identities", + )?; + + assert!(matches!( + error, + SegmentWriteError::DuplicateRecordIdentity { identity } + if identity == duplicate.identity() + )); + Ok(()) +} + +#[test] +fn changed_materialized_bytes_are_refused_before_a_write() -> Result<(), Box> { + let prefix = reusable_prefix()?; + let mut changed = prefix.clone(); + let tail = changed.last_mut().ok_or("missing prefix tail")?; + *tail ^= 1; + let storage = MemoryResumeStorage::available(&changed); + let probe = storage.probe(); + + let error = require_error( + execute_recovery_segment_resume(storage, resume_request(&prefix)?), + "changed bytes must not be resumed", + )?; + + assert!(matches!( + error, + RecoverySegmentResumeError::Admission { .. } + )); + assert_eq!(*probe.borrow(), changed); + Ok(()) +} + +#[test] +fn storage_failure_returns_no_resumable_stage() -> Result<(), Box> { + let prefix = reusable_prefix()?; + let storage = MemoryResumeStorage::failing(&prefix); + + let error = require_error( + execute_recovery_segment_resume(storage, resume_request(&prefix)?), + "storage failure must prevent continuation", + )?; + + assert!(matches!(error, RecoverySegmentResumeError::Open { .. })); + Ok(()) +} diff --git a/tests/recovery_segment_resume/planning_laws.rs b/tests/recovery_segment_resume/planning_laws.rs new file mode 100644 index 0000000..9d26431 --- /dev/null +++ b/tests/recovery_segment_resume/planning_laws.rs @@ -0,0 +1,111 @@ +//! Reusable-segment recovery planning laws. + +use std::error::Error; + +use keep::{ + LayoutEntryLimit, RecoverySegmentResumePlanError, RecoveryStage, SegmentReadPolicy, + SegmentRecordLimit, plan_recovery_segment_resume, +}; + +use super::{ + HEAD_HEX, SEGMENT_HEADER_LENGTH, SEGMENT_HEX, assessment, fixture, maximum_policy, + reusable_prefix, +}; +use crate::support::require_error; + +#[test] +fn only_exact_reusable_segment_assessment_produces_a_resume_request() -> Result<(), Box> +{ + let encoded = reusable_prefix()?; + let assessed = assessment(RecoveryStage::Segment, &encoded)?; + + let request = plan_recovery_segment_resume(&assessed, maximum_policy())?; + + assert_eq!(request.evidence(), assessed.evidence()); + assert_eq!(request.record_count(), 1); + assert_eq!(request.length().get(), u64::try_from(encoded.len())?); + assert_eq!(request.policy(), maximum_policy()); + Ok(()) +} + +#[test] +fn truncated_segment_cannot_enter_reusable_resume() -> Result<(), Box> { + let encoded = reusable_prefix()?; + let truncated = encoded + .get(..SEGMENT_HEADER_LENGTH - 1) + .ok_or("missing truncation")?; + let assessed = assessment(RecoveryStage::Segment, truncated)?; + + let error = require_error( + plan_recovery_segment_resume(&assessed, maximum_policy()), + "truncated stage must not produce a resume request", + )?; + + assert_eq!( + error, + RecoverySegmentResumePlanError::NotReusable { + stage: RecoveryStage::Segment, + } + ); + Ok(()) +} + +#[test] +fn complete_segment_cannot_enter_reusable_resume() -> Result<(), Box> { + let encoded = fixture(SEGMENT_HEX)?; + let assessed = assessment(RecoveryStage::Segment, &encoded)?; + + let error = require_error( + plan_recovery_segment_resume(&assessed, maximum_policy()), + "complete segment must not produce a resume request", + )?; + + assert_eq!( + error, + RecoverySegmentResumePlanError::NotReusable { + stage: RecoveryStage::Segment, + } + ); + Ok(()) +} + +#[test] +fn policy_below_the_admitted_record_count_is_refused() -> Result<(), Box> { + let encoded = reusable_prefix()?; + let assessed = assessment(RecoveryStage::Segment, &encoded)?; + let maximum = SegmentRecordLimit::new(0)?; + let policy = SegmentReadPolicy::new(maximum, LayoutEntryLimit::MAXIMUM); + + let error = require_error( + plan_recovery_segment_resume(&assessed, policy), + "policy below the admitted prefix must be refused", + )?; + + assert_eq!( + error, + RecoverySegmentResumePlanError::RecordLimit { + maximum, + observed: 1, + } + ); + Ok(()) +} + +#[test] +fn a_non_segment_stage_cannot_enter_reusable_resume() -> Result<(), Box> { + let encoded = fixture(HEAD_HEX)?; + let assessed = assessment(RecoveryStage::NextHead, &encoded)?; + + let error = require_error( + plan_recovery_segment_resume(&assessed, maximum_policy()), + "head.next must not produce a segment resume request", + )?; + + assert_eq!( + error, + RecoverySegmentResumePlanError::NotSegment { + stage: RecoveryStage::NextHead, + } + ); + Ok(()) +} diff --git a/tests/recovery_segment_resume/storage_double.rs b/tests/recovery_segment_resume/storage_double.rs new file mode 100644 index 0000000..caf6efb --- /dev/null +++ b/tests/recovery_segment_resume/storage_double.rs @@ -0,0 +1,81 @@ +//! This module owns the in-memory reusable-stage storage double. + +use std::cell::RefCell; +use std::io::{self, Write}; +use std::rc::Rc; + +use keep::{ + OpenedReusableSegment, RecoverySegmentResumeRequest, RecoverySegmentResumeStorage, + RecoverySegmentResumeStorageError, SegmentStage, +}; + +/// In-memory storage that either returns one exact prefix or an injected error. +pub struct MemoryResumeStorage { + encoded: Box<[u8]>, + probe: Rc>>, + failure: Option, +} + +impl MemoryResumeStorage { + /// Constructs available storage containing `encoded`. + pub fn available(encoded: &[u8]) -> Self { + Self { + encoded: encoded.into(), + probe: Rc::new(RefCell::new(encoded.to_vec())), + failure: None, + } + } + + /// Constructs storage that refuses reopening before returning a stage. + pub fn failing(encoded: &[u8]) -> Self { + Self { + encoded: encoded.into(), + probe: Rc::new(RefCell::new(encoded.to_vec())), + failure: Some(io::Error::other("injected resume failure")), + } + } + + /// Returns shared observation of all stage bytes. + pub fn probe(&self) -> Rc>> { + Rc::clone(&self.probe) + } +} + +/// Append-only in-memory stage positioned after its preloaded prefix. +pub struct MemoryResumeStage { + probe: Rc>>, +} + +impl Write for MemoryResumeStage { + fn write(&mut self, bytes: &[u8]) -> io::Result { + self.probe.borrow_mut().extend_from_slice(bytes); + Ok(bytes.len()) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +impl SegmentStage for MemoryResumeStage { + fn synchronize(&mut self) -> io::Result<()> { + Ok(()) + } +} + +impl RecoverySegmentResumeStorage for MemoryResumeStorage { + type Stage = MemoryResumeStage; + + fn open_reusable( + self, + _request: RecoverySegmentResumeRequest, + ) -> Result, RecoverySegmentResumeStorageError> { + if let Some(source) = self.failure { + return Err(RecoverySegmentResumeStorageError::storage(source)); + } + Ok(OpenedReusableSegment::new( + MemoryResumeStage { probe: self.probe }, + self.encoded, + )) + } +} From e4c83f69e01de7ca9fb7e0f09ac4383c1beaad41 Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 13:16:12 -0700 Subject: [PATCH 22/49] Add: Resume recovery segments on filesystem --- CHANGELOG.md | 6 + README.md | 11 +- docs/formats/segment-store-v1/recovery.md | 17 ++ docs/formats/segment-store-v1/requirements.md | 1 + ...stem_recovery_segment_resume_open_error.rs | 87 +++++++++ ...esystem_recovery_segment_resume_storage.rs | 90 +++++++++ ...ilesystem_recovery_segment_resume_tests.rs | 67 +++++++ .../fixture.rs | 101 ++++++++++ .../refusal_laws.rs | 163 +++++++++++++++++ .../filesystem_recovery_segment_resumer.rs | 64 +++++++ .../filesystem_recovery_segment_stage.rs | 43 +++++ src/adapters/filesystem_recovery_stage.rs | 68 ++++++- .../filesystem_recovery_stage_error.rs | 120 ++++++------ ...filesystem_recovery_stage_error_display.rs | 172 ++++++++++++++++++ ...lesystem_recovery_stage_materialization.rs | 66 +++++++ src/adapters/mod.rs | 11 ++ src/lib.rs | 50 ++--- 17 files changed, 1037 insertions(+), 100 deletions(-) create mode 100644 src/adapters/filesystem_recovery_segment_resume_open_error.rs create mode 100644 src/adapters/filesystem_recovery_segment_resume_storage.rs create mode 100644 src/adapters/filesystem_recovery_segment_resume_tests.rs create mode 100644 src/adapters/filesystem_recovery_segment_resume_tests/fixture.rs create mode 100644 src/adapters/filesystem_recovery_segment_resume_tests/refusal_laws.rs create mode 100644 src/adapters/filesystem_recovery_segment_resumer.rs create mode 100644 src/adapters/filesystem_recovery_segment_stage.rs create mode 100644 src/adapters/filesystem_recovery_stage_error_display.rs create mode 100644 src/adapters/filesystem_recovery_stage_materialization.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index d5e4319..3104890 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,12 @@ after its public API and format compatibility policies are established. authority, re-admits the materialized prefix against saved evidence, rebuilds digest and duplicate-identity state, and returns the ordinary append-only stage without rewriting admitted bytes. +- Filesystem reusable-segment recovery now retains pinned root, namespace, and + writer-lock authority in the returned stage; reopens `current.seg` read-write + without following links or truncation; bounds, materializes, and re-admits + its exact prefix; verifies the final entry and append position; and refuses + missing, changed, linked, replaced, or namespace-drifted evidence before + writing. - Complete caller-supplied catalog and candidate-head stages now distinguish exact fixed-header, declared-body, or fixed-width truncation from canonical bytes. Complete-looking corruption and oversized stages remain typed diff --git a/README.md b/README.md index 772bca5..658366e 100644 --- a/README.md +++ b/README.md @@ -81,6 +81,10 @@ An exact reusable segment assessment can authorize storage-independent continuation: the executor consumes writer authority, re-admits the complete bounded prefix, rebuilds digest and duplicate-identity state, and returns the ordinary append-only stage without rewriting admitted bytes. +`FilesystemRecoverySegmentResumer` implements that contract with pinned +namespaces and writer authority, no-follow read-write reopening, exact bounded +materialization, final entry and namespace revalidation, and an append +position equal to the admitted prefix length. Exact truncation assessments can authorize durable, evidence-bound discard. Complete segment and catalog assessments can authorize verified immutable-pool @@ -95,10 +99,9 @@ authority, reconstructs the complete current and candidate views without following links, verifies namespace and stage identity, synchronizes and reverifies the exact candidate, atomically replaces `HEAD`, and synchronizes the root. An already-finalized retry requires `head.next` to be absent. -Process-death injection, the filesystem binding for reusable-stage -continuation, retention, compaction, and garbage collection remain planned. -Presence in the reference CAS does not claim retention, crash recovery, or -durability. +Process-death injection, retention, compaction, and garbage collection remain +planned. Presence in the reference CAS does not claim retention, crash +recovery, or durability. ```rust use keep::BlobId; diff --git a/docs/formats/segment-store-v1/recovery.md b/docs/formats/segment-store-v1/recovery.md index 2365a3d..7247175 100644 --- a/docs/formats/segment-store-v1/recovery.md +++ b/docs/formats/segment-store-v1/recovery.md @@ -121,6 +121,23 @@ canonical seal. The admitted prefix is never rewritten. A storage adapter that cannot prove the writable object, canonical entry, exact materialized bytes, and end position agree must refuse before returning the stage. +`FilesystemRecoverySegmentResumer` binds this contract to the admitted +filesystem profile. It pins the root and all protocol directories, acquires +`writer.lock`, and is consumed by execution. The reopened `current.seg` is a +regular read-write file opened without following links or truncation. Its +complete protocol-bounded bytes are fingerprinted, materialized, and +re-admitted; the handle and canonical entry retain one file identity and exact +length, the pinned namespaces are reverified, and the handle is positioned at +the admitted append boundary before handoff. + +The returned `FilesystemRecoverySegmentStage` owns the pinned authority and +writer lock. Zero-record and nonempty prefixes both enter the same append and +seal state machine. Missing stages, symbolic links, changed fingerprints, +entry replacement, namespace replacement, allocation refusal, read failure, +or position disagreement return typed failures without writing stage bytes. +Dropping an unsealed resumed stage preserves its current bytes for another +explicit recovery decision. + ## Complete a durable stage The recovery plan may bind one fully verified `current.seg` or `current.cat` diff --git a/docs/formats/segment-store-v1/requirements.md b/docs/formats/segment-store-v1/requirements.md index c63c629..e521ff6 100644 --- a/docs/formats/segment-store-v1/requirements.md +++ b/docs/formats/segment-store-v1/requirements.md @@ -127,6 +127,7 @@ continuation or process-death injection. | `KEEP-RECOVERY-017` | Next-head finalization plans only from an exact complete `head.next` assessment and its matching complete transitive catalog snapshot, admits only generation one over an uninitialized root or the exact successor of an expected current snapshot, synchronizes a ready candidate before atomic replacement, accepts an already-finalized retry, and returns a receipt only after root synchronization | Snapshot-coordinate, transition, candidate-sync, operation-order, fault-stop, and post-replacement retry matrix | `tests/recovery_next_head_finalization.rs`, `tests/recovery_next_head_finalization/*.rs` | Implemented in #17 | | `KEEP-RECOVERY-018` | Filesystem next-head finalization retains root and `writer.lock` authority, pins every protocol directory, revalidates namespace identity and exact stage evidence around bounded complete current and candidate loads, synchronizes and reverifies the exact candidate before atomic replacement, refuses current drift, missing or reappeared candidates, links, corrupt transitive views, and namespace replacement, and returns only after root synchronization | Initial and successor finalization, exact retry, candidate-sync, evidence-drift, missing, link, corruption, namespace-replacement, current-drift, and writer-exclusion matrix | `src/adapters/filesystem_recovery_next_head_finalization_tests.rs`, `src/adapters/filesystem_recovery_next_head_finalization_tests/*.rs` | Implemented in #17 | | `KEEP-RECOVERY-019` | Reusable-segment continuation plans only from an exact reusable `current.seg` assessment within the selected record policy, consumes the storage authority that reopens the stage, re-admits the complete materialized prefix against prior evidence, rebuilds digest and duplicate-identity state, and returns the ordinary append-only stage without rewriting admitted bytes | Reusable-only planning, policy refusal, changed-evidence, storage-failure, duplicate-identity, append, seal, and independent decode matrix | `tests/recovery_segment_resume.rs`, `tests/recovery_segment_resume/*.rs` | Implemented in #17 | +| `KEEP-RECOVERY-020` | Filesystem reusable-segment continuation retains root and `writer.lock` authority in the returned stage, pins every protocol directory, opens `current.seg` read-write without following links or truncation, bounds and re-admits its complete bytes, positions the handle at the exact validated append boundary, refuses missing, changed, linked, replaced, or namespace-drifted evidence, and preserves the prefix on append or empty seal | Empty and nonempty continuation, writer exclusion, missing, changed evidence, link, namespace replacement, writable-handoff replacement, append, seal, and independent decode matrix | `src/adapters/filesystem_recovery_segment_resume_tests.rs`, `src/adapters/filesystem_recovery_segment_resume_tests/*.rs` | Implemented in #17 | diff --git a/src/adapters/filesystem_recovery_segment_resume_open_error.rs b/src/adapters/filesystem_recovery_segment_resume_open_error.rs new file mode 100644 index 0000000..ae024c8 --- /dev/null +++ b/src/adapters/filesystem_recovery_segment_resume_open_error.rs @@ -0,0 +1,87 @@ +//! This module owns filesystem segment-continuation authority failures. + +use std::error::Error; +use std::fmt; +use std::io; + +use super::{ + FilesystemRecoveryStageDiscardOpenError, RecoveryInventoryError, WriterLockAcquireError, +}; + +/// Why a pinned writer-authorized segment resumer could not be opened. +#[derive(Debug)] +pub enum FilesystemRecoverySegmentResumeOpenError { + /// The store root did not satisfy the supported platform profile. + Platform { + /// Exact platform-admission failure. + source: io::Error, + }, + /// Exclusive writer authority could not be acquired. + WriterLock { + /// Exact writer-lock acquisition refusal. + source: WriterLockAcquireError, + }, + /// The locked root capability could not be cloned for recovery inventory. + CloneRoot { + /// Exact root-capability clone failure. + source: io::Error, + }, + /// One pinned protocol namespace could not be admitted. + Namespace { + /// Exact recovery-namespace admission refusal. + source: RecoveryInventoryError, + }, +} + +impl From for FilesystemRecoverySegmentResumeOpenError { + fn from(source: FilesystemRecoveryStageDiscardOpenError) -> Self { + match source { + FilesystemRecoveryStageDiscardOpenError::Platform { source } => { + Self::Platform { source } + } + FilesystemRecoveryStageDiscardOpenError::WriterLock { source } => { + Self::WriterLock { source } + } + FilesystemRecoveryStageDiscardOpenError::CloneRoot { source } => { + Self::CloneRoot { source } + } + FilesystemRecoveryStageDiscardOpenError::Namespace { source } => { + Self::Namespace { source } + } + } + } +} + +impl fmt::Display for FilesystemRecoverySegmentResumeOpenError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Platform { source } => { + write!( + formatter, + "segment continuation platform was refused: {source}" + ) + } + Self::WriterLock { source } => write!( + formatter, + "segment continuation writer lock was refused: {source}" + ), + Self::CloneRoot { source } => { + write!(formatter, "locked recovery root clone failed: {source}") + } + Self::Namespace { source } => write!( + formatter, + "segment continuation namespace was refused: {source}" + ), + } + } +} + +impl Error for FilesystemRecoverySegmentResumeOpenError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Platform { source } | Self::CloneRoot { source } => Some(source), + Self::WriterLock { source } => Some(source), + Self::Namespace { source } => Some(source), + } + } +} diff --git a/src/adapters/filesystem_recovery_segment_resume_storage.rs b/src/adapters/filesystem_recovery_segment_resume_storage.rs new file mode 100644 index 0000000..1b8ba76 --- /dev/null +++ b/src/adapters/filesystem_recovery_segment_resume_storage.rs @@ -0,0 +1,90 @@ +//! This module owns filesystem execution of reusable segment continuation. + +use std::io; + +use super::{ + FilesystemRecoverySegmentResumer, FilesystemRecoverySegmentStage, FilesystemRecoveryStageError, + OpenedReusableSegment, RecoverySegmentResumeRequest, RecoverySegmentResumeStorage, + RecoverySegmentResumeStorageError, RecoveryStage, RecoveryStageNamespacePhase, + filesystem_recovery_stage, +}; + +impl RecoverySegmentResumeStorage for FilesystemRecoverySegmentResumer { + type Stage = FilesystemRecoverySegmentStage; + + fn open_reusable( + self, + request: RecoverySegmentResumeRequest, + ) -> Result, RecoverySegmentResumeStorageError> { + #[cfg(test)] + let result = { + let mut storage = self; + if let Some(before_handoff) = storage.before_handoff.take() { + return open_with(storage, request, before_handoff); + } + open_with(storage, request, || {}) + }; + #[cfg(not(test))] + let result = open_with(self, request, || {}); + result + } +} + +fn open_with( + resumer: FilesystemRecoverySegmentResumer, + request: RecoverySegmentResumeRequest, + before_handoff: F, +) -> Result, RecoverySegmentResumeStorageError> +where + F: FnOnce(), +{ + let inventory = &resumer.discarder.inventory; + inventory + .verify_stage_namespaces( + RecoveryStage::Segment, + RecoveryStageNamespacePhase::BeforeObservation, + ) + .map_err(stage_error)?; + let directory = inventory.stage_directory(RecoveryStage::Segment); + let mut observed = + match filesystem_recovery_stage::observe_writable_segment_with(directory, || {}) { + Ok(observed) => observed, + Err(FilesystemRecoveryStageError::Open { source, .. }) + if source.kind() == io::ErrorKind::NotFound => + { + return Err(RecoverySegmentResumeStorageError::Missing { request }); + } + Err(source) => return Err(stage_error(source)), + }; + let actual = observed.evidence(); + if actual != request.evidence() { + return Err(RecoverySegmentResumeStorageError::EvidenceMismatch { + expected: request.evidence(), + observed: actual, + }); + } + let encoded = observed + .materialize_and_position(RecoveryStage::Segment) + .map_err(stage_error)?; + before_handoff(); + inventory + .verify_stage_namespaces( + RecoveryStage::Segment, + RecoveryStageNamespacePhase::AfterObservation, + ) + .map_err(stage_error)?; + observed + .verify( + directory, + RecoveryStage::Segment.file_name(), + RecoveryStage::Segment, + ) + .map_err(stage_error)?; + let file = observed.into_file(); + let stage = FilesystemRecoverySegmentStage::new(file, resumer.discarder); + Ok(OpenedReusableSegment::new(stage, encoded)) +} + +fn stage_error(source: FilesystemRecoveryStageError) -> RecoverySegmentResumeStorageError { + RecoverySegmentResumeStorageError::storage(io::Error::other(source)) +} diff --git a/src/adapters/filesystem_recovery_segment_resume_tests.rs b/src/adapters/filesystem_recovery_segment_resume_tests.rs new file mode 100644 index 0000000..229dc03 --- /dev/null +++ b/src/adapters/filesystem_recovery_segment_resume_tests.rs @@ -0,0 +1,67 @@ +//! Pinned-filesystem reusable-segment continuation laws. + +use std::error::Error; +use std::fs; + +use super::{ + AdmittedSegment, AdmittedSegmentRecord, FilesystemRecoverySegmentResumeOpenError, + WriterLockAcquireError, execute_recovery_segment_resume, +}; + +mod fixture; +mod refusal_laws; + +use fixture::{ResumeFixture, empty_prefix, maximum_policy, resume_request, reusable_prefix}; + +#[test] +fn empty_prefix_resumes_and_seals_as_an_empty_segment() -> Result<(), Box> { + let fixture = ResumeFixture::new("filesystem-empty-segment-resume")?; + let prefix = empty_prefix()?; + fs::write(fixture.stage_path(), &prefix)?; + + let resumed = execute_recovery_segment_resume(fixture.resumer()?, resume_request(&prefix)?)?; + let sealed = resumed.seal()?; + assert_eq!(sealed.record_count(), 0); + let _closed = sealed.close(); + + let observed = fs::read(fixture.stage_path())?; + let admitted = AdmittedSegment::decode(&observed, maximum_policy())?; + assert!(observed.starts_with(&prefix)); + assert_eq!(admitted.record_count(), 0); + fixture.remove()?; + Ok(()) +} + +#[test] +fn exact_prefix_resumes_seals_and_retains_writer_authority() -> Result<(), Box> { + let fixture = ResumeFixture::new("filesystem-segment-resume")?; + let prefix = reusable_prefix()?; + fs::write(fixture.stage_path(), &prefix)?; + let authority = fixture.resumer()?; + + let resumed_stage = execute_recovery_segment_resume(authority, resume_request(&prefix)?)?; + let second = fixture + .resumer() + .err() + .ok_or("resumed stage did not retain writer authority")?; + let sealed = resumed_stage + .append(AdmittedSegmentRecord::for_chunk(&[1])?)? + .seal()?; + + assert!(matches!( + second, + FilesystemRecoverySegmentResumeOpenError::WriterLock { + source: WriterLockAcquireError::Busy + } + )); + assert_eq!(sealed.record_count(), 2); + let _closed = sealed.close(); + + let observed = fs::read(fixture.stage_path())?; + let admitted = AdmittedSegment::decode(&observed, maximum_policy())?; + assert!(observed.starts_with(&prefix)); + assert_eq!(admitted.record_count(), 2); + drop(fixture.resumer()?); + fixture.remove()?; + Ok(()) +} diff --git a/src/adapters/filesystem_recovery_segment_resume_tests/fixture.rs b/src/adapters/filesystem_recovery_segment_resume_tests/fixture.rs new file mode 100644 index 0000000..05cd890 --- /dev/null +++ b/src/adapters/filesystem_recovery_segment_resume_tests/fixture.rs @@ -0,0 +1,101 @@ +//! Deterministic initialized-store fixture for filesystem segment continuation. + +use std::error::Error; +use std::fs; +use std::path::{Path, PathBuf}; + +use crate::LayoutEntryLimit; + +use super::super::{ + FilesystemRecoverySegmentResumeOpenError, FilesystemRecoverySegmentResumer, + RecoverySegmentResumeRequest, RecoveryStage, RecoveryStageMetadata, SegmentReadPolicy, + SegmentRecordLimit, admit_recovery_stage_bytes, assess_recovery_stage, + filesystem_test_sandbox::TestDirectory, fingerprint_recovery_stage, + plan_recovery_segment_resume, test_support::decode_hex, +}; + +const SEGMENT_HEX: &str = + include_str!("../../../conformance/segment-store/v1/one-zero-segment.hex"); +const SEGMENT_SEAL_LENGTH: usize = 128; +const SEGMENT_HEADER_LENGTH: usize = 64; + +pub(super) fn reusable_prefix() -> Result, Box> { + let mut encoded = decode_hex( + SEGMENT_HEX + .strip_suffix('\n') + .ok_or("recovery fixture must end in one LF")?, + )?; + let length = encoded + .len() + .checked_sub(SEGMENT_SEAL_LENGTH) + .ok_or("segment fixture is shorter than its seal")?; + encoded.truncate(length); + Ok(encoded) +} + +pub(super) fn empty_prefix() -> Result, Box> { + let mut encoded = reusable_prefix()?; + encoded.truncate(SEGMENT_HEADER_LENGTH); + Ok(encoded) +} + +pub(super) fn resume_request(bytes: &[u8]) -> Result> { + let length = u64::try_from(bytes.len())?; + let observed = fingerprint_recovery_stage( + RecoveryStageMetadata::new(RecoveryStage::Segment, length)?, + bytes, + )?; + let admitted = admit_recovery_stage_bytes(RecoveryStage::Segment, observed, bytes)?; + let assessed = assess_recovery_stage(&admitted, maximum_policy())?; + Ok(plan_recovery_segment_resume(&assessed, maximum_policy())?) +} + +pub(super) const fn maximum_policy() -> SegmentReadPolicy { + SegmentReadPolicy::new(SegmentRecordLimit::MAXIMUM, LayoutEntryLimit::MAXIMUM) +} + +pub(super) struct ResumeFixture { + directory: TestDirectory, +} + +impl ResumeFixture { + pub(super) fn new(name: &str) -> Result> { + let directory = TestDirectory::create(name)?; + fs::write(directory.path().join("writer.lock"), [])?; + for name in ["staging", "segments", "catalogs"] { + fs::create_dir(directory.path().join(name))?; + } + Ok(Self { directory }) + } + + pub(super) fn root(&self) -> &Path { + self.directory.path() + } + + pub(super) fn stage_path(&self) -> PathBuf { + self.root().join("staging/current.seg") + } + + pub(super) fn resumer( + &self, + ) -> Result { + FilesystemRecoverySegmentResumer::open_unchecked_for_tests(self.root()) + } + + pub(super) fn resumer_before_handoff( + &self, + before_handoff: F, + ) -> Result + where + F: FnOnce() + 'static, + { + FilesystemRecoverySegmentResumer::open_unchecked_for_tests_before_handoff( + self.root(), + before_handoff, + ) + } + + pub(super) fn remove(self) -> std::io::Result<()> { + self.directory.remove() + } +} diff --git a/src/adapters/filesystem_recovery_segment_resume_tests/refusal_laws.rs b/src/adapters/filesystem_recovery_segment_resume_tests/refusal_laws.rs new file mode 100644 index 0000000..d49f972 --- /dev/null +++ b/src/adapters/filesystem_recovery_segment_resume_tests/refusal_laws.rs @@ -0,0 +1,163 @@ +//! Filesystem continuation evidence and namespace refusal laws. + +use std::error::Error; +use std::fs; +use std::os::unix::fs::symlink; + +use super::super::{ + FilesystemRecoveryStageError, RecoverySegmentResumeError, RecoverySegmentResumeStorageError, + RecoveryStage, RecoveryStageNamespacePhase, execute_recovery_segment_resume, +}; +use super::{ResumeFixture, resume_request, reusable_prefix}; + +#[test] +fn absent_stage_is_a_typed_refusal() -> Result<(), Box> { + let fixture = ResumeFixture::new("filesystem-segment-resume-missing")?; + let prefix = reusable_prefix()?; + + let error = execute_recovery_segment_resume(fixture.resumer()?, resume_request(&prefix)?) + .err() + .ok_or("absent stage unexpectedly resumed")?; + + assert!(matches!( + error, + RecoverySegmentResumeError::Open { + source: RecoverySegmentResumeStorageError::Missing { .. } + } + )); + fixture.remove()?; + Ok(()) +} + +#[test] +fn changed_stage_is_refused_before_a_write() -> Result<(), Box> { + let fixture = ResumeFixture::new("filesystem-segment-resume-changed")?; + let prefix = reusable_prefix()?; + let request = resume_request(&prefix)?; + let mut changed = prefix; + let tail = changed.last_mut().ok_or("missing segment tail")?; + *tail ^= 1; + fs::write(fixture.stage_path(), &changed)?; + + let error = execute_recovery_segment_resume(fixture.resumer()?, request) + .err() + .ok_or("changed stage unexpectedly resumed")?; + + assert!(matches!( + error, + RecoverySegmentResumeError::Open { + source: RecoverySegmentResumeStorageError::EvidenceMismatch { .. } + } + )); + assert_eq!(fs::read(fixture.stage_path())?, changed); + fixture.remove()?; + Ok(()) +} + +#[test] +fn symbolic_stage_is_never_followed() -> Result<(), Box> { + let fixture = ResumeFixture::new("filesystem-segment-resume-link")?; + let prefix = reusable_prefix()?; + let target = fixture.root().join("target.seg"); + fs::write(&target, &prefix)?; + symlink(&target, fixture.stage_path())?; + + let error = execute_recovery_segment_resume(fixture.resumer()?, resume_request(&prefix)?) + .err() + .ok_or("symbolic stage unexpectedly resumed")?; + + assert!(stage_error_matches(&error, |source| { + matches!( + source, + FilesystemRecoveryStageError::Open { + stage: RecoveryStage::Segment, + .. + } + ) + })); + assert_eq!(fs::read(target)?, prefix); + fixture.remove()?; + Ok(()) +} + +#[test] +fn replacement_at_the_writable_handoff_is_preserved_and_refused() -> Result<(), Box> { + let fixture = ResumeFixture::new("filesystem-segment-resume-replaced")?; + let prefix = reusable_prefix()?; + fs::write(fixture.stage_path(), &prefix)?; + let stage_path = fixture.stage_path(); + let retained = fixture.root().join("retained.seg"); + let replacement = b"replacement".to_vec(); + let replacement_for_hook = replacement.clone(); + let stage_for_hook = stage_path.clone(); + let resumer = fixture.resumer_before_handoff(move || { + let _renamed = fs::rename(&stage_for_hook, &retained); + let _written = fs::write(&stage_for_hook, replacement_for_hook); + })?; + + let error = execute_recovery_segment_resume(resumer, resume_request(&prefix)?) + .err() + .ok_or("replaced stage unexpectedly resumed")?; + + assert!(stage_error_matches(&error, |source| { + matches!( + source, + FilesystemRecoveryStageError::Replaced { + stage: RecoveryStage::Segment + } + ) + })); + assert_eq!(fs::read(stage_path)?, replacement); + fixture.remove()?; + Ok(()) +} + +#[test] +fn replaced_staging_namespace_is_refused_before_stage_open() -> Result<(), Box> { + let fixture = ResumeFixture::new("filesystem-segment-resume-namespace")?; + let prefix = reusable_prefix()?; + fs::write(fixture.stage_path(), &prefix)?; + let resumer = fixture.resumer()?; + fs::rename( + fixture.root().join("staging"), + fixture.root().join("old-staging"), + )?; + fs::create_dir(fixture.root().join("staging"))?; + fs::write(fixture.stage_path(), &prefix)?; + + let error = execute_recovery_segment_resume(resumer, resume_request(&prefix)?) + .err() + .ok_or("replaced namespace unexpectedly resumed")?; + + assert!(stage_error_matches(&error, |source| { + matches!( + source, + FilesystemRecoveryStageError::Namespace { + stage: RecoveryStage::Segment, + phase: RecoveryStageNamespacePhase::BeforeObservation, + .. + } + ) + })); + fixture.remove()?; + Ok(()) +} + +fn stage_error_matches( + error: &RecoverySegmentResumeError, + predicate: impl FnOnce(&FilesystemRecoveryStageError) -> bool, +) -> bool { + let RecoverySegmentResumeError::Open { + source: RecoverySegmentResumeStorageError::Storage { source }, + } = error + else { + return false; + }; + let Some(source) = source + .get_ref() + .and_then(|source| source.downcast_ref::()) + else { + return false; + }; + predicate(source) +} diff --git a/src/adapters/filesystem_recovery_segment_resumer.rs b/src/adapters/filesystem_recovery_segment_resumer.rs new file mode 100644 index 0000000..9789c1d --- /dev/null +++ b/src/adapters/filesystem_recovery_segment_resumer.rs @@ -0,0 +1,64 @@ +//! This module owns pinned writer authority for filesystem segment continuation. + +use std::path::Path; + +use super::{FilesystemRecoverySegmentResumeOpenError, FilesystemRecoveryStageDiscarder}; + +/// Writer-authorized pinned filesystem adapter for reusable segment recovery. +/// +/// Opening proves the supported platform, pins and exclusively locks the store +/// root and `writer.lock`, then pins all three protocol child directories +/// without following links. Execution consumes this value so the returned +/// writable stage retains that authority until it is sealed or dropped. +#[must_use] +pub struct FilesystemRecoverySegmentResumer { + pub(super) discarder: FilesystemRecoveryStageDiscarder, + #[cfg(test)] + pub(super) before_handoff: Option>, +} + +impl FilesystemRecoverySegmentResumer { + /// Opens an initialized supported store for explicit segment continuation. + /// + /// The call performs no protocol mutation. + /// + /// # Errors + /// + /// Returns [`FilesystemRecoverySegmentResumeOpenError`] on platform, + /// writer-authority, root-clone, or namespace admission failure. + pub fn open(store_root: &Path) -> Result { + FilesystemRecoveryStageDiscarder::open(store_root) + .map(Self::from_discarder) + .map_err(Into::into) + } + + const fn from_discarder(discarder: FilesystemRecoveryStageDiscarder) -> Self { + Self { + discarder, + #[cfg(test)] + before_handoff: None, + } + } + + #[cfg(test)] + pub(super) fn open_unchecked_for_tests( + store_root: &Path, + ) -> Result { + FilesystemRecoveryStageDiscarder::open_unchecked_for_tests(store_root) + .map(Self::from_discarder) + .map_err(Into::into) + } + + #[cfg(test)] + pub(super) fn open_unchecked_for_tests_before_handoff( + store_root: &Path, + before_handoff: F, + ) -> Result + where + F: FnOnce() + 'static, + { + let mut resumer = Self::open_unchecked_for_tests(store_root)?; + resumer.before_handoff = Some(Box::new(before_handoff)); + Ok(resumer) + } +} diff --git a/src/adapters/filesystem_recovery_segment_stage.rs b/src/adapters/filesystem_recovery_segment_stage.rs new file mode 100644 index 0000000..aba13a0 --- /dev/null +++ b/src/adapters/filesystem_recovery_segment_stage.rs @@ -0,0 +1,43 @@ +//! This module owns a resumed writer-authorized filesystem segment stage. + +use std::io::{self, Write}; + +use cap_std::fs::File; + +use super::{FilesystemRecoveryStageDiscarder, SegmentStage}; + +/// Writable `current.seg` reopened from one exact reusable prefix. +/// +/// The stage owns the pinned root, protocol namespaces, and exclusive writer +/// lock for its full lifetime. Dropping it preserves `current.seg` for a later +/// explicit recovery decision. It does not synchronize the staging directory, +/// publish immutable bytes, or select a catalog generation. +pub struct FilesystemRecoverySegmentStage { + file: File, + _authority: FilesystemRecoveryStageDiscarder, +} + +impl FilesystemRecoverySegmentStage { + pub(super) const fn new(file: File, authority: FilesystemRecoveryStageDiscarder) -> Self { + Self { + file, + _authority: authority, + } + } +} + +impl Write for FilesystemRecoverySegmentStage { + fn write(&mut self, bytes: &[u8]) -> io::Result { + self.file.write(bytes) + } + + fn flush(&mut self) -> io::Result<()> { + self.file.flush() + } +} + +impl SegmentStage for FilesystemRecoverySegmentStage { + fn synchronize(&mut self) -> io::Result<()> { + self.file.sync_all() + } +} diff --git a/src/adapters/filesystem_recovery_stage.rs b/src/adapters/filesystem_recovery_stage.rs index 8cf45b0..3b1d534 100644 --- a/src/adapters/filesystem_recovery_stage.rs +++ b/src/adapters/filesystem_recovery_stage.rs @@ -5,7 +5,7 @@ use cap_std::fs::{Dir, File, Metadata, OpenOptions}; use super::{ FilesystemRecoveryStageError, RecoveryStage, RecoveryStageEvidence, RecoveryStageLength, - RecoveryStageMetadata, fingerprint_recovery_stage, + RecoveryStageMetadata, filesystem_recovery_stage_materialization, fingerprint_recovery_stage, }; #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -51,6 +51,28 @@ impl ObservedRecoveryStage { verify_opened_handle(&self.file, stage, &self.admitted)?; verify_current_entry(directory, name, stage, &self.admitted) } + + pub(super) fn materialize_and_position( + &mut self, + stage: RecoveryStage, + ) -> Result, FilesystemRecoveryStageError> { + let length = self.admitted.metadata.length(); + filesystem_recovery_stage_materialization::read_and_position(&mut self.file, stage, length) + } + + pub(super) fn verify( + &self, + directory: &Dir, + name: &str, + stage: RecoveryStage, + ) -> Result<(), FilesystemRecoveryStageError> { + verify_opened_handle(&self.file, stage, &self.admitted)?; + verify_current_entry(directory, name, stage, &self.admitted) + } + + pub(super) fn into_file(self) -> File { + self.file + } } pub(super) fn fingerprint( @@ -97,7 +119,36 @@ pub(super) fn observe_named_with( where F: FnOnce(), { - let mut file = open_stage(directory, name, stage)?; + observe_named_with_options(directory, name, stage, &read_options(), after_open) +} + +pub(super) fn observe_writable_segment_with( + directory: &Dir, + after_open: F, +) -> Result +where + F: FnOnce(), +{ + observe_named_with_options( + directory, + RecoveryStage::Segment.file_name(), + RecoveryStage::Segment, + &read_write_options(), + after_open, + ) +} + +fn observe_named_with_options( + directory: &Dir, + name: &str, + stage: RecoveryStage, + options: &OpenOptions, + after_open: F, +) -> Result +where + F: FnOnce(), +{ + let mut file = open_stage(directory, name, stage, options)?; let admitted = admit_stage(&file, stage)?; after_open(); let evidence = fingerprint_recovery_stage(admitted.metadata, &mut file) @@ -116,9 +167,10 @@ fn open_stage( directory: &Dir, name: &str, stage: RecoveryStage, + options: &OpenOptions, ) -> Result { directory - .open_with(name, &read_options()) + .open_with(name, options) .map_err(|source| FilesystemRecoveryStageError::Open { stage, source }) } @@ -191,3 +243,13 @@ fn read_options() -> OpenOptions { options.read(true).follow(FollowSymlinks::No).nonblock(true); options } + +fn read_write_options() -> OpenOptions { + let mut options = OpenOptions::new(); + options + .read(true) + .write(true) + .follow(FollowSymlinks::No) + .nonblock(true); + options +} diff --git a/src/adapters/filesystem_recovery_stage_error.rs b/src/adapters/filesystem_recovery_stage_error.rs index 2b5915f..fc33030 100644 --- a/src/adapters/filesystem_recovery_stage_error.rs +++ b/src/adapters/filesystem_recovery_stage_error.rs @@ -1,7 +1,7 @@ //! This module owns capability-relative recovery-stage observation failures. +use std::collections::TryReserveError; use std::error::Error; -use std::fmt; use std::io; use super::{ @@ -82,6 +82,47 @@ pub enum FilesystemRecoveryStageError { /// Length observed during or after reading. observed: u64, }, + /// The admitted stage length cannot be represented for materialization. + MaterializeAddressSpace { + /// Fixed stage being materialized. + stage: RecoveryStage, + /// Admitted byte count that exceeds the host address space. + byte_count: u64, + }, + /// Memory for exact bounded materialization could not be reserved. + MaterializeAllocation { + /// Fixed stage being materialized. + stage: RecoveryStage, + /// Exact admitted byte count requested. + byte_count: u64, + /// Allocation reservation failure. + source: TryReserveError, + }, + /// The writable stage could not be rewound or its position inspected. + Position { + /// Fixed stage being positioned. + stage: RecoveryStage, + /// Exact underlying positioning failure. + source: io::Error, + }, + /// Exact prefix materialization failed. + Materialize { + /// Fixed stage being materialized. + stage: RecoveryStage, + /// Exact admitted byte count requested. + expected: RecoveryStageLength, + /// Exact underlying read failure. + source: io::Error, + }, + /// The writable handle is not positioned at the admitted append boundary. + PositionMismatch { + /// Fixed stage being positioned. + stage: RecoveryStage, + /// Exact admitted append boundary. + expected: RecoveryStageLength, + /// Observed writable-handle position. + observed: u64, + }, } /// Namespace-verification position around stage observation. @@ -93,72 +134,6 @@ pub enum RecoveryStageNamespacePhase { AfterObservation, } -impl fmt::Display for FilesystemRecoveryStageError { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::Namespace { - stage, - phase, - source, - } => write!( - formatter, - "{stage} namespace verification failed {phase}: {source}" - ), - Self::Open { stage, source } => { - write!(formatter, "failed to open recovery stage {stage}: {source}") - } - Self::Inspect { stage, source } => { - write!( - formatter, - "failed to inspect recovery stage {stage}: {source}" - ) - } - Self::NonRegular { stage } => { - write!(formatter, "recovery stage {stage} is not a regular file") - } - Self::MetadataAdmission { stage, source, .. } => write!( - formatter, - "recovery stage {stage} metadata was refused: {source}" - ), - Self::Fingerprint { stage, source, .. } => write!( - formatter, - "recovery stage {stage} fingerprint failed: {source}" - ), - Self::Synchronize { stage, source } => { - write!( - formatter, - "failed to synchronize recovery stage {stage}: {source}" - ) - } - Self::VerifyEntry { stage, source } => write!( - formatter, - "failed to verify recovery stage entry {stage}: {source}" - ), - Self::Replaced { stage } => { - write!(formatter, "recovery stage {stage} changed file identity") - } - Self::LengthChanged { - stage, - expected, - observed, - } => write!( - formatter, - "recovery stage {stage} changed length from {} to {observed}", - expected.get() - ), - } - } -} - -impl fmt::Display for RecoveryStageNamespacePhase { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter.write_str(match self { - Self::BeforeObservation => "before observation", - Self::AfterObservation => "after observation", - }) - } -} - impl Error for FilesystemRecoveryStageError { fn source(&self) -> Option<&(dyn Error + 'static)> { match self { @@ -166,10 +141,17 @@ impl Error for FilesystemRecoveryStageError { Self::Open { source, .. } | Self::Inspect { source, .. } | Self::Synchronize { source, .. } - | Self::VerifyEntry { source, .. } => Some(source), + | Self::VerifyEntry { source, .. } + | Self::Position { source, .. } + | Self::Materialize { source, .. } => Some(source), Self::MetadataAdmission { source, .. } => Some(source), Self::Fingerprint { source, .. } => Some(source), - Self::NonRegular { .. } | Self::Replaced { .. } | Self::LengthChanged { .. } => None, + Self::MaterializeAllocation { source, .. } => Some(source), + Self::NonRegular { .. } + | Self::Replaced { .. } + | Self::LengthChanged { .. } + | Self::MaterializeAddressSpace { .. } + | Self::PositionMismatch { .. } => None, } } } diff --git a/src/adapters/filesystem_recovery_stage_error_display.rs b/src/adapters/filesystem_recovery_stage_error_display.rs new file mode 100644 index 0000000..c3e2a84 --- /dev/null +++ b/src/adapters/filesystem_recovery_stage_error_display.rs @@ -0,0 +1,172 @@ +//! This module owns filesystem recovery-stage error rendering. + +use std::fmt; + +use super::{FilesystemRecoveryStageError, RecoveryStageNamespacePhase}; + +impl fmt::Display for FilesystemRecoveryStageError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::MaterializeAddressSpace { .. } + | Self::MaterializeAllocation { .. } + | Self::Position { .. } + | Self::Materialize { .. } + | Self::PositionMismatch { .. } => fmt_materialization(self, formatter), + Self::Namespace { .. } + | Self::Open { .. } + | Self::Inspect { .. } + | Self::NonRegular { .. } + | Self::MetadataAdmission { .. } + | Self::Fingerprint { .. } + | Self::Synchronize { .. } + | Self::VerifyEntry { .. } + | Self::Replaced { .. } + | Self::LengthChanged { .. } => fmt_observation(self, formatter), + } + } +} + +fn fmt_observation( + error: &FilesystemRecoveryStageError, + formatter: &mut fmt::Formatter<'_>, +) -> fmt::Result { + match error { + FilesystemRecoveryStageError::Namespace { .. } + | FilesystemRecoveryStageError::Open { .. } + | FilesystemRecoveryStageError::Inspect { .. } + | FilesystemRecoveryStageError::NonRegular { .. } + | FilesystemRecoveryStageError::MetadataAdmission { .. } => { + fmt_open_admission(error, formatter) + } + FilesystemRecoveryStageError::Fingerprint { .. } + | FilesystemRecoveryStageError::Synchronize { .. } + | FilesystemRecoveryStageError::VerifyEntry { .. } + | FilesystemRecoveryStageError::Replaced { .. } + | FilesystemRecoveryStageError::LengthChanged { .. } => fmt_verification(error, formatter), + _ => fmt_materialization(error, formatter), + } +} + +fn fmt_open_admission( + error: &FilesystemRecoveryStageError, + formatter: &mut fmt::Formatter<'_>, +) -> fmt::Result { + match error { + FilesystemRecoveryStageError::Namespace { + stage, + phase, + source, + } => write!( + formatter, + "{stage} namespace verification failed {phase}: {source}" + ), + FilesystemRecoveryStageError::Open { stage, source } => { + write!(formatter, "failed to open recovery stage {stage}: {source}") + } + FilesystemRecoveryStageError::Inspect { stage, source } => { + write!( + formatter, + "failed to inspect recovery stage {stage}: {source}" + ) + } + FilesystemRecoveryStageError::NonRegular { stage } => { + write!(formatter, "recovery stage {stage} is not a regular file") + } + FilesystemRecoveryStageError::MetadataAdmission { stage, source } => { + write!( + formatter, + "recovery stage {stage} metadata was refused: {source}" + ) + } + _ => fmt_materialization(error, formatter), + } +} + +fn fmt_verification( + error: &FilesystemRecoveryStageError, + formatter: &mut fmt::Formatter<'_>, +) -> fmt::Result { + match error { + FilesystemRecoveryStageError::Fingerprint { stage, source } => { + write!( + formatter, + "recovery stage {stage} fingerprint failed: {source}" + ) + } + FilesystemRecoveryStageError::Synchronize { stage, source } => { + write!( + formatter, + "failed to synchronize recovery stage {stage}: {source}" + ) + } + FilesystemRecoveryStageError::VerifyEntry { stage, source } => write!( + formatter, + "failed to verify recovery stage entry {stage}: {source}" + ), + FilesystemRecoveryStageError::Replaced { stage } => { + write!(formatter, "recovery stage {stage} changed file identity") + } + FilesystemRecoveryStageError::LengthChanged { + stage, + expected, + observed, + } => write!( + formatter, + "recovery stage {stage} changed length from {} to {observed}", + expected.get() + ), + _ => fmt_materialization(error, formatter), + } +} + +fn fmt_materialization( + error: &FilesystemRecoveryStageError, + formatter: &mut fmt::Formatter<'_>, +) -> fmt::Result { + match error { + FilesystemRecoveryStageError::MaterializeAddressSpace { stage, byte_count } => write!( + formatter, + "recovery stage {stage} byte count {byte_count} exceeds the host address space" + ), + FilesystemRecoveryStageError::MaterializeAllocation { + stage, byte_count, .. + } => write!( + formatter, + "cannot reserve {byte_count} bytes for recovery stage {stage}" + ), + FilesystemRecoveryStageError::Position { stage, source } => { + write!( + formatter, + "cannot position recovery stage {stage}: {source}" + ) + } + FilesystemRecoveryStageError::Materialize { + stage, + expected, + source, + } => write!( + formatter, + "cannot materialize {} bytes from recovery stage {stage}: {source}", + expected.get() + ), + FilesystemRecoveryStageError::PositionMismatch { + stage, + expected, + observed, + } => write!( + formatter, + "recovery stage {stage} position is {observed}, expected {}", + expected.get() + ), + _ => fmt_observation(error, formatter), + } +} + +impl fmt::Display for RecoveryStageNamespacePhase { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(match self { + Self::BeforeObservation => "before observation", + Self::AfterObservation => "after observation", + }) + } +} diff --git a/src/adapters/filesystem_recovery_stage_materialization.rs b/src/adapters/filesystem_recovery_stage_materialization.rs new file mode 100644 index 0000000..124bb20 --- /dev/null +++ b/src/adapters/filesystem_recovery_stage_materialization.rs @@ -0,0 +1,66 @@ +//! This module owns exact writable recovery-stage materialization. + +use std::io::{Read, Seek, SeekFrom}; + +use cap_std::fs::File; + +use super::{FilesystemRecoveryStageError, RecoveryStage, RecoveryStageLength}; + +pub(super) fn read_and_position( + file: &mut File, + stage: RecoveryStage, + length: RecoveryStageLength, +) -> Result, FilesystemRecoveryStageError> { + let mut encoded = allocate(stage, length)?; + file.seek(SeekFrom::Start(0)) + .map_err(|source| FilesystemRecoveryStageError::Position { stage, source })?; + file.read_exact(&mut encoded) + .map_err(|source| FilesystemRecoveryStageError::Materialize { + stage, + expected: length, + source, + })?; + verify_position(file, stage, length)?; + Ok(encoded.into_boxed_slice()) +} + +fn allocate( + stage: RecoveryStage, + length: RecoveryStageLength, +) -> Result, FilesystemRecoveryStageError> { + let host_length = usize::try_from(length.get()).map_err(|_source| { + FilesystemRecoveryStageError::MaterializeAddressSpace { + stage, + byte_count: length.get(), + } + })?; + let mut encoded = Vec::new(); + encoded.try_reserve_exact(host_length).map_err(|source| { + FilesystemRecoveryStageError::MaterializeAllocation { + stage, + byte_count: length.get(), + source, + } + })?; + encoded.resize(host_length, 0); + Ok(encoded) +} + +fn verify_position( + file: &mut File, + stage: RecoveryStage, + expected: RecoveryStageLength, +) -> Result<(), FilesystemRecoveryStageError> { + let observed = file + .stream_position() + .map_err(|source| FilesystemRecoveryStageError::Position { stage, source })?; + if observed == expected.get() { + Ok(()) + } else { + Err(FilesystemRecoveryStageError::PositionMismatch { + stage, + expected, + observed, + }) + } +} diff --git a/src/adapters/mod.rs b/src/adapters/mod.rs index 829bd0a..8258b4b 100644 --- a/src/adapters/mod.rs +++ b/src/adapters/mod.rs @@ -91,6 +91,12 @@ mod filesystem_recovery_next_head_finalization_storage; #[cfg(all(test, unix))] mod filesystem_recovery_next_head_finalization_tests; mod filesystem_recovery_next_head_finalizer; +mod filesystem_recovery_segment_resume_open_error; +mod filesystem_recovery_segment_resume_storage; +#[cfg(all(test, unix))] +mod filesystem_recovery_segment_resume_tests; +mod filesystem_recovery_segment_resumer; +mod filesystem_recovery_segment_stage; mod filesystem_recovery_stage; mod filesystem_recovery_stage_completer; mod filesystem_recovery_stage_completion_open_error; @@ -104,6 +110,8 @@ mod filesystem_recovery_stage_discard_storage; mod filesystem_recovery_stage_discard_tests; mod filesystem_recovery_stage_discarder; mod filesystem_recovery_stage_error; +mod filesystem_recovery_stage_error_display; +mod filesystem_recovery_stage_materialization; mod filesystem_recovery_stage_sync; #[cfg(all(test, unix))] mod filesystem_recovery_stage_tests; @@ -331,6 +339,9 @@ pub use filesystem_platform_admission::FilesystemPlatformAdmission; pub use filesystem_recovery_inventory_reader::FilesystemRecoveryInventoryReader; pub use filesystem_recovery_next_head_finalization_open_error::FilesystemRecoveryNextHeadFinalizationOpenError; pub use filesystem_recovery_next_head_finalizer::FilesystemRecoveryNextHeadFinalizer; +pub use filesystem_recovery_segment_resume_open_error::FilesystemRecoverySegmentResumeOpenError; +pub use filesystem_recovery_segment_resumer::FilesystemRecoverySegmentResumer; +pub use filesystem_recovery_segment_stage::FilesystemRecoverySegmentStage; pub use filesystem_recovery_stage_completer::FilesystemRecoveryStageCompleter; pub use filesystem_recovery_stage_completion_open_error::FilesystemRecoveryStageCompletionOpenError; pub use filesystem_recovery_stage_discard_open_error::FilesystemRecoveryStageDiscardOpenError; diff --git a/src/lib.rs b/src/lib.rs index 7f305b5..fc0ccfd 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -19,9 +19,10 @@ //! truncated-stage discard, and complete-stage valid-orphan recovery are //! explicit. Exact next-head finalization now has a storage-independent //! contract and a pinned writer-authorized filesystem adapter. Reusable-stage -//! continuation has a storage-independent planning and execution boundary. -//! Its filesystem binding, retention, and garbage collection remain -//! intentionally absent until their contracts have executable specifications. +//! continuation has a storage-independent planning and execution boundary plus +//! a pinned writer-authorized filesystem adapter. Retention and garbage +//! collection remain intentionally absent until their contracts have +//! executable specifications. #[cfg(test)] extern crate self as keep; @@ -48,27 +49,28 @@ pub use adapters::{ FilesystemCatalogPublicationError, FilesystemCatalogPublisher, FilesystemCatalogSnapshot, FilesystemPlatformAdmission, FilesystemRecoveryInventoryReader, FilesystemRecoveryNextHeadFinalizationOpenError, FilesystemRecoveryNextHeadFinalizer, - FilesystemRecoveryStageCompleter, FilesystemRecoveryStageCompletionOpenError, - FilesystemRecoveryStageDiscardOpenError, FilesystemRecoveryStageDiscarder, - FilesystemRecoveryStageError, FilesystemSegmentStage, FilesystemWriterLock, LayoutDecodeError, - LayoutDecodePolicy, LayoutEncodeError, LayoutIdBinaryParseError, LayoutIdTextParseError, - OpenedReusableSegment, PublicationHeadDecodeError, RecoveryCatalogStage, - RecoveryCatalogStageError, RecoveryEntryName, RecoveryEntryNameError, RecoveryEntryRole, - RecoveryInventory, RecoveryInventoryEntry, RecoveryInventoryError, RecoveryInventoryLimit, - RecoveryInventoryLimitError, RecoveryInventoryOperation, RecoveryInventoryStorage, - RecoveryNameClassificationError, RecoveryNameManifest, RecoveryNamedEntry, RecoveryNamespace, - RecoveryNextHeadFinalizationError, RecoveryNextHeadFinalizationOutcome, - RecoveryNextHeadFinalizationPlanError, RecoveryNextHeadFinalizationReadiness, - RecoveryNextHeadFinalizationReceipt, RecoveryNextHeadFinalizationRequest, - RecoveryNextHeadFinalizationStorage, RecoveryNextHeadFinalizationStorageError, - RecoveryNextHeadFinalizationTarget, RecoveryNextHeadStage, RecoveryNextHeadStageError, - RecoveryPoolNameError, RecoveryRequiredEntry, RecoverySegmentResumeError, - RecoverySegmentResumePlanError, RecoverySegmentResumeRequest, RecoverySegmentResumeStorage, - RecoverySegmentResumeStorageError, RecoverySegmentStage, RecoverySegmentStageError, - RecoverySegmentTruncation, RecoveryStage, RecoveryStageAssessment, - RecoveryStageAssessmentError, RecoveryStageByteAdmissionError, RecoveryStageCompletionError, - RecoveryStageCompletionPlanError, RecoveryStageCompletionPool, RecoveryStageCompletionReceipt, - RecoveryStageCompletionRequest, RecoveryStageCompletionStorage, + FilesystemRecoverySegmentResumeOpenError, FilesystemRecoverySegmentResumer, + FilesystemRecoverySegmentStage, FilesystemRecoveryStageCompleter, + FilesystemRecoveryStageCompletionOpenError, FilesystemRecoveryStageDiscardOpenError, + FilesystemRecoveryStageDiscarder, FilesystemRecoveryStageError, FilesystemSegmentStage, + FilesystemWriterLock, LayoutDecodeError, LayoutDecodePolicy, LayoutEncodeError, + LayoutIdBinaryParseError, LayoutIdTextParseError, OpenedReusableSegment, + PublicationHeadDecodeError, RecoveryCatalogStage, RecoveryCatalogStageError, RecoveryEntryName, + RecoveryEntryNameError, RecoveryEntryRole, RecoveryInventory, RecoveryInventoryEntry, + RecoveryInventoryError, RecoveryInventoryLimit, RecoveryInventoryLimitError, + RecoveryInventoryOperation, RecoveryInventoryStorage, RecoveryNameClassificationError, + RecoveryNameManifest, RecoveryNamedEntry, RecoveryNamespace, RecoveryNextHeadFinalizationError, + RecoveryNextHeadFinalizationOutcome, RecoveryNextHeadFinalizationPlanError, + RecoveryNextHeadFinalizationReadiness, RecoveryNextHeadFinalizationReceipt, + RecoveryNextHeadFinalizationRequest, RecoveryNextHeadFinalizationStorage, + RecoveryNextHeadFinalizationStorageError, RecoveryNextHeadFinalizationTarget, + RecoveryNextHeadStage, RecoveryNextHeadStageError, RecoveryPoolNameError, + RecoveryRequiredEntry, RecoverySegmentResumeError, RecoverySegmentResumePlanError, + RecoverySegmentResumeRequest, RecoverySegmentResumeStorage, RecoverySegmentResumeStorageError, + RecoverySegmentStage, RecoverySegmentStageError, RecoverySegmentTruncation, RecoveryStage, + RecoveryStageAssessment, RecoveryStageAssessmentError, RecoveryStageByteAdmissionError, + RecoveryStageCompletionError, RecoveryStageCompletionPlanError, RecoveryStageCompletionPool, + RecoveryStageCompletionReceipt, RecoveryStageCompletionRequest, RecoveryStageCompletionStorage, RecoveryStageCompletionStorageError, RecoveryStageCompletionTarget, RecoveryStageDiscardError, RecoveryStageDiscardOutcome, RecoveryStageDiscardPlanError, RecoveryStageDiscardReason, RecoveryStageDiscardReceipt, RecoveryStageDiscardRequest, RecoveryStageDiscardStorage, From 18313671d849bf6c42ef511e1e50de23ee771b6d Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 13:25:33 -0700 Subject: [PATCH 23/49] Add: Define deterministic crash matrix cases --- xtask/src/durability_crash_case.rs | 81 +++++++++++++++ xtask/src/durability_crash_case_error.rs | 41 ++++++++ xtask/src/durability_crash_occurrence.rs | 22 +++++ xtask/src/durability_crash_point.rs | 8 ++ xtask/src/durability_crash_position.rs | 35 +++++++ xtask/src/lib.rs | 16 +++ xtask/tests/durability_crash_case_contract.rs | 98 +++++++++++++++++++ 7 files changed, 301 insertions(+) create mode 100644 xtask/src/durability_crash_case.rs create mode 100644 xtask/src/durability_crash_case_error.rs create mode 100644 xtask/src/durability_crash_occurrence.rs create mode 100644 xtask/src/durability_crash_position.rs create mode 100644 xtask/tests/durability_crash_case_contract.rs diff --git a/xtask/src/durability_crash_case.rs b/xtask/src/durability_crash_case.rs new file mode 100644 index 0000000..3b267d4 --- /dev/null +++ b/xtask/src/durability_crash_case.rs @@ -0,0 +1,81 @@ +//! This module owns validated coordinates in the deterministic crash matrix. + +use crate::{ + DurabilityCrashCaseError, DurabilityCrashOccurrence, DurabilityCrashPoint, + DurabilityCrashPosition, +}; + +/// One validated process-death coordinate in the durability crash matrix. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct DurabilityCrashCase { + point: DurabilityCrashPoint, + position: DurabilityCrashPosition, + occurrence: Option, +} + +impl DurabilityCrashCase { + /// Creates a coordinate after validating occurrence ownership. + /// + /// # Errors + /// + /// Returns [`DurabilityCrashCaseError::MissingOccurrence`] when a repeated + /// transition lacks an occurrence, or + /// [`DurabilityCrashCaseError::UnexpectedOccurrence`] when a non-repeated + /// transition receives one. + pub const fn new( + point: DurabilityCrashPoint, + position: DurabilityCrashPosition, + occurrence: Option, + ) -> Result { + match (point.occurrence_counted(), occurrence) { + (true, None) => Err(DurabilityCrashCaseError::MissingOccurrence { point }), + (false, Some(observed)) => { + Err(DurabilityCrashCaseError::UnexpectedOccurrence { point, observed }) + } + (_, occurrence) => Ok(Self { + point, + position, + occurrence, + }), + } + } + + /// Returns every canonical case in point-major, position-minor order. + pub fn all() -> impl Iterator { + DurabilityCrashPoint::ALL.into_iter().flat_map(|point| { + DurabilityCrashPosition::ALL + .into_iter() + .map(move |position| Self::canonical(point, position)) + }) + } + + /// Returns the transition targeted by this coordinate. + #[must_use] + pub const fn point(self) -> DurabilityCrashPoint { + self.point + } + + /// Returns the process-death position targeted by this coordinate. + #[must_use] + pub const fn position(self) -> DurabilityCrashPosition { + self.position + } + + /// Returns the occurrence coordinate when the transition repeats. + #[must_use] + pub const fn occurrence(self) -> Option { + self.occurrence + } + + const fn canonical(point: DurabilityCrashPoint, position: DurabilityCrashPosition) -> Self { + Self { + point, + position, + occurrence: if point.occurrence_counted() { + Some(DurabilityCrashOccurrence::FIRST) + } else { + None + }, + } + } +} diff --git a/xtask/src/durability_crash_case_error.rs b/xtask/src/durability_crash_case_error.rs new file mode 100644 index 0000000..13c053e --- /dev/null +++ b/xtask/src/durability_crash_case_error.rs @@ -0,0 +1,41 @@ +//! This module owns crash-matrix coordinate validation failures. + +use std::error::Error; +use std::fmt; + +use crate::{DurabilityCrashOccurrence, DurabilityCrashPoint}; + +/// A failure to construct a valid crash-matrix coordinate. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum DurabilityCrashCaseError { + /// A repeated durability transition lacks its occurrence coordinate. + MissingOccurrence { + /// The repeated transition missing its coordinate. + point: DurabilityCrashPoint, + }, + /// A non-repeated transition received an occurrence coordinate. + UnexpectedOccurrence { + /// The transition that cannot accept an occurrence. + point: DurabilityCrashPoint, + /// The rejected coordinate. + observed: DurabilityCrashOccurrence, + }, +} + +impl fmt::Display for DurabilityCrashCaseError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::MissingOccurrence { point } => { + write!(formatter, "{} requires an occurrence", point.identifier()) + } + Self::UnexpectedOccurrence { point, observed } => write!( + formatter, + "{} cannot accept occurrence {}", + point.identifier(), + observed.get() + ), + } + } +} + +impl Error for DurabilityCrashCaseError {} diff --git a/xtask/src/durability_crash_occurrence.rs b/xtask/src/durability_crash_occurrence.rs new file mode 100644 index 0000000..9bafee1 --- /dev/null +++ b/xtask/src/durability_crash_occurrence.rs @@ -0,0 +1,22 @@ +//! This module owns repeated crash-point occurrence coordinates. + +/// A zero-based occurrence coordinate for a repeated durability transition. +#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] +pub struct DurabilityCrashOccurrence(u32); + +impl DurabilityCrashOccurrence { + /// The first occurrence of a repeated durability transition. + pub const FIRST: Self = Self(0); + + /// Creates an occurrence from its zero-based coordinate. + #[must_use] + pub const fn new(value: u32) -> Self { + Self(value) + } + + /// Returns the zero-based coordinate. + #[must_use] + pub const fn get(self) -> u32 { + self.0 + } +} diff --git a/xtask/src/durability_crash_point.rs b/xtask/src/durability_crash_point.rs index 6a0ca81..2ed8d6a 100644 --- a/xtask/src/durability_crash_point.rs +++ b/xtask/src/durability_crash_point.rs @@ -130,6 +130,14 @@ impl DurabilityCrashPoint { Self::SynchronizeRootAfterInitialization, ]; + /// Parses one exact stable crash identifier. + #[must_use] + pub fn from_identifier(identifier: &str) -> Option { + Self::ALL + .into_iter() + .find(|point| point.identifier() == identifier) + } + /// Returns the durable protocol sequence containing this boundary. #[must_use] pub const fn sequence(self) -> DurabilityCrashSequence { diff --git a/xtask/src/durability_crash_position.rs b/xtask/src/durability_crash_position.rs new file mode 100644 index 0000000..4843c0e --- /dev/null +++ b/xtask/src/durability_crash_position.rs @@ -0,0 +1,35 @@ +//! This module owns process-death positions around one durability transition. + +/// The process-death position relative to one durability transition. +#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] +pub enum DurabilityCrashPosition { + /// Process death occurs before the transition begins. + Before, + /// Process death occurs while the transition is incomplete. + During, + /// Process death occurs after the transition completes. + After, +} + +impl DurabilityCrashPosition { + /// Every position in canonical matrix order. + pub const ALL: [Self; 3] = [Self::Before, Self::During, Self::After]; + + /// Returns the stable identifier used by the crash-matrix protocol. + #[must_use] + pub const fn identifier(self) -> &'static str { + match self { + Self::Before => "before", + Self::During => "during", + Self::After => "after", + } + } + + /// Parses one exact crash-position identifier. + #[must_use] + pub fn from_identifier(identifier: &str) -> Option { + Self::ALL + .into_iter() + .find(|position| position.identifier() == identifier) + } +} diff --git a/xtask/src/lib.rs b/xtask/src/lib.rs index 5f164b7..f2d9786 100644 --- a/xtask/src/lib.rs +++ b/xtask/src/lib.rs @@ -26,10 +26,18 @@ mod repository_json; pub mod protocol_admission; +#[cfg(feature = "repository-tasks")] +mod durability_crash_case; +#[cfg(feature = "repository-tasks")] +mod durability_crash_case_error; +#[cfg(feature = "repository-tasks")] +mod durability_crash_occurrence; #[cfg(feature = "repository-tasks")] mod durability_crash_point; #[cfg(feature = "repository-tasks")] mod durability_crash_point_identity; +#[cfg(feature = "repository-tasks")] +mod durability_crash_position; #[cfg(test)] #[allow( @@ -38,8 +46,16 @@ mod durability_crash_point_identity; )] mod test_directory; +#[cfg(feature = "repository-tasks")] +pub use durability_crash_case::DurabilityCrashCase; +#[cfg(feature = "repository-tasks")] +pub use durability_crash_case_error::DurabilityCrashCaseError; +#[cfg(feature = "repository-tasks")] +pub use durability_crash_occurrence::DurabilityCrashOccurrence; #[cfg(feature = "repository-tasks")] pub use durability_crash_point::{DurabilityCrashPoint, DurabilityCrashSequence}; +#[cfg(feature = "repository-tasks")] +pub use durability_crash_position::DurabilityCrashPosition; /// Whether one bounded Golden File Worldline production parser admitted input. #[cfg(feature = "golden-protocol-fuzz")] diff --git a/xtask/tests/durability_crash_case_contract.rs b/xtask/tests/durability_crash_case_contract.rs new file mode 100644 index 0000000..d6297d4 --- /dev/null +++ b/xtask/tests/durability_crash_case_contract.rs @@ -0,0 +1,98 @@ +//! Canonical deterministic crash-matrix coordinate laws. + +use std::error::Error; + +use xtask::{ + DurabilityCrashCase, DurabilityCrashCaseError, DurabilityCrashOccurrence, DurabilityCrashPoint, + DurabilityCrashPosition, +}; + +#[test] +fn every_crash_point_has_exactly_three_ordered_process_death_cases() -> Result<(), Box> { + let cases: Vec<_> = DurabilityCrashCase::all().collect(); + let expected = DurabilityCrashPoint::ALL + .len() + .checked_mul(DurabilityCrashPosition::ALL.len()) + .ok_or("crash-matrix case count overflow")?; + + assert_eq!(cases.len(), expected); + for (point_index, point) in DurabilityCrashPoint::ALL.into_iter().enumerate() { + for (position_index, position) in DurabilityCrashPosition::ALL.into_iter().enumerate() { + let index = point_index + .checked_mul(DurabilityCrashPosition::ALL.len()) + .and_then(|base| base.checked_add(position_index)) + .ok_or("crash-matrix index overflow")?; + let case = cases.get(index).ok_or("missing canonical crash case")?; + assert_eq!(case.point(), point); + assert_eq!(case.position(), position); + assert_eq!( + case.occurrence(), + point + .occurrence_counted() + .then_some(DurabilityCrashOccurrence::FIRST) + ); + } + } + Ok(()) +} + +#[test] +fn occurrence_coordinates_exist_only_for_record_append() -> Result<(), Box> { + let occurrence = DurabilityCrashOccurrence::new(7); + + let counted = DurabilityCrashCase::new( + DurabilityCrashPoint::AppendSegmentRecord, + DurabilityCrashPosition::During, + Some(occurrence), + )?; + assert_eq!(counted.occurrence(), Some(occurrence)); + + let missing = DurabilityCrashCase::new( + DurabilityCrashPoint::AppendSegmentRecord, + DurabilityCrashPosition::During, + None, + ); + assert_eq!( + missing, + Err(DurabilityCrashCaseError::MissingOccurrence { + point: DurabilityCrashPoint::AppendSegmentRecord, + }) + ); + + let unexpected = DurabilityCrashCase::new( + DurabilityCrashPoint::WriteSegmentHeader, + DurabilityCrashPosition::During, + Some(occurrence), + ); + assert_eq!( + unexpected, + Err(DurabilityCrashCaseError::UnexpectedOccurrence { + point: DurabilityCrashPoint::WriteSegmentHeader, + observed: occurrence, + }) + ); + Ok(()) +} + +#[test] +fn identifiers_and_positions_round_trip_without_aliases() -> Result<(), Box> { + for point in DurabilityCrashPoint::ALL { + assert_eq!( + DurabilityCrashPoint::from_identifier(point.identifier()), + Some(point) + ); + } + assert_eq!( + DurabilityCrashPoint::from_identifier("KEEP-CRASH-000"), + None + ); + + for position in DurabilityCrashPosition::ALL { + assert_eq!( + DurabilityCrashPosition::from_identifier(position.identifier()), + Some(position) + ); + } + assert_eq!(DurabilityCrashPosition::from_identifier("between"), None); + Ok(()) +} From 6e3ada021ec11ff22ddcdd2b191d7821c4d05949 Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 13:34:04 -0700 Subject: [PATCH 24/49] Add: Kill crash cases at deterministic readiness --- xtask/src/bounded_process.rs | 9 + xtask/src/bounded_process/process_group.rs | 7 +- xtask/src/durability_crash_matrix.rs | 70 ++++++ xtask/src/durability_crash_matrix/child.rs | 53 +++++ xtask/src/durability_crash_matrix/error.rs | 107 ++++++++++ xtask/src/durability_crash_matrix/process.rs | 199 ++++++++++++++++++ xtask/src/main.rs | 13 ++ xtask/src/task_error.rs | 11 + xtask/tests/durability_crash_case_contract.rs | 3 +- .../durability_crash_process_contract.rs | 24 +++ 10 files changed, 491 insertions(+), 5 deletions(-) create mode 100644 xtask/src/durability_crash_matrix.rs create mode 100644 xtask/src/durability_crash_matrix/child.rs create mode 100644 xtask/src/durability_crash_matrix/error.rs create mode 100644 xtask/src/durability_crash_matrix/process.rs create mode 100644 xtask/tests/durability_crash_process_contract.rs diff --git a/xtask/src/bounded_process.rs b/xtask/src/bounded_process.rs index 628b3c5..58fb224 100644 --- a/xtask/src/bounded_process.rs +++ b/xtask/src/bounded_process.rs @@ -20,6 +20,7 @@ pub(crate) use capture_limit::CaptureLimits; use deadline::ProcessDeadline; pub(crate) use error::ProcessError; use interrupt::InterruptGuard; +pub(crate) use process_group::ProcessGroup; use reader::ReaderWorker; /// The completed child status and any output retained by the selected mode. @@ -66,6 +67,14 @@ pub(crate) fn status( }) } +/// Spawns a child in a dedicated process group. +pub(crate) fn spawn_in_process_group( + command: &mut Command, +) -> Result { + command.process_group(0); + command.spawn() +} + #[cfg(test)] #[path = "bounded_process/tests.rs"] mod tests; diff --git a/xtask/src/bounded_process/process_group.rs b/xtask/src/bounded_process/process_group.rs index 78f264d..4a976b5 100644 --- a/xtask/src/bounded_process/process_group.rs +++ b/xtask/src/bounded_process/process_group.rs @@ -14,14 +14,15 @@ use rustix::io::Errno; use rustix::process::{Pid, Signal, kill_process_group}; /// The dedicated process-group identity established for one spawned child. -pub(super) struct ProcessGroup(Pid); +#[derive(Clone, Copy)] +pub(crate) struct ProcessGroup(Pid); impl ProcessGroup { /// Admits the child's nonzero operating-system identifier as a group ID. /// /// Conversion fails when the unsigned child ID does not fit the platform's /// signed PID representation or when the observed ID is zero. - pub(super) fn for_child(child: &Child) -> Result { + pub(crate) fn for_child(child: &Child) -> Result { let raw = i32::try_from(child.id()) .map_err(|source| io::Error::new(io::ErrorKind::InvalidData, source))?; let pid = Pid::from_raw(raw) @@ -33,7 +34,7 @@ impl ProcessGroup { /// /// An absent group is already terminated and succeeds. Other operating /// system errors are preserved. - pub(super) fn terminate(self) -> Result<(), io::Error> { + pub(crate) fn terminate(self) -> Result<(), io::Error> { match kill_process_group(self.0, Signal::KILL) { Ok(()) | Err(Errno::SRCH) => Ok(()), Err(source) => Err(source.into()), diff --git a/xtask/src/durability_crash_matrix.rs b/xtask/src/durability_crash_matrix.rs new file mode 100644 index 0000000..aad4aae --- /dev/null +++ b/xtask/src/durability_crash_matrix.rs @@ -0,0 +1,70 @@ +//! This module owns deterministic subprocess durability crash-matrix execution. + +mod child; +mod error; +mod process; + +use std::ffi::{OsStr, OsString}; +use std::path::Path; + +pub(crate) use error::DurabilityCrashMatrixError; +use xtask::{ + DurabilityCrashCase, DurabilityCrashOccurrence, DurabilityCrashPoint, DurabilityCrashPosition, +}; + +const CASE_ARGUMENT: &str = "--case"; + +pub(crate) fn run( + repository_root: &Path, + mut arguments: impl Iterator, +) -> Result<(), DurabilityCrashMatrixError> { + let flag = arguments.next().ok_or(DurabilityCrashMatrixError::Usage)?; + if flag != OsStr::new(CASE_ARGUMENT) { + return Err(DurabilityCrashMatrixError::Usage); + } + let case = parse_case(&mut arguments)?; + refuse_extra(&mut arguments)?; + process::run(repository_root, case) +} + +pub(crate) fn run_child( + mut arguments: impl Iterator, +) -> Result<(), DurabilityCrashMatrixError> { + let case = parse_case(&mut arguments)?; + let case_root = arguments.next().ok_or(DurabilityCrashMatrixError::Usage)?; + let readiness_socket = arguments.next().ok_or(DurabilityCrashMatrixError::Usage)?; + refuse_extra(&mut arguments)?; + child::run(case, Path::new(&case_root), Path::new(&readiness_socket)) +} + +fn parse_case( + arguments: &mut impl Iterator, +) -> Result { + let point_argument = arguments.next().ok_or(DurabilityCrashMatrixError::Usage)?; + let point_text = point_argument + .to_str() + .ok_or(DurabilityCrashMatrixError::InvalidPointEncoding)?; + let point = DurabilityCrashPoint::from_identifier(point_text) + .ok_or_else(|| DurabilityCrashMatrixError::UnknownPoint(point_text.into()))?; + let position_argument = arguments.next().ok_or(DurabilityCrashMatrixError::Usage)?; + let position_text = position_argument + .to_str() + .ok_or(DurabilityCrashMatrixError::InvalidPositionEncoding)?; + let position = DurabilityCrashPosition::from_identifier(position_text) + .ok_or_else(|| DurabilityCrashMatrixError::UnknownPosition(position_text.into()))?; + let occurrence = point + .occurrence_counted() + .then_some(DurabilityCrashOccurrence::FIRST); + DurabilityCrashCase::new(point, position, occurrence) + .map_err(DurabilityCrashMatrixError::InvalidCase) +} + +fn refuse_extra( + arguments: &mut impl Iterator, +) -> Result<(), DurabilityCrashMatrixError> { + if arguments.next().is_some() { + Err(DurabilityCrashMatrixError::Usage) + } else { + Ok(()) + } +} diff --git a/xtask/src/durability_crash_matrix/child.rs b/xtask/src/durability_crash_matrix/child.rs new file mode 100644 index 0000000..36d4305 --- /dev/null +++ b/xtask/src/durability_crash_matrix/child.rs @@ -0,0 +1,53 @@ +//! This module owns the crash child readiness boundary. + +use std::fs::OpenOptions; +use std::io::{Read, Write}; +use std::os::unix::net::UnixStream; +use std::path::Path; + +use super::DurabilityCrashMatrixError; +use xtask::DurabilityCrashCase; + +const READY: u8 = b'r'; + +pub(super) fn run( + case: DurabilityCrashCase, + case_root: &Path, + readiness_socket: &Path, +) -> Result<(), DurabilityCrashMatrixError> { + write_marker(case, case_root)?; + let mut stream = UnixStream::connect(readiness_socket) + .map_err(|source| DurabilityCrashMatrixError::io("connect readiness socket", source))?; + stream + .write_all(&[READY]) + .map_err(|source| DurabilityCrashMatrixError::io("signal crash readiness", source))?; + let mut release = [0_u8; 1]; + stream + .read_exact(&mut release) + .map_err(|source| DurabilityCrashMatrixError::io("await process termination", source)) +} + +fn write_marker( + case: DurabilityCrashCase, + case_root: &Path, +) -> Result<(), DurabilityCrashMatrixError> { + let path = case_root.join("prepared-case"); + let mut file = OpenOptions::new() + .write(true) + .create_new(true) + .open(path) + .map_err(|source| DurabilityCrashMatrixError::io("create crash-case marker", source))?; + file.write_all(&marker(case)) + .map_err(|source| DurabilityCrashMatrixError::io("write crash-case marker", source))?; + file.sync_all() + .map_err(|source| DurabilityCrashMatrixError::io("synchronize crash-case marker", source)) +} + +pub(super) fn marker(case: DurabilityCrashCase) -> Vec { + format!( + "{}\t{}\n", + case.point().identifier(), + case.position().identifier() + ) + .into_bytes() +} diff --git a/xtask/src/durability_crash_matrix/error.rs b/xtask/src/durability_crash_matrix/error.rs new file mode 100644 index 0000000..d3ff4be --- /dev/null +++ b/xtask/src/durability_crash_matrix/error.rs @@ -0,0 +1,107 @@ +//! This module owns deterministic crash-matrix execution failures. + +use std::error::Error; +use std::fmt; +use std::io; +use std::time::Duration; + +use xtask::DurabilityCrashCaseError; + +pub(crate) enum DurabilityCrashMatrixError { + ChildExitedEarly { + code: Option, + }, + ChildSurvivedTermination { + code: Option, + }, + InvalidCase(DurabilityCrashCaseError), + InvalidPointEncoding, + InvalidPositionEncoding, + InvalidReadinessSignal { + observed: u8, + }, + Io { + action: &'static str, + source: io::Error, + }, + StateMismatch, + Timeout { + duration: Duration, + }, + UnknownPoint(String), + UnknownPosition(String), + Usage, +} + +impl DurabilityCrashMatrixError { + pub(crate) const fn io(action: &'static str, source: io::Error) -> Self { + Self::Io { action, source } + } +} + +impl fmt::Debug for DurabilityCrashMatrixError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(self, formatter) + } +} + +impl fmt::Display for DurabilityCrashMatrixError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::ChildExitedEarly { code } => { + write!( + formatter, + "crash child exited before readiness with code {code:?}" + ) + } + Self::ChildSurvivedTermination { code } => { + write!( + formatter, + "crash child survived termination with code {code:?}" + ) + } + Self::InvalidCase(error) => write!(formatter, "invalid crash case: {error}"), + Self::InvalidPointEncoding => formatter.write_str("crash point is not valid Unicode"), + Self::InvalidPositionEncoding => { + formatter.write_str("crash position is not valid Unicode") + } + Self::InvalidReadinessSignal { observed } => { + write!(formatter, "crash child sent readiness byte {observed}") + } + Self::Io { action, .. } => write!(formatter, "cannot {action}"), + Self::StateMismatch => { + formatter.write_str("crash child durable marker does not match its case") + } + Self::Timeout { duration } => { + write!(formatter, "crash child exceeded its {duration:?} deadline") + } + Self::UnknownPoint(point) => write!(formatter, "unknown crash point `{point}`"), + Self::UnknownPosition(position) => { + write!(formatter, "unknown crash position `{position}`") + } + Self::Usage => formatter.write_str( + "usage: cargo xtask durability-crash-matrix \ + --case ", + ), + } + } +} + +impl Error for DurabilityCrashMatrixError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::InvalidCase(error) => Some(error), + Self::Io { source, .. } => Some(source), + Self::ChildExitedEarly { .. } + | Self::ChildSurvivedTermination { .. } + | Self::InvalidPointEncoding + | Self::InvalidPositionEncoding + | Self::InvalidReadinessSignal { .. } + | Self::StateMismatch + | Self::Timeout { .. } + | Self::UnknownPoint(_) + | Self::UnknownPosition(_) + | Self::Usage => None, + } + } +} diff --git a/xtask/src/durability_crash_matrix/process.rs b/xtask/src/durability_crash_matrix/process.rs new file mode 100644 index 0000000..5dee30d --- /dev/null +++ b/xtask/src/durability_crash_matrix/process.rs @@ -0,0 +1,199 @@ +//! This module owns deadline-bounded crash-child execution and termination. + +use std::fs; +use std::io::Read; +use std::os::unix::net::{UnixListener, UnixStream}; +use std::os::unix::process::ExitStatusExt; +use std::path::Path; +use std::process::{Child, Command}; +use std::time::{Duration, Instant}; + +use crate::bounded_process::ProcessGroup; +use crate::test_directory::TestDirectory; + +use super::DurabilityCrashMatrixError; +use super::child::marker; +use xtask::DurabilityCrashCase; + +const DEADLINE: Duration = Duration::from_secs(10); +const READY: u8 = b'r'; + +pub(super) fn run( + repository_root: &Path, + case: DurabilityCrashCase, +) -> Result<(), DurabilityCrashMatrixError> { + let directory = TestDirectory::create("durability-crash") + .map_err(|source| DurabilityCrashMatrixError::io("create crash-case directory", source))?; + let socket_path = directory.path().join("ready.sock"); + let listener = listener(&socket_path)?; + let deadline = Instant::now() + .checked_add(DEADLINE) + .ok_or(DurabilityCrashMatrixError::Timeout { duration: DEADLINE })?; + let (mut child, group) = spawn(repository_root, directory.path(), &socket_path, case)?; + let readiness = wait_for_ready(&listener, &mut child, deadline); + let termination = terminate(group, &mut child); + readiness?; + termination?; + verify_marker(directory.path(), case)?; + drop(listener); + fs::remove_file(&socket_path) + .map_err(|source| DurabilityCrashMatrixError::io("remove readiness socket", source))?; + directory + .close() + .map_err(|source| DurabilityCrashMatrixError::io("remove crash-case directory", source)) +} + +fn listener(path: &Path) -> Result { + let listener = UnixListener::bind(path) + .map_err(|source| DurabilityCrashMatrixError::io("bind readiness socket", source))?; + listener + .set_nonblocking(true) + .map_err(|source| DurabilityCrashMatrixError::io("configure readiness socket", source))?; + Ok(listener) +} + +fn spawn( + repository_root: &Path, + case_root: &Path, + socket_path: &Path, + case: DurabilityCrashCase, +) -> Result<(Child, ProcessGroup), DurabilityCrashMatrixError> { + let executable = std::env::current_exe() + .map_err(|source| DurabilityCrashMatrixError::io("resolve xtask executable", source))?; + let mut command = Command::new(executable); + command + .current_dir(repository_root) + .arg("__durability-crash-child") + .arg(case.point().identifier()) + .arg(case.position().identifier()) + .arg(case_root) + .arg(socket_path); + let mut child = crate::bounded_process::spawn_in_process_group(&mut command) + .map_err(|source| DurabilityCrashMatrixError::io("spawn crash child", source))?; + match ProcessGroup::for_child(&child) { + Ok(group) => Ok((child, group)), + Err(source) => { + let admission = + DurabilityCrashMatrixError::io("admit crash-child process group", source); + child + .kill() + .map_err(|source| DurabilityCrashMatrixError::io("kill crash child", source))?; + child + .wait() + .map_err(|source| DurabilityCrashMatrixError::io("reap crash child", source))?; + Err(admission) + } + } +} + +fn wait_for_ready( + listener: &UnixListener, + child: &mut Child, + deadline: Instant, +) -> Result<(), DurabilityCrashMatrixError> { + loop { + match listener.accept() { + Ok((stream, _address)) => return read_ready(stream, child, deadline), + Err(source) if source.kind() == std::io::ErrorKind::WouldBlock => { + admit_running(child)?; + admit_deadline(deadline)?; + std::thread::yield_now(); + } + Err(source) => { + return Err(DurabilityCrashMatrixError::io( + "accept crash-child readiness", + source, + )); + } + } + } +} + +fn read_ready( + mut stream: UnixStream, + child: &mut Child, + deadline: Instant, +) -> Result<(), DurabilityCrashMatrixError> { + stream + .set_nonblocking(true) + .map_err(|source| DurabilityCrashMatrixError::io("configure child readiness", source))?; + let mut signal = [0_u8; 1]; + loop { + match stream.read_exact(&mut signal) { + Ok(()) if signal == [READY] => return Ok(()), + Ok(()) => { + return Err(DurabilityCrashMatrixError::InvalidReadinessSignal { + observed: signal[0], + }); + } + Err(source) if source.kind() == std::io::ErrorKind::WouldBlock => { + admit_running(child)?; + admit_deadline(deadline)?; + std::thread::yield_now(); + } + Err(source) => { + return Err(DurabilityCrashMatrixError::io( + "read crash-child readiness", + source, + )); + } + } + } +} + +fn admit_running(child: &mut Child) -> Result<(), DurabilityCrashMatrixError> { + child + .try_wait() + .map_err(|source| DurabilityCrashMatrixError::io("poll crash child", source))? + .map_or(Ok(()), |status| { + Err(DurabilityCrashMatrixError::ChildExitedEarly { + code: status.code(), + }) + }) +} + +fn admit_deadline(deadline: Instant) -> Result<(), DurabilityCrashMatrixError> { + if Instant::now() >= deadline { + Err(DurabilityCrashMatrixError::Timeout { duration: DEADLINE }) + } else { + Ok(()) + } +} + +fn terminate(group: ProcessGroup, child: &mut Child) -> Result<(), DurabilityCrashMatrixError> { + if let Err(group_error) = group.terminate() { + child + .kill() + .map_err(|source| DurabilityCrashMatrixError::io("kill crash child", source))?; + child + .wait() + .map_err(|source| DurabilityCrashMatrixError::io("reap crash child", source))?; + return Err(DurabilityCrashMatrixError::io( + "terminate crash-child process group", + group_error, + )); + } + let status = child + .wait() + .map_err(|source| DurabilityCrashMatrixError::io("reap crash child", source))?; + if status.signal().is_some() { + Ok(()) + } else { + Err(DurabilityCrashMatrixError::ChildSurvivedTermination { + code: status.code(), + }) + } +} + +fn verify_marker( + case_root: &Path, + case: DurabilityCrashCase, +) -> Result<(), DurabilityCrashMatrixError> { + let observed = fs::read(case_root.join("prepared-case")) + .map_err(|source| DurabilityCrashMatrixError::io("read crash-case marker", source))?; + if observed == marker(case) { + Ok(()) + } else { + Err(DurabilityCrashMatrixError::StateMismatch) + } +} diff --git a/xtask/src/main.rs b/xtask/src/main.rs index cfa6908..cee0e5f 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -19,6 +19,11 @@ mod diagnostic; reason = "the command and task-error boundaries are sibling consumers" )] mod documentation_integrity; +#[allow( + clippy::redundant_pub_crate, + reason = "the command and task-error boundaries are sibling consumers" +)] +mod durability_crash_matrix; #[allow( clippy::redundant_pub_crate, reason = "the external digest witness is shared by sibling repository tasks" @@ -105,6 +110,14 @@ fn run(mut arguments: impl Iterator) -> Result<(), TaskError> { fuzz_campaign::run(repository_root, arguments, &mut output)?; return Ok(()); } + if command == "durability-crash-matrix" { + durability_crash_matrix::run(repository_root, arguments)?; + return Ok(()); + } + if command == "__durability-crash-child" { + durability_crash_matrix::run_child(arguments)?; + return Ok(()); + } refuse_extra(&mut arguments)?; match command.as_str() { "benchmark-baseline" => { diff --git a/xtask/src/task_error.rs b/xtask/src/task_error.rs index 2df59ae..b79072d 100644 --- a/xtask/src/task_error.rs +++ b/xtask/src/task_error.rs @@ -6,6 +6,7 @@ use std::fmt; use crate::benchmark_baseline::BenchmarkBaselineError; use crate::diagnostic::escaped_controls; use crate::documentation_integrity::DocumentationError; +use crate::durability_crash_matrix::DurabilityCrashMatrixError; use crate::fuzz_campaign::FuzzCampaignError; use crate::fuzz_seed_corpus::FuzzSeedError; use crate::golden_file_worldline::GoldenError; @@ -16,6 +17,7 @@ pub(super) enum TaskError { BenchmarkBaseline(BenchmarkBaselineError), Conformance(ConformanceError), Documentation(DocumentationError), + DurabilityCrashMatrix(DurabilityCrashMatrixError), FuzzCampaign(FuzzCampaignError), FuzzSeed(FuzzSeedError), Golden(GoldenError), @@ -40,6 +42,7 @@ impl fmt::Display for TaskError { Self::BenchmarkBaseline(error) => write!(formatter, "{error}"), Self::Conformance(error) => write!(formatter, "{error}"), Self::Documentation(error) => write!(formatter, "{error}"), + Self::DurabilityCrashMatrix(error) => write!(formatter, "{error}"), Self::FuzzCampaign(error) => write!(formatter, "{error}"), Self::FuzzSeed(error) => write!(formatter, "{error}"), Self::Golden(error) => write!(formatter, "{error}"), @@ -66,6 +69,7 @@ impl fmt::Display for TaskError { ", @@ -80,6 +84,7 @@ impl Error for TaskError { Self::BenchmarkBaseline(error) => Some(error), Self::Conformance(error) => Some(error), Self::Documentation(error) => Some(error), + Self::DurabilityCrashMatrix(error) => Some(error), Self::FuzzCampaign(error) => Some(error), Self::FuzzSeed(error) => Some(error), Self::Golden(error) => Some(error), @@ -112,6 +117,12 @@ impl From for TaskError { } } +impl From for TaskError { + fn from(error: DurabilityCrashMatrixError) -> Self { + Self::DurabilityCrashMatrix(error) + } +} + impl From for TaskError { fn from(error: FuzzCampaignError) -> Self { Self::FuzzCampaign(error) diff --git a/xtask/tests/durability_crash_case_contract.rs b/xtask/tests/durability_crash_case_contract.rs index d6297d4..396a9bb 100644 --- a/xtask/tests/durability_crash_case_contract.rs +++ b/xtask/tests/durability_crash_case_contract.rs @@ -75,7 +75,7 @@ fn occurrence_coordinates_exist_only_for_record_append() -> Result<(), Box Result<(), Box> { +fn identifiers_and_positions_round_trip_without_aliases() { for point in DurabilityCrashPoint::ALL { assert_eq!( DurabilityCrashPoint::from_identifier(point.identifier()), @@ -94,5 +94,4 @@ fn identifiers_and_positions_round_trip_without_aliases() -> Result<(), Box Result<(), Box> { + let output = Command::new(env!("CARGO_BIN_EXE_xtask")) + .args([ + "durability-crash-matrix", + "--case", + "KEEP-CRASH-001", + "during", + ]) + .output()?; + + assert!( + output.status.success(), + "selected crash case failed: {output:?}" + ); + assert_eq!(output.stdout, b""); + assert_eq!(output.stderr, b""); + Ok(()) +} From 92d42825b31638fca528f8c300944fcb4f30dd7f Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 13:36:55 -0700 Subject: [PATCH 25/49] Fix: Retain crash readiness through termination --- xtask/src/durability_crash_matrix.rs | 7 ++++++- xtask/src/durability_crash_matrix/process.rs | 10 +++++----- xtask/tests/durability_crash_process_contract.rs | 15 +++++++++++++++ 3 files changed, 26 insertions(+), 6 deletions(-) diff --git a/xtask/src/durability_crash_matrix.rs b/xtask/src/durability_crash_matrix.rs index aad4aae..958a113 100644 --- a/xtask/src/durability_crash_matrix.rs +++ b/xtask/src/durability_crash_matrix.rs @@ -18,7 +18,12 @@ pub(crate) fn run( repository_root: &Path, mut arguments: impl Iterator, ) -> Result<(), DurabilityCrashMatrixError> { - let flag = arguments.next().ok_or(DurabilityCrashMatrixError::Usage)?; + let Some(flag) = arguments.next() else { + for case in DurabilityCrashCase::all() { + process::run(repository_root, case)?; + } + return Ok(()); + }; if flag != OsStr::new(CASE_ARGUMENT) { return Err(DurabilityCrashMatrixError::Usage); } diff --git a/xtask/src/durability_crash_matrix/process.rs b/xtask/src/durability_crash_matrix/process.rs index 5dee30d..c216c1d 100644 --- a/xtask/src/durability_crash_matrix/process.rs +++ b/xtask/src/durability_crash_matrix/process.rs @@ -30,9 +30,9 @@ pub(super) fn run( .checked_add(DEADLINE) .ok_or(DurabilityCrashMatrixError::Timeout { duration: DEADLINE })?; let (mut child, group) = spawn(repository_root, directory.path(), &socket_path, case)?; - let readiness = wait_for_ready(&listener, &mut child, deadline); + let readiness_stream = wait_for_ready(&listener, &mut child, deadline); let termination = terminate(group, &mut child); - readiness?; + let _readiness_stream = readiness_stream?; termination?; verify_marker(directory.path(), case)?; drop(listener); @@ -90,7 +90,7 @@ fn wait_for_ready( listener: &UnixListener, child: &mut Child, deadline: Instant, -) -> Result<(), DurabilityCrashMatrixError> { +) -> Result { loop { match listener.accept() { Ok((stream, _address)) => return read_ready(stream, child, deadline), @@ -113,14 +113,14 @@ fn read_ready( mut stream: UnixStream, child: &mut Child, deadline: Instant, -) -> Result<(), DurabilityCrashMatrixError> { +) -> Result { stream .set_nonblocking(true) .map_err(|source| DurabilityCrashMatrixError::io("configure child readiness", source))?; let mut signal = [0_u8; 1]; loop { match stream.read_exact(&mut signal) { - Ok(()) if signal == [READY] => return Ok(()), + Ok(()) if signal == [READY] => return Ok(stream), Ok(()) => { return Err(DurabilityCrashMatrixError::InvalidReadinessSignal { observed: signal[0], diff --git a/xtask/tests/durability_crash_process_contract.rs b/xtask/tests/durability_crash_process_contract.rs index 04183c3..10f552a 100644 --- a/xtask/tests/durability_crash_process_contract.rs +++ b/xtask/tests/durability_crash_process_contract.rs @@ -22,3 +22,18 @@ fn one_selected_case_reaches_readiness_and_survives_process_death() -> Result<() assert_eq!(output.stderr, b""); Ok(()) } + +#[test] +fn complete_matrix_terminates_all_canonical_cases() -> Result<(), Box> { + let output = Command::new(env!("CARGO_BIN_EXE_xtask")) + .arg("durability-crash-matrix") + .output()?; + + assert!( + output.status.success(), + "complete crash matrix failed: {output:?}" + ); + assert_eq!(output.stdout, b""); + assert_eq!(output.stderr, b""); + Ok(()) +} From bfdd443571632f9c4801a5db322f3503d54efcec Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 13:54:42 -0700 Subject: [PATCH 26/49] Add: Verify Golden Worldline crash states --- Cargo.lock | 1 + xtask/Cargo.toml | 2 + xtask/src/durability_crash_matrix.rs | 13 +- xtask/src/durability_crash_matrix/child.rs | 9 +- xtask/src/durability_crash_matrix/error.rs | 102 ++++++++- xtask/src/durability_crash_matrix/process.rs | 2 + xtask/src/durability_crash_matrix/restart.rs | 89 ++++++++ .../restart/expectation.rs | 91 ++++++++ .../restart/expectation/sequence.rs | 144 +++++++++++++ .../restart/expectation/steps.rs | 147 +++++++++++++ .../restart/semantic.rs | 202 ++++++++++++++++++ xtask/src/durability_crash_matrix/state.rs | 202 ++++++++++++++++++ .../durability_crash_matrix/state/catalog.rs | 84 ++++++++ .../durability_crash_matrix/state/fixture.rs | 77 +++++++ .../src/durability_crash_matrix/state/head.rs | 79 +++++++ .../state/initialization.rs | 60 ++++++ .../durability_crash_matrix/state/recovery.rs | 60 ++++++ .../durability_crash_matrix/state/segment.rs | 103 +++++++++ 18 files changed, 1458 insertions(+), 9 deletions(-) create mode 100644 xtask/src/durability_crash_matrix/restart.rs create mode 100644 xtask/src/durability_crash_matrix/restart/expectation.rs create mode 100644 xtask/src/durability_crash_matrix/restart/expectation/sequence.rs create mode 100644 xtask/src/durability_crash_matrix/restart/expectation/steps.rs create mode 100644 xtask/src/durability_crash_matrix/restart/semantic.rs create mode 100644 xtask/src/durability_crash_matrix/state.rs create mode 100644 xtask/src/durability_crash_matrix/state/catalog.rs create mode 100644 xtask/src/durability_crash_matrix/state/fixture.rs create mode 100644 xtask/src/durability_crash_matrix/state/head.rs create mode 100644 xtask/src/durability_crash_matrix/state/initialization.rs create mode 100644 xtask/src/durability_crash_matrix/state/recovery.rs create mode 100644 xtask/src/durability_crash_matrix/state/segment.rs diff --git a/Cargo.lock b/Cargo.lock index b2e76f4..f5b15c6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -737,6 +737,7 @@ dependencies = [ "blake3", "cap-fs-ext", "cap-std", + "keep", "md-5", "repository-process-spawn", "rustix", diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index b5bad5c..8c88380 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -14,6 +14,7 @@ repository-tasks = [ "dep:blake3", "dep:cap-fs-ext", "dep:cap-std", + "dep:keep", "dep:md-5", "dep:repository-process-spawn", "dep:rustix", @@ -28,6 +29,7 @@ blake3 = { version = "=1.8.5", default-features = false, features = ["pure", "st # Capability-relative, no-follow opens close source-scan replacement races. cap-fs-ext = { version = "=4.0.2", default-features = false, features = ["std"], optional = true } cap-std = { version = "=4.0.2", default-features = false, optional = true } +keep = { path = "..", optional = true } # Pure Rust MD5 regenerates the public Gear-table recipe; it is not an identity primitive. md-5 = { version = "=0.11.0", default-features = false, optional = true } # Dedicated unsafe boundary sets exact child working directories by descriptor. diff --git a/xtask/src/durability_crash_matrix.rs b/xtask/src/durability_crash_matrix.rs index 958a113..7242eba 100644 --- a/xtask/src/durability_crash_matrix.rs +++ b/xtask/src/durability_crash_matrix.rs @@ -3,6 +3,8 @@ mod child; mod error; mod process; +mod restart; +mod state; use std::ffi::{OsStr, OsString}; use std::path::Path; @@ -20,7 +22,7 @@ pub(crate) fn run( ) -> Result<(), DurabilityCrashMatrixError> { let Some(flag) = arguments.next() else { for case in DurabilityCrashCase::all() { - process::run(repository_root, case)?; + run_case(repository_root, case)?; } return Ok(()); }; @@ -29,7 +31,7 @@ pub(crate) fn run( } let case = parse_case(&mut arguments)?; refuse_extra(&mut arguments)?; - process::run(repository_root, case) + run_case(repository_root, case) } pub(crate) fn run_child( @@ -73,3 +75,10 @@ fn refuse_extra( Ok(()) } } + +fn run_case( + repository_root: &Path, + case: DurabilityCrashCase, +) -> Result<(), DurabilityCrashMatrixError> { + process::run(repository_root, case).map_err(|source| source.at_case(case)) +} diff --git a/xtask/src/durability_crash_matrix/child.rs b/xtask/src/durability_crash_matrix/child.rs index 36d4305..6d72dbd 100644 --- a/xtask/src/durability_crash_matrix/child.rs +++ b/xtask/src/durability_crash_matrix/child.rs @@ -1,11 +1,12 @@ //! This module owns the crash child readiness boundary. use std::fs::OpenOptions; -use std::io::{Read, Write}; +use std::io::Write; use std::os::unix::net::UnixStream; use std::path::Path; use super::DurabilityCrashMatrixError; +use super::state; use xtask::DurabilityCrashCase; const READY: u8 = b'r'; @@ -15,16 +16,14 @@ pub(super) fn run( case_root: &Path, readiness_socket: &Path, ) -> Result<(), DurabilityCrashMatrixError> { + let prepared = state::prepare(case, case_root)?; write_marker(case, case_root)?; let mut stream = UnixStream::connect(readiness_socket) .map_err(|source| DurabilityCrashMatrixError::io("connect readiness socket", source))?; stream .write_all(&[READY]) .map_err(|source| DurabilityCrashMatrixError::io("signal crash readiness", source))?; - let mut release = [0_u8; 1]; - stream - .read_exact(&mut release) - .map_err(|source| DurabilityCrashMatrixError::io("await process termination", source)) + prepared.await_process_death(&mut stream) } fn write_marker( diff --git a/xtask/src/durability_crash_matrix/error.rs b/xtask/src/durability_crash_matrix/error.rs index d3ff4be..9a22e7f 100644 --- a/xtask/src/durability_crash_matrix/error.rs +++ b/xtask/src/durability_crash_matrix/error.rs @@ -5,15 +5,37 @@ use std::fmt; use std::io; use std::time::Duration; -use xtask::DurabilityCrashCaseError; +use keep::WriterLockAcquireError; +use xtask::protocol_admission::HexError; +use xtask::{ + DurabilityCrashCase, DurabilityCrashCaseError, DurabilityCrashPoint, DurabilityCrashPosition, +}; pub(crate) enum DurabilityCrashMatrixError { + Case { + point: DurabilityCrashPoint, + position: DurabilityCrashPosition, + source: Box, + }, ChildExitedEarly { code: Option, }, ChildSurvivedTermination { code: Option, }, + Fixture { + artifact: &'static str, + source: HexError, + }, + FixtureLength { + artifact: &'static str, + expected: usize, + observed: usize, + }, + FixtureRange, + FixtureTerminator { + artifact: &'static str, + }, InvalidCase(DurabilityCrashCaseError), InvalidPointEncoding, InvalidPositionEncoding, @@ -24,6 +46,11 @@ pub(crate) enum DurabilityCrashMatrixError { action: &'static str, source: io::Error, }, + MissingActiveFile, + NonUnicodeStatePath, + PointSequenceMismatch { + point: DurabilityCrashPoint, + }, StateMismatch, Timeout { duration: Duration, @@ -31,12 +58,25 @@ pub(crate) enum DurabilityCrashMatrixError { UnknownPoint(String), UnknownPosition(String), Usage, + Verification { + phase: &'static str, + source: Box, + }, + WriterLock(WriterLockAcquireError), } impl DurabilityCrashMatrixError { pub(crate) const fn io(action: &'static str, source: io::Error) -> Self { Self::Io { action, source } } + + pub(crate) fn at_case(self, case: DurabilityCrashCase) -> Self { + Self::Case { + point: case.point(), + position: case.position(), + source: Box::new(self), + } + } } impl fmt::Debug for DurabilityCrashMatrixError { @@ -48,6 +88,16 @@ impl fmt::Debug for DurabilityCrashMatrixError { impl fmt::Display for DurabilityCrashMatrixError { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { match self { + Self::Case { + point, + position, + source, + } => write!( + formatter, + "{} {}: {source}", + point.identifier(), + position.identifier() + ), Self::ChildExitedEarly { code } => { write!( formatter, @@ -60,6 +110,24 @@ impl fmt::Display for DurabilityCrashMatrixError { "crash child survived termination with code {code:?}" ) } + Self::Fixture { artifact, .. } => { + write!(formatter, "cannot decode {artifact} crash fixture") + } + Self::FixtureLength { + artifact, + expected, + observed, + } => write!( + formatter, + "{artifact} crash fixture has length {observed}, expected {expected}" + ), + Self::FixtureRange => formatter.write_str("crash fixture range is invalid"), + Self::FixtureTerminator { artifact } => { + write!( + formatter, + "{artifact} crash fixture lacks its final line feed" + ) + } Self::InvalidCase(error) => write!(formatter, "invalid crash case: {error}"), Self::InvalidPointEncoding => formatter.write_str("crash point is not valid Unicode"), Self::InvalidPositionEncoding => { @@ -69,8 +137,19 @@ impl fmt::Display for DurabilityCrashMatrixError { write!(formatter, "crash child sent readiness byte {observed}") } Self::Io { action, .. } => write!(formatter, "cannot {action}"), + Self::MissingActiveFile => { + formatter.write_str("crash sequence has no active staged artifact") + } + Self::NonUnicodeStatePath => { + formatter.write_str("post-crash store path is not valid Unicode") + } + Self::PointSequenceMismatch { point } => write!( + formatter, + "{} is outside the selected crash sequence", + point.identifier() + ), Self::StateMismatch => { - formatter.write_str("crash child durable marker does not match its case") + formatter.write_str("post-crash store state does not match its case") } Self::Timeout { duration } => { write!(formatter, "crash child exceeded its {duration:?} deadline") @@ -83,6 +162,15 @@ impl fmt::Display for DurabilityCrashMatrixError { "usage: cargo xtask durability-crash-matrix \ --case ", ), + Self::Verification { phase, source } => { + write!( + formatter, + "post-crash verification failed while attempting to {phase}: {source}" + ) + } + Self::WriterLock(source) => { + write!(formatter, "cannot acquire crash-case writer lock: {source}") + } } } } @@ -90,13 +178,23 @@ impl fmt::Display for DurabilityCrashMatrixError { impl Error for DurabilityCrashMatrixError { fn source(&self) -> Option<&(dyn Error + 'static)> { match self { + Self::Case { source, .. } => Some(source.as_ref()), + Self::Fixture { source, .. } => Some(source), Self::InvalidCase(error) => Some(error), Self::Io { source, .. } => Some(source), + Self::Verification { source, .. } => Some(source.as_ref()), + Self::WriterLock(source) => Some(source), Self::ChildExitedEarly { .. } | Self::ChildSurvivedTermination { .. } + | Self::FixtureLength { .. } + | Self::FixtureRange + | Self::FixtureTerminator { .. } | Self::InvalidPointEncoding | Self::InvalidPositionEncoding | Self::InvalidReadinessSignal { .. } + | Self::MissingActiveFile + | Self::NonUnicodeStatePath + | Self::PointSequenceMismatch { .. } | Self::StateMismatch | Self::Timeout { .. } | Self::UnknownPoint(_) diff --git a/xtask/src/durability_crash_matrix/process.rs b/xtask/src/durability_crash_matrix/process.rs index c216c1d..0faef21 100644 --- a/xtask/src/durability_crash_matrix/process.rs +++ b/xtask/src/durability_crash_matrix/process.rs @@ -13,6 +13,7 @@ use crate::test_directory::TestDirectory; use super::DurabilityCrashMatrixError; use super::child::marker; +use super::restart; use xtask::DurabilityCrashCase; const DEADLINE: Duration = Duration::from_secs(10); @@ -35,6 +36,7 @@ pub(super) fn run( let _readiness_stream = readiness_stream?; termination?; verify_marker(directory.path(), case)?; + restart::verify(&directory.path().join("store"), case)?; drop(listener); fs::remove_file(&socket_path) .map_err(|source| DurabilityCrashMatrixError::io("remove readiness socket", source))?; diff --git a/xtask/src/durability_crash_matrix/restart.rs b/xtask/src/durability_crash_matrix/restart.rs new file mode 100644 index 0000000..fa3226a --- /dev/null +++ b/xtask/src/durability_crash_matrix/restart.rs @@ -0,0 +1,89 @@ +//! This module owns independent post-process-death store verification. + +mod expectation; +mod semantic; + +use std::collections::BTreeSet; +use std::fs; +use std::os::unix::fs::MetadataExt; +use std::path::{Path, PathBuf}; + +use super::DurabilityCrashMatrixError; +use super::state::fixture::GoldenFixture; +use expectation::ExpectedStoreState; +use xtask::DurabilityCrashCase; + +pub(super) fn verify( + store_root: &Path, + case: DurabilityCrashCase, +) -> Result<(), DurabilityCrashMatrixError> { + let expected = ExpectedStoreState::for_case(case)?; + let observed_paths = inventory(store_root)?; + if observed_paths != expected.paths() { + return Err(DurabilityCrashMatrixError::StateMismatch); + } + let segment = GoldenFixture::segment()?; + let catalog = GoldenFixture::catalog()?; + let head = GoldenFixture::head()?; + for (relative, bytes) in expected.artifacts() { + let observed = fs::read(store_root.join(relative)) + .map_err(|source| DurabilityCrashMatrixError::io("read crash artifact", source))?; + let expected_bytes = bytes.resolve(&segment, &catalog, &head)?; + if observed != expected_bytes { + return Err(DurabilityCrashMatrixError::StateMismatch); + } + } + if let Some((source, target)) = expected.hard_link() { + verify_hard_link(store_root, source, target)?; + } + semantic::verify(store_root, &expected)?; + Ok(()) +} + +fn inventory(store_root: &Path) -> Result, DurabilityCrashMatrixError> { + let mut paths = BTreeSet::new(); + let mut pending = vec![PathBuf::new()]; + while let Some(relative_parent) = pending.pop() { + let parent = store_root.join(&relative_parent); + let entries = fs::read_dir(parent) + .map_err(|source| DurabilityCrashMatrixError::io("inventory crash store", source))?; + for entry in entries { + let entry = entry.map_err(|source| { + DurabilityCrashMatrixError::io("read crash-store entry", source) + })?; + let relative = relative_parent.join(entry.file_name()); + let text = relative + .to_str() + .ok_or(DurabilityCrashMatrixError::NonUnicodeStatePath)? + .into(); + if !paths.insert(text) { + return Err(DurabilityCrashMatrixError::StateMismatch); + } + let file_type = entry.file_type().map_err(|source| { + DurabilityCrashMatrixError::io("inspect crash-store entry", source) + })?; + if file_type.is_dir() { + pending.push(relative); + } + } + } + Ok(paths) +} + +fn verify_hard_link( + store_root: &Path, + source: &str, + target: &str, +) -> Result<(), DurabilityCrashMatrixError> { + let source_metadata = fs::metadata(store_root.join(source)) + .map_err(|error| DurabilityCrashMatrixError::io("inspect crash source link", error))?; + let target_metadata = fs::metadata(store_root.join(target)) + .map_err(|error| DurabilityCrashMatrixError::io("inspect crash target link", error))?; + if source_metadata.dev() == target_metadata.dev() + && source_metadata.ino() == target_metadata.ino() + { + Ok(()) + } else { + Err(DurabilityCrashMatrixError::StateMismatch) + } +} diff --git a/xtask/src/durability_crash_matrix/restart/expectation.rs b/xtask/src/durability_crash_matrix/restart/expectation.rs new file mode 100644 index 0000000..7b11ada --- /dev/null +++ b/xtask/src/durability_crash_matrix/restart/expectation.rs @@ -0,0 +1,91 @@ +//! This module owns the independent expected crash-state model. + +mod sequence; +mod steps; + +use std::collections::{BTreeMap, BTreeSet}; + +use super::super::DurabilityCrashMatrixError; +use super::super::state::fixture::GoldenFixture; +use xtask::{DurabilityCrashCase, DurabilityCrashSequence}; + +pub(super) const WRITER_LOCK: &str = "writer.lock"; +pub(super) const STAGING: &str = "staging"; +pub(super) const SEGMENTS: &str = "segments"; +pub(super) const CATALOGS: &str = "catalogs"; +pub(super) const SEGMENT_STAGE: &str = "staging/current.seg"; +pub(super) const CATALOG_STAGE: &str = "staging/current.cat"; +pub(super) const NEXT_HEAD: &str = "head.next"; +pub(super) const HEAD: &str = "HEAD"; + +pub(super) enum ArtifactBytes { + Empty, + Segment(usize), + Catalog(usize), + Head(usize), +} + +impl ArtifactBytes { + pub(super) fn resolve<'a>( + &self, + segment: &'a GoldenFixture, + catalog: &'a GoldenFixture, + head: &'a GoldenFixture, + ) -> Result<&'a [u8], DurabilityCrashMatrixError> { + match self { + Self::Empty => Ok(&[]), + Self::Segment(end) => segment.prefix(*end), + Self::Catalog(end) => catalog.prefix(*end), + Self::Head(end) => head.prefix(*end), + } + } +} + +pub(super) struct ExpectedStoreState { + directories: BTreeSet<&'static str>, + artifacts: BTreeMap<&'static str, ArtifactBytes>, + hard_link: Option<(&'static str, &'static str)>, +} + +impl ExpectedStoreState { + pub(super) fn for_case(case: DurabilityCrashCase) -> Result { + match case.point().sequence() { + DurabilityCrashSequence::Segment => sequence::segment(case), + DurabilityCrashSequence::Catalog => sequence::catalog(case), + DurabilityCrashSequence::Head => sequence::head(case), + DurabilityCrashSequence::RecoveryDiscard => sequence::recovery(case), + DurabilityCrashSequence::Initialization => sequence::initialization(case), + } + } + + pub(super) fn paths(&self) -> BTreeSet { + self.directories + .iter() + .copied() + .chain(self.artifacts.keys().copied()) + .map(Into::into) + .collect() + } + + pub(super) const fn artifacts(&self) -> &BTreeMap<&'static str, ArtifactBytes> { + &self.artifacts + } + + pub(super) fn artifact(&self, path: &str) -> Option<&ArtifactBytes> { + self.artifacts.get(path) + } + + pub(super) const fn hard_link(&self) -> Option<(&'static str, &'static str)> { + self.hard_link + } + + fn initialized() -> Self { + let directories = [STAGING, SEGMENTS, CATALOGS].into_iter().collect(); + let artifacts = std::iter::once((WRITER_LOCK, ArtifactBytes::Empty)).collect(); + Self { + directories, + artifacts, + hard_link: None, + } + } +} diff --git a/xtask/src/durability_crash_matrix/restart/expectation/sequence.rs b/xtask/src/durability_crash_matrix/restart/expectation/sequence.rs new file mode 100644 index 0000000..1d0d83c --- /dev/null +++ b/xtask/src/durability_crash_matrix/restart/expectation/sequence.rs @@ -0,0 +1,144 @@ +//! This module owns expected states for each durable protocol sequence. + +use std::collections::{BTreeMap, BTreeSet}; + +use super::steps::{ + catalog_bytes, catalog_step, completed_step, head_bytes, head_step, initialization_step, + interrupted_segment_bytes, segment_bytes, segment_step, +}; +use super::{ + ArtifactBytes, CATALOG_STAGE, CATALOGS, ExpectedStoreState, HEAD, NEXT_HEAD, SEGMENT_STAGE, + SEGMENTS, STAGING, WRITER_LOCK, +}; +use crate::durability_crash_matrix::DurabilityCrashMatrixError; +use crate::durability_crash_matrix::state::fixture::{CATALOG_POOL_PATH, SEGMENT_POOL_PATH}; +use xtask::{DurabilityCrashCase, DurabilityCrashPoint, DurabilityCrashPosition}; + +pub(super) fn segment( + case: DurabilityCrashCase, +) -> Result { + let mut state = ExpectedStoreState::initialized(); + let completed = completed_step(segment_step(case.point())?, case.position(), case.point()); + if let Some(bytes) = interrupted_segment_bytes(case) { + state.artifacts.insert(SEGMENT_STAGE, bytes); + } else if let Some(step) = completed { + if step <= 9 { + state.artifacts.insert(SEGMENT_STAGE, segment_bytes(step)); + } + if step >= 8 { + state + .artifacts + .insert(SEGMENT_POOL_PATH, ArtifactBytes::Segment(337)); + } + if (8..=9).contains(&step) { + state.hard_link = Some((SEGMENT_STAGE, SEGMENT_POOL_PATH)); + } + } + Ok(state) +} + +pub(super) fn catalog( + case: DurabilityCrashCase, +) -> Result { + let mut state = ExpectedStoreState::initialized(); + state + .artifacts + .insert(SEGMENT_POOL_PATH, ArtifactBytes::Segment(337)); + let completed = completed_step(catalog_step(case.point())?, case.position(), case.point()); + if case.point() == DurabilityCrashPoint::WriteCatalog + && case.position() == DurabilityCrashPosition::During + { + state + .artifacts + .insert(CATALOG_STAGE, ArtifactBytes::Catalog(176)); + } else if let Some(step) = completed { + if step <= 5 { + state.artifacts.insert(CATALOG_STAGE, catalog_bytes(step)); + } + if step >= 4 { + state + .artifacts + .insert(CATALOG_POOL_PATH, ArtifactBytes::Catalog(352)); + } + if (4..=5).contains(&step) { + state.hard_link = Some((CATALOG_STAGE, CATALOG_POOL_PATH)); + } + } + Ok(state) +} + +pub(super) fn head( + case: DurabilityCrashCase, +) -> Result { + let mut state = ExpectedStoreState::initialized(); + state + .artifacts + .insert(SEGMENT_POOL_PATH, ArtifactBytes::Segment(337)); + state + .artifacts + .insert(CATALOG_POOL_PATH, ArtifactBytes::Catalog(352)); + let completed = completed_step(head_step(case.point())?, case.position(), case.point()); + if case.point() == DurabilityCrashPoint::WriteHead + && case.position() == DurabilityCrashPosition::During + { + state.artifacts.insert(NEXT_HEAD, ArtifactBytes::Head(64)); + } else if let Some(step) = completed { + if step <= 3 { + state.artifacts.insert(NEXT_HEAD, head_bytes(step)); + } else { + state.artifacts.insert(HEAD, ArtifactBytes::Head(128)); + } + } + Ok(state) +} + +pub(super) fn recovery( + case: DurabilityCrashCase, +) -> Result { + let mut state = ExpectedStoreState::initialized(); + match case.point() { + DurabilityCrashPoint::RemoveRecoveryStage + if case.position() == DurabilityCrashPosition::Before => + { + state + .artifacts + .insert(SEGMENT_STAGE, ArtifactBytes::Segment(32)); + } + DurabilityCrashPoint::RemoveRecoveryHead + if case.position() == DurabilityCrashPosition::Before => + { + state.artifacts.insert(NEXT_HEAD, ArtifactBytes::Head(64)); + } + DurabilityCrashPoint::RemoveRecoveryStage + | DurabilityCrashPoint::SynchronizeStagingAfterRecovery + | DurabilityCrashPoint::RemoveRecoveryHead + | DurabilityCrashPoint::SynchronizeRootAfterRecovery => {} + point => return Err(DurabilityCrashMatrixError::PointSequenceMismatch { point }), + } + Ok(state) +} + +pub(super) fn initialization( + case: DurabilityCrashCase, +) -> Result { + let step = initialization_step(case.point())?; + let completed = completed_step(step, case.position(), case.point()); + let mut state = ExpectedStoreState { + directories: BTreeSet::new(), + artifacts: BTreeMap::new(), + hard_link: None, + }; + if let Some(completed) = completed { + state.artifacts.insert(WRITER_LOCK, ArtifactBytes::Empty); + if completed >= 1 { + state.directories.insert(STAGING); + } + if completed >= 2 { + state.directories.insert(SEGMENTS); + } + if completed >= 3 { + state.directories.insert(CATALOGS); + } + } + Ok(state) +} diff --git a/xtask/src/durability_crash_matrix/restart/expectation/steps.rs b/xtask/src/durability_crash_matrix/restart/expectation/steps.rs new file mode 100644 index 0000000..19d6a6e --- /dev/null +++ b/xtask/src/durability_crash_matrix/restart/expectation/steps.rs @@ -0,0 +1,147 @@ +//! This module owns independent crash-point ordering and byte-state rules. + +use super::ArtifactBytes; +use crate::durability_crash_matrix::DurabilityCrashMatrixError; +use xtask::{DurabilityCrashCase, DurabilityCrashPoint, DurabilityCrashPosition}; + +pub(super) fn completed_step( + step: usize, + position: DurabilityCrashPosition, + point: DurabilityCrashPoint, +) -> Option { + if position == DurabilityCrashPosition::After + || (position == DurabilityCrashPosition::During && atomic(point)) + { + Some(step) + } else { + step.checked_sub(1) + } +} + +const fn atomic(point: DurabilityCrashPoint) -> bool { + matches!( + point, + DurabilityCrashPoint::CreateSegmentStage + | DurabilityCrashPoint::LinkSegment + | DurabilityCrashPoint::RemoveSegmentStage + | DurabilityCrashPoint::CreateCatalogStage + | DurabilityCrashPoint::LinkCatalog + | DurabilityCrashPoint::RemoveCatalogStage + | DurabilityCrashPoint::CreateHeadStage + | DurabilityCrashPoint::ReplaceHead + | DurabilityCrashPoint::RemoveRecoveryStage + | DurabilityCrashPoint::RemoveRecoveryHead + | DurabilityCrashPoint::OpenAndLockWriterFile + | DurabilityCrashPoint::CreateStagingDirectory + | DurabilityCrashPoint::CreateSegmentPoolDirectory + | DurabilityCrashPoint::CreateCatalogPoolDirectory + ) +} + +pub(super) fn interrupted_segment_bytes(case: DurabilityCrashCase) -> Option { + if case.position() != DurabilityCrashPosition::During { + return None; + } + match case.point() { + DurabilityCrashPoint::WriteSegmentHeader => Some(ArtifactBytes::Segment(32)), + DurabilityCrashPoint::AppendSegmentRecord => Some(ArtifactBytes::Segment(136)), + DurabilityCrashPoint::AppendSegmentSeal => Some(ArtifactBytes::Segment(273)), + _ => None, + } +} + +pub(super) const fn segment_bytes(step: usize) -> ArtifactBytes { + match step { + 0 => ArtifactBytes::Empty, + 1 => ArtifactBytes::Segment(64), + 2..=4 => ArtifactBytes::Segment(209), + _ => ArtifactBytes::Segment(337), + } +} + +pub(super) const fn catalog_bytes(step: usize) -> ArtifactBytes { + if step == 0 { + ArtifactBytes::Empty + } else { + ArtifactBytes::Catalog(352) + } +} + +pub(super) const fn head_bytes(step: usize) -> ArtifactBytes { + if step == 0 { + ArtifactBytes::Empty + } else { + ArtifactBytes::Head(128) + } +} + +pub(super) fn segment_step( + point: DurabilityCrashPoint, +) -> Result { + const POINTS: [DurabilityCrashPoint; 12] = [ + DurabilityCrashPoint::CreateSegmentStage, + DurabilityCrashPoint::WriteSegmentHeader, + DurabilityCrashPoint::AppendSegmentRecord, + DurabilityCrashPoint::FlushSegmentRecordPrefix, + DurabilityCrashPoint::SynchronizeSegmentRecordPrefix, + DurabilityCrashPoint::AppendSegmentSeal, + DurabilityCrashPoint::FlushSealedSegment, + DurabilityCrashPoint::SynchronizeSealedSegment, + DurabilityCrashPoint::LinkSegment, + DurabilityCrashPoint::SynchronizeSegmentPool, + DurabilityCrashPoint::RemoveSegmentStage, + DurabilityCrashPoint::SynchronizeStagingAfterSegment, + ]; + step(POINTS, point) +} + +pub(super) fn catalog_step( + point: DurabilityCrashPoint, +) -> Result { + const POINTS: [DurabilityCrashPoint; 8] = [ + DurabilityCrashPoint::CreateCatalogStage, + DurabilityCrashPoint::WriteCatalog, + DurabilityCrashPoint::FlushCatalog, + DurabilityCrashPoint::SynchronizeCatalog, + DurabilityCrashPoint::LinkCatalog, + DurabilityCrashPoint::SynchronizeCatalogPool, + DurabilityCrashPoint::RemoveCatalogStage, + DurabilityCrashPoint::SynchronizeStagingAfterCatalog, + ]; + step(POINTS, point) +} + +pub(super) fn head_step(point: DurabilityCrashPoint) -> Result { + const POINTS: [DurabilityCrashPoint; 6] = [ + DurabilityCrashPoint::CreateHeadStage, + DurabilityCrashPoint::WriteHead, + DurabilityCrashPoint::FlushHead, + DurabilityCrashPoint::SynchronizeHead, + DurabilityCrashPoint::ReplaceHead, + DurabilityCrashPoint::SynchronizeRootAfterHead, + ]; + step(POINTS, point) +} + +pub(super) fn initialization_step( + point: DurabilityCrashPoint, +) -> Result { + const POINTS: [DurabilityCrashPoint; 5] = [ + DurabilityCrashPoint::OpenAndLockWriterFile, + DurabilityCrashPoint::CreateStagingDirectory, + DurabilityCrashPoint::CreateSegmentPoolDirectory, + DurabilityCrashPoint::CreateCatalogPoolDirectory, + DurabilityCrashPoint::SynchronizeRootAfterInitialization, + ]; + step(POINTS, point) +} + +fn step( + points: [DurabilityCrashPoint; N], + point: DurabilityCrashPoint, +) -> Result { + points + .into_iter() + .position(|candidate| candidate == point) + .ok_or(DurabilityCrashMatrixError::PointSequenceMismatch { point }) +} diff --git a/xtask/src/durability_crash_matrix/restart/semantic.rs b/xtask/src/durability_crash_matrix/restart/semantic.rs new file mode 100644 index 0000000..4efc17f --- /dev/null +++ b/xtask/src/durability_crash_matrix/restart/semantic.rs @@ -0,0 +1,202 @@ +//! This module owns production semantic restart checks for crash states. + +use std::error::Error; +use std::fs; +use std::path::Path; + +use keep::{ + AdmittedSegment, CatalogRestartByteLimit, CatalogRestartPolicy, ChecksummedCatalog, ChunkId, + FilesystemCatalogSnapshot, FilesystemWriterLock, LayoutEntryLimit, RecoveryCatalogStage, + RecoveryNextHeadStage, RecoverySegmentStage, SegmentReadPolicy, SegmentRecordIdentity, + SegmentRecordLimit, classify_recovery_catalog_stage, classify_recovery_next_head_stage, + classify_recovery_segment_stage, +}; + +use super::expectation::{ + ArtifactBytes, CATALOG_STAGE, ExpectedStoreState, HEAD, NEXT_HEAD, SEGMENT_STAGE, WRITER_LOCK, +}; +use crate::durability_crash_matrix::DurabilityCrashMatrixError; +use crate::durability_crash_matrix::state::fixture::{CATALOG_POOL_PATH, SEGMENT_POOL_PATH}; + +const RESTART_BYTE_LIMIT: u64 = 1_048_576; + +pub(super) fn verify( + store_root: &Path, + expected: &ExpectedStoreState, +) -> Result<(), DurabilityCrashMatrixError> { + verify_writer_release(store_root, expected)?; + verify_segment_stage(store_root, expected)?; + verify_catalog_stage(store_root, expected)?; + verify_next_head(store_root, expected)?; + verify_immutable_artifacts(store_root, expected)?; + verify_published_snapshot(store_root, expected) +} + +fn verify_writer_release( + store_root: &Path, + expected: &ExpectedStoreState, +) -> Result<(), DurabilityCrashMatrixError> { + if expected.artifact(WRITER_LOCK).is_none() { + return Ok(()); + } + let lock = FilesystemWriterLock::try_acquire(store_root) + .map_err(|source| verification("reacquire writer lock after process death", source))?; + drop(lock); + Ok(()) +} + +fn verify_segment_stage( + store_root: &Path, + expected: &ExpectedStoreState, +) -> Result<(), DurabilityCrashMatrixError> { + let Some(bytes) = expected.artifact(SEGMENT_STAGE) else { + return Ok(()); + }; + let encoded = fs::read(store_root.join(SEGMENT_STAGE)) + .map_err(|source| DurabilityCrashMatrixError::io("read recovery segment stage", source))?; + let observed = match classify_recovery_segment_stage(&encoded, segment_policy()) + .map_err(|source| verification("classify recovery segment stage", source))? + { + RecoverySegmentStage::Reusable(_) => "reusable", + RecoverySegmentStage::Complete(_) => "complete", + RecoverySegmentStage::Truncated(_) => "truncated", + }; + let expected_class = match bytes { + ArtifactBytes::Segment(64 | 209) => "reusable", + ArtifactBytes::Segment(337) => "complete", + ArtifactBytes::Empty | ArtifactBytes::Segment(_) => "truncated", + ArtifactBytes::Catalog(_) | ArtifactBytes::Head(_) => { + return Err(DurabilityCrashMatrixError::StateMismatch); + } + }; + require_class(observed, expected_class) +} + +fn verify_catalog_stage( + store_root: &Path, + expected: &ExpectedStoreState, +) -> Result<(), DurabilityCrashMatrixError> { + let Some(bytes) = expected.artifact(CATALOG_STAGE) else { + return Ok(()); + }; + let encoded = fs::read(store_root.join(CATALOG_STAGE)) + .map_err(|source| DurabilityCrashMatrixError::io("read recovery catalog stage", source))?; + let observed = match classify_recovery_catalog_stage(&encoded) + .map_err(|source| verification("classify recovery catalog stage", source))? + { + RecoveryCatalogStage::Complete(_) => "complete", + RecoveryCatalogStage::HeaderTruncated { .. } + | RecoveryCatalogStage::BodyTruncated { .. } => "truncated", + }; + let expected_class = match bytes { + ArtifactBytes::Catalog(352) => "complete", + ArtifactBytes::Empty | ArtifactBytes::Catalog(_) => "truncated", + ArtifactBytes::Segment(_) | ArtifactBytes::Head(_) => { + return Err(DurabilityCrashMatrixError::StateMismatch); + } + }; + require_class(observed, expected_class) +} + +fn verify_next_head( + store_root: &Path, + expected: &ExpectedStoreState, +) -> Result<(), DurabilityCrashMatrixError> { + let Some(bytes) = expected.artifact(NEXT_HEAD) else { + return Ok(()); + }; + let encoded = fs::read(store_root.join(NEXT_HEAD)) + .map_err(|source| DurabilityCrashMatrixError::io("read recovery next head", source))?; + let observed = match classify_recovery_next_head_stage(&encoded) + .map_err(|source| verification("classify recovery next head", source))? + { + RecoveryNextHeadStage::Complete(_) => "complete", + RecoveryNextHeadStage::Truncated { .. } => "truncated", + }; + let expected_class = match bytes { + ArtifactBytes::Head(128) => "complete", + ArtifactBytes::Empty | ArtifactBytes::Head(_) => "truncated", + ArtifactBytes::Segment(_) | ArtifactBytes::Catalog(_) => { + return Err(DurabilityCrashMatrixError::StateMismatch); + } + }; + require_class(observed, expected_class) +} + +fn verify_immutable_artifacts( + store_root: &Path, + expected: &ExpectedStoreState, +) -> Result<(), DurabilityCrashMatrixError> { + let Some(_) = expected.artifact(SEGMENT_POOL_PATH) else { + return Ok(()); + }; + let segment_bytes = fs::read(store_root.join(SEGMENT_POOL_PATH)) + .map_err(|source| DurabilityCrashMatrixError::io("read immutable segment", source))?; + let segment = AdmittedSegment::decode(&segment_bytes, segment_policy()) + .map_err(|source| verification("admit immutable segment after restart", source))?; + if expected.artifact(CATALOG_POOL_PATH).is_none() { + return Ok(()); + } + let catalog_bytes = fs::read(store_root.join(CATALOG_POOL_PATH)) + .map_err(|source| DurabilityCrashMatrixError::io("read immutable catalog", source))?; + let _catalog = ChecksummedCatalog::decode(&catalog_bytes) + .map_err(|source| verification("decode immutable catalog after restart", source))? + .admit(&[segment]) + .map_err(|source| verification("admit immutable catalog after restart", source))?; + Ok(()) +} + +fn verify_published_snapshot( + store_root: &Path, + expected: &ExpectedStoreState, +) -> Result<(), DurabilityCrashMatrixError> { + if expected.artifact(HEAD).is_none() { + return Ok(()); + } + let loaded = FilesystemCatalogSnapshot::load(store_root, restart_policy()?) + .map_err(|source| verification("load published restart snapshot", source))?; + if loaded.generation().get() != 1 { + return Err(DurabilityCrashMatrixError::StateMismatch); + } + let snapshot = loaded + .snapshot() + .map_err(|source| verification("admit published restart snapshot", source))?; + let chunk = ChunkId::hash_bytes(&[0]) + .map_err(|source| verification("construct Golden Worldline chunk identity", source))?; + let record = snapshot + .record(SegmentRecordIdentity::Chunk(chunk)) + .ok_or(DurabilityCrashMatrixError::StateMismatch)?; + if record.payload() == [0] { + Ok(()) + } else { + Err(DurabilityCrashMatrixError::StateMismatch) + } +} + +const fn segment_policy() -> SegmentReadPolicy { + SegmentReadPolicy::new(SegmentRecordLimit::MAXIMUM, LayoutEntryLimit::MAXIMUM) +} + +fn restart_policy() -> Result { + let byte_limit = CatalogRestartByteLimit::new(RESTART_BYTE_LIMIT) + .map_err(|source| verification("construct restart byte limit", source))?; + Ok(CatalogRestartPolicy::new(segment_policy(), byte_limit)) +} + +fn require_class( + observed: &'static str, + expected: &'static str, +) -> Result<(), DurabilityCrashMatrixError> { + if observed == expected { + Ok(()) + } else { + Err(DurabilityCrashMatrixError::StateMismatch) + } +} + +fn verification(phase: &'static str, source: impl Error + 'static) -> DurabilityCrashMatrixError { + DurabilityCrashMatrixError::Verification { + phase, + source: Box::new(source), + } +} diff --git a/xtask/src/durability_crash_matrix/state.rs b/xtask/src/durability_crash_matrix/state.rs new file mode 100644 index 0000000..a0d98f6 --- /dev/null +++ b/xtask/src/durability_crash_matrix/state.rs @@ -0,0 +1,202 @@ +//! This module owns one crash child's retained filesystem state. + +mod catalog; +pub(super) mod fixture; +mod head; +mod initialization; +mod recovery; +mod segment; + +use std::fs::{self, File, OpenOptions}; +use std::io::{Read, Write}; +use std::ops::Range; +use std::os::unix::net::UnixStream; +use std::path::{Path, PathBuf}; + +use keep::FilesystemWriterLock; +use xtask::{DurabilityCrashCase, DurabilityCrashSequence}; + +use super::DurabilityCrashMatrixError; +use fixture::GoldenFixture; + +const STORE_DIRECTORY: &str = "store"; + +pub(super) struct PreparedCrashState { + state: StoreState, +} + +impl PreparedCrashState { + pub(super) fn await_process_death( + self, + stream: &mut UnixStream, + ) -> Result<(), DurabilityCrashMatrixError> { + let retained_root = &self.state.root; + let mut release = [0_u8; 1]; + let result = stream + .read_exact(&mut release) + .map_err(|source| DurabilityCrashMatrixError::io("await process termination", source)); + let _ = retained_root; + self.state.finish(result) + } +} + +pub(super) fn prepare( + case: DurabilityCrashCase, + case_root: &Path, +) -> Result { + let mut state = StoreState::create(case_root)?; + match case.point().sequence() { + DurabilityCrashSequence::Segment => segment::prepare(&mut state, case)?, + DurabilityCrashSequence::Catalog => catalog::prepare(&mut state, case)?, + DurabilityCrashSequence::Head => head::prepare(&mut state, case)?, + DurabilityCrashSequence::RecoveryDiscard => recovery::prepare(&mut state, case)?, + DurabilityCrashSequence::Initialization => initialization::prepare(&mut state, case)?, + } + Ok(PreparedCrashState { state }) +} + +struct StoreState { + root: PathBuf, + active_file: Option, + writer_lock: Option, +} + +impl StoreState { + fn create(case_root: &Path) -> Result { + let root = case_root.join(STORE_DIRECTORY); + fs::create_dir(&root) + .map_err(|source| DurabilityCrashMatrixError::io("create crash store root", source))?; + Ok(Self { + root, + active_file: None, + writer_lock: None, + }) + } + + fn initialize(&mut self) -> Result<(), DurabilityCrashMatrixError> { + self.create_writer_lock()?; + self.create_directory("staging")?; + self.create_directory("segments")?; + self.create_directory("catalogs")?; + self.acquire_writer_lock() + } + + fn create_writer_lock(&self) -> Result<(), DurabilityCrashMatrixError> { + let file = OpenOptions::new() + .write(true) + .create_new(true) + .open(self.root.join("writer.lock")) + .map_err(|source| DurabilityCrashMatrixError::io("create writer lock", source))?; + file.sync_all() + .map_err(|source| DurabilityCrashMatrixError::io("synchronize writer lock", source)) + } + + fn acquire_writer_lock(&mut self) -> Result<(), DurabilityCrashMatrixError> { + let lock = FilesystemWriterLock::try_acquire(&self.root) + .map_err(DurabilityCrashMatrixError::WriterLock)?; + self.writer_lock = Some(lock); + Ok(()) + } + + fn create_directory(&self, relative: &str) -> Result<(), DurabilityCrashMatrixError> { + fs::create_dir(self.root.join(relative)) + .map_err(|source| DurabilityCrashMatrixError::io("create protocol directory", source)) + } + + fn create_stage(&mut self, relative: &str) -> Result<(), DurabilityCrashMatrixError> { + let file = OpenOptions::new() + .read(true) + .write(true) + .create_new(true) + .open(self.root.join(relative)) + .map_err(|source| DurabilityCrashMatrixError::io("create fixed stage", source))?; + self.active_file = Some(file); + Ok(()) + } + + fn write_range( + &mut self, + fixture: &GoldenFixture, + range: Range, + ) -> Result<(), DurabilityCrashMatrixError> { + let bytes = fixture.range(range)?; + self.active_file()? + .write_all(bytes) + .map_err(|source| DurabilityCrashMatrixError::io("write staged artifact", source)) + } + + fn flush(&mut self) -> Result<(), DurabilityCrashMatrixError> { + self.active_file()? + .flush() + .map_err(|source| DurabilityCrashMatrixError::io("flush staged artifact", source)) + } + + fn synchronize_file(&self) -> Result<(), DurabilityCrashMatrixError> { + self.active_file_ref()? + .sync_all() + .map_err(|source| DurabilityCrashMatrixError::io("synchronize staged artifact", source)) + } + + fn link(&self, source: &str, target: &str) -> Result<(), DurabilityCrashMatrixError> { + fs::hard_link(self.root.join(source), self.root.join(target)) + .map_err(|source| DurabilityCrashMatrixError::io("link immutable artifact", source)) + } + + fn synchronize_directory(&self, relative: &str) -> Result<(), DurabilityCrashMatrixError> { + File::open(self.root.join(relative)) + .and_then(|directory| directory.sync_all()) + .map_err(|source| { + DurabilityCrashMatrixError::io("synchronize protocol directory", source) + }) + } + + fn remove(&self, relative: &str) -> Result<(), DurabilityCrashMatrixError> { + fs::remove_file(self.root.join(relative)) + .map_err(|source| DurabilityCrashMatrixError::io("remove protocol stage", source)) + } + + fn rename(&self, source: &str, target: &str) -> Result<(), DurabilityCrashMatrixError> { + fs::rename(self.root.join(source), self.root.join(target)) + .map_err(|source| DurabilityCrashMatrixError::io("replace publication head", source)) + } + + fn write_immutable( + &self, + relative: &str, + fixture: &GoldenFixture, + ) -> Result<(), DurabilityCrashMatrixError> { + let path = self.root.join(relative); + let mut file = OpenOptions::new() + .write(true) + .create_new(true) + .open(path) + .map_err(|source| { + DurabilityCrashMatrixError::io("create immutable artifact", source) + })?; + file.write_all(fixture.bytes()) + .and_then(|()| file.sync_all()) + .map_err(|source| DurabilityCrashMatrixError::io("write immutable artifact", source)) + } + + fn active_file(&mut self) -> Result<&mut File, DurabilityCrashMatrixError> { + self.active_file + .as_mut() + .ok_or(DurabilityCrashMatrixError::MissingActiveFile) + } + + fn active_file_ref(&self) -> Result<&File, DurabilityCrashMatrixError> { + self.active_file + .as_ref() + .ok_or(DurabilityCrashMatrixError::MissingActiveFile) + } + + fn finish( + self, + result: Result<(), DurabilityCrashMatrixError>, + ) -> Result<(), DurabilityCrashMatrixError> { + drop(self.active_file); + drop(self.writer_lock); + drop(self.root); + result + } +} diff --git a/xtask/src/durability_crash_matrix/state/catalog.rs b/xtask/src/durability_crash_matrix/state/catalog.rs new file mode 100644 index 0000000..b296670 --- /dev/null +++ b/xtask/src/durability_crash_matrix/state/catalog.rs @@ -0,0 +1,84 @@ +//! This module owns Golden File Worldline catalog crash-state construction. + +use xtask::{DurabilityCrashCase, DurabilityCrashPoint, DurabilityCrashPosition}; + +use super::fixture::{CATALOG_POOL_PATH, GoldenFixture, SEGMENT_POOL_PATH}; +use super::{DurabilityCrashMatrixError, StoreState}; + +const STEPS: [DurabilityCrashPoint; 8] = [ + DurabilityCrashPoint::CreateCatalogStage, + DurabilityCrashPoint::WriteCatalog, + DurabilityCrashPoint::FlushCatalog, + DurabilityCrashPoint::SynchronizeCatalog, + DurabilityCrashPoint::LinkCatalog, + DurabilityCrashPoint::SynchronizeCatalogPool, + DurabilityCrashPoint::RemoveCatalogStage, + DurabilityCrashPoint::SynchronizeStagingAfterCatalog, +]; + +pub(super) fn prepare( + state: &mut StoreState, + case: DurabilityCrashCase, +) -> Result<(), DurabilityCrashMatrixError> { + state.initialize()?; + state.write_immutable(SEGMENT_POOL_PATH, &GoldenFixture::segment()?)?; + let fixture = GoldenFixture::catalog()?; + for step in STEPS { + let position = if step == case.point() { + case.position() + } else { + DurabilityCrashPosition::After + }; + apply(state, &fixture, step, position)?; + if step == case.point() { + return Ok(()); + } + } + Err(DurabilityCrashMatrixError::PointSequenceMismatch { + point: case.point(), + }) +} + +fn apply( + state: &mut StoreState, + fixture: &GoldenFixture, + point: DurabilityCrashPoint, + position: DurabilityCrashPosition, +) -> Result<(), DurabilityCrashMatrixError> { + if position == DurabilityCrashPosition::Before { + return Ok(()); + } + match point { + DurabilityCrashPoint::CreateCatalogStage => state.create_stage("staging/current.cat"), + DurabilityCrashPoint::WriteCatalog => { + let end = if position == DurabilityCrashPosition::During { + 176 + } else { + 352 + }; + state.write_range(fixture, 0..end) + } + DurabilityCrashPoint::FlushCatalog => after(position, || state.flush()), + DurabilityCrashPoint::SynchronizeCatalog => after(position, || state.synchronize_file()), + DurabilityCrashPoint::LinkCatalog => state.link("staging/current.cat", CATALOG_POOL_PATH), + DurabilityCrashPoint::SynchronizeCatalogPool => { + after(position, || state.synchronize_directory("catalogs")) + } + DurabilityCrashPoint::RemoveCatalogStage => state.remove("staging/current.cat"), + DurabilityCrashPoint::SynchronizeStagingAfterCatalog => { + after(position, || state.synchronize_directory("staging")) + } + _ => Err(DurabilityCrashMatrixError::PointSequenceMismatch { point }), + } +} + +fn after( + position: DurabilityCrashPosition, + operation: impl FnOnce() -> Result<(), DurabilityCrashMatrixError>, +) -> Result<(), DurabilityCrashMatrixError> { + if position == DurabilityCrashPosition::After { + operation() + } else { + Ok(()) + } +} diff --git a/xtask/src/durability_crash_matrix/state/fixture.rs b/xtask/src/durability_crash_matrix/state/fixture.rs new file mode 100644 index 0000000..2ab5324 --- /dev/null +++ b/xtask/src/durability_crash_matrix/state/fixture.rs @@ -0,0 +1,77 @@ +//! This module owns admitted Golden File Worldline crash fixtures. + +use std::ops::Range; + +use xtask::protocol_admission::{EmptyHex, decode_lower_hex}; + +use super::super::DurabilityCrashMatrixError; + +const SEGMENT_HEX: &str = + include_str!("../../../../conformance/segment-store/v1/one-zero-segment.hex"); +const CATALOG_HEX: &str = + include_str!("../../../../conformance/segment-store/v1/one-zero-catalog.hex"); +const HEAD_HEX: &str = include_str!("../../../../conformance/segment-store/v1/one-zero-head.hex"); + +pub(in crate::durability_crash_matrix) const SEGMENT_POOL_PATH: &str = + "segments/b7542dced2ab770894a14d1d04b066e3a899942602c5986d35ba6df6c1a35cfc.seg"; +pub(in crate::durability_crash_matrix) const CATALOG_POOL_PATH: &str = "catalogs/0000000000000001-04b82519b0399baefd0b9c0f32a871052e4c47e3a00226ab03b21661470f7320.cat"; + +pub(in crate::durability_crash_matrix) struct GoldenFixture { + bytes: Vec, +} + +impl GoldenFixture { + pub(in crate::durability_crash_matrix) fn segment() -> Result + { + Self::decode("segment", SEGMENT_HEX, 337) + } + + pub(in crate::durability_crash_matrix) fn catalog() -> Result + { + Self::decode("catalog", CATALOG_HEX, 352) + } + + pub(in crate::durability_crash_matrix) fn head() -> Result { + Self::decode("head", HEAD_HEX, 128) + } + + pub(in crate::durability_crash_matrix) fn bytes(&self) -> &[u8] { + &self.bytes + } + + pub(in crate::durability_crash_matrix) fn range( + &self, + range: Range, + ) -> Result<&[u8], DurabilityCrashMatrixError> { + self.bytes + .get(range) + .ok_or(DurabilityCrashMatrixError::FixtureRange) + } + + pub(in crate::durability_crash_matrix) fn prefix( + &self, + end: usize, + ) -> Result<&[u8], DurabilityCrashMatrixError> { + self.range(0..end) + } + + fn decode( + artifact: &'static str, + encoded: &str, + length: usize, + ) -> Result { + let hex = encoded + .strip_suffix('\n') + .ok_or(DurabilityCrashMatrixError::FixtureTerminator { artifact })?; + let bytes = decode_lower_hex(hex, length, EmptyHex::Refuse) + .map_err(|source| DurabilityCrashMatrixError::Fixture { artifact, source })?; + if bytes.len() != length { + return Err(DurabilityCrashMatrixError::FixtureLength { + artifact, + expected: length, + observed: bytes.len(), + }); + } + Ok(Self { bytes }) + } +} diff --git a/xtask/src/durability_crash_matrix/state/head.rs b/xtask/src/durability_crash_matrix/state/head.rs new file mode 100644 index 0000000..e666b2e --- /dev/null +++ b/xtask/src/durability_crash_matrix/state/head.rs @@ -0,0 +1,79 @@ +//! This module owns Golden File Worldline publication-head crash states. + +use xtask::{DurabilityCrashCase, DurabilityCrashPoint, DurabilityCrashPosition}; + +use super::fixture::{CATALOG_POOL_PATH, GoldenFixture, SEGMENT_POOL_PATH}; +use super::{DurabilityCrashMatrixError, StoreState}; + +const STEPS: [DurabilityCrashPoint; 6] = [ + DurabilityCrashPoint::CreateHeadStage, + DurabilityCrashPoint::WriteHead, + DurabilityCrashPoint::FlushHead, + DurabilityCrashPoint::SynchronizeHead, + DurabilityCrashPoint::ReplaceHead, + DurabilityCrashPoint::SynchronizeRootAfterHead, +]; + +pub(super) fn prepare( + state: &mut StoreState, + case: DurabilityCrashCase, +) -> Result<(), DurabilityCrashMatrixError> { + state.initialize()?; + state.write_immutable(SEGMENT_POOL_PATH, &GoldenFixture::segment()?)?; + state.write_immutable(CATALOG_POOL_PATH, &GoldenFixture::catalog()?)?; + let fixture = GoldenFixture::head()?; + for step in STEPS { + let position = if step == case.point() { + case.position() + } else { + DurabilityCrashPosition::After + }; + apply(state, &fixture, step, position)?; + if step == case.point() { + return Ok(()); + } + } + Err(DurabilityCrashMatrixError::PointSequenceMismatch { + point: case.point(), + }) +} + +fn apply( + state: &mut StoreState, + fixture: &GoldenFixture, + point: DurabilityCrashPoint, + position: DurabilityCrashPosition, +) -> Result<(), DurabilityCrashMatrixError> { + if position == DurabilityCrashPosition::Before { + return Ok(()); + } + match point { + DurabilityCrashPoint::CreateHeadStage => state.create_stage("head.next"), + DurabilityCrashPoint::WriteHead => { + let end = if position == DurabilityCrashPosition::During { + 64 + } else { + 128 + }; + state.write_range(fixture, 0..end) + } + DurabilityCrashPoint::FlushHead => after(position, || state.flush()), + DurabilityCrashPoint::SynchronizeHead => after(position, || state.synchronize_file()), + DurabilityCrashPoint::ReplaceHead => state.rename("head.next", "HEAD"), + DurabilityCrashPoint::SynchronizeRootAfterHead => { + after(position, || state.synchronize_directory(".")) + } + _ => Err(DurabilityCrashMatrixError::PointSequenceMismatch { point }), + } +} + +fn after( + position: DurabilityCrashPosition, + operation: impl FnOnce() -> Result<(), DurabilityCrashMatrixError>, +) -> Result<(), DurabilityCrashMatrixError> { + if position == DurabilityCrashPosition::After { + operation() + } else { + Ok(()) + } +} diff --git a/xtask/src/durability_crash_matrix/state/initialization.rs b/xtask/src/durability_crash_matrix/state/initialization.rs new file mode 100644 index 0000000..1d59e39 --- /dev/null +++ b/xtask/src/durability_crash_matrix/state/initialization.rs @@ -0,0 +1,60 @@ +//! This module owns crash-safe initialization state construction. + +use xtask::{DurabilityCrashCase, DurabilityCrashPoint, DurabilityCrashPosition}; + +use super::{DurabilityCrashMatrixError, StoreState}; + +const STEPS: [DurabilityCrashPoint; 5] = [ + DurabilityCrashPoint::OpenAndLockWriterFile, + DurabilityCrashPoint::CreateStagingDirectory, + DurabilityCrashPoint::CreateSegmentPoolDirectory, + DurabilityCrashPoint::CreateCatalogPoolDirectory, + DurabilityCrashPoint::SynchronizeRootAfterInitialization, +]; + +pub(super) fn prepare( + state: &mut StoreState, + case: DurabilityCrashCase, +) -> Result<(), DurabilityCrashMatrixError> { + for step in STEPS { + let position = if step == case.point() { + case.position() + } else { + DurabilityCrashPosition::After + }; + apply(state, step, position)?; + if step == case.point() { + return Ok(()); + } + } + Err(DurabilityCrashMatrixError::PointSequenceMismatch { + point: case.point(), + }) +} + +fn apply( + state: &mut StoreState, + point: DurabilityCrashPoint, + position: DurabilityCrashPosition, +) -> Result<(), DurabilityCrashMatrixError> { + if position == DurabilityCrashPosition::Before { + return Ok(()); + } + match point { + DurabilityCrashPoint::OpenAndLockWriterFile => { + state.create_writer_lock()?; + state.acquire_writer_lock() + } + DurabilityCrashPoint::CreateStagingDirectory => state.create_directory("staging"), + DurabilityCrashPoint::CreateSegmentPoolDirectory => state.create_directory("segments"), + DurabilityCrashPoint::CreateCatalogPoolDirectory => state.create_directory("catalogs"), + DurabilityCrashPoint::SynchronizeRootAfterInitialization => { + if position == DurabilityCrashPosition::After { + state.synchronize_directory(".") + } else { + Ok(()) + } + } + _ => Err(DurabilityCrashMatrixError::PointSequenceMismatch { point }), + } +} diff --git a/xtask/src/durability_crash_matrix/state/recovery.rs b/xtask/src/durability_crash_matrix/state/recovery.rs new file mode 100644 index 0000000..d943efc --- /dev/null +++ b/xtask/src/durability_crash_matrix/state/recovery.rs @@ -0,0 +1,60 @@ +//! This module owns explicit-discard crash-state construction. + +use xtask::{DurabilityCrashCase, DurabilityCrashPoint, DurabilityCrashPosition}; + +use super::fixture::GoldenFixture; +use super::{DurabilityCrashMatrixError, StoreState}; + +pub(super) fn prepare( + state: &mut StoreState, + case: DurabilityCrashCase, +) -> Result<(), DurabilityCrashMatrixError> { + state.initialize()?; + match case.point() { + DurabilityCrashPoint::RemoveRecoveryStage + | DurabilityCrashPoint::SynchronizeStagingAfterRecovery => { + prepare_segment_discard(state, case) + } + DurabilityCrashPoint::RemoveRecoveryHead + | DurabilityCrashPoint::SynchronizeRootAfterRecovery => prepare_head_discard(state, case), + point => Err(DurabilityCrashMatrixError::PointSequenceMismatch { point }), + } +} + +fn prepare_segment_discard( + state: &mut StoreState, + case: DurabilityCrashCase, +) -> Result<(), DurabilityCrashMatrixError> { + state.create_stage("staging/current.seg")?; + state.write_range(&GoldenFixture::segment()?, 0..32)?; + if case.point() == DurabilityCrashPoint::RemoveRecoveryStage { + if case.position() != DurabilityCrashPosition::Before { + state.remove("staging/current.seg")?; + } + return Ok(()); + } + state.remove("staging/current.seg")?; + if case.position() == DurabilityCrashPosition::After { + state.synchronize_directory("staging")?; + } + Ok(()) +} + +fn prepare_head_discard( + state: &mut StoreState, + case: DurabilityCrashCase, +) -> Result<(), DurabilityCrashMatrixError> { + state.create_stage("head.next")?; + state.write_range(&GoldenFixture::head()?, 0..64)?; + if case.point() == DurabilityCrashPoint::RemoveRecoveryHead { + if case.position() != DurabilityCrashPosition::Before { + state.remove("head.next")?; + } + return Ok(()); + } + state.remove("head.next")?; + if case.position() == DurabilityCrashPosition::After { + state.synchronize_directory(".")?; + } + Ok(()) +} diff --git a/xtask/src/durability_crash_matrix/state/segment.rs b/xtask/src/durability_crash_matrix/state/segment.rs new file mode 100644 index 0000000..75f968b --- /dev/null +++ b/xtask/src/durability_crash_matrix/state/segment.rs @@ -0,0 +1,103 @@ +//! This module owns Golden File Worldline segment crash-state construction. + +use xtask::{DurabilityCrashCase, DurabilityCrashPoint, DurabilityCrashPosition}; + +use super::fixture::{GoldenFixture, SEGMENT_POOL_PATH}; +use super::{DurabilityCrashMatrixError, StoreState}; + +const STEPS: [DurabilityCrashPoint; 12] = [ + DurabilityCrashPoint::CreateSegmentStage, + DurabilityCrashPoint::WriteSegmentHeader, + DurabilityCrashPoint::AppendSegmentRecord, + DurabilityCrashPoint::FlushSegmentRecordPrefix, + DurabilityCrashPoint::SynchronizeSegmentRecordPrefix, + DurabilityCrashPoint::AppendSegmentSeal, + DurabilityCrashPoint::FlushSealedSegment, + DurabilityCrashPoint::SynchronizeSealedSegment, + DurabilityCrashPoint::LinkSegment, + DurabilityCrashPoint::SynchronizeSegmentPool, + DurabilityCrashPoint::RemoveSegmentStage, + DurabilityCrashPoint::SynchronizeStagingAfterSegment, +]; + +pub(super) fn prepare( + state: &mut StoreState, + case: DurabilityCrashCase, +) -> Result<(), DurabilityCrashMatrixError> { + state.initialize()?; + let fixture = GoldenFixture::segment()?; + for step in STEPS { + let position = if step == case.point() { + case.position() + } else { + DurabilityCrashPosition::After + }; + apply(state, &fixture, step, position)?; + if step == case.point() { + return Ok(()); + } + } + Err(DurabilityCrashMatrixError::PointSequenceMismatch { + point: case.point(), + }) +} + +fn apply( + state: &mut StoreState, + fixture: &GoldenFixture, + point: DurabilityCrashPoint, + position: DurabilityCrashPosition, +) -> Result<(), DurabilityCrashMatrixError> { + if position == DurabilityCrashPosition::Before { + return Ok(()); + } + match point { + DurabilityCrashPoint::CreateSegmentStage => state.create_stage("staging/current.seg"), + DurabilityCrashPoint::WriteSegmentHeader => { + let end = interrupted_end(position, 32, 64); + state.write_range(fixture, 0..end) + } + DurabilityCrashPoint::AppendSegmentRecord => { + let end = interrupted_end(position, 136, 209); + state.write_range(fixture, 64..end) + } + DurabilityCrashPoint::FlushSegmentRecordPrefix + | DurabilityCrashPoint::FlushSealedSegment => after(position, || state.flush()), + DurabilityCrashPoint::SynchronizeSegmentRecordPrefix + | DurabilityCrashPoint::SynchronizeSealedSegment => { + after(position, || state.synchronize_file()) + } + DurabilityCrashPoint::AppendSegmentSeal => { + let end = interrupted_end(position, 273, 337); + state.write_range(fixture, 209..end) + } + DurabilityCrashPoint::LinkSegment => state.link("staging/current.seg", SEGMENT_POOL_PATH), + DurabilityCrashPoint::SynchronizeSegmentPool => { + after(position, || state.synchronize_directory("segments")) + } + DurabilityCrashPoint::RemoveSegmentStage => state.remove("staging/current.seg"), + DurabilityCrashPoint::SynchronizeStagingAfterSegment => { + after(position, || state.synchronize_directory("staging")) + } + _ => Err(DurabilityCrashMatrixError::PointSequenceMismatch { point }), + } +} + +fn interrupted_end(position: DurabilityCrashPosition, during: usize, after: usize) -> usize { + if position == DurabilityCrashPosition::During { + during + } else { + after + } +} + +fn after( + position: DurabilityCrashPosition, + operation: impl FnOnce() -> Result<(), DurabilityCrashMatrixError>, +) -> Result<(), DurabilityCrashMatrixError> { + if position == DurabilityCrashPosition::After { + operation() + } else { + Ok(()) + } +} From fbb26678ec5706a8172cbb8828306b2f1e8e6e39 Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 13:56:25 -0700 Subject: [PATCH 27/49] Add: Run crash matrix in CI profiles --- .github/workflows/ci.yml | 6 ++++++ .../protocol_conformance/workflow_tests.rs | 19 +++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d4602df..3135ee7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,6 +33,12 @@ jobs: - name: Check Golden File Worldline corpus run: cargo xtask golden-file-worldline-check + - name: Run process-death crash matrix (debug) + run: cargo xtask durability-crash-matrix + + - name: Run process-death crash matrix (optimized) + run: cargo run --quiet --release --locked --package xtask -- durability-crash-matrix + - name: Check protocol conformance corpora run: cargo xtask conformance-check diff --git a/xtask/src/protocol_conformance/workflow_tests.rs b/xtask/src/protocol_conformance/workflow_tests.rs index dfa7553..eac7030 100644 --- a/xtask/src/protocol_conformance/workflow_tests.rs +++ b/xtask/src/protocol_conformance/workflow_tests.rs @@ -9,6 +9,9 @@ const CHUNK_GUIDE: &str = include_str!("../../../conformance/chunk-id/v1/README. const CI_WORKFLOW: &str = include_str!("../../../.github/workflows/ci.yml"); const COMMAND: &str = "cargo xtask conformance-check"; const CI_RUN_STEP: &str = "run: cargo xtask conformance-check"; +const CRASH_MATRIX_DEBUG_STEP: &str = "run: cargo xtask durability-crash-matrix"; +const CRASH_MATRIX_RELEASE_STEP: &str = + "run: cargo run --quiet --release --locked --package xtask -- durability-crash-matrix"; #[test] fn ci_and_living_guides_route_both_corpora_through_rust() { @@ -36,6 +39,22 @@ fn a_commented_command_is_not_ci_execution() { )); } +#[test] +fn ci_executes_the_complete_crash_matrix_in_debug_and_optimized_profiles() { + assert_eq!(exact_run_step_count(CI_WORKFLOW, CRASH_MATRIX_DEBUG_STEP), 1); + assert_eq!( + exact_run_step_count(CI_WORKFLOW, CRASH_MATRIX_RELEASE_STEP), + 1 + ); +} + +fn exact_run_step_count(workflow: &str, command: &str) -> usize { + workflow + .lines() + .filter(|line| line.trim() == command) + .count() +} + #[test] fn superseded_conformance_python_programs_are_absent() { let root = Path::new(env!("CARGO_MANIFEST_DIR")) From c0984e5e2b3c87991f55f32f29d52c462cbbd883 Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 14:00:10 -0700 Subject: [PATCH 28/49] Fix: Format crash matrix workflow contract --- xtask/src/protocol_conformance/workflow_tests.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/xtask/src/protocol_conformance/workflow_tests.rs b/xtask/src/protocol_conformance/workflow_tests.rs index eac7030..012781f 100644 --- a/xtask/src/protocol_conformance/workflow_tests.rs +++ b/xtask/src/protocol_conformance/workflow_tests.rs @@ -41,7 +41,10 @@ fn a_commented_command_is_not_ci_execution() { #[test] fn ci_executes_the_complete_crash_matrix_in_debug_and_optimized_profiles() { - assert_eq!(exact_run_step_count(CI_WORKFLOW, CRASH_MATRIX_DEBUG_STEP), 1); + assert_eq!( + exact_run_step_count(CI_WORKFLOW, CRASH_MATRIX_DEBUG_STEP), + 1 + ); assert_eq!( exact_run_step_count(CI_WORKFLOW, CRASH_MATRIX_RELEASE_STEP), 1 From a90b2f05e4748d1556e6072c99c7361549539c8d Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 14:00:15 -0700 Subject: [PATCH 29/49] Docs: Record the process-death crash matrix --- CHANGELOG.md | 6 +++ README.md | 19 +++++++- conformance/segment-store/v1/README.md | 9 ++-- docs/formats/segment-store-v1/recovery.md | 43 +++++++++++++++++++ docs/formats/segment-store-v1/requirements.md | 6 ++- xtask/tests/durability_crash_documentation.rs | 25 +++++++++++ 6 files changed, 101 insertions(+), 7 deletions(-) create mode 100644 xtask/tests/durability_crash_documentation.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 3104890..897493e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,12 @@ after its public API and format compatibility policies are established. ### Changed +- Repository crash-matrix execution now terminates isolated writer process + groups at all 105 canonical before/during/after coordinates, retains open + writer and stage authority until termination, and verifies exact Golden File + Worldline namespaces, bytes, hard links, released locks, recovery + classifications, immutable artifacts, and published visible state after + restart. CI runs the complete matrix in debug and optimized profiles. - Production filesystem initialization now admits only the documented writable, non-casefolded Linux ext4 profile, refuses ambiguous root namespaces before mutation, completes the canonical directory shape idempotently, retains diff --git a/README.md b/README.md index 658366e..99944b9 100644 --- a/README.md +++ b/README.md @@ -99,10 +99,25 @@ authority, reconstructs the complete current and candidate views without following links, verifies namespace and stage identity, synchronizes and reverifies the exact candidate, atomically replaces `HEAD`, and synchronizes the root. An already-finalized retry requires `head.next` to be absent. -Process-death injection, retention, compaction, and garbage collection remain -planned. Presence in the reference CAS does not claim retention, crash +The repository-owned process-death matrix executes all 105 +`KEEP-CRASH-001`–`KEEP-CRASH-035` before/during/after coordinates in isolated +process groups. Restart verification compares the exact Golden File Worldline +namespace and bytes, checks hard-link identity and writer-lock release, runs +the production recovery classifiers and immutable-artifact admission, and +reconstructs the exact published generation and visible one-zero chunk when +`HEAD` exists. This matrix proves application process-death behavior; it does +not simulate host power loss. Retention, compaction, and garbage collection +remain planned. Presence in the reference CAS does not claim retention, crash recovery, or durability. +Run the complete debug-profile matrix: + +```bash +cargo xtask durability-crash-matrix +``` + +CI also runs the command through an optimized `xtask` build. + ```rust use keep::BlobId; diff --git a/conformance/segment-store/v1/README.md b/conformance/segment-store/v1/README.md index 558a54e..f507da2 100644 --- a/conformance/segment-store/v1/README.md +++ b/conformance/segment-store/v1/README.md @@ -105,6 +105,9 @@ complete validation. A `truncated-tail-or-reusable-stage` result, for example, means the observed prefix decides between those two typed outcomes; it does not authorize truncation. -The table is a planned oracle for issue #17. Until a production recovery -adapter and crash harness exist, its evidence status is specification, not -implementation. +The issue #17 harness executes 105 canonical process-death cases from this +table. It terminates an isolated writer process group, compares +the exact restarted namespace and bytes against an independent expected-state +model, and exercises the production recovery classifiers and Golden File +Worldline restart loader. The harness proves application process-death +behavior; host power loss remains outside its claim. diff --git a/docs/formats/segment-store-v1/recovery.md b/docs/formats/segment-store-v1/recovery.md index 7247175..6d17b58 100644 --- a/docs/formats/segment-store-v1/recovery.md +++ b/docs/formats/segment-store-v1/recovery.md @@ -96,6 +96,49 @@ same crash points and must be idempotent. It may not silently promote the newest artifact, truncate to the last plausible boundary, rewrite a checksum, delete a valid orphan, or select by timestamp. +## Process-death crash matrix + +Run the repository-owned matrix: + +```bash +cargo xtask durability-crash-matrix +``` + +The command executes the three ordered positions for each stable +`KEEP-CRASH-001`–`KEEP-CRASH-035` point: 105 canonical cases. Each case owns a +fresh filesystem store and an isolated child process group. The child retains +the writer lock and any open staged artifact, prepares the point-selected +state, sends one readiness byte over a Unix socket, and waits on the retained +connection. The parent keeps the accepted socket open, sends `SIGKILL` to the +complete process group, reaps the child, and only then begins restart +inspection. A ten-second deadline bounds failure handling; successful +synchronization does not depend on elapsed time, sleeps, or test ordering. + +The state constructor and expected-state model are separate implementations. +Restart inspection proves the complete path set and exact Golden File +Worldline bytes. It additionally verifies hard-link identity at link +transitions, reacquires `writer.lock` after process death, classifies +`current.seg`, `current.cat`, and `head.next` through the production recovery +classifiers, admits immutable segment and catalog bytes through the production +decoders, and loads the exact generation-1 snapshot when `HEAD` is present. +That snapshot must expose the one-zero chunk with the exact payload `00`. + +For byte writes, a `during` case retains a deterministic proper prefix and its +open file handle at termination. Atomic create, hard-link, unlink, and rename +operations admit their completed namespace state because no torn namespace +operation is representable through the documented filesystem contract. +Flush and synchronization cases preserve the exact application-visible bytes +on both sides of the durability call. The matrix proves process-death +recovery; it does not simulate host power loss, torn media writes, or a +filesystem that violates the admitted atomicity contract. + +CI runs the complete command once through the debug `xtask` binary and once +through an optimized `xtask` binary. To isolate one coordinate locally, run: + +```bash +cargo xtask durability-crash-matrix --case KEEP-CRASH-006 during +``` + ## Resume a reusable segment The public semantic boundary admits only diff --git a/docs/formats/segment-store-v1/requirements.md b/docs/formats/segment-store-v1/requirements.md index e521ff6..eea28f2 100644 --- a/docs/formats/segment-store-v1/requirements.md +++ b/docs/formats/segment-store-v1/requirements.md @@ -101,8 +101,9 @@ the filesystem completer now binds that transition to pinned writer-authorized storage. A complete next-head assessment and exact transitive catalog snapshot may now authorize a transition-checked finalization through a semantic storage port, and the filesystem finalizer binds that transition to pinned -writer-authorized storage. These slices do not yet claim reusable-stage -continuation or process-death injection. +writer-authorized storage. These slices now include reusable-stage continuation +and the complete process-death crash matrix. Retention, compaction, garbage +collection, and host-power-loss simulation remain outside issue #17. @@ -128,6 +129,7 @@ continuation or process-death injection. | `KEEP-RECOVERY-018` | Filesystem next-head finalization retains root and `writer.lock` authority, pins every protocol directory, revalidates namespace identity and exact stage evidence around bounded complete current and candidate loads, synchronizes and reverifies the exact candidate before atomic replacement, refuses current drift, missing or reappeared candidates, links, corrupt transitive views, and namespace replacement, and returns only after root synchronization | Initial and successor finalization, exact retry, candidate-sync, evidence-drift, missing, link, corruption, namespace-replacement, current-drift, and writer-exclusion matrix | `src/adapters/filesystem_recovery_next_head_finalization_tests.rs`, `src/adapters/filesystem_recovery_next_head_finalization_tests/*.rs` | Implemented in #17 | | `KEEP-RECOVERY-019` | Reusable-segment continuation plans only from an exact reusable `current.seg` assessment within the selected record policy, consumes the storage authority that reopens the stage, re-admits the complete materialized prefix against prior evidence, rebuilds digest and duplicate-identity state, and returns the ordinary append-only stage without rewriting admitted bytes | Reusable-only planning, policy refusal, changed-evidence, storage-failure, duplicate-identity, append, seal, and independent decode matrix | `tests/recovery_segment_resume.rs`, `tests/recovery_segment_resume/*.rs` | Implemented in #17 | | `KEEP-RECOVERY-020` | Filesystem reusable-segment continuation retains root and `writer.lock` authority in the returned stage, pins every protocol directory, opens `current.seg` read-write without following links or truncation, bounds and re-admits its complete bytes, positions the handle at the exact validated append boundary, refuses missing, changed, linked, replaced, or namespace-drifted evidence, and preserves the prefix on append or empty seal | Empty and nonempty continuation, writer exclusion, missing, changed evidence, link, namespace replacement, writable-handoff replacement, append, seal, and independent decode matrix | `src/adapters/filesystem_recovery_segment_resume_tests.rs`, `src/adapters/filesystem_recovery_segment_resume_tests/*.rs` | Implemented in #17 | +| `KEEP-RECOVERY-021` | Every crash point has exact before, during, and after coordinates; a deadline-bounded parent receives readiness over a Unix socket, retains that socket, terminates the isolated child process group, and independently verifies the exact Golden File Worldline namespace, bytes, hard-link identity, released writer lock, recovery-stage class, immutable-artifact admission, generation, and visible chunk after restart | Ordered 105-case model, independent expected-state model, production recovery classifiers, production restart loader, and explicit debug/optimized CI commands | `xtask/tests/durability_crash_case_contract.rs`, `xtask/tests/durability_crash_process_contract.rs`, `xtask/src/durability_crash_matrix/`, `.github/workflows/ci.yml` | Implemented in #17 | diff --git a/xtask/tests/durability_crash_documentation.rs b/xtask/tests/durability_crash_documentation.rs new file mode 100644 index 0000000..92c6916 --- /dev/null +++ b/xtask/tests/durability_crash_documentation.rs @@ -0,0 +1,25 @@ +//! Documentation truth laws for the process-death crash matrix. + +const ROOT_README: &str = include_str!("../../README.md"); +const RECOVERY: &str = include_str!("../../docs/formats/segment-store-v1/recovery.md"); +const REQUIREMENTS: &str = include_str!("../../docs/formats/segment-store-v1/requirements.md"); +const CORPUS_README: &str = include_str!("../../conformance/segment-store/v1/README.md"); + +#[test] +fn living_documentation_routes_the_complete_crash_matrix_and_its_limits() { + for (document, claim) in [ + (ROOT_README, "cargo xtask durability-crash-matrix"), + (RECOVERY, "## Process-death crash matrix"), + (REQUIREMENTS, "`KEEP-RECOVERY-021`"), + (CORPUS_README, "105 canonical process-death cases"), + ] { + assert!( + document.contains(claim), + "missing crash-matrix documentation claim: {claim}" + ); + } + assert!(!ROOT_README.contains( + "Process-death injection, retention, compaction, and garbage collection remain planned." + )); + assert!(RECOVERY.contains("does not simulate host power loss")); +} From 55b87c9d104f7a95311e0543c64bda345fda4043 Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 14:17:39 -0700 Subject: [PATCH 30/49] Fix: Report precise crash-state mismatches --- xtask/src/durability_crash_matrix/error.rs | 167 +++++------- .../durability_crash_matrix/error/display.rs | 243 ++++++++++++++++++ xtask/src/durability_crash_matrix/process.rs | 6 +- xtask/src/durability_crash_matrix/restart.rs | 30 ++- .../restart/expectation.rs | 9 + .../restart/semantic.rs | 60 ++++- 6 files changed, 398 insertions(+), 117 deletions(-) create mode 100644 xtask/src/durability_crash_matrix/error/display.rs diff --git a/xtask/src/durability_crash_matrix/error.rs b/xtask/src/durability_crash_matrix/error.rs index 9a22e7f..60f40e8 100644 --- a/xtask/src/durability_crash_matrix/error.rs +++ b/xtask/src/durability_crash_matrix/error.rs @@ -1,7 +1,9 @@ //! This module owns deterministic crash-matrix execution failures. +mod display; + +use std::collections::BTreeSet; use std::error::Error; -use std::fmt; use std::io; use std::time::Duration; @@ -17,6 +19,18 @@ pub(crate) enum DurabilityCrashMatrixError { position: DurabilityCrashPosition, source: Box, }, + ArtifactBytesMismatch { + artifact: &'static str, + expected_length: usize, + observed_length: usize, + offset: usize, + expected: Option, + observed: Option, + }, + ArtifactClassificationMismatch { + expected: &'static str, + observed: &'static str, + }, ChildExitedEarly { code: Option, }, @@ -42,19 +56,45 @@ pub(crate) enum DurabilityCrashMatrixError { InvalidReadinessSignal { observed: u8, }, + InventoryMismatch { + expected: BTreeSet, + observed: BTreeSet, + }, Io { action: &'static str, source: io::Error, }, + HardLinkIdentityMismatch { + source: &'static str, + target: &'static str, + source_device: u64, + source_inode: u64, + target_device: u64, + target_inode: u64, + }, MissingActiveFile, + MissingVisibleRecord { + record: &'static str, + }, NonUnicodeStatePath, PointSequenceMismatch { point: DurabilityCrashPoint, }, - StateMismatch, + RepeatedInventoryPath { + path: String, + }, + SnapshotGenerationMismatch { + expected: u64, + observed: u64, + }, Timeout { duration: Duration, }, + UnexpectedArtifactKind { + artifact: &'static str, + expected: &'static str, + observed: &'static str, + }, UnknownPoint(String), UnknownPosition(String), Usage, @@ -66,6 +106,22 @@ pub(crate) enum DurabilityCrashMatrixError { } impl DurabilityCrashMatrixError { + pub(crate) fn artifact_bytes(artifact: &'static str, expected: &[u8], observed: &[u8]) -> Self { + let offset = expected + .iter() + .zip(observed) + .position(|(expected, observed)| expected != observed) + .unwrap_or_else(|| expected.len().min(observed.len())); + Self::ArtifactBytesMismatch { + artifact, + expected_length: expected.len(), + observed_length: observed.len(), + offset, + expected: expected.get(offset).copied(), + observed: observed.get(offset).copied(), + } + } + pub(crate) const fn io(action: &'static str, source: io::Error) -> Self { Self::Io { action, source } } @@ -79,102 +135,6 @@ impl DurabilityCrashMatrixError { } } -impl fmt::Debug for DurabilityCrashMatrixError { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt::Display::fmt(self, formatter) - } -} - -impl fmt::Display for DurabilityCrashMatrixError { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::Case { - point, - position, - source, - } => write!( - formatter, - "{} {}: {source}", - point.identifier(), - position.identifier() - ), - Self::ChildExitedEarly { code } => { - write!( - formatter, - "crash child exited before readiness with code {code:?}" - ) - } - Self::ChildSurvivedTermination { code } => { - write!( - formatter, - "crash child survived termination with code {code:?}" - ) - } - Self::Fixture { artifact, .. } => { - write!(formatter, "cannot decode {artifact} crash fixture") - } - Self::FixtureLength { - artifact, - expected, - observed, - } => write!( - formatter, - "{artifact} crash fixture has length {observed}, expected {expected}" - ), - Self::FixtureRange => formatter.write_str("crash fixture range is invalid"), - Self::FixtureTerminator { artifact } => { - write!( - formatter, - "{artifact} crash fixture lacks its final line feed" - ) - } - Self::InvalidCase(error) => write!(formatter, "invalid crash case: {error}"), - Self::InvalidPointEncoding => formatter.write_str("crash point is not valid Unicode"), - Self::InvalidPositionEncoding => { - formatter.write_str("crash position is not valid Unicode") - } - Self::InvalidReadinessSignal { observed } => { - write!(formatter, "crash child sent readiness byte {observed}") - } - Self::Io { action, .. } => write!(formatter, "cannot {action}"), - Self::MissingActiveFile => { - formatter.write_str("crash sequence has no active staged artifact") - } - Self::NonUnicodeStatePath => { - formatter.write_str("post-crash store path is not valid Unicode") - } - Self::PointSequenceMismatch { point } => write!( - formatter, - "{} is outside the selected crash sequence", - point.identifier() - ), - Self::StateMismatch => { - formatter.write_str("post-crash store state does not match its case") - } - Self::Timeout { duration } => { - write!(formatter, "crash child exceeded its {duration:?} deadline") - } - Self::UnknownPoint(point) => write!(formatter, "unknown crash point `{point}`"), - Self::UnknownPosition(position) => { - write!(formatter, "unknown crash position `{position}`") - } - Self::Usage => formatter.write_str( - "usage: cargo xtask durability-crash-matrix \ - --case ", - ), - Self::Verification { phase, source } => { - write!( - formatter, - "post-crash verification failed while attempting to {phase}: {source}" - ) - } - Self::WriterLock(source) => { - write!(formatter, "cannot acquire crash-case writer lock: {source}") - } - } - } -} - impl Error for DurabilityCrashMatrixError { fn source(&self) -> Option<&(dyn Error + 'static)> { match self { @@ -184,7 +144,9 @@ impl Error for DurabilityCrashMatrixError { Self::Io { source, .. } => Some(source), Self::Verification { source, .. } => Some(source.as_ref()), Self::WriterLock(source) => Some(source), - Self::ChildExitedEarly { .. } + Self::ArtifactBytesMismatch { .. } + | Self::ArtifactClassificationMismatch { .. } + | Self::ChildExitedEarly { .. } | Self::ChildSurvivedTermination { .. } | Self::FixtureLength { .. } | Self::FixtureRange @@ -192,11 +154,16 @@ impl Error for DurabilityCrashMatrixError { | Self::InvalidPointEncoding | Self::InvalidPositionEncoding | Self::InvalidReadinessSignal { .. } + | Self::InventoryMismatch { .. } + | Self::HardLinkIdentityMismatch { .. } | Self::MissingActiveFile + | Self::MissingVisibleRecord { .. } | Self::NonUnicodeStatePath | Self::PointSequenceMismatch { .. } - | Self::StateMismatch + | Self::RepeatedInventoryPath { .. } + | Self::SnapshotGenerationMismatch { .. } | Self::Timeout { .. } + | Self::UnexpectedArtifactKind { .. } | Self::UnknownPoint(_) | Self::UnknownPosition(_) | Self::Usage => None, diff --git a/xtask/src/durability_crash_matrix/error/display.rs b/xtask/src/durability_crash_matrix/error/display.rs new file mode 100644 index 0000000..34a06d0 --- /dev/null +++ b/xtask/src/durability_crash_matrix/error/display.rs @@ -0,0 +1,243 @@ +//! This module owns human-readable crash-matrix failure rendering. + +use std::fmt; + +use super::DurabilityCrashMatrixError; + +impl fmt::Debug for DurabilityCrashMatrixError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(self, formatter) + } +} + +impl fmt::Display for DurabilityCrashMatrixError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::ArtifactBytesMismatch { .. } + | Self::ArtifactClassificationMismatch { .. } + | Self::HardLinkIdentityMismatch { .. } + | Self::InventoryMismatch { .. } + | Self::MissingVisibleRecord { .. } + | Self::RepeatedInventoryPath { .. } + | Self::SnapshotGenerationMismatch { .. } + | Self::UnexpectedArtifactKind { .. } => format_state(self, formatter), + Self::ChildExitedEarly { .. } + | Self::ChildSurvivedTermination { .. } + | Self::InvalidReadinessSignal { .. } + | Self::Timeout { .. } => format_process(self, formatter), + Self::Fixture { .. } + | Self::FixtureLength { .. } + | Self::FixtureRange + | Self::FixtureTerminator { .. } => format_fixture(self, formatter), + Self::Case { .. } + | Self::InvalidCase(_) + | Self::InvalidPointEncoding + | Self::InvalidPositionEncoding + | Self::UnknownPoint(_) + | Self::UnknownPosition(_) + | Self::Usage => format_command(self, formatter), + Self::Io { .. } + | Self::MissingActiveFile + | Self::NonUnicodeStatePath + | Self::PointSequenceMismatch { .. } + | Self::Verification { .. } + | Self::WriterLock(_) => format_boundary(self, formatter), + } + } +} + +fn format_state( + error: &DurabilityCrashMatrixError, + formatter: &mut fmt::Formatter<'_>, +) -> fmt::Result { + match error { + DurabilityCrashMatrixError::ArtifactBytesMismatch { + artifact, + expected_length, + observed_length, + offset, + expected, + observed, + } => write!( + formatter, + "post-crash artifact `{artifact}` differs at byte {offset}: \ + expected {expected:?} within {expected_length} bytes, \ + observed {observed:?} within {observed_length} bytes" + ), + DurabilityCrashMatrixError::ArtifactClassificationMismatch { expected, observed } => { + write!( + formatter, + "post-crash artifact classification mismatch: \ + expected `{expected}`, observed `{observed}`" + ) + } + DurabilityCrashMatrixError::HardLinkIdentityMismatch { .. } => { + format_hard_link(error, formatter) + } + DurabilityCrashMatrixError::InventoryMismatch { expected, observed } => write!( + formatter, + "post-crash path inventory mismatch: expected {expected:?}, observed {observed:?}" + ), + DurabilityCrashMatrixError::MissingVisibleRecord { record } => { + write!( + formatter, + "post-crash snapshot lacks visible record `{record}`" + ) + } + DurabilityCrashMatrixError::RepeatedInventoryPath { path } => { + write!(formatter, "post-crash inventory repeated path `{path}`") + } + DurabilityCrashMatrixError::SnapshotGenerationMismatch { expected, observed } => write!( + formatter, + "post-crash snapshot generation is {observed}, expected {expected}" + ), + DurabilityCrashMatrixError::UnexpectedArtifactKind { + artifact, + expected, + observed, + } => write!( + formatter, + "post-crash model assigned `{artifact}` kind `{observed}`, expected `{expected}`" + ), + _ => Err(fmt::Error), + } +} + +fn format_hard_link( + error: &DurabilityCrashMatrixError, + formatter: &mut fmt::Formatter<'_>, +) -> fmt::Result { + let DurabilityCrashMatrixError::HardLinkIdentityMismatch { + source, + target, + source_device, + source_inode, + target_device, + target_inode, + } = error + else { + return Err(fmt::Error); + }; + write!( + formatter, + "post-crash hard-link identity mismatch between `{source}` \ + ({source_device}:{source_inode}) and `{target}` \ + ({target_device}:{target_inode})" + ) +} + +fn format_process( + error: &DurabilityCrashMatrixError, + formatter: &mut fmt::Formatter<'_>, +) -> fmt::Result { + match error { + DurabilityCrashMatrixError::ChildExitedEarly { code } => write!( + formatter, + "crash child exited before readiness with code {code:?}" + ), + DurabilityCrashMatrixError::ChildSurvivedTermination { code } => write!( + formatter, + "crash child survived termination with code {code:?}" + ), + DurabilityCrashMatrixError::InvalidReadinessSignal { observed } => { + write!(formatter, "crash child sent readiness byte {observed}") + } + DurabilityCrashMatrixError::Timeout { duration } => { + write!(formatter, "crash child exceeded its {duration:?} deadline") + } + _ => Err(fmt::Error), + } +} + +fn format_fixture( + error: &DurabilityCrashMatrixError, + formatter: &mut fmt::Formatter<'_>, +) -> fmt::Result { + match error { + DurabilityCrashMatrixError::Fixture { artifact, .. } => { + write!(formatter, "cannot decode {artifact} crash fixture") + } + DurabilityCrashMatrixError::FixtureLength { + artifact, + expected, + observed, + } => write!( + formatter, + "{artifact} crash fixture has length {observed}, expected {expected}" + ), + DurabilityCrashMatrixError::FixtureRange => { + formatter.write_str("crash fixture range is invalid") + } + DurabilityCrashMatrixError::FixtureTerminator { artifact } => write!( + formatter, + "{artifact} crash fixture lacks its final line feed" + ), + _ => Err(fmt::Error), + } +} + +fn format_command( + error: &DurabilityCrashMatrixError, + formatter: &mut fmt::Formatter<'_>, +) -> fmt::Result { + match error { + DurabilityCrashMatrixError::Case { + point, + position, + source, + } => write!( + formatter, + "{} {}: {source}", + point.identifier(), + position.identifier() + ), + DurabilityCrashMatrixError::InvalidCase(error) => { + write!(formatter, "invalid crash case: {error}") + } + DurabilityCrashMatrixError::InvalidPointEncoding => { + formatter.write_str("crash point is not valid Unicode") + } + DurabilityCrashMatrixError::InvalidPositionEncoding => { + formatter.write_str("crash position is not valid Unicode") + } + DurabilityCrashMatrixError::UnknownPoint(point) => { + write!(formatter, "unknown crash point `{point}`") + } + DurabilityCrashMatrixError::UnknownPosition(position) => { + write!(formatter, "unknown crash position `{position}`") + } + DurabilityCrashMatrixError::Usage => formatter.write_str( + "usage: cargo xtask durability-crash-matrix \ + --case ", + ), + _ => Err(fmt::Error), + } +} + +fn format_boundary( + error: &DurabilityCrashMatrixError, + formatter: &mut fmt::Formatter<'_>, +) -> fmt::Result { + match error { + DurabilityCrashMatrixError::Io { action, .. } => write!(formatter, "cannot {action}"), + DurabilityCrashMatrixError::MissingActiveFile => { + formatter.write_str("crash sequence has no active staged artifact") + } + DurabilityCrashMatrixError::NonUnicodeStatePath => { + formatter.write_str("post-crash store path is not valid Unicode") + } + DurabilityCrashMatrixError::PointSequenceMismatch { point } => write!( + formatter, + "{} is outside the selected crash sequence", + point.identifier() + ), + DurabilityCrashMatrixError::Verification { phase, source } => write!( + formatter, + "post-crash verification failed while attempting to {phase}: {source}" + ), + DurabilityCrashMatrixError::WriterLock(source) => { + write!(formatter, "cannot acquire crash-case writer lock: {source}") + } + _ => Err(fmt::Error), + } +} diff --git a/xtask/src/durability_crash_matrix/process.rs b/xtask/src/durability_crash_matrix/process.rs index 0faef21..8375ad6 100644 --- a/xtask/src/durability_crash_matrix/process.rs +++ b/xtask/src/durability_crash_matrix/process.rs @@ -196,6 +196,10 @@ fn verify_marker( if observed == marker(case) { Ok(()) } else { - Err(DurabilityCrashMatrixError::StateMismatch) + Err(DurabilityCrashMatrixError::artifact_bytes( + "prepared-case", + &marker(case), + &observed, + )) } } diff --git a/xtask/src/durability_crash_matrix/restart.rs b/xtask/src/durability_crash_matrix/restart.rs index fa3226a..d9be317 100644 --- a/xtask/src/durability_crash_matrix/restart.rs +++ b/xtask/src/durability_crash_matrix/restart.rs @@ -20,7 +20,10 @@ pub(super) fn verify( let expected = ExpectedStoreState::for_case(case)?; let observed_paths = inventory(store_root)?; if observed_paths != expected.paths() { - return Err(DurabilityCrashMatrixError::StateMismatch); + return Err(DurabilityCrashMatrixError::InventoryMismatch { + expected: expected.paths(), + observed: observed_paths, + }); } let segment = GoldenFixture::segment()?; let catalog = GoldenFixture::catalog()?; @@ -30,7 +33,11 @@ pub(super) fn verify( .map_err(|source| DurabilityCrashMatrixError::io("read crash artifact", source))?; let expected_bytes = bytes.resolve(&segment, &catalog, &head)?; if observed != expected_bytes { - return Err(DurabilityCrashMatrixError::StateMismatch); + return Err(DurabilityCrashMatrixError::artifact_bytes( + relative, + expected_bytes, + &observed, + )); } } if let Some((source, target)) = expected.hard_link() { @@ -57,7 +64,11 @@ fn inventory(store_root: &Path) -> Result, DurabilityCrashMatri .ok_or(DurabilityCrashMatrixError::NonUnicodeStatePath)? .into(); if !paths.insert(text) { - return Err(DurabilityCrashMatrixError::StateMismatch); + let path = relative + .to_str() + .ok_or(DurabilityCrashMatrixError::NonUnicodeStatePath)? + .into(); + return Err(DurabilityCrashMatrixError::RepeatedInventoryPath { path }); } let file_type = entry.file_type().map_err(|source| { DurabilityCrashMatrixError::io("inspect crash-store entry", source) @@ -72,8 +83,8 @@ fn inventory(store_root: &Path) -> Result, DurabilityCrashMatri fn verify_hard_link( store_root: &Path, - source: &str, - target: &str, + source: &'static str, + target: &'static str, ) -> Result<(), DurabilityCrashMatrixError> { let source_metadata = fs::metadata(store_root.join(source)) .map_err(|error| DurabilityCrashMatrixError::io("inspect crash source link", error))?; @@ -84,6 +95,13 @@ fn verify_hard_link( { Ok(()) } else { - Err(DurabilityCrashMatrixError::StateMismatch) + Err(DurabilityCrashMatrixError::HardLinkIdentityMismatch { + source, + target, + source_device: source_metadata.dev(), + source_inode: source_metadata.ino(), + target_device: target_metadata.dev(), + target_inode: target_metadata.ino(), + }) } } diff --git a/xtask/src/durability_crash_matrix/restart/expectation.rs b/xtask/src/durability_crash_matrix/restart/expectation.rs index 7b11ada..aeae457 100644 --- a/xtask/src/durability_crash_matrix/restart/expectation.rs +++ b/xtask/src/durability_crash_matrix/restart/expectation.rs @@ -26,6 +26,15 @@ pub(super) enum ArtifactBytes { } impl ArtifactBytes { + pub(super) const fn kind(&self) -> &'static str { + match self { + Self::Empty => "empty", + Self::Segment(_) => "segment", + Self::Catalog(_) => "catalog", + Self::Head(_) => "head", + } + } + pub(super) fn resolve<'a>( &self, segment: &'a GoldenFixture, diff --git a/xtask/src/durability_crash_matrix/restart/semantic.rs b/xtask/src/durability_crash_matrix/restart/semantic.rs index 4efc17f..32323c6 100644 --- a/xtask/src/durability_crash_matrix/restart/semantic.rs +++ b/xtask/src/durability_crash_matrix/restart/semantic.rs @@ -66,7 +66,11 @@ fn verify_segment_stage( ArtifactBytes::Segment(337) => "complete", ArtifactBytes::Empty | ArtifactBytes::Segment(_) => "truncated", ArtifactBytes::Catalog(_) | ArtifactBytes::Head(_) => { - return Err(DurabilityCrashMatrixError::StateMismatch); + return Err(DurabilityCrashMatrixError::UnexpectedArtifactKind { + artifact: SEGMENT_STAGE, + expected: "segment", + observed: bytes.kind(), + }); } }; require_class(observed, expected_class) @@ -92,7 +96,11 @@ fn verify_catalog_stage( ArtifactBytes::Catalog(352) => "complete", ArtifactBytes::Empty | ArtifactBytes::Catalog(_) => "truncated", ArtifactBytes::Segment(_) | ArtifactBytes::Head(_) => { - return Err(DurabilityCrashMatrixError::StateMismatch); + return Err(DurabilityCrashMatrixError::UnexpectedArtifactKind { + artifact: CATALOG_STAGE, + expected: "catalog", + observed: bytes.kind(), + }); } }; require_class(observed, expected_class) @@ -117,7 +125,11 @@ fn verify_next_head( ArtifactBytes::Head(128) => "complete", ArtifactBytes::Empty | ArtifactBytes::Head(_) => "truncated", ArtifactBytes::Segment(_) | ArtifactBytes::Catalog(_) => { - return Err(DurabilityCrashMatrixError::StateMismatch); + return Err(DurabilityCrashMatrixError::UnexpectedArtifactKind { + artifact: NEXT_HEAD, + expected: "head", + observed: bytes.kind(), + }); } }; require_class(observed, expected_class) @@ -155,21 +167,31 @@ fn verify_published_snapshot( } let loaded = FilesystemCatalogSnapshot::load(store_root, restart_policy()?) .map_err(|source| verification("load published restart snapshot", source))?; - if loaded.generation().get() != 1 { - return Err(DurabilityCrashMatrixError::StateMismatch); + let observed_generation = loaded.generation().get(); + if observed_generation != 1 { + return Err(DurabilityCrashMatrixError::SnapshotGenerationMismatch { + expected: 1, + observed: observed_generation, + }); } let snapshot = loaded .snapshot() .map_err(|source| verification("admit published restart snapshot", source))?; let chunk = ChunkId::hash_bytes(&[0]) .map_err(|source| verification("construct Golden Worldline chunk identity", source))?; - let record = snapshot - .record(SegmentRecordIdentity::Chunk(chunk)) - .ok_or(DurabilityCrashMatrixError::StateMismatch)?; + let record = snapshot.record(SegmentRecordIdentity::Chunk(chunk)).ok_or( + DurabilityCrashMatrixError::MissingVisibleRecord { + record: "one-zero chunk", + }, + )?; if record.payload() == [0] { Ok(()) } else { - Err(DurabilityCrashMatrixError::StateMismatch) + Err(DurabilityCrashMatrixError::artifact_bytes( + "visible one-zero record payload", + &[0], + record.payload(), + )) } } @@ -190,7 +212,7 @@ fn require_class( if observed == expected { Ok(()) } else { - Err(DurabilityCrashMatrixError::StateMismatch) + Err(DurabilityCrashMatrixError::ArtifactClassificationMismatch { expected, observed }) } } @@ -200,3 +222,21 @@ fn verification(phase: &'static str, source: impl Error + 'static) -> Durability source: Box::new(source), } } + +#[cfg(test)] +mod tests { + use super::require_class; + + #[test] + fn classification_mismatch_names_expected_and_observed() -> Result<(), &'static str> { + let error = require_class("truncated", "complete") + .err() + .ok_or("mismatched classes were accepted")?; + + assert_eq!( + error.to_string(), + "post-crash artifact classification mismatch: expected `complete`, observed `truncated`" + ); + Ok(()) + } +} From c8cdc84a4a98d70ca3c4aac0c78b2da151e7fa12 Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 14:20:10 -0700 Subject: [PATCH 31/49] Fix: Freeze the crash matrix CLI command --- xtask/tests/cli_contract.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xtask/tests/cli_contract.rs b/xtask/tests/cli_contract.rs index 4be148d..aa03303 100644 --- a/xtask/tests/cli_contract.rs +++ b/xtask/tests/cli_contract.rs @@ -90,7 +90,7 @@ fn missing_command_returns_the_versioned_usage_contract() -> Result<(), io::Erro \n" ); From 88d447a08b46941dca22fc9467bdab2a45a4a8ff Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 14:21:40 -0700 Subject: [PATCH 32/49] Fix: Track the crash process-group contract --- xtask/tests/process_policy_contract.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/xtask/tests/process_policy_contract.rs b/xtask/tests/process_policy_contract.rs index f99edbc..e0da193 100644 --- a/xtask/tests/process_policy_contract.rs +++ b/xtask/tests/process_policy_contract.rs @@ -218,9 +218,9 @@ fn bounded_process_support_documents_every_exported_contract() -> Result<(), Str require_docs( BOUNDED_PROCESS_GROUP, &[ - "pub(super) struct ProcessGroup", - " pub(super) fn for_child(", - " pub(super) fn terminate(", + "pub(crate) struct ProcessGroup", + " pub(crate) fn for_child(", + " pub(crate) fn terminate(", ], )?; require_docs( From b0ea83904e42b4392997f3e84855eb5315f0b328 Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 14:24:51 -0700 Subject: [PATCH 33/49] Fix: Gate crash contracts on repository tasks --- xtask/tests/durability_crash_case_contract.rs | 2 ++ xtask/tests/durability_crash_documentation.rs | 2 ++ xtask/tests/durability_crash_process_contract.rs | 2 ++ 3 files changed, 6 insertions(+) diff --git a/xtask/tests/durability_crash_case_contract.rs b/xtask/tests/durability_crash_case_contract.rs index 396a9bb..87b671e 100644 --- a/xtask/tests/durability_crash_case_contract.rs +++ b/xtask/tests/durability_crash_case_contract.rs @@ -1,5 +1,7 @@ //! Canonical deterministic crash-matrix coordinate laws. +#![cfg(feature = "repository-tasks")] + use std::error::Error; use xtask::{ diff --git a/xtask/tests/durability_crash_documentation.rs b/xtask/tests/durability_crash_documentation.rs index 92c6916..8949f2d 100644 --- a/xtask/tests/durability_crash_documentation.rs +++ b/xtask/tests/durability_crash_documentation.rs @@ -1,5 +1,7 @@ //! Documentation truth laws for the process-death crash matrix. +#![cfg(feature = "repository-tasks")] + const ROOT_README: &str = include_str!("../../README.md"); const RECOVERY: &str = include_str!("../../docs/formats/segment-store-v1/recovery.md"); const REQUIREMENTS: &str = include_str!("../../docs/formats/segment-store-v1/requirements.md"); diff --git a/xtask/tests/durability_crash_process_contract.rs b/xtask/tests/durability_crash_process_contract.rs index 10f552a..0121431 100644 --- a/xtask/tests/durability_crash_process_contract.rs +++ b/xtask/tests/durability_crash_process_contract.rs @@ -1,5 +1,7 @@ //! Deterministic subprocess process-death laws. +#![cfg(feature = "repository-tasks")] + use std::error::Error; use std::process::Command; From c47a6f6b45417350c2da0a791526876e9f304688 Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 14:40:02 -0700 Subject: [PATCH 34/49] Fix: Lock writer roots on Linux --- src/adapters/filesystem_writer_lock.rs | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/adapters/filesystem_writer_lock.rs b/src/adapters/filesystem_writer_lock.rs index 4270198..4a17db0 100644 --- a/src/adapters/filesystem_writer_lock.rs +++ b/src/adapters/filesystem_writer_lock.rs @@ -7,6 +7,8 @@ use std::path::Path; use cap_fs_ext::{FollowSymlinks, MetadataExt, OpenOptionsFollowExt, OpenOptionsSyncExt}; use cap_std::ambient_authority; use cap_std::fs::{Dir, Metadata, OpenOptions}; +use rustix::fs::{FlockOperation, flock}; +use rustix::io::Errno; use super::{WriterLockAcquireError, WriterLockAcquirePhase}; @@ -123,10 +125,24 @@ fn acquire_root(directory: &Dir) -> Result { .try_clone() .map_err(|source| WriterLockAcquireError::io(WriterLockAcquirePhase::AcquireRoot, source))? .into_std_file(); - acquire_lock(&file, WriterLockAcquirePhase::AcquireRoot)?; + acquire_root_lock(&file)?; Ok(file) } +fn acquire_root_lock(file: &File) -> Result<(), WriterLockAcquireError> { + // The pinned directory handle is read-only. Linux POSIX record locks reject + // an exclusive lock on that handle, while `flock` locks the directory inode + // without requiring write access to the directory file description. + match flock(file, FlockOperation::NonBlockingLockExclusive) { + Ok(()) => Ok(()), + Err(source) if source == Errno::WOULDBLOCK => Err(WriterLockAcquireError::Busy), + Err(source) => Err(WriterLockAcquireError::io( + WriterLockAcquirePhase::AcquireRoot, + source.into(), + )), + } +} + fn acquire_lock(file: &File, phase: WriterLockAcquirePhase) -> Result<(), WriterLockAcquireError> { match file.try_lock() { Ok(()) => Ok(()), From 8514705470b8b9e4da215ef0735f1763a4a5fa12 Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 14:51:56 -0700 Subject: [PATCH 35/49] Fix: Open a lockable writer root handle --- src/adapters/filesystem_writer_lock.rs | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/src/adapters/filesystem_writer_lock.rs b/src/adapters/filesystem_writer_lock.rs index 4a17db0..c3a9844 100644 --- a/src/adapters/filesystem_writer_lock.rs +++ b/src/adapters/filesystem_writer_lock.rs @@ -7,7 +7,7 @@ use std::path::Path; use cap_fs_ext::{FollowSymlinks, MetadataExt, OpenOptionsFollowExt, OpenOptionsSyncExt}; use cap_std::ambient_authority; use cap_std::fs::{Dir, Metadata, OpenOptions}; -use rustix::fs::{FlockOperation, flock}; +use rustix::fs::{FlockOperation, Mode, OFlags, flock, openat}; use rustix::io::Errno; use super::{WriterLockAcquireError, WriterLockAcquirePhase}; @@ -121,18 +121,24 @@ impl FilesystemWriterLock { } fn acquire_root(directory: &Dir) -> Result { - let file = directory - .try_clone() - .map_err(|source| WriterLockAcquireError::io(WriterLockAcquirePhase::AcquireRoot, source))? - .into_std_file(); + let descriptor = openat( + directory, + ".", + OFlags::RDONLY | OFlags::DIRECTORY | OFlags::CLOEXEC | OFlags::NOFOLLOW, + Mode::empty(), + ) + .map_err(|source| { + WriterLockAcquireError::io(WriterLockAcquirePhase::AcquireRoot, source.into()) + })?; + let file = File::from(descriptor); acquire_root_lock(&file)?; Ok(file) } fn acquire_root_lock(file: &File) -> Result<(), WriterLockAcquireError> { - // The pinned directory handle is read-only. Linux POSIX record locks reject - // an exclusive lock on that handle, while `flock` locks the directory inode - // without requiring write access to the directory file description. + // Linux capability directories use `O_PATH`, which cannot be locked. Open a + // read-only descriptor to the same pinned inode before reaching this step; + // `flock` locks that directory descriptor without requiring write access. match flock(file, FlockOperation::NonBlockingLockExclusive) { Ok(()) => Ok(()), Err(source) if source == Errno::WOULDBLOCK => Err(WriterLockAcquireError::Busy), From 1f9da851b9f631348e533da47eadca6053385adc Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 15:26:36 -0700 Subject: [PATCH 36/49] Fix: Reopen root for durable synchronization --- src/adapters/filesystem_initialization_storage.rs | 4 +++- src/adapters/filesystem_store_initializer_tests.rs | 12 ++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/adapters/filesystem_initialization_storage.rs b/src/adapters/filesystem_initialization_storage.rs index 06c4184..ca264fa 100644 --- a/src/adapters/filesystem_initialization_storage.rs +++ b/src/adapters/filesystem_initialization_storage.rs @@ -95,6 +95,8 @@ impl StoreInitializationStorage for FilesystemInitializationStorage { "root synchronization requires writer authority", )); } - self.directory.try_clone()?.into_std_file().sync_all() + sync_capable_directory::open(&self.directory, ".")? + .into_std_file() + .sync_all() } } diff --git a/src/adapters/filesystem_store_initializer_tests.rs b/src/adapters/filesystem_store_initializer_tests.rs index 8c6563b..5dae865 100644 --- a/src/adapters/filesystem_store_initializer_tests.rs +++ b/src/adapters/filesystem_store_initializer_tests.rs @@ -26,6 +26,18 @@ fn empty_namespace_is_admitted_only_with_the_complete_root_shape() -> Result<(), Ok(()) } +#[cfg(target_os = "linux")] +#[test] +fn linux_initializer_synchronizes_an_opath_root() -> Result<(), Box> { + let sandbox = TestDirectory::create("store-initialization-opath-sync")?; + + let admission = FilesystemPlatformAdmission::initialize_unchecked_for_tests(sandbox.path())?; + + drop(admission); + sandbox.remove()?; + Ok(()) +} + #[test] fn partial_canonical_namespace_is_completed_without_replacing_evidence() -> Result<(), Box> { From 090d0b0fece370404c003983f3cf03801cba5627 Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 15:30:02 -0700 Subject: [PATCH 37/49] Fix: Reopen capability directories before synchronization --- src/adapters/filesystem_catalog_artifact.rs | 26 ++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/src/adapters/filesystem_catalog_artifact.rs b/src/adapters/filesystem_catalog_artifact.rs index 2c9e7db..0b9aab6 100644 --- a/src/adapters/filesystem_catalog_artifact.rs +++ b/src/adapters/filesystem_catalog_artifact.rs @@ -10,6 +10,7 @@ use super::segment_header::MAXIMUM_SEGMENT_LENGTH; use super::{ AdmittedSegment, CatalogRestartArtifact, CatalogRestartError, CatalogRestartPhase, ChecksummedCatalog, FilesystemCatalogPublicationError, SegmentReadPolicy, catalog_restart_io, + sync_capable_directory, }; use crate::CatalogLength; @@ -24,7 +25,9 @@ pub(super) fn create_exclusive(directory: &Dir, name: &str) -> io::Result } pub(super) fn synchronize_directory(directory: &Dir) -> io::Result<()> { - directory.try_clone()?.into_std_file().sync_all() + sync_capable_directory::open(directory, ".")? + .into_std_file() + .sync_all() } pub(super) fn link_without_replacement( @@ -150,3 +153,24 @@ fn require_exact_bytes( )) } } + +#[cfg(all(test, target_os = "linux"))] +mod tests { + use std::error::Error; + + use cap_std::{ambient_authority, fs::Dir}; + + use super::super::filesystem_test_sandbox::TestDirectory; + + #[test] + fn directory_synchronization_reopens_an_opath_capability() -> Result<(), Box> { + let sandbox = TestDirectory::create("catalog-artifact-opath-sync")?; + let directory = Dir::open_ambient_dir(sandbox.path(), ambient_authority())?; + + super::synchronize_directory(&directory)?; + + drop(directory); + sandbox.remove()?; + Ok(()) + } +} From 1fc1ad0ff1826028fc2a97c4bf104e06ca983088 Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 15:35:00 -0700 Subject: [PATCH 38/49] Fix: Exercise production protocols in crash tests --- CHANGELOG.md | 6 +- Cargo.toml | 4 + README.md | 19 +- docs/formats/segment-store-v1/recovery.md | 36 ++- docs/formats/segment-store-v1/requirements.md | 2 +- src/adapters/filesystem_catalog_catalog.rs | 18 +- src/adapters/filesystem_catalog_head.rs | 19 +- src/adapters/filesystem_catalog_publisher.rs | 52 ++++ .../filesystem_initialization_storage.rs | 12 +- src/adapters/filesystem_platform_admission.rs | 5 + .../filesystem_recovery_stage_discarder.rs | 23 +- src/adapters/mod.rs | 4 + .../repository_initialization_storage.rs | 67 +++++ src/adapters/sealed_segment.rs | 16 ++ src/lib.rs | 3 + xtask/Cargo.toml | 1 + xtask/src/durability_crash_matrix.rs | 2 +- xtask/src/durability_crash_matrix/child.rs | 12 +- xtask/src/durability_crash_matrix/error.rs | 5 - .../durability_crash_matrix/error/display.rs | 10 +- .../production_protocol.rs | 65 +++++ .../production_protocol/control.rs | 66 +++++ .../{state => production_protocol}/fixture.rs | 2 +- .../production_protocol/initialization.rs | 50 ++++ .../initialization_storage.rs | 63 ++++ .../production_protocol/publication.rs | 71 +++++ .../publication_storage.rs | 270 ++++++++++++++++++ .../production_protocol/recovery.rs | 115 ++++++++ .../production_protocol/recovery_storage.rs | 62 ++++ .../production_protocol/segment_stage.rs | 133 +++++++++ xtask/src/durability_crash_matrix/restart.rs | 2 +- .../restart/expectation.rs | 2 +- .../restart/expectation/sequence.rs | 4 +- .../restart/semantic.rs | 4 +- xtask/src/durability_crash_matrix/state.rs | 202 ------------- .../durability_crash_matrix/state/catalog.rs | 84 ------ .../src/durability_crash_matrix/state/head.rs | 79 ----- .../state/initialization.rs | 60 ---- .../durability_crash_matrix/state/recovery.rs | 60 ---- .../durability_crash_matrix/state/segment.rs | 103 ------- .../durability_crash_production_contract.rs | 37 +++ 41 files changed, 1202 insertions(+), 648 deletions(-) create mode 100644 src/adapters/repository_initialization_storage.rs create mode 100644 xtask/src/durability_crash_matrix/production_protocol.rs create mode 100644 xtask/src/durability_crash_matrix/production_protocol/control.rs rename xtask/src/durability_crash_matrix/{state => production_protocol}/fixture.rs (97%) create mode 100644 xtask/src/durability_crash_matrix/production_protocol/initialization.rs create mode 100644 xtask/src/durability_crash_matrix/production_protocol/initialization_storage.rs create mode 100644 xtask/src/durability_crash_matrix/production_protocol/publication.rs create mode 100644 xtask/src/durability_crash_matrix/production_protocol/publication_storage.rs create mode 100644 xtask/src/durability_crash_matrix/production_protocol/recovery.rs create mode 100644 xtask/src/durability_crash_matrix/production_protocol/recovery_storage.rs create mode 100644 xtask/src/durability_crash_matrix/production_protocol/segment_stage.rs delete mode 100644 xtask/src/durability_crash_matrix/state.rs delete mode 100644 xtask/src/durability_crash_matrix/state/catalog.rs delete mode 100644 xtask/src/durability_crash_matrix/state/head.rs delete mode 100644 xtask/src/durability_crash_matrix/state/initialization.rs delete mode 100644 xtask/src/durability_crash_matrix/state/recovery.rs delete mode 100644 xtask/src/durability_crash_matrix/state/segment.rs create mode 100644 xtask/tests/durability_crash_production_contract.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 897493e..6df9dd4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,8 +12,10 @@ after its public API and format compatibility policies are established. - Repository crash-matrix execution now terminates isolated writer process groups at all 105 canonical before/during/after coordinates, retains open - writer and stage authority until termination, and verifies exact Golden File - Worldline namespaces, bytes, hard links, released locks, recovery + writer and stage authority until termination, executes production + initialization, segment-writing, catalog-publication, and recovery-discard + protocols through fault-injecting port decorators, and verifies exact Golden + File Worldline namespaces, bytes, hard links, released locks, recovery classifications, immutable artifacts, and published visible state after restart. CI runs the complete matrix in debug and optimized profiles. - Production filesystem initialization now admits only the documented writable, diff --git a/Cargo.toml b/Cargo.toml index 5c4d601..230d7cf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,6 +12,10 @@ keywords = ["cas", "storage", "content-addressed", "deduplication"] categories = ["data-structures", "filesystem"] publish = false +[features] +default = [] +repository-tasks = [] + [dependencies] blake3 = { version = "=1.8.5", default-features = false, features = ["pure", "std"] } cap-fs-ext = { version = "=4.0.2", default-features = false, features = ["std"] } diff --git a/README.md b/README.md index 99944b9..b55d3e2 100644 --- a/README.md +++ b/README.md @@ -101,14 +101,17 @@ reverifies the exact candidate, atomically replaces `HEAD`, and synchronizes the root. An already-finalized retry requires `head.next` to be absent. The repository-owned process-death matrix executes all 105 `KEEP-CRASH-001`–`KEEP-CRASH-035` before/during/after coordinates in isolated -process groups. Restart verification compares the exact Golden File Worldline -namespace and bytes, checks hard-link identity and writer-lock release, runs -the production recovery classifiers and immutable-artifact admission, and -reconstructs the exact published generation and visible one-zero chunk when -`HEAD` exists. This matrix proves application process-death behavior; it does -not simulate host power loss. Retention, compaction, and garbage collection -remain planned. Presence in the reference CAS does not claim retention, crash -recovery, or durability. +process groups. Crash children execute the production initialization, +segment-writing, catalog-publication, and recovery-discard protocols through +fault-injecting port decorators; they do not synthesize the target namespace. +Restart verification compares the exact Golden File Worldline namespace and +bytes, checks hard-link identity and writer-lock release, runs the production +recovery classifiers and immutable-artifact admission, and reconstructs the +exact published generation and visible one-zero chunk when `HEAD` exists. This +matrix proves application process-death behavior; it does not simulate host +power loss. Retention, compaction, and garbage collection remain planned. +Presence in the reference CAS does not claim retention, crash recovery, or +durability. Run the complete debug-profile matrix: diff --git a/docs/formats/segment-store-v1/recovery.md b/docs/formats/segment-store-v1/recovery.md index 6d17b58..ff60a9f 100644 --- a/docs/formats/segment-store-v1/recovery.md +++ b/docs/formats/segment-store-v1/recovery.md @@ -107,21 +107,27 @@ cargo xtask durability-crash-matrix The command executes the three ordered positions for each stable `KEEP-CRASH-001`–`KEEP-CRASH-035` point: 105 canonical cases. Each case owns a fresh filesystem store and an isolated child process group. The child retains -the writer lock and any open staged artifact, prepares the point-selected -state, sends one readiness byte over a Unix socket, and waits on the retained -connection. The parent keeps the accepted socket open, sends `SIGKILL` to the -complete process group, reaps the child, and only then begins restart -inspection. A ten-second deadline bounds failure handling; successful -synchronization does not depend on elapsed time, sleeps, or test ordering. - -The state constructor and expected-state model are separate implementations. -Restart inspection proves the complete path set and exact Golden File -Worldline bytes. It additionally verifies hard-link identity at link -transitions, reacquires `writer.lock` after process death, classifies -`current.seg`, `current.cat`, and `head.next` through the production recovery -classifiers, admits immutable segment and catalog bytes through the production -decoders, and loads the exact generation-1 snapshot when `HEAD` is present. -That snapshot must expose the one-zero chunk with the exact payload `00`. +the writer lock and any open staged artifact while it executes the production +initialization, segment-writing, catalog-publication, or recovery-discard +protocol. A fault-injecting port decorator sends one readiness byte at the +selected semantic boundary and waits on the retained connection. The parent +keeps the accepted socket open, sends `SIGKILL` to the complete process group, +reaps the child, and only then begins restart inspection. A ten-second +deadline bounds failure handling; successful synchronization does not depend +on elapsed time, sleeps, or test ordering. + +The production protocol driver and expected-state model are separate +implementations. The driver delegates every target mutation to the ordinary +filesystem adapters. Repository-only partial-write methods share the same +catalog and head stage writers and stop at deterministic proper prefixes; +they do not hand-construct a replacement namespace. Restart inspection proves +the complete path set and exact Golden File Worldline bytes. It additionally +verifies hard-link identity at link transitions, reacquires `writer.lock` +after process death, classifies `current.seg`, `current.cat`, and `head.next` +through the production recovery classifiers, admits immutable segment and +catalog bytes through the production decoders, and loads the exact +generation-1 snapshot when `HEAD` is present. That snapshot must expose the +one-zero chunk with the exact payload `00`. For byte writes, a `during` case retains a deterministic proper prefix and its open file handle at termination. Atomic create, hard-link, unlink, and rename diff --git a/docs/formats/segment-store-v1/requirements.md b/docs/formats/segment-store-v1/requirements.md index eea28f2..184048c 100644 --- a/docs/formats/segment-store-v1/requirements.md +++ b/docs/formats/segment-store-v1/requirements.md @@ -129,7 +129,7 @@ collection, and host-power-loss simulation remain outside issue #17. | `KEEP-RECOVERY-018` | Filesystem next-head finalization retains root and `writer.lock` authority, pins every protocol directory, revalidates namespace identity and exact stage evidence around bounded complete current and candidate loads, synchronizes and reverifies the exact candidate before atomic replacement, refuses current drift, missing or reappeared candidates, links, corrupt transitive views, and namespace replacement, and returns only after root synchronization | Initial and successor finalization, exact retry, candidate-sync, evidence-drift, missing, link, corruption, namespace-replacement, current-drift, and writer-exclusion matrix | `src/adapters/filesystem_recovery_next_head_finalization_tests.rs`, `src/adapters/filesystem_recovery_next_head_finalization_tests/*.rs` | Implemented in #17 | | `KEEP-RECOVERY-019` | Reusable-segment continuation plans only from an exact reusable `current.seg` assessment within the selected record policy, consumes the storage authority that reopens the stage, re-admits the complete materialized prefix against prior evidence, rebuilds digest and duplicate-identity state, and returns the ordinary append-only stage without rewriting admitted bytes | Reusable-only planning, policy refusal, changed-evidence, storage-failure, duplicate-identity, append, seal, and independent decode matrix | `tests/recovery_segment_resume.rs`, `tests/recovery_segment_resume/*.rs` | Implemented in #17 | | `KEEP-RECOVERY-020` | Filesystem reusable-segment continuation retains root and `writer.lock` authority in the returned stage, pins every protocol directory, opens `current.seg` read-write without following links or truncation, bounds and re-admits its complete bytes, positions the handle at the exact validated append boundary, refuses missing, changed, linked, replaced, or namespace-drifted evidence, and preserves the prefix on append or empty seal | Empty and nonempty continuation, writer exclusion, missing, changed evidence, link, namespace replacement, writable-handoff replacement, append, seal, and independent decode matrix | `src/adapters/filesystem_recovery_segment_resume_tests.rs`, `src/adapters/filesystem_recovery_segment_resume_tests/*.rs` | Implemented in #17 | -| `KEEP-RECOVERY-021` | Every crash point has exact before, during, and after coordinates; a deadline-bounded parent receives readiness over a Unix socket, retains that socket, terminates the isolated child process group, and independently verifies the exact Golden File Worldline namespace, bytes, hard-link identity, released writer lock, recovery-stage class, immutable-artifact admission, generation, and visible chunk after restart | Ordered 105-case model, independent expected-state model, production recovery classifiers, production restart loader, and explicit debug/optimized CI commands | `xtask/tests/durability_crash_case_contract.rs`, `xtask/tests/durability_crash_process_contract.rs`, `xtask/src/durability_crash_matrix/`, `.github/workflows/ci.yml` | Implemented in #17 | +| `KEEP-RECOVERY-021` | Every crash point has exact before, during, and after coordinates; each child executes the production initialization, segment-writing, catalog-publication, or recovery-discard protocol through a fault-injecting port decorator; a deadline-bounded parent receives readiness over a Unix socket, retains that socket, terminates the isolated child process group, and independently verifies the exact Golden File Worldline namespace, bytes, hard-link identity, released writer lock, recovery-stage class, immutable-artifact admission, generation, and visible chunk after restart | Production protocol driver, ordered 105-case model, independent expected-state model, production recovery classifiers, production restart loader, and explicit debug/optimized CI commands | `xtask/tests/durability_crash_production_contract.rs`, `xtask/tests/durability_crash_case_contract.rs`, `xtask/tests/durability_crash_process_contract.rs`, `xtask/src/durability_crash_matrix/`, `.github/workflows/ci.yml` | Implemented in #17 | diff --git a/src/adapters/filesystem_catalog_catalog.rs b/src/adapters/filesystem_catalog_catalog.rs index 5648a2c..023e98e 100644 --- a/src/adapters/filesystem_catalog_catalog.rs +++ b/src/adapters/filesystem_catalog_catalog.rs @@ -23,7 +23,23 @@ pub(super) fn write( publisher: &mut FilesystemCatalogPublisher, catalog: &CanonicalCatalog, ) -> io::Result<()> { - stage_mut(publisher)?.write_all(catalog.encoded()) + write_bytes(publisher, catalog.encoded()) +} + +#[cfg(feature = "repository-tasks")] +pub(super) fn write_prefix( + publisher: &mut FilesystemCatalogPublisher, + catalog: &CanonicalCatalog, + prefix: usize, +) -> io::Result<()> { + let bytes = catalog.encoded().get(..prefix).ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidInput, "catalog prefix exceeds bytes") + })?; + write_bytes(publisher, bytes) +} + +fn write_bytes(publisher: &mut FilesystemCatalogPublisher, bytes: &[u8]) -> io::Result<()> { + stage_mut(publisher)?.write_all(bytes) } pub(super) fn flush(publisher: &mut FilesystemCatalogPublisher) -> io::Result<()> { diff --git a/src/adapters/filesystem_catalog_head.rs b/src/adapters/filesystem_catalog_head.rs index 1c465e8..27df44f 100644 --- a/src/adapters/filesystem_catalog_head.rs +++ b/src/adapters/filesystem_catalog_head.rs @@ -23,7 +23,24 @@ pub(super) fn write( publisher: &mut FilesystemCatalogPublisher, head: &CanonicalPublicationHead, ) -> io::Result<()> { - stage_mut(publisher)?.write_all(head.encoded()) + write_bytes(publisher, head.encoded()) +} + +#[cfg(feature = "repository-tasks")] +pub(super) fn write_prefix( + publisher: &mut FilesystemCatalogPublisher, + head: &CanonicalPublicationHead, + prefix: usize, +) -> io::Result<()> { + let bytes = head + .encoded() + .get(..prefix) + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "head prefix exceeds bytes"))?; + write_bytes(publisher, bytes) +} + +fn write_bytes(publisher: &mut FilesystemCatalogPublisher, bytes: &[u8]) -> io::Result<()> { + stage_mut(publisher)?.write_all(bytes) } pub(super) fn flush(publisher: &mut FilesystemCatalogPublisher) -> io::Result<()> { diff --git a/src/adapters/filesystem_catalog_publisher.rs b/src/adapters/filesystem_catalog_publisher.rs index 8abaa31..83518c9 100644 --- a/src/adapters/filesystem_catalog_publisher.rs +++ b/src/adapters/filesystem_catalog_publisher.rs @@ -10,6 +10,8 @@ use super::{ FilesystemSegmentStage, FilesystemWriterLock, SealedSegment, SegmentPublication, SegmentPublicationError, SegmentStageCreateError, sync_capable_directory, }; +#[cfg(feature = "repository-tasks")] +use super::{CanonicalCatalog, CanonicalPublicationHead, filesystem_catalog_catalog}; pub(super) const CURRENT_SEGMENT: &str = "current.seg"; pub(super) const CURRENT_CATALOG: &str = "current.cat"; @@ -74,6 +76,26 @@ impl FilesystemCatalogPublisher { }) } + /// Opens a publisher without the production platform-profile proof. + /// + /// Repository process-death tests use this after executing the production + /// initialization protocol through [`crate::RepositoryInitializationStorage`]. + /// + /// # Errors + /// + /// Returns the same pinned-directory admission failures as [`Self::open`]. + #[cfg(feature = "repository-tasks")] + #[doc(hidden)] + pub fn open_unchecked_for_repository_tasks( + lock: FilesystemWriterLock, + policy: CatalogRestartPolicy, + ) -> io::Result { + Self::open( + FilesystemPlatformAdmission::unchecked_for_repository_tasks(lock), + policy, + ) + } + #[cfg(test)] pub(super) fn open_unchecked_for_tests( lock: FilesystemWriterLock, @@ -128,4 +150,34 @@ impl FilesystemCatalogPublisher { authority, ) } + + /// Writes a strict prefix through the production catalog-stage adapter. + /// + /// # Errors + /// + /// Returns the exact missing-stage, prefix-bound, or write failure. + #[cfg(feature = "repository-tasks")] + #[doc(hidden)] + pub fn write_catalog_prefix_for_repository_tasks( + &mut self, + catalog: &CanonicalCatalog, + prefix: usize, + ) -> io::Result<()> { + filesystem_catalog_catalog::write_prefix(self, catalog, prefix) + } + + /// Writes a strict prefix through the production head-stage adapter. + /// + /// # Errors + /// + /// Returns the exact missing-stage, prefix-bound, or write failure. + #[cfg(feature = "repository-tasks")] + #[doc(hidden)] + pub fn write_head_prefix_for_repository_tasks( + &mut self, + head: &CanonicalPublicationHead, + prefix: usize, + ) -> io::Result<()> { + super::filesystem_catalog_head::write_prefix(self, head, prefix) + } } diff --git a/src/adapters/filesystem_initialization_storage.rs b/src/adapters/filesystem_initialization_storage.rs index ca264fa..833b47e 100644 --- a/src/adapters/filesystem_initialization_storage.rs +++ b/src/adapters/filesystem_initialization_storage.rs @@ -3,7 +3,7 @@ use std::io; use std::path::Path; -#[cfg(test)] +#[cfg(any(test, feature = "repository-tasks"))] use cap_std::ambient_authority; use cap_std::fs::Dir; @@ -32,6 +32,16 @@ impl FilesystemInitializationStorage { #[cfg(test)] pub(super) fn admit_unchecked_for_tests(store_root: &Path) -> io::Result { + Self::admit_unchecked(store_root) + } + + #[cfg(feature = "repository-tasks")] + pub(super) fn admit_unchecked_for_repository_tasks(store_root: &Path) -> io::Result { + Self::admit_unchecked(store_root) + } + + #[cfg(any(test, feature = "repository-tasks"))] + fn admit_unchecked(store_root: &Path) -> io::Result { let directory = Dir::open_ambient_dir(store_root, ambient_authority())?; Ok(Self { directory, diff --git a/src/adapters/filesystem_platform_admission.rs b/src/adapters/filesystem_platform_admission.rs index a44e8ec..626dab3 100644 --- a/src/adapters/filesystem_platform_admission.rs +++ b/src/adapters/filesystem_platform_admission.rs @@ -21,6 +21,11 @@ impl FilesystemPlatformAdmission { Self { lock } } + #[cfg(feature = "repository-tasks")] + pub(super) const fn unchecked_for_repository_tasks(lock: FilesystemWriterLock) -> Self { + Self { lock } + } + pub(super) fn into_lock(self) -> FilesystemWriterLock { self.lock } diff --git a/src/adapters/filesystem_recovery_stage_discarder.rs b/src/adapters/filesystem_recovery_stage_discarder.rs index 9cc71bd..e29cb92 100644 --- a/src/adapters/filesystem_recovery_stage_discarder.rs +++ b/src/adapters/filesystem_recovery_stage_discarder.rs @@ -2,7 +2,7 @@ use std::path::Path; -#[cfg(test)] +#[cfg(any(test, feature = "repository-tasks"))] use cap_std::ambient_authority; use cap_std::fs::Dir; @@ -10,7 +10,7 @@ use super::{ FilesystemRecoveryInventoryReader, FilesystemRecoveryStageDiscardOpenError, FilesystemWriterLock, filesystem_platform_profile, }; -#[cfg(test)] +#[cfg(any(test, feature = "repository-tasks"))] use super::{RecoveryInventoryError, RecoveryInventoryOperation, RecoveryNamespace}; /// Writer-authorized pinned filesystem adapter for exact stage discard. @@ -44,6 +44,25 @@ impl FilesystemRecoveryStageDiscarder { pub(super) fn open_unchecked_for_tests( store_root: &Path, ) -> Result { + Self::open_unchecked(store_root) + } + + /// Opens repository crash-test storage without the production platform + /// profile probe. + /// + /// # Errors + /// + /// Returns the exact root, writer-lock, or namespace admission failure. + #[cfg(feature = "repository-tasks")] + #[doc(hidden)] + pub fn open_unchecked_for_repository_tasks( + store_root: &Path, + ) -> Result { + Self::open_unchecked(store_root) + } + + #[cfg(any(test, feature = "repository-tasks"))] + fn open_unchecked(store_root: &Path) -> Result { let root = Dir::open_ambient_dir(store_root, ambient_authority()).map_err(|source| { FilesystemRecoveryStageDiscardOpenError::Namespace { source: RecoveryInventoryError::io( diff --git a/src/adapters/mod.rs b/src/adapters/mod.rs index 8258b4b..748ac12 100644 --- a/src/adapters/mod.rs +++ b/src/adapters/mod.rs @@ -227,6 +227,8 @@ mod recovery_stage_metadata_error; mod recovery_stage_parent; mod recovery_stage_pool_outcome; mod recovery_stage_synchronization_outcome; +#[cfg(feature = "repository-tasks")] +mod repository_initialization_storage; mod sealed_segment; mod segment_digest; mod segment_digest_builder; @@ -438,6 +440,8 @@ pub use recovery_stage_metadata_error::RecoveryStageMetadataError; pub use recovery_stage_parent::RecoveryStageParent; pub use recovery_stage_pool_outcome::RecoveryStagePoolOutcome; pub use recovery_stage_synchronization_outcome::RecoveryStageSynchronizationOutcome; +#[cfg(feature = "repository-tasks")] +pub use repository_initialization_storage::RepositoryInitializationStorage; pub use sealed_segment::SealedSegment; pub use segment_digest::SegmentDigest; pub use segment_header::SegmentHeader; diff --git a/src/adapters/repository_initialization_storage.rs b/src/adapters/repository_initialization_storage.rs new file mode 100644 index 0000000..fc82253 --- /dev/null +++ b/src/adapters/repository_initialization_storage.rs @@ -0,0 +1,67 @@ +//! This module owns repository-only access to production initialization storage. + +use std::io; +use std::path::Path; + +use super::filesystem_initialization_storage::FilesystemInitializationStorage; +use super::{FilesystemWriterLock, StoreInitializationStorage}; + +/// Repository-tooling adapter over the production filesystem initialization +/// implementation. +/// +/// This adapter bypasses the Linux ext4 profile probe so the process-death +/// harness can run on development hosts. It does not bypass namespace, +/// synchronization, writer-lock, or initialization protocol operations. +#[doc(hidden)] +pub struct RepositoryInitializationStorage { + inner: FilesystemInitializationStorage, +} + +impl RepositoryInitializationStorage { + /// Opens `store_root` without applying the production platform profile. + /// + /// # Errors + /// + /// Returns the exact ambient root-open failure. + pub fn admit_unchecked(store_root: &Path) -> io::Result { + FilesystemInitializationStorage::admit_unchecked_for_repository_tasks(store_root) + .map(|inner| Self { inner }) + } + + /// Consumes completed initialization storage and returns retained writer + /// authority. + /// + /// # Errors + /// + /// Returns [`io::ErrorKind::Other`] when initialization did not acquire + /// writer authority. + pub fn into_writer_lock(self) -> io::Result { + self.inner.into_lock() + } +} + +impl StoreInitializationStorage for RepositoryInitializationStorage { + fn admit_platform(&mut self) -> io::Result<()> { + self.inner.admit_platform() + } + + fn open_and_lock_writer_file(&mut self) -> io::Result<()> { + self.inner.open_and_lock_writer_file() + } + + fn admit_staging_directory(&mut self) -> io::Result<()> { + self.inner.admit_staging_directory() + } + + fn admit_segment_pool_directory(&mut self) -> io::Result<()> { + self.inner.admit_segment_pool_directory() + } + + fn admit_catalog_pool_directory(&mut self) -> io::Result<()> { + self.inner.admit_catalog_pool_directory() + } + + fn synchronize_root(&mut self) -> io::Result<()> { + self.inner.synchronize_root() + } +} diff --git a/src/adapters/sealed_segment.rs b/src/adapters/sealed_segment.rs index de68d20..70b5b98 100644 --- a/src/adapters/sealed_segment.rs +++ b/src/adapters/sealed_segment.rs @@ -53,6 +53,22 @@ where self.digest } + /// Replaces a repository-only storage decorator while preserving sealed + /// metadata. + /// + /// This supports transparent storage-port decorators. The mapping does not + /// change, revalidate, or publish the sealed bytes; authority-bound + /// adapters still validate the returned stage before publication. + #[cfg(feature = "repository-tasks")] + #[doc(hidden)] + pub fn map_stage(self, map: impl FnOnce(S) -> T) -> SealedSegment + where + T: SegmentStage, + { + let (stage, record_count, segment_length, digest) = self.into_parts(); + SealedSegment::admitted(map(stage), record_count, segment_length, digest) + } + pub(super) fn into_parts(self) -> (S, u32, u64, SegmentDigest) { let Self { _stage: stage, diff --git a/src/lib.rs b/src/lib.rs index fc0ccfd..3d3fce8 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -35,6 +35,9 @@ mod layout; mod profile; mod reference; +#[cfg(feature = "repository-tasks")] +#[doc(hidden)] +pub use adapters::RepositoryInitializationStorage; pub use adapters::{ AdmittedCatalog, AdmittedRecoveryStageBytes, AdmittedSegment, AdmittedSegmentRecord, BlobIdBinaryParseError, BlobIdTextParseError, CanonicalCatalog, CanonicalLayoutRecord, diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index 8c88380..a108967 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -15,6 +15,7 @@ repository-tasks = [ "dep:cap-fs-ext", "dep:cap-std", "dep:keep", + "keep/repository-tasks", "dep:md-5", "dep:repository-process-spawn", "dep:rustix", diff --git a/xtask/src/durability_crash_matrix.rs b/xtask/src/durability_crash_matrix.rs index 7242eba..aca8546 100644 --- a/xtask/src/durability_crash_matrix.rs +++ b/xtask/src/durability_crash_matrix.rs @@ -3,8 +3,8 @@ mod child; mod error; mod process; +mod production_protocol; mod restart; -mod state; use std::ffi::{OsStr, OsString}; use std::path::Path; diff --git a/xtask/src/durability_crash_matrix/child.rs b/xtask/src/durability_crash_matrix/child.rs index 6d72dbd..4c66485 100644 --- a/xtask/src/durability_crash_matrix/child.rs +++ b/xtask/src/durability_crash_matrix/child.rs @@ -6,24 +6,18 @@ use std::os::unix::net::UnixStream; use std::path::Path; use super::DurabilityCrashMatrixError; -use super::state; +use super::production_protocol; use xtask::DurabilityCrashCase; -const READY: u8 = b'r'; - pub(super) fn run( case: DurabilityCrashCase, case_root: &Path, readiness_socket: &Path, ) -> Result<(), DurabilityCrashMatrixError> { - let prepared = state::prepare(case, case_root)?; write_marker(case, case_root)?; - let mut stream = UnixStream::connect(readiness_socket) + let stream = UnixStream::connect(readiness_socket) .map_err(|source| DurabilityCrashMatrixError::io("connect readiness socket", source))?; - stream - .write_all(&[READY]) - .map_err(|source| DurabilityCrashMatrixError::io("signal crash readiness", source))?; - prepared.await_process_death(&mut stream) + production_protocol::run(case, case_root, stream) } fn write_marker( diff --git a/xtask/src/durability_crash_matrix/error.rs b/xtask/src/durability_crash_matrix/error.rs index 60f40e8..d0a7ec4 100644 --- a/xtask/src/durability_crash_matrix/error.rs +++ b/xtask/src/durability_crash_matrix/error.rs @@ -7,7 +7,6 @@ use std::error::Error; use std::io; use std::time::Duration; -use keep::WriterLockAcquireError; use xtask::protocol_admission::HexError; use xtask::{ DurabilityCrashCase, DurabilityCrashCaseError, DurabilityCrashPoint, DurabilityCrashPosition, @@ -72,7 +71,6 @@ pub(crate) enum DurabilityCrashMatrixError { target_device: u64, target_inode: u64, }, - MissingActiveFile, MissingVisibleRecord { record: &'static str, }, @@ -102,7 +100,6 @@ pub(crate) enum DurabilityCrashMatrixError { phase: &'static str, source: Box, }, - WriterLock(WriterLockAcquireError), } impl DurabilityCrashMatrixError { @@ -143,7 +140,6 @@ impl Error for DurabilityCrashMatrixError { Self::InvalidCase(error) => Some(error), Self::Io { source, .. } => Some(source), Self::Verification { source, .. } => Some(source.as_ref()), - Self::WriterLock(source) => Some(source), Self::ArtifactBytesMismatch { .. } | Self::ArtifactClassificationMismatch { .. } | Self::ChildExitedEarly { .. } @@ -156,7 +152,6 @@ impl Error for DurabilityCrashMatrixError { | Self::InvalidReadinessSignal { .. } | Self::InventoryMismatch { .. } | Self::HardLinkIdentityMismatch { .. } - | Self::MissingActiveFile | Self::MissingVisibleRecord { .. } | Self::NonUnicodeStatePath | Self::PointSequenceMismatch { .. } diff --git a/xtask/src/durability_crash_matrix/error/display.rs b/xtask/src/durability_crash_matrix/error/display.rs index 34a06d0..b9a215d 100644 --- a/xtask/src/durability_crash_matrix/error/display.rs +++ b/xtask/src/durability_crash_matrix/error/display.rs @@ -37,11 +37,9 @@ impl fmt::Display for DurabilityCrashMatrixError { | Self::UnknownPosition(_) | Self::Usage => format_command(self, formatter), Self::Io { .. } - | Self::MissingActiveFile | Self::NonUnicodeStatePath | Self::PointSequenceMismatch { .. } - | Self::Verification { .. } - | Self::WriterLock(_) => format_boundary(self, formatter), + | Self::Verification { .. } => format_boundary(self, formatter), } } } @@ -220,9 +218,6 @@ fn format_boundary( ) -> fmt::Result { match error { DurabilityCrashMatrixError::Io { action, .. } => write!(formatter, "cannot {action}"), - DurabilityCrashMatrixError::MissingActiveFile => { - formatter.write_str("crash sequence has no active staged artifact") - } DurabilityCrashMatrixError::NonUnicodeStatePath => { formatter.write_str("post-crash store path is not valid Unicode") } @@ -235,9 +230,6 @@ fn format_boundary( formatter, "post-crash verification failed while attempting to {phase}: {source}" ), - DurabilityCrashMatrixError::WriterLock(source) => { - write!(formatter, "cannot acquire crash-case writer lock: {source}") - } _ => Err(fmt::Error), } } diff --git a/xtask/src/durability_crash_matrix/production_protocol.rs b/xtask/src/durability_crash_matrix/production_protocol.rs new file mode 100644 index 0000000..6453a94 --- /dev/null +++ b/xtask/src/durability_crash_matrix/production_protocol.rs @@ -0,0 +1,65 @@ +//! This module owns real production protocol execution for crash children. + +mod control; +pub(super) mod fixture; +mod initialization; +mod initialization_storage; +mod publication; +mod publication_storage; +mod recovery; +mod recovery_storage; +mod segment_stage; + +use std::error::Error; +use std::fs; +use std::os::unix::net::UnixStream; +use std::path::{Path, PathBuf}; + +use control::CrashControl; +use xtask::{DurabilityCrashCase, DurabilityCrashSequence}; + +use super::DurabilityCrashMatrixError; + +const STORE_DIRECTORY: &str = "store"; + +pub(super) fn run( + case: DurabilityCrashCase, + case_root: &Path, + readiness: UnixStream, +) -> Result<(), DurabilityCrashMatrixError> { + let store_root = create_store_root(case_root)?; + let mut control = CrashControl::new(case, readiness); + match case.point().sequence() { + DurabilityCrashSequence::Initialization => { + initialization::run(&store_root, &mut control)?; + } + DurabilityCrashSequence::Segment + | DurabilityCrashSequence::Catalog + | DurabilityCrashSequence::Head => { + publication::run(&store_root, &mut control)?; + } + DurabilityCrashSequence::RecoveryDiscard => { + recovery::run(&store_root, &mut control)?; + } + } + Err(DurabilityCrashMatrixError::PointSequenceMismatch { + point: case.point(), + }) +} + +fn create_store_root(case_root: &Path) -> Result { + let store_root = case_root.join(STORE_DIRECTORY); + fs::create_dir(&store_root) + .map_err(|source| DurabilityCrashMatrixError::io("create crash store root", source))?; + Ok(store_root) +} + +pub(super) fn verification( + phase: &'static str, + source: impl Error + 'static, +) -> DurabilityCrashMatrixError { + DurabilityCrashMatrixError::Verification { + phase, + source: Box::new(source), + } +} diff --git a/xtask/src/durability_crash_matrix/production_protocol/control.rs b/xtask/src/durability_crash_matrix/production_protocol/control.rs new file mode 100644 index 0000000..9d96a91 --- /dev/null +++ b/xtask/src/durability_crash_matrix/production_protocol/control.rs @@ -0,0 +1,66 @@ +//! This module owns the process-death gate at production protocol boundaries. + +use std::io::{self, Read, Write}; +use std::os::unix::net::UnixStream; + +use xtask::{DurabilityCrashCase, DurabilityCrashPoint, DurabilityCrashPosition}; + +const READY: u8 = b'r'; + +#[derive(Clone, Copy, Eq, PartialEq)] +pub(super) enum DuringTiming { + Before, + After, +} + +pub(super) struct CrashControl { + case: DurabilityCrashCase, + readiness: UnixStream, +} + +impl CrashControl { + pub(super) const fn new(case: DurabilityCrashCase, readiness: UnixStream) -> Self { + Self { case, readiness } + } + + pub(super) fn before( + &mut self, + point: DurabilityCrashPoint, + during: DuringTiming, + ) -> io::Result<()> { + let position = self.position(point); + if position == Some(DurabilityCrashPosition::Before) + || position == Some(DurabilityCrashPosition::During) && during == DuringTiming::Before + { + return self.await_process_death(); + } + Ok(()) + } + + pub(super) fn after( + &mut self, + point: DurabilityCrashPoint, + during: DuringTiming, + ) -> io::Result<()> { + let position = self.position(point); + if position == Some(DurabilityCrashPosition::After) + || position == Some(DurabilityCrashPosition::During) && during == DuringTiming::After + { + return self.await_process_death(); + } + Ok(()) + } + + pub(super) fn position(&self, point: DurabilityCrashPoint) -> Option { + (self.case.point() == point).then(|| self.case.position()) + } + + pub(super) fn await_process_death(&mut self) -> io::Result<()> { + self.readiness.write_all(&[READY])?; + let mut unexpected = [0_u8; 1]; + self.readiness.read_exact(&mut unexpected)?; + Err(io::Error::other( + "crash controller resumed a child selected for process death", + )) + } +} diff --git a/xtask/src/durability_crash_matrix/state/fixture.rs b/xtask/src/durability_crash_matrix/production_protocol/fixture.rs similarity index 97% rename from xtask/src/durability_crash_matrix/state/fixture.rs rename to xtask/src/durability_crash_matrix/production_protocol/fixture.rs index 2ab5324..728f7dd 100644 --- a/xtask/src/durability_crash_matrix/state/fixture.rs +++ b/xtask/src/durability_crash_matrix/production_protocol/fixture.rs @@ -1,4 +1,4 @@ -//! This module owns admitted Golden File Worldline crash fixtures. +//! This module owns admitted Golden File Worldline production crash fixtures. use std::ops::Range; diff --git a/xtask/src/durability_crash_matrix/production_protocol/initialization.rs b/xtask/src/durability_crash_matrix/production_protocol/initialization.rs new file mode 100644 index 0000000..6bd5466 --- /dev/null +++ b/xtask/src/durability_crash_matrix/production_protocol/initialization.rs @@ -0,0 +1,50 @@ +//! This module owns execution of the production initialization protocol. + +use std::path::Path; + +use keep::{ + CatalogRestartByteLimit, CatalogRestartPolicy, FilesystemCatalogPublisher, LayoutEntryLimit, + RepositoryInitializationStorage, SegmentReadPolicy, SegmentRecordLimit, initialize_store, +}; + +use super::control::CrashControl; +use super::initialization_storage::CrashInitializationStorage; +use super::{DurabilityCrashMatrixError, verification}; + +const RESTART_BYTE_LIMIT: u64 = 1_048_576; + +pub(super) fn run( + store_root: &Path, + control: &mut CrashControl, +) -> Result<(), DurabilityCrashMatrixError> { + let storage = RepositoryInitializationStorage::admit_unchecked(store_root) + .map_err(|source| DurabilityCrashMatrixError::io("open initialization storage", source))?; + let mut storage = CrashInitializationStorage::new(storage, control); + initialize_store(&mut storage) + .map(|_receipt| ()) + .map_err(|source| verification("execute production store initialization", source)) +} + +pub(super) fn publisher( + store_root: &Path, +) -> Result { + let mut storage = RepositoryInitializationStorage::admit_unchecked(store_root) + .map_err(|source| DurabilityCrashMatrixError::io("open initialization storage", source))?; + let _receipt = initialize_store(&mut storage) + .map_err(|source| verification("initialize production crash store", source))?; + let lock = storage.into_writer_lock().map_err(|source| { + DurabilityCrashMatrixError::io("retain initialized writer lock", source) + })?; + FilesystemCatalogPublisher::open_unchecked_for_repository_tasks(lock, restart_policy()?) + .map_err(|source| DurabilityCrashMatrixError::io("open crash catalog publisher", source)) +} + +pub(super) fn restart_policy() -> Result { + let byte_limit = CatalogRestartByteLimit::new(RESTART_BYTE_LIMIT) + .map_err(|source| verification("construct crash restart byte limit", source))?; + Ok(CatalogRestartPolicy::new(segment_policy(), byte_limit)) +} + +pub(super) const fn segment_policy() -> SegmentReadPolicy { + SegmentReadPolicy::new(SegmentRecordLimit::MAXIMUM, LayoutEntryLimit::MAXIMUM) +} diff --git a/xtask/src/durability_crash_matrix/production_protocol/initialization_storage.rs b/xtask/src/durability_crash_matrix/production_protocol/initialization_storage.rs new file mode 100644 index 0000000..4967fa0 --- /dev/null +++ b/xtask/src/durability_crash_matrix/production_protocol/initialization_storage.rs @@ -0,0 +1,63 @@ +//! This module owns crash injection around production store initialization. + +use std::io; + +use keep::{RepositoryInitializationStorage, StoreInitializationStorage}; +use xtask::DurabilityCrashPoint; + +use super::control::{CrashControl, DuringTiming}; + +pub(super) struct CrashInitializationStorage<'control> { + inner: RepositoryInitializationStorage, + control: &'control mut CrashControl, +} + +impl<'control> CrashInitializationStorage<'control> { + pub(super) const fn new( + inner: RepositoryInitializationStorage, + control: &'control mut CrashControl, + ) -> Self { + Self { inner, control } + } +} + +impl StoreInitializationStorage for CrashInitializationStorage<'_> { + fn admit_platform(&mut self) -> io::Result<()> { + self.inner.admit_platform() + } + + fn open_and_lock_writer_file(&mut self) -> io::Result<()> { + let point = DurabilityCrashPoint::OpenAndLockWriterFile; + self.control.before(point, DuringTiming::After)?; + self.inner.open_and_lock_writer_file()?; + self.control.after(point, DuringTiming::After) + } + + fn admit_staging_directory(&mut self) -> io::Result<()> { + let point = DurabilityCrashPoint::CreateStagingDirectory; + self.control.before(point, DuringTiming::After)?; + self.inner.admit_staging_directory()?; + self.control.after(point, DuringTiming::After) + } + + fn admit_segment_pool_directory(&mut self) -> io::Result<()> { + let point = DurabilityCrashPoint::CreateSegmentPoolDirectory; + self.control.before(point, DuringTiming::After)?; + self.inner.admit_segment_pool_directory()?; + self.control.after(point, DuringTiming::After) + } + + fn admit_catalog_pool_directory(&mut self) -> io::Result<()> { + let point = DurabilityCrashPoint::CreateCatalogPoolDirectory; + self.control.before(point, DuringTiming::After)?; + self.inner.admit_catalog_pool_directory()?; + self.control.after(point, DuringTiming::After) + } + + fn synchronize_root(&mut self) -> io::Result<()> { + let point = DurabilityCrashPoint::SynchronizeRootAfterInitialization; + self.control.before(point, DuringTiming::Before)?; + self.inner.synchronize_root()?; + self.control.after(point, DuringTiming::Before) + } +} diff --git a/xtask/src/durability_crash_matrix/production_protocol/publication.rs b/xtask/src/durability_crash_matrix/production_protocol/publication.rs new file mode 100644 index 0000000..0f2a4f6 --- /dev/null +++ b/xtask/src/durability_crash_matrix/production_protocol/publication.rs @@ -0,0 +1,71 @@ +//! This module owns execution of the production publication protocols. + +use std::fs; +use std::path::Path; + +use keep::{ + AdmittedSegment, AdmittedSegmentRecord, CanonicalCatalog, CatalogGeneration, + CatalogPublicationExpectation, SegmentRecordLimit, StagedSegment, publish_catalog_generation, +}; +use xtask::DurabilityCrashPoint; + +use super::control::{CrashControl, DuringTiming}; +use super::initialization; +use super::publication_storage::CrashPublicationStorage; +use super::segment_stage::CrashSegmentStage; +use super::{DurabilityCrashMatrixError, verification}; + +pub(super) fn run( + store_root: &Path, + control: &mut CrashControl, +) -> Result<(), DurabilityCrashMatrixError> { + let publisher = initialization::publisher(store_root)?; + let point = DurabilityCrashPoint::CreateSegmentStage; + control + .before(point, DuringTiming::After) + .map_err(crash_gate)?; + let stage = publisher + .create_segment_stage() + .map_err(|source| verification("create production segment stage", source))?; + control + .after(point, DuringTiming::After) + .map_err(crash_gate)?; + + let stage = CrashSegmentStage::new(stage, control); + let record = AdmittedSegmentRecord::for_chunk(&[0]) + .map_err(|source| verification("admit crash segment record", source))?; + let sealed = StagedSegment::begin(stage, SegmentRecordLimit::MAXIMUM) + .map_err(|source| verification("write production segment header", source))? + .append(record) + .map_err(|source| verification("append production segment record", source))? + .seal() + .map_err(|source| verification("seal production segment", source))? + .map_stage(CrashSegmentStage::into_inner); + + let segment_bytes = fs::read(store_root.join("staging/current.seg")) + .map_err(|source| DurabilityCrashMatrixError::io("read sealed crash segment", source))?; + let segment = AdmittedSegment::decode(&segment_bytes, initialization::segment_policy()) + .map_err(|source| verification("admit sealed crash segment", source))?; + let segments = [segment]; + let generation = CatalogGeneration::new(1) + .map_err(|source| verification("construct crash catalog generation", source))?; + let catalog = CanonicalCatalog::from_segments(generation, None, &segments) + .map_err(|source| verification("encode crash catalog", source))?; + let selection = publisher + .select_segment(sealed, &segments[0]) + .map_err(|source| verification("select production segment stage", source))?; + let mut storage = CrashPublicationStorage::new(publisher, control); + publish_catalog_generation( + &mut storage, + CatalogPublicationExpectation::uninitialized(), + selection, + &catalog, + &segments, + ) + .map(|_receipt| ()) + .map_err(|source| verification("execute production catalog publication", source)) +} + +const fn crash_gate(source: std::io::Error) -> DurabilityCrashMatrixError { + DurabilityCrashMatrixError::io("await production crash boundary", source) +} diff --git a/xtask/src/durability_crash_matrix/production_protocol/publication_storage.rs b/xtask/src/durability_crash_matrix/production_protocol/publication_storage.rs new file mode 100644 index 0000000..5520644 --- /dev/null +++ b/xtask/src/durability_crash_matrix/production_protocol/publication_storage.rs @@ -0,0 +1,270 @@ +//! This module owns crash injection around production catalog publication. + +use std::io; + +use keep::{ + AdmittedSegment, CanonicalCatalog, CanonicalPublicationHead, CatalogPublicationExpectation, + CatalogPublicationReadiness, CatalogPublicationStorage, CatalogSnapshot, ChecksummedCatalog, + FilesystemCatalogPublisher, SegmentPublication, +}; +use xtask::{DurabilityCrashPoint, DurabilityCrashPosition}; + +use super::control::{CrashControl, DuringTiming}; + +const CATALOG_INTERRUPTION: usize = 176; +const HEAD_INTERRUPTION: usize = 64; + +pub(super) struct CrashPublicationStorage<'control> { + inner: FilesystemCatalogPublisher, + control: &'control mut CrashControl, +} + +impl<'control> CrashPublicationStorage<'control> { + pub(super) const fn new( + inner: FilesystemCatalogPublisher, + control: &'control mut CrashControl, + ) -> Self { + Self { inner, control } + } +} + +impl CatalogPublicationStorage for CrashPublicationStorage<'_> { + fn verify_current( + &mut self, + expected: CatalogPublicationExpectation, + candidate: &CatalogSnapshot<'_, '_, '_>, + segment: &SegmentPublication<'_, '_>, + ) -> io::Result { + self.inner.verify_current(expected, candidate, segment) + } + + fn link_segment(&mut self, segment: &AdmittedSegment<'_>) -> io::Result<()> { + execute( + &mut self.inner, + self.control, + DurabilityCrashPoint::LinkSegment, + DuringTiming::After, + |inner| inner.link_segment(segment), + ) + } + + fn verify_segment_pool(&mut self, segment: &AdmittedSegment<'_>) -> io::Result<()> { + self.inner.verify_segment_pool(segment) + } + + fn synchronize_segments(&mut self) -> io::Result<()> { + execute( + &mut self.inner, + self.control, + DurabilityCrashPoint::SynchronizeSegmentPool, + DuringTiming::Before, + FilesystemCatalogPublisher::synchronize_segments, + ) + } + + fn remove_segment_stage(&mut self) -> io::Result<()> { + execute( + &mut self.inner, + self.control, + DurabilityCrashPoint::RemoveSegmentStage, + DuringTiming::After, + FilesystemCatalogPublisher::remove_segment_stage, + ) + } + + fn synchronize_staging_after_segment(&mut self) -> io::Result<()> { + execute( + &mut self.inner, + self.control, + DurabilityCrashPoint::SynchronizeStagingAfterSegment, + DuringTiming::Before, + FilesystemCatalogPublisher::synchronize_staging_after_segment, + ) + } + + fn create_catalog_stage(&mut self) -> io::Result<()> { + execute( + &mut self.inner, + self.control, + DurabilityCrashPoint::CreateCatalogStage, + DuringTiming::After, + FilesystemCatalogPublisher::create_catalog_stage, + ) + } + + fn write_catalog(&mut self, catalog: &CanonicalCatalog) -> io::Result<()> { + execute_write( + &mut self.inner, + self.control, + DurabilityCrashPoint::WriteCatalog, + |inner| inner.write_catalog(catalog), + |inner| inner.write_catalog_prefix_for_repository_tasks(catalog, CATALOG_INTERRUPTION), + ) + } + + fn flush_catalog(&mut self) -> io::Result<()> { + execute( + &mut self.inner, + self.control, + DurabilityCrashPoint::FlushCatalog, + DuringTiming::Before, + FilesystemCatalogPublisher::flush_catalog, + ) + } + + fn synchronize_catalog(&mut self) -> io::Result<()> { + execute( + &mut self.inner, + self.control, + DurabilityCrashPoint::SynchronizeCatalog, + DuringTiming::Before, + FilesystemCatalogPublisher::synchronize_catalog, + ) + } + + fn link_catalog(&mut self, catalog: ChecksummedCatalog<'_>) -> io::Result<()> { + execute( + &mut self.inner, + self.control, + DurabilityCrashPoint::LinkCatalog, + DuringTiming::After, + |inner| inner.link_catalog(catalog), + ) + } + + fn verify_catalog_pool(&mut self, catalog: ChecksummedCatalog<'_>) -> io::Result<()> { + self.inner.verify_catalog_pool(catalog) + } + + fn synchronize_catalogs(&mut self) -> io::Result<()> { + execute( + &mut self.inner, + self.control, + DurabilityCrashPoint::SynchronizeCatalogPool, + DuringTiming::Before, + FilesystemCatalogPublisher::synchronize_catalogs, + ) + } + + fn remove_catalog_stage(&mut self) -> io::Result<()> { + execute( + &mut self.inner, + self.control, + DurabilityCrashPoint::RemoveCatalogStage, + DuringTiming::After, + FilesystemCatalogPublisher::remove_catalog_stage, + ) + } + + fn synchronize_staging_after_catalog(&mut self) -> io::Result<()> { + execute( + &mut self.inner, + self.control, + DurabilityCrashPoint::SynchronizeStagingAfterCatalog, + DuringTiming::Before, + FilesystemCatalogPublisher::synchronize_staging_after_catalog, + ) + } + + fn create_head_stage(&mut self) -> io::Result<()> { + execute( + &mut self.inner, + self.control, + DurabilityCrashPoint::CreateHeadStage, + DuringTiming::After, + FilesystemCatalogPublisher::create_head_stage, + ) + } + + fn write_head(&mut self, head: &CanonicalPublicationHead) -> io::Result<()> { + execute_write( + &mut self.inner, + self.control, + DurabilityCrashPoint::WriteHead, + |inner| inner.write_head(head), + |inner| inner.write_head_prefix_for_repository_tasks(head, HEAD_INTERRUPTION), + ) + } + + fn flush_head(&mut self) -> io::Result<()> { + execute( + &mut self.inner, + self.control, + DurabilityCrashPoint::FlushHead, + DuringTiming::Before, + FilesystemCatalogPublisher::flush_head, + ) + } + + fn synchronize_head(&mut self) -> io::Result<()> { + execute( + &mut self.inner, + self.control, + DurabilityCrashPoint::SynchronizeHead, + DuringTiming::Before, + FilesystemCatalogPublisher::synchronize_head, + ) + } + + fn verify_head_view( + &mut self, + head: &CanonicalPublicationHead, + snapshot: &CatalogSnapshot<'_, '_, '_>, + ) -> io::Result<()> { + self.inner.verify_head_view(head, snapshot) + } + + fn replace_head(&mut self) -> io::Result<()> { + execute( + &mut self.inner, + self.control, + DurabilityCrashPoint::ReplaceHead, + DuringTiming::After, + FilesystemCatalogPublisher::replace_head, + ) + } + + fn synchronize_root(&mut self) -> io::Result<()> { + execute( + &mut self.inner, + self.control, + DurabilityCrashPoint::SynchronizeRootAfterHead, + DuringTiming::Before, + FilesystemCatalogPublisher::synchronize_root, + ) + } +} + +fn execute( + inner: &mut FilesystemCatalogPublisher, + control: &mut CrashControl, + point: DurabilityCrashPoint, + during: DuringTiming, + operation: impl FnOnce(&mut FilesystemCatalogPublisher) -> io::Result, +) -> io::Result { + control.before(point, during)?; + let result = operation(inner)?; + control.after(point, during)?; + Ok(result) +} + +fn execute_write( + inner: &mut FilesystemCatalogPublisher, + control: &mut CrashControl, + point: DurabilityCrashPoint, + complete: impl FnOnce(&mut FilesystemCatalogPublisher) -> io::Result<()>, + interrupted: impl FnOnce(&mut FilesystemCatalogPublisher) -> io::Result<()>, +) -> io::Result<()> { + match control.position(point) { + None => complete(inner), + Some(DurabilityCrashPosition::Before) => control.await_process_death(), + Some(DurabilityCrashPosition::During) => { + interrupted(inner)?; + control.await_process_death() + } + Some(DurabilityCrashPosition::After) => { + complete(inner)?; + control.await_process_death() + } + } +} diff --git a/xtask/src/durability_crash_matrix/production_protocol/recovery.rs b/xtask/src/durability_crash_matrix/production_protocol/recovery.rs new file mode 100644 index 0000000..c87ba5d --- /dev/null +++ b/xtask/src/durability_crash_matrix/production_protocol/recovery.rs @@ -0,0 +1,115 @@ +//! This module owns execution of the production recovery-discard protocol. + +use std::fs; +use std::io::Write; +use std::path::Path; + +use keep::{ + AdmittedSegment, CanonicalCatalog, CanonicalPublicationHead, CatalogGeneration, + CatalogPublicationStorage, FilesystemRecoveryStageDiscarder, RecoveryStage, + RecoveryStageDiscardRequest, RecoveryStageMetadata, admit_recovery_stage_bytes, + assess_recovery_stage, execute_recovery_stage_discard, fingerprint_recovery_stage, + plan_recovery_stage_discard, +}; +use xtask::DurabilityCrashPoint; + +use super::control::CrashControl; +use super::fixture::GoldenFixture; +use super::initialization; +use super::recovery_storage::CrashRecoveryStorage; +use super::{DurabilityCrashMatrixError, verification}; + +const SEGMENT_INTERRUPTION: usize = 32; +const HEAD_INTERRUPTION: usize = 64; + +pub(super) fn run( + store_root: &Path, + control: &mut CrashControl, +) -> Result<(), DurabilityCrashMatrixError> { + let stage = if targets_segment(control) { + prepare_segment(store_root)? + } else { + prepare_head(store_root)? + }; + let request = discard_request(store_root, stage)?; + let storage = FilesystemRecoveryStageDiscarder::open_unchecked_for_repository_tasks(store_root) + .map_err(|source| verification("open production recovery discarder", source))?; + let mut storage = CrashRecoveryStorage::new(storage, control); + execute_recovery_stage_discard(&mut storage, request) + .map(|_receipt| ()) + .map_err(|source| verification("execute production recovery discard", source)) +} + +fn targets_segment(control: &CrashControl) -> bool { + control + .position(DurabilityCrashPoint::RemoveRecoveryStage) + .is_some() + || control + .position(DurabilityCrashPoint::SynchronizeStagingAfterRecovery) + .is_some() +} + +fn prepare_segment(store_root: &Path) -> Result { + let publisher = initialization::publisher(store_root)?; + let mut stage = publisher + .create_segment_stage() + .map_err(|source| verification("create recovery segment precondition", source))?; + let fixture = GoldenFixture::segment()?; + stage + .write_all(fixture.prefix(SEGMENT_INTERRUPTION)?) + .map_err(|source| { + DurabilityCrashMatrixError::io("write recovery segment precondition", source) + })?; + drop(stage); + drop(publisher); + Ok(RecoveryStage::Segment) +} + +fn prepare_head(store_root: &Path) -> Result { + let mut publisher = initialization::publisher(store_root)?; + let segment_fixture = GoldenFixture::segment()?; + let segment = + AdmittedSegment::decode(segment_fixture.bytes(), initialization::segment_policy()) + .map_err(|source| verification("admit recovery head segment", source))?; + let segments = [segment]; + let generation = CatalogGeneration::new(1) + .map_err(|source| verification("construct recovery head generation", source))?; + let catalog = CanonicalCatalog::from_segments(generation, None, &segments) + .map_err(|source| verification("encode recovery head catalog", source))?; + let head = CanonicalPublicationHead::for_catalog(catalog.checksummed()); + publisher.create_head_stage().map_err(|source| { + DurabilityCrashMatrixError::io("create recovery head precondition", source) + })?; + publisher + .write_head_prefix_for_repository_tasks(&head, HEAD_INTERRUPTION) + .map_err(|source| { + DurabilityCrashMatrixError::io("write recovery head precondition", source) + })?; + drop(publisher); + Ok(RecoveryStage::NextHead) +} + +fn discard_request( + store_root: &Path, + stage: RecoveryStage, +) -> Result { + let path = match stage { + RecoveryStage::Segment => store_root.join("staging/current.seg"), + RecoveryStage::Catalog => store_root.join("staging/current.cat"), + RecoveryStage::NextHead => store_root.join("head.next"), + }; + let bytes = fs::read(path) + .map_err(|source| DurabilityCrashMatrixError::io("read recovery precondition", source))?; + let length = u64::try_from(bytes.len()) + .map_err(|source| verification("convert recovery precondition length", source))?; + let metadata = RecoveryStageMetadata::new(stage, length) + .map_err(|source| verification("admit recovery precondition metadata", source))?; + let evidence = fingerprint_recovery_stage(metadata, bytes.as_slice()) + .map_err(|source| verification("fingerprint recovery precondition", source))?; + let admitted = admit_recovery_stage_bytes(stage, evidence, &bytes) + .map_err(|source| verification("admit recovery precondition bytes", source))?; + let assessment = assess_recovery_stage(&admitted, initialization::segment_policy()) + .map_err(|source| verification("assess recovery precondition", source))?; + plan_recovery_stage_discard(&assessment) + .map_err(|source| verification("plan production recovery discard", source)) +} diff --git a/xtask/src/durability_crash_matrix/production_protocol/recovery_storage.rs b/xtask/src/durability_crash_matrix/production_protocol/recovery_storage.rs new file mode 100644 index 0000000..8e79282 --- /dev/null +++ b/xtask/src/durability_crash_matrix/production_protocol/recovery_storage.rs @@ -0,0 +1,62 @@ +//! This module owns crash injection around production recovery discard. + +use std::io; + +use keep::{ + FilesystemRecoveryStageDiscarder, RecoveryStage, RecoveryStageDiscardOutcome, + RecoveryStageDiscardStorage, RecoveryStageDiscardStorageError, RecoveryStageEvidence, + RecoveryStageParent, +}; +use xtask::DurabilityCrashPoint; + +use super::control::{CrashControl, DuringTiming}; + +pub(super) struct CrashRecoveryStorage<'control> { + inner: FilesystemRecoveryStageDiscarder, + control: &'control mut CrashControl, +} + +impl<'control> CrashRecoveryStorage<'control> { + pub(super) const fn new( + inner: FilesystemRecoveryStageDiscarder, + control: &'control mut CrashControl, + ) -> Self { + Self { inner, control } + } +} + +impl RecoveryStageDiscardStorage for CrashRecoveryStorage<'_> { + fn remove_if_matching( + &mut self, + expected: RecoveryStageEvidence, + ) -> Result { + let point = match expected.stage() { + RecoveryStage::NextHead => DurabilityCrashPoint::RemoveRecoveryHead, + RecoveryStage::Segment | RecoveryStage::Catalog => { + DurabilityCrashPoint::RemoveRecoveryStage + } + }; + self.control + .before(point, DuringTiming::After) + .map_err(storage_error)?; + let outcome = self.inner.remove_if_matching(expected)?; + self.control + .after(point, DuringTiming::After) + .map_err(storage_error)?; + Ok(outcome) + } + + fn synchronize_parent(&mut self, parent: RecoveryStageParent) -> io::Result<()> { + let point = match parent { + RecoveryStageParent::Staging => DurabilityCrashPoint::SynchronizeStagingAfterRecovery, + RecoveryStageParent::Root => DurabilityCrashPoint::SynchronizeRootAfterRecovery, + }; + self.control.before(point, DuringTiming::Before)?; + self.inner.synchronize_parent(parent)?; + self.control.after(point, DuringTiming::Before) + } +} + +const fn storage_error(source: io::Error) -> RecoveryStageDiscardStorageError { + RecoveryStageDiscardStorageError::Storage { source } +} diff --git a/xtask/src/durability_crash_matrix/production_protocol/segment_stage.rs b/xtask/src/durability_crash_matrix/production_protocol/segment_stage.rs new file mode 100644 index 0000000..68cf9d9 --- /dev/null +++ b/xtask/src/durability_crash_matrix/production_protocol/segment_stage.rs @@ -0,0 +1,133 @@ +//! This module owns crash injection around the production segment-stage port. + +use std::io::{self, Write}; + +use keep::SegmentStage; +use xtask::{DurabilityCrashPoint, DurabilityCrashPosition}; + +use super::control::{CrashControl, DuringTiming}; + +const HEADER_END: usize = 64; +const HEADER_INTERRUPTION: usize = 32; +const RECORD_END: usize = 209; +const RECORD_INTERRUPTION: usize = 136; +const SEAL_END: usize = 337; +const SEAL_INTERRUPTION: usize = 273; + +pub(super) struct CrashSegmentStage<'control, S> { + inner: S, + control: &'control mut CrashControl, + bytes_written: usize, +} + +impl<'control, S> CrashSegmentStage<'control, S> { + pub(super) const fn new(inner: S, control: &'control mut CrashControl) -> Self { + Self { + inner, + control, + bytes_written: 0, + } + } + + pub(super) fn into_inner(self) -> S { + self.inner + } + + fn write_boundary(&self) -> io::Result<(DurabilityCrashPoint, usize, usize)> { + match self.bytes_written { + 0..HEADER_END => Ok(( + DurabilityCrashPoint::WriteSegmentHeader, + HEADER_INTERRUPTION, + HEADER_END, + )), + HEADER_END..RECORD_END => Ok(( + DurabilityCrashPoint::AppendSegmentRecord, + RECORD_INTERRUPTION, + RECORD_END, + )), + RECORD_END..SEAL_END => Ok(( + DurabilityCrashPoint::AppendSegmentSeal, + SEAL_INTERRUPTION, + SEAL_END, + )), + _ => Err(io::Error::other( + "segment stage wrote beyond the canonical crash fixture", + )), + } + } + + fn durability_point( + &self, + prefix: DurabilityCrashPoint, + sealed: DurabilityCrashPoint, + ) -> io::Result { + match self.bytes_written { + RECORD_END => Ok(prefix), + SEAL_END => Ok(sealed), + _ => Err(io::Error::other( + "segment durability operation occurred at an unknown length", + )), + } + } +} + +impl Write for CrashSegmentStage<'_, S> +where + S: SegmentStage, +{ + fn write(&mut self, bytes: &[u8]) -> io::Result { + let (point, interruption, end) = self.write_boundary()?; + let position = self.control.position(point); + if position == Some(DurabilityCrashPosition::Before) { + self.control.await_process_death()?; + } + let limit = if position == Some(DurabilityCrashPosition::During) { + interruption + } else { + end + }; + let remaining = limit + .checked_sub(self.bytes_written) + .ok_or_else(|| io::Error::other("segment crash boundary moved backward"))?; + let allowed = remaining.min(bytes.len()); + let prefix = bytes + .get(..allowed) + .ok_or_else(|| io::Error::other("segment write prefix exceeded input"))?; + let written = self.inner.write(prefix)?; + self.bytes_written = self + .bytes_written + .checked_add(written) + .ok_or_else(|| io::Error::other("segment write count overflowed"))?; + if position == Some(DurabilityCrashPosition::During) && self.bytes_written == interruption + || position == Some(DurabilityCrashPosition::After) && self.bytes_written == end + { + self.control.await_process_death()?; + } + Ok(written) + } + + fn flush(&mut self) -> io::Result<()> { + let point = self.durability_point( + DurabilityCrashPoint::FlushSegmentRecordPrefix, + DurabilityCrashPoint::FlushSealedSegment, + )?; + self.control.before(point, DuringTiming::Before)?; + self.inner.flush()?; + self.control.after(point, DuringTiming::Before) + } +} + +impl SegmentStage for CrashSegmentStage<'_, S> +where + S: SegmentStage, +{ + fn synchronize(&mut self) -> io::Result<()> { + let point = self.durability_point( + DurabilityCrashPoint::SynchronizeSegmentRecordPrefix, + DurabilityCrashPoint::SynchronizeSealedSegment, + )?; + self.control.before(point, DuringTiming::Before)?; + self.inner.synchronize()?; + self.control.after(point, DuringTiming::Before) + } +} diff --git a/xtask/src/durability_crash_matrix/restart.rs b/xtask/src/durability_crash_matrix/restart.rs index d9be317..423efc8 100644 --- a/xtask/src/durability_crash_matrix/restart.rs +++ b/xtask/src/durability_crash_matrix/restart.rs @@ -9,7 +9,7 @@ use std::os::unix::fs::MetadataExt; use std::path::{Path, PathBuf}; use super::DurabilityCrashMatrixError; -use super::state::fixture::GoldenFixture; +use super::production_protocol::fixture::GoldenFixture; use expectation::ExpectedStoreState; use xtask::DurabilityCrashCase; diff --git a/xtask/src/durability_crash_matrix/restart/expectation.rs b/xtask/src/durability_crash_matrix/restart/expectation.rs index aeae457..1b6be39 100644 --- a/xtask/src/durability_crash_matrix/restart/expectation.rs +++ b/xtask/src/durability_crash_matrix/restart/expectation.rs @@ -6,7 +6,7 @@ mod steps; use std::collections::{BTreeMap, BTreeSet}; use super::super::DurabilityCrashMatrixError; -use super::super::state::fixture::GoldenFixture; +use super::super::production_protocol::fixture::GoldenFixture; use xtask::{DurabilityCrashCase, DurabilityCrashSequence}; pub(super) const WRITER_LOCK: &str = "writer.lock"; diff --git a/xtask/src/durability_crash_matrix/restart/expectation/sequence.rs b/xtask/src/durability_crash_matrix/restart/expectation/sequence.rs index 1d0d83c..1679b4f 100644 --- a/xtask/src/durability_crash_matrix/restart/expectation/sequence.rs +++ b/xtask/src/durability_crash_matrix/restart/expectation/sequence.rs @@ -11,7 +11,9 @@ use super::{ SEGMENTS, STAGING, WRITER_LOCK, }; use crate::durability_crash_matrix::DurabilityCrashMatrixError; -use crate::durability_crash_matrix::state::fixture::{CATALOG_POOL_PATH, SEGMENT_POOL_PATH}; +use crate::durability_crash_matrix::production_protocol::fixture::{ + CATALOG_POOL_PATH, SEGMENT_POOL_PATH, +}; use xtask::{DurabilityCrashCase, DurabilityCrashPoint, DurabilityCrashPosition}; pub(super) fn segment( diff --git a/xtask/src/durability_crash_matrix/restart/semantic.rs b/xtask/src/durability_crash_matrix/restart/semantic.rs index 32323c6..8d5238f 100644 --- a/xtask/src/durability_crash_matrix/restart/semantic.rs +++ b/xtask/src/durability_crash_matrix/restart/semantic.rs @@ -16,7 +16,9 @@ use super::expectation::{ ArtifactBytes, CATALOG_STAGE, ExpectedStoreState, HEAD, NEXT_HEAD, SEGMENT_STAGE, WRITER_LOCK, }; use crate::durability_crash_matrix::DurabilityCrashMatrixError; -use crate::durability_crash_matrix::state::fixture::{CATALOG_POOL_PATH, SEGMENT_POOL_PATH}; +use crate::durability_crash_matrix::production_protocol::fixture::{ + CATALOG_POOL_PATH, SEGMENT_POOL_PATH, +}; const RESTART_BYTE_LIMIT: u64 = 1_048_576; diff --git a/xtask/src/durability_crash_matrix/state.rs b/xtask/src/durability_crash_matrix/state.rs deleted file mode 100644 index a0d98f6..0000000 --- a/xtask/src/durability_crash_matrix/state.rs +++ /dev/null @@ -1,202 +0,0 @@ -//! This module owns one crash child's retained filesystem state. - -mod catalog; -pub(super) mod fixture; -mod head; -mod initialization; -mod recovery; -mod segment; - -use std::fs::{self, File, OpenOptions}; -use std::io::{Read, Write}; -use std::ops::Range; -use std::os::unix::net::UnixStream; -use std::path::{Path, PathBuf}; - -use keep::FilesystemWriterLock; -use xtask::{DurabilityCrashCase, DurabilityCrashSequence}; - -use super::DurabilityCrashMatrixError; -use fixture::GoldenFixture; - -const STORE_DIRECTORY: &str = "store"; - -pub(super) struct PreparedCrashState { - state: StoreState, -} - -impl PreparedCrashState { - pub(super) fn await_process_death( - self, - stream: &mut UnixStream, - ) -> Result<(), DurabilityCrashMatrixError> { - let retained_root = &self.state.root; - let mut release = [0_u8; 1]; - let result = stream - .read_exact(&mut release) - .map_err(|source| DurabilityCrashMatrixError::io("await process termination", source)); - let _ = retained_root; - self.state.finish(result) - } -} - -pub(super) fn prepare( - case: DurabilityCrashCase, - case_root: &Path, -) -> Result { - let mut state = StoreState::create(case_root)?; - match case.point().sequence() { - DurabilityCrashSequence::Segment => segment::prepare(&mut state, case)?, - DurabilityCrashSequence::Catalog => catalog::prepare(&mut state, case)?, - DurabilityCrashSequence::Head => head::prepare(&mut state, case)?, - DurabilityCrashSequence::RecoveryDiscard => recovery::prepare(&mut state, case)?, - DurabilityCrashSequence::Initialization => initialization::prepare(&mut state, case)?, - } - Ok(PreparedCrashState { state }) -} - -struct StoreState { - root: PathBuf, - active_file: Option, - writer_lock: Option, -} - -impl StoreState { - fn create(case_root: &Path) -> Result { - let root = case_root.join(STORE_DIRECTORY); - fs::create_dir(&root) - .map_err(|source| DurabilityCrashMatrixError::io("create crash store root", source))?; - Ok(Self { - root, - active_file: None, - writer_lock: None, - }) - } - - fn initialize(&mut self) -> Result<(), DurabilityCrashMatrixError> { - self.create_writer_lock()?; - self.create_directory("staging")?; - self.create_directory("segments")?; - self.create_directory("catalogs")?; - self.acquire_writer_lock() - } - - fn create_writer_lock(&self) -> Result<(), DurabilityCrashMatrixError> { - let file = OpenOptions::new() - .write(true) - .create_new(true) - .open(self.root.join("writer.lock")) - .map_err(|source| DurabilityCrashMatrixError::io("create writer lock", source))?; - file.sync_all() - .map_err(|source| DurabilityCrashMatrixError::io("synchronize writer lock", source)) - } - - fn acquire_writer_lock(&mut self) -> Result<(), DurabilityCrashMatrixError> { - let lock = FilesystemWriterLock::try_acquire(&self.root) - .map_err(DurabilityCrashMatrixError::WriterLock)?; - self.writer_lock = Some(lock); - Ok(()) - } - - fn create_directory(&self, relative: &str) -> Result<(), DurabilityCrashMatrixError> { - fs::create_dir(self.root.join(relative)) - .map_err(|source| DurabilityCrashMatrixError::io("create protocol directory", source)) - } - - fn create_stage(&mut self, relative: &str) -> Result<(), DurabilityCrashMatrixError> { - let file = OpenOptions::new() - .read(true) - .write(true) - .create_new(true) - .open(self.root.join(relative)) - .map_err(|source| DurabilityCrashMatrixError::io("create fixed stage", source))?; - self.active_file = Some(file); - Ok(()) - } - - fn write_range( - &mut self, - fixture: &GoldenFixture, - range: Range, - ) -> Result<(), DurabilityCrashMatrixError> { - let bytes = fixture.range(range)?; - self.active_file()? - .write_all(bytes) - .map_err(|source| DurabilityCrashMatrixError::io("write staged artifact", source)) - } - - fn flush(&mut self) -> Result<(), DurabilityCrashMatrixError> { - self.active_file()? - .flush() - .map_err(|source| DurabilityCrashMatrixError::io("flush staged artifact", source)) - } - - fn synchronize_file(&self) -> Result<(), DurabilityCrashMatrixError> { - self.active_file_ref()? - .sync_all() - .map_err(|source| DurabilityCrashMatrixError::io("synchronize staged artifact", source)) - } - - fn link(&self, source: &str, target: &str) -> Result<(), DurabilityCrashMatrixError> { - fs::hard_link(self.root.join(source), self.root.join(target)) - .map_err(|source| DurabilityCrashMatrixError::io("link immutable artifact", source)) - } - - fn synchronize_directory(&self, relative: &str) -> Result<(), DurabilityCrashMatrixError> { - File::open(self.root.join(relative)) - .and_then(|directory| directory.sync_all()) - .map_err(|source| { - DurabilityCrashMatrixError::io("synchronize protocol directory", source) - }) - } - - fn remove(&self, relative: &str) -> Result<(), DurabilityCrashMatrixError> { - fs::remove_file(self.root.join(relative)) - .map_err(|source| DurabilityCrashMatrixError::io("remove protocol stage", source)) - } - - fn rename(&self, source: &str, target: &str) -> Result<(), DurabilityCrashMatrixError> { - fs::rename(self.root.join(source), self.root.join(target)) - .map_err(|source| DurabilityCrashMatrixError::io("replace publication head", source)) - } - - fn write_immutable( - &self, - relative: &str, - fixture: &GoldenFixture, - ) -> Result<(), DurabilityCrashMatrixError> { - let path = self.root.join(relative); - let mut file = OpenOptions::new() - .write(true) - .create_new(true) - .open(path) - .map_err(|source| { - DurabilityCrashMatrixError::io("create immutable artifact", source) - })?; - file.write_all(fixture.bytes()) - .and_then(|()| file.sync_all()) - .map_err(|source| DurabilityCrashMatrixError::io("write immutable artifact", source)) - } - - fn active_file(&mut self) -> Result<&mut File, DurabilityCrashMatrixError> { - self.active_file - .as_mut() - .ok_or(DurabilityCrashMatrixError::MissingActiveFile) - } - - fn active_file_ref(&self) -> Result<&File, DurabilityCrashMatrixError> { - self.active_file - .as_ref() - .ok_or(DurabilityCrashMatrixError::MissingActiveFile) - } - - fn finish( - self, - result: Result<(), DurabilityCrashMatrixError>, - ) -> Result<(), DurabilityCrashMatrixError> { - drop(self.active_file); - drop(self.writer_lock); - drop(self.root); - result - } -} diff --git a/xtask/src/durability_crash_matrix/state/catalog.rs b/xtask/src/durability_crash_matrix/state/catalog.rs deleted file mode 100644 index b296670..0000000 --- a/xtask/src/durability_crash_matrix/state/catalog.rs +++ /dev/null @@ -1,84 +0,0 @@ -//! This module owns Golden File Worldline catalog crash-state construction. - -use xtask::{DurabilityCrashCase, DurabilityCrashPoint, DurabilityCrashPosition}; - -use super::fixture::{CATALOG_POOL_PATH, GoldenFixture, SEGMENT_POOL_PATH}; -use super::{DurabilityCrashMatrixError, StoreState}; - -const STEPS: [DurabilityCrashPoint; 8] = [ - DurabilityCrashPoint::CreateCatalogStage, - DurabilityCrashPoint::WriteCatalog, - DurabilityCrashPoint::FlushCatalog, - DurabilityCrashPoint::SynchronizeCatalog, - DurabilityCrashPoint::LinkCatalog, - DurabilityCrashPoint::SynchronizeCatalogPool, - DurabilityCrashPoint::RemoveCatalogStage, - DurabilityCrashPoint::SynchronizeStagingAfterCatalog, -]; - -pub(super) fn prepare( - state: &mut StoreState, - case: DurabilityCrashCase, -) -> Result<(), DurabilityCrashMatrixError> { - state.initialize()?; - state.write_immutable(SEGMENT_POOL_PATH, &GoldenFixture::segment()?)?; - let fixture = GoldenFixture::catalog()?; - for step in STEPS { - let position = if step == case.point() { - case.position() - } else { - DurabilityCrashPosition::After - }; - apply(state, &fixture, step, position)?; - if step == case.point() { - return Ok(()); - } - } - Err(DurabilityCrashMatrixError::PointSequenceMismatch { - point: case.point(), - }) -} - -fn apply( - state: &mut StoreState, - fixture: &GoldenFixture, - point: DurabilityCrashPoint, - position: DurabilityCrashPosition, -) -> Result<(), DurabilityCrashMatrixError> { - if position == DurabilityCrashPosition::Before { - return Ok(()); - } - match point { - DurabilityCrashPoint::CreateCatalogStage => state.create_stage("staging/current.cat"), - DurabilityCrashPoint::WriteCatalog => { - let end = if position == DurabilityCrashPosition::During { - 176 - } else { - 352 - }; - state.write_range(fixture, 0..end) - } - DurabilityCrashPoint::FlushCatalog => after(position, || state.flush()), - DurabilityCrashPoint::SynchronizeCatalog => after(position, || state.synchronize_file()), - DurabilityCrashPoint::LinkCatalog => state.link("staging/current.cat", CATALOG_POOL_PATH), - DurabilityCrashPoint::SynchronizeCatalogPool => { - after(position, || state.synchronize_directory("catalogs")) - } - DurabilityCrashPoint::RemoveCatalogStage => state.remove("staging/current.cat"), - DurabilityCrashPoint::SynchronizeStagingAfterCatalog => { - after(position, || state.synchronize_directory("staging")) - } - _ => Err(DurabilityCrashMatrixError::PointSequenceMismatch { point }), - } -} - -fn after( - position: DurabilityCrashPosition, - operation: impl FnOnce() -> Result<(), DurabilityCrashMatrixError>, -) -> Result<(), DurabilityCrashMatrixError> { - if position == DurabilityCrashPosition::After { - operation() - } else { - Ok(()) - } -} diff --git a/xtask/src/durability_crash_matrix/state/head.rs b/xtask/src/durability_crash_matrix/state/head.rs deleted file mode 100644 index e666b2e..0000000 --- a/xtask/src/durability_crash_matrix/state/head.rs +++ /dev/null @@ -1,79 +0,0 @@ -//! This module owns Golden File Worldline publication-head crash states. - -use xtask::{DurabilityCrashCase, DurabilityCrashPoint, DurabilityCrashPosition}; - -use super::fixture::{CATALOG_POOL_PATH, GoldenFixture, SEGMENT_POOL_PATH}; -use super::{DurabilityCrashMatrixError, StoreState}; - -const STEPS: [DurabilityCrashPoint; 6] = [ - DurabilityCrashPoint::CreateHeadStage, - DurabilityCrashPoint::WriteHead, - DurabilityCrashPoint::FlushHead, - DurabilityCrashPoint::SynchronizeHead, - DurabilityCrashPoint::ReplaceHead, - DurabilityCrashPoint::SynchronizeRootAfterHead, -]; - -pub(super) fn prepare( - state: &mut StoreState, - case: DurabilityCrashCase, -) -> Result<(), DurabilityCrashMatrixError> { - state.initialize()?; - state.write_immutable(SEGMENT_POOL_PATH, &GoldenFixture::segment()?)?; - state.write_immutable(CATALOG_POOL_PATH, &GoldenFixture::catalog()?)?; - let fixture = GoldenFixture::head()?; - for step in STEPS { - let position = if step == case.point() { - case.position() - } else { - DurabilityCrashPosition::After - }; - apply(state, &fixture, step, position)?; - if step == case.point() { - return Ok(()); - } - } - Err(DurabilityCrashMatrixError::PointSequenceMismatch { - point: case.point(), - }) -} - -fn apply( - state: &mut StoreState, - fixture: &GoldenFixture, - point: DurabilityCrashPoint, - position: DurabilityCrashPosition, -) -> Result<(), DurabilityCrashMatrixError> { - if position == DurabilityCrashPosition::Before { - return Ok(()); - } - match point { - DurabilityCrashPoint::CreateHeadStage => state.create_stage("head.next"), - DurabilityCrashPoint::WriteHead => { - let end = if position == DurabilityCrashPosition::During { - 64 - } else { - 128 - }; - state.write_range(fixture, 0..end) - } - DurabilityCrashPoint::FlushHead => after(position, || state.flush()), - DurabilityCrashPoint::SynchronizeHead => after(position, || state.synchronize_file()), - DurabilityCrashPoint::ReplaceHead => state.rename("head.next", "HEAD"), - DurabilityCrashPoint::SynchronizeRootAfterHead => { - after(position, || state.synchronize_directory(".")) - } - _ => Err(DurabilityCrashMatrixError::PointSequenceMismatch { point }), - } -} - -fn after( - position: DurabilityCrashPosition, - operation: impl FnOnce() -> Result<(), DurabilityCrashMatrixError>, -) -> Result<(), DurabilityCrashMatrixError> { - if position == DurabilityCrashPosition::After { - operation() - } else { - Ok(()) - } -} diff --git a/xtask/src/durability_crash_matrix/state/initialization.rs b/xtask/src/durability_crash_matrix/state/initialization.rs deleted file mode 100644 index 1d59e39..0000000 --- a/xtask/src/durability_crash_matrix/state/initialization.rs +++ /dev/null @@ -1,60 +0,0 @@ -//! This module owns crash-safe initialization state construction. - -use xtask::{DurabilityCrashCase, DurabilityCrashPoint, DurabilityCrashPosition}; - -use super::{DurabilityCrashMatrixError, StoreState}; - -const STEPS: [DurabilityCrashPoint; 5] = [ - DurabilityCrashPoint::OpenAndLockWriterFile, - DurabilityCrashPoint::CreateStagingDirectory, - DurabilityCrashPoint::CreateSegmentPoolDirectory, - DurabilityCrashPoint::CreateCatalogPoolDirectory, - DurabilityCrashPoint::SynchronizeRootAfterInitialization, -]; - -pub(super) fn prepare( - state: &mut StoreState, - case: DurabilityCrashCase, -) -> Result<(), DurabilityCrashMatrixError> { - for step in STEPS { - let position = if step == case.point() { - case.position() - } else { - DurabilityCrashPosition::After - }; - apply(state, step, position)?; - if step == case.point() { - return Ok(()); - } - } - Err(DurabilityCrashMatrixError::PointSequenceMismatch { - point: case.point(), - }) -} - -fn apply( - state: &mut StoreState, - point: DurabilityCrashPoint, - position: DurabilityCrashPosition, -) -> Result<(), DurabilityCrashMatrixError> { - if position == DurabilityCrashPosition::Before { - return Ok(()); - } - match point { - DurabilityCrashPoint::OpenAndLockWriterFile => { - state.create_writer_lock()?; - state.acquire_writer_lock() - } - DurabilityCrashPoint::CreateStagingDirectory => state.create_directory("staging"), - DurabilityCrashPoint::CreateSegmentPoolDirectory => state.create_directory("segments"), - DurabilityCrashPoint::CreateCatalogPoolDirectory => state.create_directory("catalogs"), - DurabilityCrashPoint::SynchronizeRootAfterInitialization => { - if position == DurabilityCrashPosition::After { - state.synchronize_directory(".") - } else { - Ok(()) - } - } - _ => Err(DurabilityCrashMatrixError::PointSequenceMismatch { point }), - } -} diff --git a/xtask/src/durability_crash_matrix/state/recovery.rs b/xtask/src/durability_crash_matrix/state/recovery.rs deleted file mode 100644 index d943efc..0000000 --- a/xtask/src/durability_crash_matrix/state/recovery.rs +++ /dev/null @@ -1,60 +0,0 @@ -//! This module owns explicit-discard crash-state construction. - -use xtask::{DurabilityCrashCase, DurabilityCrashPoint, DurabilityCrashPosition}; - -use super::fixture::GoldenFixture; -use super::{DurabilityCrashMatrixError, StoreState}; - -pub(super) fn prepare( - state: &mut StoreState, - case: DurabilityCrashCase, -) -> Result<(), DurabilityCrashMatrixError> { - state.initialize()?; - match case.point() { - DurabilityCrashPoint::RemoveRecoveryStage - | DurabilityCrashPoint::SynchronizeStagingAfterRecovery => { - prepare_segment_discard(state, case) - } - DurabilityCrashPoint::RemoveRecoveryHead - | DurabilityCrashPoint::SynchronizeRootAfterRecovery => prepare_head_discard(state, case), - point => Err(DurabilityCrashMatrixError::PointSequenceMismatch { point }), - } -} - -fn prepare_segment_discard( - state: &mut StoreState, - case: DurabilityCrashCase, -) -> Result<(), DurabilityCrashMatrixError> { - state.create_stage("staging/current.seg")?; - state.write_range(&GoldenFixture::segment()?, 0..32)?; - if case.point() == DurabilityCrashPoint::RemoveRecoveryStage { - if case.position() != DurabilityCrashPosition::Before { - state.remove("staging/current.seg")?; - } - return Ok(()); - } - state.remove("staging/current.seg")?; - if case.position() == DurabilityCrashPosition::After { - state.synchronize_directory("staging")?; - } - Ok(()) -} - -fn prepare_head_discard( - state: &mut StoreState, - case: DurabilityCrashCase, -) -> Result<(), DurabilityCrashMatrixError> { - state.create_stage("head.next")?; - state.write_range(&GoldenFixture::head()?, 0..64)?; - if case.point() == DurabilityCrashPoint::RemoveRecoveryHead { - if case.position() != DurabilityCrashPosition::Before { - state.remove("head.next")?; - } - return Ok(()); - } - state.remove("head.next")?; - if case.position() == DurabilityCrashPosition::After { - state.synchronize_directory(".")?; - } - Ok(()) -} diff --git a/xtask/src/durability_crash_matrix/state/segment.rs b/xtask/src/durability_crash_matrix/state/segment.rs deleted file mode 100644 index 75f968b..0000000 --- a/xtask/src/durability_crash_matrix/state/segment.rs +++ /dev/null @@ -1,103 +0,0 @@ -//! This module owns Golden File Worldline segment crash-state construction. - -use xtask::{DurabilityCrashCase, DurabilityCrashPoint, DurabilityCrashPosition}; - -use super::fixture::{GoldenFixture, SEGMENT_POOL_PATH}; -use super::{DurabilityCrashMatrixError, StoreState}; - -const STEPS: [DurabilityCrashPoint; 12] = [ - DurabilityCrashPoint::CreateSegmentStage, - DurabilityCrashPoint::WriteSegmentHeader, - DurabilityCrashPoint::AppendSegmentRecord, - DurabilityCrashPoint::FlushSegmentRecordPrefix, - DurabilityCrashPoint::SynchronizeSegmentRecordPrefix, - DurabilityCrashPoint::AppendSegmentSeal, - DurabilityCrashPoint::FlushSealedSegment, - DurabilityCrashPoint::SynchronizeSealedSegment, - DurabilityCrashPoint::LinkSegment, - DurabilityCrashPoint::SynchronizeSegmentPool, - DurabilityCrashPoint::RemoveSegmentStage, - DurabilityCrashPoint::SynchronizeStagingAfterSegment, -]; - -pub(super) fn prepare( - state: &mut StoreState, - case: DurabilityCrashCase, -) -> Result<(), DurabilityCrashMatrixError> { - state.initialize()?; - let fixture = GoldenFixture::segment()?; - for step in STEPS { - let position = if step == case.point() { - case.position() - } else { - DurabilityCrashPosition::After - }; - apply(state, &fixture, step, position)?; - if step == case.point() { - return Ok(()); - } - } - Err(DurabilityCrashMatrixError::PointSequenceMismatch { - point: case.point(), - }) -} - -fn apply( - state: &mut StoreState, - fixture: &GoldenFixture, - point: DurabilityCrashPoint, - position: DurabilityCrashPosition, -) -> Result<(), DurabilityCrashMatrixError> { - if position == DurabilityCrashPosition::Before { - return Ok(()); - } - match point { - DurabilityCrashPoint::CreateSegmentStage => state.create_stage("staging/current.seg"), - DurabilityCrashPoint::WriteSegmentHeader => { - let end = interrupted_end(position, 32, 64); - state.write_range(fixture, 0..end) - } - DurabilityCrashPoint::AppendSegmentRecord => { - let end = interrupted_end(position, 136, 209); - state.write_range(fixture, 64..end) - } - DurabilityCrashPoint::FlushSegmentRecordPrefix - | DurabilityCrashPoint::FlushSealedSegment => after(position, || state.flush()), - DurabilityCrashPoint::SynchronizeSegmentRecordPrefix - | DurabilityCrashPoint::SynchronizeSealedSegment => { - after(position, || state.synchronize_file()) - } - DurabilityCrashPoint::AppendSegmentSeal => { - let end = interrupted_end(position, 273, 337); - state.write_range(fixture, 209..end) - } - DurabilityCrashPoint::LinkSegment => state.link("staging/current.seg", SEGMENT_POOL_PATH), - DurabilityCrashPoint::SynchronizeSegmentPool => { - after(position, || state.synchronize_directory("segments")) - } - DurabilityCrashPoint::RemoveSegmentStage => state.remove("staging/current.seg"), - DurabilityCrashPoint::SynchronizeStagingAfterSegment => { - after(position, || state.synchronize_directory("staging")) - } - _ => Err(DurabilityCrashMatrixError::PointSequenceMismatch { point }), - } -} - -fn interrupted_end(position: DurabilityCrashPosition, during: usize, after: usize) -> usize { - if position == DurabilityCrashPosition::During { - during - } else { - after - } -} - -fn after( - position: DurabilityCrashPosition, - operation: impl FnOnce() -> Result<(), DurabilityCrashMatrixError>, -) -> Result<(), DurabilityCrashMatrixError> { - if position == DurabilityCrashPosition::After { - operation() - } else { - Ok(()) - } -} diff --git a/xtask/tests/durability_crash_production_contract.rs b/xtask/tests/durability_crash_production_contract.rs new file mode 100644 index 0000000..7c5cbf6 --- /dev/null +++ b/xtask/tests/durability_crash_production_contract.rs @@ -0,0 +1,37 @@ +//! Production-protocol ownership laws for the process-death crash matrix. + +#![cfg(feature = "repository-tasks")] + +use std::error::Error; +use std::fs; +use std::path::PathBuf; + +#[test] +fn crash_children_execute_every_claimed_production_protocol() -> Result<(), Box> { + let source_root = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("src"); + let child = fs::read_to_string(source_root.join("durability_crash_matrix/child.rs"))?; + let protocol = [ + "durability_crash_matrix/production_protocol.rs", + "durability_crash_matrix/production_protocol/initialization.rs", + "durability_crash_matrix/production_protocol/publication.rs", + "durability_crash_matrix/production_protocol/recovery.rs", + ] + .into_iter() + .map(|path| fs::read_to_string(source_root.join(path))) + .collect::>()?; + + assert!(child.contains("production_protocol::run(")); + assert!(!child.contains("state::prepare(")); + for required in [ + "StagedSegment::begin(", + "publish_catalog_generation(", + "initialize_store(", + "execute_recovery_stage_discard(", + ] { + assert!( + protocol.contains(required), + "crash protocol does not execute {required}" + ); + } + Ok(()) +} From 3deee9d0c19944aefd536b80c65e78c1199b46e7 Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 15:43:53 -0700 Subject: [PATCH 39/49] Fix: Revalidate recovery stages before removal --- src/adapters/filesystem_recovery_stage.rs | 9 ++- ...lesystem_recovery_stage_discard_storage.rs | 61 ++++++++++++++++--- ...filesystem_recovery_stage_discard_tests.rs | 31 ++++++++++ 3 files changed, 91 insertions(+), 10 deletions(-) diff --git a/src/adapters/filesystem_recovery_stage.rs b/src/adapters/filesystem_recovery_stage.rs index 3b1d534..982a5cb 100644 --- a/src/adapters/filesystem_recovery_stage.rs +++ b/src/adapters/filesystem_recovery_stage.rs @@ -79,7 +79,14 @@ pub(super) fn fingerprint( directory: &Dir, stage: RecoveryStage, ) -> Result { - fingerprint_named(directory, stage.file_name(), stage) + Ok(observe(directory, stage)?.evidence()) +} + +pub(super) fn observe( + directory: &Dir, + stage: RecoveryStage, +) -> Result { + observe_named(directory, stage.file_name(), stage) } pub(super) fn fingerprint_named( diff --git a/src/adapters/filesystem_recovery_stage_discard_storage.rs b/src/adapters/filesystem_recovery_stage_discard_storage.rs index 5f9b03f..398507e 100644 --- a/src/adapters/filesystem_recovery_stage_discard_storage.rs +++ b/src/adapters/filesystem_recovery_stage_discard_storage.rs @@ -4,6 +4,7 @@ use std::io; use cap_std::fs::Dir; +use super::filesystem_recovery_stage::ObservedRecoveryStage; use super::{ FilesystemRecoveryInventoryReader, FilesystemRecoveryStageDiscarder, FilesystemRecoveryStageError, RecoveryStage, RecoveryStageDiscardOutcome, @@ -20,7 +21,8 @@ impl RecoveryStageDiscardStorage for FilesystemRecoveryStageDiscarder { remove_with( &self.inventory, expected, - filesystem_recovery_stage::fingerprint, + filesystem_recovery_stage::observe, + || {}, ) } @@ -39,9 +41,36 @@ impl FilesystemRecoveryStageDiscarder { where F: FnOnce(), { - remove_with(&self.inventory, expected, |directory, stage| { - filesystem_recovery_stage::fingerprint_with(directory, stage, after_open) - }) + remove_with( + &self.inventory, + expected, + |directory, stage| { + filesystem_recovery_stage::observe_named_with( + directory, + stage.file_name(), + stage, + after_open, + ) + }, + || {}, + ) + } + + #[cfg(test)] + pub(super) fn remove_if_matching_after_observation_with( + &self, + expected: RecoveryStageEvidence, + before_remove: F, + ) -> Result + where + F: FnOnce(), + { + remove_with( + &self.inventory, + expected, + filesystem_recovery_stage::observe, + before_remove, + ) } } @@ -49,16 +78,23 @@ pub(super) fn remove_if_matching( inventory: &FilesystemRecoveryInventoryReader, expected: RecoveryStageEvidence, ) -> Result { - remove_with(inventory, expected, filesystem_recovery_stage::fingerprint) + remove_with( + inventory, + expected, + filesystem_recovery_stage::observe, + || {}, + ) } -fn remove_with( +fn remove_with( inventory: &FilesystemRecoveryInventoryReader, expected: RecoveryStageEvidence, observe: F, + before_remove: G, ) -> Result where - F: FnOnce(&Dir, RecoveryStage) -> Result, + F: FnOnce(&Dir, RecoveryStage) -> Result, + G: FnOnce(), { let stage = expected.stage(); inventory @@ -72,12 +108,19 @@ where return Ok(RecoveryStageDiscardOutcome::AlreadyAbsent); } let observed = observe(directory, stage).map_err(stage_error)?; - if observed != expected { - return Err(RecoveryStageDiscardStorageError::EvidenceMismatch { expected, observed }); + if observed.evidence() != expected { + return Err(RecoveryStageDiscardStorageError::EvidenceMismatch { + expected, + observed: observed.evidence(), + }); } inventory .verify_stage_namespaces(stage, RecoveryStageNamespacePhase::AfterObservation) .map_err(stage_error)?; + before_remove(); + observed + .verify(directory, stage.file_name(), stage) + .map_err(stage_error)?; directory .remove_file(stage.file_name()) .map_err(storage_error)?; diff --git a/src/adapters/filesystem_recovery_stage_discard_tests.rs b/src/adapters/filesystem_recovery_stage_discard_tests.rs index e0bf5ca..78097dd 100644 --- a/src/adapters/filesystem_recovery_stage_discard_tests.rs +++ b/src/adapters/filesystem_recovery_stage_discard_tests.rs @@ -136,6 +136,37 @@ fn replacement_after_open_refuses_without_removing_the_new_entry() -> Result<(), Ok(()) } +#[test] +fn replacement_after_observation_refuses_without_removing_the_new_entry() +-> Result<(), Box> { + let fixture = DiscardFixture::new("filesystem-stage-discard-final-handoff")?; + let stage_path = fixture.stage_path(RecoveryStage::Segment); + let retained_path = fixture.root().join("retained-observed-stage"); + fs::write(&stage_path, b"old")?; + let request = request(RecoveryStage::Segment, b"old")?; + let discarder = fixture.discarder()?; + let mut hook_result = Ok(()); + + let result = discarder.remove_if_matching_after_observation_with(request.evidence(), || { + hook_result = + fs::rename(&stage_path, &retained_path).and_then(|()| fs::write(&stage_path, b"new")); + }); + + hook_result?; + let error = result.err().ok_or("replacement at handoff was removed")?; + assert!(matches!( + filesystem_stage_source(&error)?, + FilesystemRecoveryStageError::Replaced { + stage: RecoveryStage::Segment, + } + )); + assert_eq!(fs::read(&stage_path)?, b"new"); + assert_eq!(fs::read(&retained_path)?, b"old"); + drop(discarder); + fixture.remove()?; + Ok(()) +} + fn filesystem_stage_source( error: &RecoveryStageDiscardStorageError, ) -> Result<&FilesystemRecoveryStageError, &'static str> { From ecf4f7e8f190f22974bead9eee7e8d6a1a467f32 Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 15:50:55 -0700 Subject: [PATCH 40/49] Fix: Reopen published stores through platform admission --- CHANGELOG.md | 3 ++ README.md | 19 ++++--- docs/formats/segment-store-v1/recovery.md | 10 ++++ docs/formats/segment-store-v1/requirements.md | 2 +- src/adapters/filesystem_catalog_publisher.rs | 5 +- .../filesystem_catalog_publisher_tests.rs | 9 ++-- .../filesystem_initialization_namespace.rs | 54 +++++++++++++++---- .../filesystem_platform_admission_error.rs | 46 ++++++++++++++++ src/adapters/filesystem_store_initializer.rs | 51 +++++++++++++++++- .../filesystem_store_initializer_tests.rs | 52 +++++++++++++++++- src/adapters/mod.rs | 2 + src/lib.rs | 49 ++++++++--------- 12 files changed, 249 insertions(+), 53 deletions(-) create mode 100644 src/adapters/filesystem_platform_admission_error.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 6df9dd4..50546e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,9 @@ after its public API and format compatibility policies are established. non-casefolded Linux ext4 profile, refuses ambiguous root namespaces before mutation, completes the canonical directory shape idempotently, retains writer authority, and returns only after synchronizing the root. +- Published filesystem stores can now reacquire writer authority without + mutation through a typed platform-admission boundary that requires the exact + initialized root shape plus a regular `HEAD`. - Writer-lock acquisition now reopens `writer.lock` after kernel locking and refuses when the resolved entry no longer has the locked device and inode. - Writer authority now also retains an advisory lock on the pinned store-root diff --git a/README.md b/README.md index b55d3e2..d88b45d 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,10 @@ unforgeable `FilesystemPlatformAdmission`. On Linux, its public initializer admits only a writable, non-casefolded ext4 root, refuses unknown or aliased namespace entries before mutation, creates or verifies the canonical `writer.lock`, `staging`, `segments`, and `catalogs` shape, and returns only -after root synchronization with the writer lock retained. +after root synchronization with the writer lock retained. After publication, +`FilesystemPlatformAdmission::reopen` reacquires the existing writer lock +without mutation and requires that exact initialized shape plus a regular +`HEAD` before returning new publisher authority. `FilesystemCatalogPublisher` retains one kernel-managed writer lock and pinned root, staging, segment-pool, and catalog-pool capabilities for the complete @@ -59,13 +62,13 @@ logical reads. The reference CAS is executable evidence for M2 storage laws, not a durable backend. Its committed state is process memory; process death loses it all. -The durable boundary can initialize and platform-admit a store only under the -documented Linux ext4 contract. Acquiring `FilesystemWriterLock` alone cannot -construct a filesystem publisher. Ambiguous crash states remain explicit -recovery work. An absent `HEAD` is admitted for first publication only when -both immutable pools are empty. The public storage-independent recovery -inventory counts all four protocol namespaces before retaining names, applies -a configurable ceiling no greater than 2,097,152 entries, and returns +The durable boundary can initialize or reopen and platform-admit a store only +under the documented Linux ext4 contract. Acquiring `FilesystemWriterLock` +alone cannot construct a filesystem publisher. Ambiguous crash states remain +explicit recovery work. An absent `HEAD` is admitted for first publication +only when both immutable pools are empty. The public storage-independent +recovery inventory counts all four protocol namespaces before retaining names, +applies a configurable ceiling no greater than 2,097,152 entries, and returns duplicate-free deterministic raw name order. `FilesystemRecoveryInventoryReader` implements that contract with pinned, no-follow namespace capabilities and pre/post identity verification on the diff --git a/docs/formats/segment-store-v1/recovery.md b/docs/formats/segment-store-v1/recovery.md index ff60a9f..7e120ae 100644 --- a/docs/formats/segment-store-v1/recovery.md +++ b/docs/formats/segment-store-v1/recovery.md @@ -36,6 +36,16 @@ coordinate, and refuses an unknown or conflicting name without artifact I/O. bounded stream, refuses metadata or observed bytes above the name-selected maximum, and returns its exact observed length and `KEEP:RECOVERY:STAGE\0` fingerprint. + +New stores obtain writer authority through +`FilesystemPlatformAdmission::initialize`. A stable published store reacquires +authority through `FilesystemPlatformAdmission::reopen`, which performs no +protocol mutation, admits the production platform, acquires the existing +writer lock, and requires exactly `writer.lock`, `staging`, `segments`, +`catalogs`, and a regular `HEAD` in the root. Missing or additional root +evidence remains a typed namespace refusal; content-level head, catalog, and +segment verification stays at the publisher and restart boundaries. + `FilesystemRecoveryInventoryReader::fingerprint_stage` binds that stream to the pinned root or staging capability, opens without following links, admits only regular files, and refuses entry replacement or length drift after diff --git a/docs/formats/segment-store-v1/requirements.md b/docs/formats/segment-store-v1/requirements.md index 184048c..45a9080 100644 --- a/docs/formats/segment-store-v1/requirements.md +++ b/docs/formats/segment-store-v1/requirements.md @@ -111,7 +111,7 @@ collection, and host-power-loss simulation remain outside issue #17. | --- | --- | --- | --- | --- | | `KEEP-RECOVERY-001` | Crash identifiers `KEEP-CRASH-001` through `KEEP-CRASH-035` form one contiguous typed vocabulary, map to the exact owning protocol sequence, and admit an occurrence counter only for record append | Ordered identifier-and-sequence ledger | `xtask/tests/durability_crash_point_contract.rs` | Implemented in #17 | | `KEEP-RECOVERY-002` | Initialization admits the platform before mutation, opens and locks the writer file, admits `staging`, `segments`, and `catalogs` in order, and returns a receipt only after root synchronization; every failed operation retains its exact phase and prevents later transitions | Fault-recording initialization port | `tests/store_initialization.rs` | Implemented in #17 | -| `KEEP-RECOVERY-003` | Production initialization admits only a writable, non-casefolded Linux ext4 root, refuses any noncanonical root entry before mutation, completes an empty or partial canonical namespace without replacing evidence, excludes a second initializer, and retains writer authority through the synchronized receipt | Capability-relative filesystem fixture and exact platform-profile classifier | `src/adapters/filesystem_store_initializer_tests.rs`, `src/adapters/filesystem_platform_profile.rs`, `tests/store_initialization.rs` | Implemented in #17 | +| `KEEP-RECOVERY-003` | Production initialization admits only a writable, non-casefolded Linux ext4 root, refuses any noncanonical root entry before mutation, completes an empty or partial canonical namespace without replacing evidence, excludes a second initializer, and retains writer authority through the synchronized receipt; published-store reopen performs no mutation, reacquires the writer lock, and requires the complete initialized root plus regular `HEAD` | Capability-relative initialization, published-reopen, and exact platform-profile matrix | `src/adapters/filesystem_store_initializer_tests.rs`, `src/adapters/filesystem_catalog_publisher_tests.rs`, `src/adapters/filesystem_platform_profile.rs`, `tests/store_initialization.rs` | Implemented in #17 | | `KEEP-RECOVERY-004` | Writer authority is returned only when the locked handle still has the exact device and inode resolved by the canonical `writer.lock` entry after kernel acquisition | Deterministic lock-entry replacement fixture | `src/adapters/filesystem_writer_lock_tests.rs` | Implemented in #17 | | `KEEP-RECOVERY-005` | Recovery counts the root and three protocol directories in fixed order before retaining names, refuses at the configured or protocol entry ceiling with the exact observed-at-least count, then returns one duplicate-free inventory sorted by namespace and raw name bytes | Fault-recording inventory port | `tests/recovery_inventory.rs` | Implemented in #17 | | `KEEP-RECOVERY-006` | Filesystem inventory pins the admitted root and protocol directories without following links, verifies child-directory identity before and after scanning, stops each count at the remaining global budget plus one, preserves raw Linux entry-name bytes, and performs no protocol mutation | Capability-relative filesystem fixture | `src/adapters/filesystem_recovery_inventory_tests.rs`, `tests/recovery_inventory.rs` | Implemented in #17 | diff --git a/src/adapters/filesystem_catalog_publisher.rs b/src/adapters/filesystem_catalog_publisher.rs index 83518c9..b4c191b 100644 --- a/src/adapters/filesystem_catalog_publisher.rs +++ b/src/adapters/filesystem_catalog_publisher.rs @@ -25,9 +25,8 @@ pub(super) const NEXT_HEAD: &str = "head.next"; /// closes open stages and directory capabilities before releasing the writer /// lock, but never publishes, removes, truncates, or repairs protocol state. /// -/// Construction consumes a [`FilesystemPlatformAdmission`] proof. Keep exposes -/// no public producer for that proof until issue #17 supplies crash-tested -/// initialization and platform admission. +/// Construction consumes a [`FilesystemPlatformAdmission`] proof created by +/// initializing a new store or reopening a completely published store. #[must_use] pub struct FilesystemCatalogPublisher { pub(super) root: Dir, diff --git a/src/adapters/filesystem_catalog_publisher_tests.rs b/src/adapters/filesystem_catalog_publisher_tests.rs index 4319f39..8ff44ae 100644 --- a/src/adapters/filesystem_catalog_publisher_tests.rs +++ b/src/adapters/filesystem_catalog_publisher_tests.rs @@ -19,8 +19,8 @@ use crate::{ AdmittedSegment, AdmittedSegmentRecord, CanonicalCatalog, CatalogGeneration, CatalogPublicationExpectation, CatalogPublicationOutcome, CatalogRestartByteLimit, CatalogRestartPolicy, FilesystemCatalogPublisher, FilesystemCatalogSnapshot, - FilesystemSegmentStage, FilesystemWriterLock, LayoutEntryLimit, SealedSegment, - SegmentPublication, SegmentReadPolicy, SegmentRecordLimit, StagedSegment, + FilesystemPlatformAdmission, FilesystemSegmentStage, FilesystemWriterLock, LayoutEntryLimit, + SealedSegment, SegmentPublication, SegmentReadPolicy, SegmentRecordLimit, StagedSegment, publish_catalog_generation, }; @@ -89,9 +89,8 @@ fn durable_publication_retry_returns_the_same_synchronized_receipt() -> Result<( )?; drop(publisher); - let lock = FilesystemWriterLock::try_acquire(store.path())?; - let mut publisher = - FilesystemCatalogPublisher::open_unchecked_for_tests(lock, restart_policy()?)?; + let admission = FilesystemPlatformAdmission::reopen_unchecked_for_tests(store.path())?; + let mut publisher = FilesystemCatalogPublisher::open(admission, restart_policy()?)?; let retry = publish_catalog_generation( &mut publisher, CatalogPublicationExpectation::uninitialized(), diff --git a/src/adapters/filesystem_initialization_namespace.rs b/src/adapters/filesystem_initialization_namespace.rs index a6844e2..2ec01d1 100644 --- a/src/adapters/filesystem_initialization_namespace.rs +++ b/src/adapters/filesystem_initialization_namespace.rs @@ -9,14 +9,31 @@ const LOCK_NAME: &str = "writer.lock"; const STAGING_NAME: &str = "staging"; const SEGMENTS_NAME: &str = "segments"; const CATALOGS_NAME: &str = "catalogs"; -const CANONICAL_ENTRY_COUNT: usize = 4; +const HEAD_NAME: &str = "HEAD"; +const INITIALIZATION_NAMES: [&str; 4] = [LOCK_NAME, STAGING_NAME, SEGMENTS_NAME, CATALOGS_NAME]; +const PUBLISHED_NAMES: [&str; 5] = [ + LOCK_NAME, + STAGING_NAME, + SEGMENTS_NAME, + CATALOGS_NAME, + HEAD_NAME, +]; pub(super) fn admit(directory: &Dir) -> io::Result<()> { admit_optional_file(directory, LOCK_NAME)?; admit_optional_directory(directory, STAGING_NAME)?; admit_optional_directory(directory, SEGMENTS_NAME)?; admit_optional_directory(directory, CATALOGS_NAME)?; - admit_membership(directory) + admit_membership(directory, &INITIALIZATION_NAMES) +} + +pub(super) fn admit_published(directory: &Dir) -> io::Result<()> { + admit_required_file(directory, LOCK_NAME)?; + admit_required_directory(directory, STAGING_NAME)?; + admit_required_directory(directory, SEGMENTS_NAME)?; + admit_required_directory(directory, CATALOGS_NAME)?; + admit_required_file(directory, HEAD_NAME)?; + admit_membership(directory, &PUBLISHED_NAMES) } fn admit_optional_file(directory: &Dir, name: &str) -> io::Result<()> { @@ -27,6 +44,27 @@ fn admit_optional_directory(directory: &Dir, name: &str) -> io::Result<()> { admit_optional_kind(directory, name, cap_std::fs::FileType::is_dir) } +fn admit_required_file(directory: &Dir, name: &str) -> io::Result<()> { + admit_required_kind(directory, name, cap_std::fs::FileType::is_file) +} + +fn admit_required_directory(directory: &Dir, name: &str) -> io::Result<()> { + admit_required_kind(directory, name, cap_std::fs::FileType::is_dir) +} + +fn admit_required_kind( + directory: &Dir, + name: &str, + expected: fn(&cap_std::fs::FileType) -> bool, +) -> io::Result<()> { + let metadata = directory.symlink_metadata(name)?; + if expected(&metadata.file_type()) { + Ok(()) + } else { + Err(ambiguous_namespace()) + } +} + fn admit_optional_kind( directory: &Dir, name: &str, @@ -40,25 +78,23 @@ fn admit_optional_kind( } } -fn admit_membership(directory: &Dir) -> io::Result<()> { +fn admit_membership(directory: &Dir, canonical_names: &[&str]) -> io::Result<()> { let mut observed = 0_usize; for entry in directory.entries()? { observed = observed.checked_add(1).ok_or_else(ambiguous_namespace)?; - if observed > CANONICAL_ENTRY_COUNT { + if observed > canonical_names.len() { return Err(ambiguous_namespace()); } let name = entry?.file_name(); - if !is_canonical(&name) { + if !is_canonical(&name, canonical_names) { return Err(ambiguous_namespace()); } } Ok(()) } -fn is_canonical(name: &OsStr) -> bool { - [LOCK_NAME, STAGING_NAME, SEGMENTS_NAME, CATALOGS_NAME] - .into_iter() - .any(|candidate| name == candidate) +fn is_canonical(name: &OsStr, canonical_names: &[&str]) -> bool { + canonical_names.iter().any(|candidate| name == *candidate) } fn ambiguous_namespace() -> io::Error { diff --git a/src/adapters/filesystem_platform_admission_error.rs b/src/adapters/filesystem_platform_admission_error.rs new file mode 100644 index 0000000..7bee25a --- /dev/null +++ b/src/adapters/filesystem_platform_admission_error.rs @@ -0,0 +1,46 @@ +//! This module owns published-store platform-admission failures. + +use std::error::Error; +use std::fmt; +use std::io; + +use super::WriterLockAcquireError; + +/// Failure to reacquire writer authority over one published filesystem store. +#[derive(Debug)] +pub enum FilesystemPlatformAdmissionError { + /// The store root does not satisfy the production platform profile. + Platform { + /// Preserved platform-admission failure. + source: io::Error, + }, + /// Exclusive writer authority could not be acquired. + WriterLock { + /// Preserved writer-lock failure. + source: WriterLockAcquireError, + }, + /// The writer-locked published namespace is incomplete or ambiguous. + Namespace { + /// Preserved namespace-admission failure. + source: io::Error, + }, +} + +impl fmt::Display for FilesystemPlatformAdmissionError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(match self { + Self::Platform { .. } => "published store platform admission failed", + Self::WriterLock { .. } => "published store writer-lock acquisition failed", + Self::Namespace { .. } => "published store namespace admission failed", + }) + } +} + +impl Error for FilesystemPlatformAdmissionError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Platform { source } | Self::Namespace { source } => Some(source), + Self::WriterLock { source } => Some(source), + } + } +} diff --git a/src/adapters/filesystem_store_initializer.rs b/src/adapters/filesystem_store_initializer.rs index 102f915..bbcc392 100644 --- a/src/adapters/filesystem_store_initializer.rs +++ b/src/adapters/filesystem_store_initializer.rs @@ -2,10 +2,16 @@ use std::path::Path; +#[cfg(test)] +use cap_std::ambient_authority; +#[cfg(test)] +use cap_std::fs::Dir; + use super::filesystem_initialization_storage::FilesystemInitializationStorage; use super::{ - FilesystemPlatformAdmission, StoreInitializationError, StoreInitializationPhase, - initialize_store, + FilesystemPlatformAdmission, FilesystemPlatformAdmissionError, FilesystemWriterLock, + StoreInitializationError, StoreInitializationPhase, filesystem_initialization_namespace, + filesystem_platform_profile, initialize_store, }; impl FilesystemPlatformAdmission { @@ -30,6 +36,25 @@ impl FilesystemPlatformAdmission { initialize_storage(storage) } + /// Reacquires writer authority over one completely published store. + /// + /// The call mutates no protocol state. It admits the production platform, + /// acquires the existing writer lock, and requires the exact published root + /// namespace: `writer.lock`, `staging`, `segments`, `catalogs`, and a + /// regular `HEAD`. Publication and restart adapters perform content-level + /// validation under the returned authority. The synchronous call may block + /// on filesystem I/O. + /// + /// # Errors + /// + /// Returns [`FilesystemPlatformAdmissionError`] with the exact platform, + /// writer-lock, or namespace boundary and preserved source. + pub fn reopen(store_root: &Path) -> Result { + let root = filesystem_platform_profile::open(store_root) + .map_err(|source| FilesystemPlatformAdmissionError::Platform { source })?; + reopen_root(root) + } + #[cfg(test)] pub(super) fn initialize_unchecked_for_tests( store_root: &Path, @@ -40,6 +65,15 @@ impl FilesystemPlatformAdmission { })?; initialize_storage(storage) } + + #[cfg(test)] + pub(super) fn reopen_unchecked_for_tests( + store_root: &Path, + ) -> Result { + let root = Dir::open_ambient_dir(store_root, ambient_authority()) + .map_err(|source| FilesystemPlatformAdmissionError::Platform { source })?; + reopen_root(root) + } } fn initialize_storage( @@ -51,3 +85,16 @@ fn initialize_storage( })?; Ok(FilesystemPlatformAdmission::initialized(lock)) } + +fn reopen_root( + root: cap_std::fs::Dir, +) -> Result { + let lock = FilesystemWriterLock::try_acquire_in(root) + .map_err(|source| FilesystemPlatformAdmissionError::WriterLock { source })?; + let directory = lock + .clone_directory() + .map_err(|source| FilesystemPlatformAdmissionError::Namespace { source })?; + filesystem_initialization_namespace::admit_published(&directory) + .map_err(|source| FilesystemPlatformAdmissionError::Namespace { source })?; + Ok(FilesystemPlatformAdmission::initialized(lock)) +} diff --git a/src/adapters/filesystem_store_initializer_tests.rs b/src/adapters/filesystem_store_initializer_tests.rs index 5dae865..9684787 100644 --- a/src/adapters/filesystem_store_initializer_tests.rs +++ b/src/adapters/filesystem_store_initializer_tests.rs @@ -4,7 +4,10 @@ use std::error::Error; use std::fs; use super::filesystem_test_sandbox::TestDirectory; -use super::{FilesystemPlatformAdmission, StoreInitializationError, StoreInitializationPhase}; +use super::{ + FilesystemPlatformAdmission, FilesystemPlatformAdmissionError, StoreInitializationError, + StoreInitializationPhase, +}; const LOCK_NAME: &str = "writer.lock"; const STAGING_NAME: &str = "staging"; @@ -107,3 +110,50 @@ fn retained_initializer_authority_excludes_a_second_initializer() -> Result<(), sandbox.remove()?; Ok(()) } + +#[test] +fn published_reopen_requires_a_complete_root_namespace() -> Result<(), Box> { + let sandbox = TestDirectory::create("store-reopen-incomplete")?; + let admission = FilesystemPlatformAdmission::initialize_unchecked_for_tests(sandbox.path())?; + drop(admission); + + let error = FilesystemPlatformAdmission::reopen_unchecked_for_tests(sandbox.path()) + .err() + .ok_or("published admission accepted a store without HEAD")?; + + assert!(matches!( + error, + FilesystemPlatformAdmissionError::Namespace { + ref source, + } if source.kind() == std::io::ErrorKind::NotFound + )); + assert!(!sandbox.path().join("HEAD").exists()); + sandbox.remove()?; + Ok(()) +} + +#[test] +fn published_reopen_refuses_unknown_root_evidence() -> Result<(), Box> { + let sandbox = TestDirectory::create("store-reopen-unknown")?; + let admission = FilesystemPlatformAdmission::initialize_unchecked_for_tests(sandbox.path())?; + drop(admission); + fs::write(sandbox.path().join("HEAD"), b"published head")?; + fs::write(sandbox.path().join("unknown"), b"retained evidence")?; + + let error = FilesystemPlatformAdmission::reopen_unchecked_for_tests(sandbox.path()) + .err() + .ok_or("published admission accepted an unknown root entry")?; + + assert!(matches!( + error, + FilesystemPlatformAdmissionError::Namespace { + ref source, + } if source.kind() == std::io::ErrorKind::InvalidData + )); + assert_eq!( + fs::read(sandbox.path().join("unknown"))?, + b"retained evidence" + ); + sandbox.remove()?; + Ok(()) +} diff --git a/src/adapters/mod.rs b/src/adapters/mod.rs index 748ac12..8244a14 100644 --- a/src/adapters/mod.rs +++ b/src/adapters/mod.rs @@ -78,6 +78,7 @@ mod filesystem_catalog_storage; mod filesystem_initialization_namespace; mod filesystem_initialization_storage; mod filesystem_platform_admission; +mod filesystem_platform_admission_error; mod filesystem_platform_profile; mod filesystem_publisher_authority; mod filesystem_recovery_inventory_reader; @@ -338,6 +339,7 @@ pub use filesystem_catalog_publication_error::FilesystemCatalogPublicationError; pub use filesystem_catalog_publisher::FilesystemCatalogPublisher; pub use filesystem_catalog_snapshot::FilesystemCatalogSnapshot; pub use filesystem_platform_admission::FilesystemPlatformAdmission; +pub use filesystem_platform_admission_error::FilesystemPlatformAdmissionError; pub use filesystem_recovery_inventory_reader::FilesystemRecoveryInventoryReader; pub use filesystem_recovery_next_head_finalization_open_error::FilesystemRecoveryNextHeadFinalizationOpenError; pub use filesystem_recovery_next_head_finalizer::FilesystemRecoveryNextHeadFinalizer; diff --git a/src/lib.rs b/src/lib.rs index 3d3fce8..84b5b78 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -50,30 +50,31 @@ pub use adapters::{ CatalogSnapshotError, CatalogSuccessor, CatalogTransitionError, ChecksummedCatalog, ChecksummedPublicationHead, ChecksummedSegmentRecord, ClosedSegment, FilesystemCatalogPublicationError, FilesystemCatalogPublisher, FilesystemCatalogSnapshot, - FilesystemPlatformAdmission, FilesystemRecoveryInventoryReader, - FilesystemRecoveryNextHeadFinalizationOpenError, FilesystemRecoveryNextHeadFinalizer, - FilesystemRecoverySegmentResumeOpenError, FilesystemRecoverySegmentResumer, - FilesystemRecoverySegmentStage, FilesystemRecoveryStageCompleter, - FilesystemRecoveryStageCompletionOpenError, FilesystemRecoveryStageDiscardOpenError, - FilesystemRecoveryStageDiscarder, FilesystemRecoveryStageError, FilesystemSegmentStage, - FilesystemWriterLock, LayoutDecodeError, LayoutDecodePolicy, LayoutEncodeError, - LayoutIdBinaryParseError, LayoutIdTextParseError, OpenedReusableSegment, - PublicationHeadDecodeError, RecoveryCatalogStage, RecoveryCatalogStageError, RecoveryEntryName, - RecoveryEntryNameError, RecoveryEntryRole, RecoveryInventory, RecoveryInventoryEntry, - RecoveryInventoryError, RecoveryInventoryLimit, RecoveryInventoryLimitError, - RecoveryInventoryOperation, RecoveryInventoryStorage, RecoveryNameClassificationError, - RecoveryNameManifest, RecoveryNamedEntry, RecoveryNamespace, RecoveryNextHeadFinalizationError, - RecoveryNextHeadFinalizationOutcome, RecoveryNextHeadFinalizationPlanError, - RecoveryNextHeadFinalizationReadiness, RecoveryNextHeadFinalizationReceipt, - RecoveryNextHeadFinalizationRequest, RecoveryNextHeadFinalizationStorage, - RecoveryNextHeadFinalizationStorageError, RecoveryNextHeadFinalizationTarget, - RecoveryNextHeadStage, RecoveryNextHeadStageError, RecoveryPoolNameError, - RecoveryRequiredEntry, RecoverySegmentResumeError, RecoverySegmentResumePlanError, - RecoverySegmentResumeRequest, RecoverySegmentResumeStorage, RecoverySegmentResumeStorageError, - RecoverySegmentStage, RecoverySegmentStageError, RecoverySegmentTruncation, RecoveryStage, - RecoveryStageAssessment, RecoveryStageAssessmentError, RecoveryStageByteAdmissionError, - RecoveryStageCompletionError, RecoveryStageCompletionPlanError, RecoveryStageCompletionPool, - RecoveryStageCompletionReceipt, RecoveryStageCompletionRequest, RecoveryStageCompletionStorage, + FilesystemPlatformAdmission, FilesystemPlatformAdmissionError, + FilesystemRecoveryInventoryReader, FilesystemRecoveryNextHeadFinalizationOpenError, + FilesystemRecoveryNextHeadFinalizer, FilesystemRecoverySegmentResumeOpenError, + FilesystemRecoverySegmentResumer, FilesystemRecoverySegmentStage, + FilesystemRecoveryStageCompleter, FilesystemRecoveryStageCompletionOpenError, + FilesystemRecoveryStageDiscardOpenError, FilesystemRecoveryStageDiscarder, + FilesystemRecoveryStageError, FilesystemSegmentStage, FilesystemWriterLock, LayoutDecodeError, + LayoutDecodePolicy, LayoutEncodeError, LayoutIdBinaryParseError, LayoutIdTextParseError, + OpenedReusableSegment, PublicationHeadDecodeError, RecoveryCatalogStage, + RecoveryCatalogStageError, RecoveryEntryName, RecoveryEntryNameError, RecoveryEntryRole, + RecoveryInventory, RecoveryInventoryEntry, RecoveryInventoryError, RecoveryInventoryLimit, + RecoveryInventoryLimitError, RecoveryInventoryOperation, RecoveryInventoryStorage, + RecoveryNameClassificationError, RecoveryNameManifest, RecoveryNamedEntry, RecoveryNamespace, + RecoveryNextHeadFinalizationError, RecoveryNextHeadFinalizationOutcome, + RecoveryNextHeadFinalizationPlanError, RecoveryNextHeadFinalizationReadiness, + RecoveryNextHeadFinalizationReceipt, RecoveryNextHeadFinalizationRequest, + RecoveryNextHeadFinalizationStorage, RecoveryNextHeadFinalizationStorageError, + RecoveryNextHeadFinalizationTarget, RecoveryNextHeadStage, RecoveryNextHeadStageError, + RecoveryPoolNameError, RecoveryRequiredEntry, RecoverySegmentResumeError, + RecoverySegmentResumePlanError, RecoverySegmentResumeRequest, RecoverySegmentResumeStorage, + RecoverySegmentResumeStorageError, RecoverySegmentStage, RecoverySegmentStageError, + RecoverySegmentTruncation, RecoveryStage, RecoveryStageAssessment, + RecoveryStageAssessmentError, RecoveryStageByteAdmissionError, RecoveryStageCompletionError, + RecoveryStageCompletionPlanError, RecoveryStageCompletionPool, RecoveryStageCompletionReceipt, + RecoveryStageCompletionRequest, RecoveryStageCompletionStorage, RecoveryStageCompletionStorageError, RecoveryStageCompletionTarget, RecoveryStageDiscardError, RecoveryStageDiscardOutcome, RecoveryStageDiscardPlanError, RecoveryStageDiscardReason, RecoveryStageDiscardReceipt, RecoveryStageDiscardRequest, RecoveryStageDiscardStorage, From 0f7fa3dff88f533d770ff12803d1574ebd6ebed5 Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 16:01:43 -0700 Subject: [PATCH 41/49] Fix: Validate protocol child filesystem profiles --- CHANGELOG.md | 10 +- README.md | 15 +- docs/formats/segment-store-v1/recovery.md | 13 +- docs/formats/segment-store-v1/requirements.md | 2 +- src/adapters/filesystem_platform_profile.rs | 154 +++++++++++++++--- 5 files changed, 154 insertions(+), 40 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 50546e8..b2f5ecb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,10 +18,12 @@ after its public API and format compatibility policies are established. File Worldline namespaces, bytes, hard links, released locks, recovery classifications, immutable artifacts, and published visible state after restart. CI runs the complete matrix in debug and optimized profiles. -- Production filesystem initialization now admits only the documented writable, - non-casefolded Linux ext4 profile, refuses ambiguous root namespaces before - mutation, completes the canonical directory shape idempotently, retains - writer authority, and returns only after synchronizing the root. +- Production filesystem initialization now admits only one documented + writable, non-casefolded Linux ext4 profile, independently applies it to + every existing protocol directory, requires each child to share the root's + device and mount identity, refuses ambiguous or foreign root namespaces + before mutation, completes the canonical directory shape idempotently, + retains writer authority, and returns only after synchronizing the root. - Published filesystem stores can now reacquire writer authority without mutation through a typed platform-admission boundary that requires the exact initialized root shape plus a regular `HEAD`. diff --git a/README.md b/README.md index d88b45d..786d137 100644 --- a/README.md +++ b/README.md @@ -38,13 +38,14 @@ creates the fixed `current.seg` stage without truncating existing evidence, and the `FilesystemSegmentStage` lifetime keeps that writer authority borrowed until the writable stage closes. Publisher construction consumes an unforgeable `FilesystemPlatformAdmission`. On Linux, its public initializer -admits only a writable, non-casefolded ext4 root, refuses unknown or aliased -namespace entries before mutation, creates or verifies the canonical -`writer.lock`, `staging`, `segments`, and `catalogs` shape, and returns only -after root synchronization with the writer lock retained. After publication, -`FilesystemPlatformAdmission::reopen` reacquires the existing writer lock -without mutation and requires that exact initialized shape plus a regular -`HEAD` before returning new publisher authority. +admits only one writable, non-casefolded ext4 store profile, requires every +existing protocol directory to share the root's filesystem and mount identity, +refuses unknown, aliased, or foreign namespace entries before mutation, creates +or verifies the canonical `writer.lock`, `staging`, `segments`, and `catalogs` +shape, and returns only after root synchronization with the writer lock +retained. After publication, `FilesystemPlatformAdmission::reopen` reacquires +the existing writer lock without mutation and requires that exact initialized +shape plus a regular `HEAD` before returning new publisher authority. `FilesystemCatalogPublisher` retains one kernel-managed writer lock and pinned root, staging, segment-pool, and catalog-pool capabilities for the complete diff --git a/docs/formats/segment-store-v1/recovery.md b/docs/formats/segment-store-v1/recovery.md index 7e120ae..edd8b22 100644 --- a/docs/formats/segment-store-v1/recovery.md +++ b/docs/formats/segment-store-v1/recovery.md @@ -348,6 +348,8 @@ The initial production adapter is supported only on Linux when it proves: - capability-relative no-follow access to regular files and directories; - an ext4 store root whose inode does not enable ext4 casefolding; +- `staging`, `segments`, and `catalogs` are independently writable, + non-casefolded ext4 directories on the root's device and mount; - a writable mount and successful file and directory synchronization calls; - atomic same-filesystem no-clobber hard-link creation; - atomic same-filesystem replacement of one regular file by another; @@ -358,11 +360,12 @@ The initial production adapter is supported only on Linux when it proves: - post-acquisition device-and-inode verification of the retained writer-lock handle. -The adapter refuses every non-ext4 filesystem, read-only mount, casefolded store -root, symlinked selected path, or platform other than Linux. A single local -host is an explicit deployment precondition: filesystem metadata cannot prove -that an administrator has not exposed one block device to another host. Shared -or multiply mounted ext4 is therefore unsupported even though the adapter +The adapter refuses every non-ext4 filesystem, read-only mount, casefolded +store or protocol directory, protocol-directory mount point, foreign +protocol-directory device, symlinked selected path, or platform other than +Linux. A single local host is an explicit deployment precondition: filesystem +metadata cannot prove that an administrator has not exposed one block device +to another host. Shared ext4 is therefore unsupported even though the adapter cannot distinguish it from a valid local mount. Windows support is deferred until an adapter and crash harness prove equivalent semantics. diff --git a/docs/formats/segment-store-v1/requirements.md b/docs/formats/segment-store-v1/requirements.md index 45a9080..ab41a7a 100644 --- a/docs/formats/segment-store-v1/requirements.md +++ b/docs/formats/segment-store-v1/requirements.md @@ -111,7 +111,7 @@ collection, and host-power-loss simulation remain outside issue #17. | --- | --- | --- | --- | --- | | `KEEP-RECOVERY-001` | Crash identifiers `KEEP-CRASH-001` through `KEEP-CRASH-035` form one contiguous typed vocabulary, map to the exact owning protocol sequence, and admit an occurrence counter only for record append | Ordered identifier-and-sequence ledger | `xtask/tests/durability_crash_point_contract.rs` | Implemented in #17 | | `KEEP-RECOVERY-002` | Initialization admits the platform before mutation, opens and locks the writer file, admits `staging`, `segments`, and `catalogs` in order, and returns a receipt only after root synchronization; every failed operation retains its exact phase and prevents later transitions | Fault-recording initialization port | `tests/store_initialization.rs` | Implemented in #17 | -| `KEEP-RECOVERY-003` | Production initialization admits only a writable, non-casefolded Linux ext4 root, refuses any noncanonical root entry before mutation, completes an empty or partial canonical namespace without replacing evidence, excludes a second initializer, and retains writer authority through the synchronized receipt; published-store reopen performs no mutation, reacquires the writer lock, and requires the complete initialized root plus regular `HEAD` | Capability-relative initialization, published-reopen, and exact platform-profile matrix | `src/adapters/filesystem_store_initializer_tests.rs`, `src/adapters/filesystem_catalog_publisher_tests.rs`, `src/adapters/filesystem_platform_profile.rs`, `tests/store_initialization.rs` | Implemented in #17 | +| `KEEP-RECOVERY-003` | Production initialization admits only one writable, non-casefolded Linux ext4 store profile; every existing protocol directory must independently satisfy that profile and share the root's device and mount identity; initialization refuses any noncanonical root entry before mutation, completes an empty or partial canonical namespace without replacing evidence, excludes a second initializer, and retains writer authority through the synchronized receipt; published-store reopen performs no mutation, reacquires the writer lock, and requires the complete initialized root plus regular `HEAD` | Capability-relative initialization, published-reopen, child-profile, and exact platform-profile matrix | `src/adapters/filesystem_store_initializer_tests.rs`, `src/adapters/filesystem_catalog_publisher_tests.rs`, `src/adapters/filesystem_platform_profile.rs`, `tests/store_initialization.rs` | Implemented in #17 | | `KEEP-RECOVERY-004` | Writer authority is returned only when the locked handle still has the exact device and inode resolved by the canonical `writer.lock` entry after kernel acquisition | Deterministic lock-entry replacement fixture | `src/adapters/filesystem_writer_lock_tests.rs` | Implemented in #17 | | `KEEP-RECOVERY-005` | Recovery counts the root and three protocol directories in fixed order before retaining names, refuses at the configured or protocol entry ceiling with the exact observed-at-least count, then returns one duplicate-free inventory sorted by namespace and raw name bytes | Fault-recording inventory port | `tests/recovery_inventory.rs` | Implemented in #17 | | `KEEP-RECOVERY-006` | Filesystem inventory pins the admitted root and protocol directories without following links, verifies child-directory identity before and after scanning, stops each count at the remaining global budget plus one, preserves raw Linux entry-name bytes, and performs no protocol mutation | Capability-relative filesystem fixture | `src/adapters/filesystem_recovery_inventory_tests.rs`, `tests/recovery_inventory.rs` | Implemented in #17 | diff --git a/src/adapters/filesystem_platform_profile.rs b/src/adapters/filesystem_platform_profile.rs index 11a29c6..1781a55 100644 --- a/src/adapters/filesystem_platform_profile.rs +++ b/src/adapters/filesystem_platform_profile.rs @@ -5,6 +5,20 @@ use std::path::Path; use cap_std::fs::Dir; +#[cfg(target_os = "linux")] +const PROTOCOL_DIRECTORIES: [&str; 3] = ["staging", "segments", "catalogs"]; + +#[cfg(target_os = "linux")] +#[derive(Clone, Copy)] +struct LinuxDirectoryProperties { + filesystem_type: rustix::fs::FsWord, + mount_flags: rustix::fs::StatVfsMountFlags, + inode_flags: u32, + device_major: u32, + device_minor: u32, + mount_id: u64, +} + #[cfg(target_os = "linux")] pub(super) fn open(store_root: &Path) -> io::Result { use std::fs::File; @@ -33,16 +47,44 @@ pub(super) fn open(_store_root: &Path) -> io::Result { #[cfg(target_os = "linux")] fn admit_linux_profile(directory: &Dir) -> io::Result<()> { - use rustix::fs::{fstatfs, fstatvfs, ioctl_getflags}; - let file = directory.try_clone()?.into_std_file(); - let filesystem = fstatfs(&file)?; - let mount = fstatvfs(&file)?; - let inode_flags = ioctl_getflags(&file)?; - admit_linux_properties(filesystem.f_type, mount.f_flag, inode_flags.bits())?; + let root = linux_directory_properties(&file)?; + admit_linux_properties(root.filesystem_type, root.mount_flags, root.inode_flags)?; + for name in PROTOCOL_DIRECTORIES { + let child = match super::sync_capable_directory::open(directory, name) { + Ok(child) => child, + Err(source) if source.kind() == io::ErrorKind::NotFound => continue, + Err(source) => return Err(source), + }; + let child = linux_directory_properties(&child.into_std_file())?; + admit_linux_child_properties(root, child)?; + } file.sync_all() } +#[cfg(target_os = "linux")] +fn linux_directory_properties(file: &std::fs::File) -> io::Result { + use rustix::fs::{AtFlags, StatxFlags, fstatfs, fstatvfs, ioctl_getflags, statx}; + + let filesystem = fstatfs(file)?; + let mount = fstatvfs(file)?; + let inode_flags = ioctl_getflags(file)?; + let required = StatxFlags::BASIC_STATS | StatxFlags::MNT_ID; + let status = statx(file, ".", AtFlags::empty(), required)?; + let observed = StatxFlags::from_bits_retain(status.stx_mask); + if !observed.contains(required) { + return Err(unsupported_linux_profile()); + } + Ok(LinuxDirectoryProperties { + filesystem_type: filesystem.f_type, + mount_flags: mount.f_flag, + inode_flags: inode_flags.bits(), + device_major: status.stx_dev_major, + device_minor: status.stx_dev_minor, + mount_id: status.stx_mnt_id, + }) +} + #[cfg(target_os = "linux")] fn admit_linux_properties( filesystem_type: rustix::fs::FsWord, @@ -59,17 +101,40 @@ fn admit_linux_properties( || mount_flags.contains(rustix::fs::StatVfsMountFlags::RDONLY) || inode_flags & EXT4_CASEFOLD_FLAG != 0 { - return Err(io::Error::new( - io::ErrorKind::Unsupported, - "store root does not satisfy the admitted local case-sensitive ext4 profile", - )); + return Err(unsupported_linux_profile()); + } + Ok(()) +} + +#[cfg(target_os = "linux")] +fn admit_linux_child_properties( + root: LinuxDirectoryProperties, + child: LinuxDirectoryProperties, +) -> io::Result<()> { + admit_linux_properties(child.filesystem_type, child.mount_flags, child.inode_flags)?; + if root.device_major != child.device_major + || root.device_minor != child.device_minor + || root.mount_id != child.mount_id + { + return Err(unsupported_linux_profile()); } Ok(()) } +#[cfg(target_os = "linux")] +fn unsupported_linux_profile() -> io::Error { + io::Error::new( + io::ErrorKind::Unsupported, + "store namespace does not satisfy one local writable case-sensitive ext4 profile", + ) +} + #[cfg(all(test, target_os = "linux"))] mod tests { - use super::admit_linux_properties; + use super::{ + LinuxDirectoryProperties, PROTOCOL_DIRECTORIES, admit_linux_child_properties, + admit_linux_properties, + }; use rustix::fs::{NFS_SUPER_MAGIC, StatVfsMountFlags}; @@ -79,21 +144,64 @@ mod tests { #[test] fn only_writable_case_sensitive_ext4_is_admitted() { assert!(admit_linux_properties(EXT4_SUPER_MAGIC, StatVfsMountFlags::empty(), 0).is_ok()); - assert!(matches!( - admit_linux_properties( - EXT4_SUPER_MAGIC, - StatVfsMountFlags::empty(), - EXT4_CASEFOLD_FLAG, - ), - Err(ref error) if error.kind() == std::io::ErrorKind::Unsupported + assert_unsupported(admit_linux_properties( + EXT4_SUPER_MAGIC, + StatVfsMountFlags::empty(), + EXT4_CASEFOLD_FLAG, )); - assert!(matches!( - admit_linux_properties(EXT4_SUPER_MAGIC, StatVfsMountFlags::RDONLY, 0), - Err(ref error) if error.kind() == std::io::ErrorKind::Unsupported + assert_unsupported(admit_linux_properties( + EXT4_SUPER_MAGIC, + StatVfsMountFlags::RDONLY, + 0, )); + assert_unsupported(admit_linux_properties( + NFS_SUPER_MAGIC, + StatVfsMountFlags::empty(), + 0, + )); + } + + #[test] + fn every_protocol_child_must_share_the_root_filesystem_and_mount() { + assert_eq!(PROTOCOL_DIRECTORIES, ["staging", "segments", "catalogs"]); + let root = properties(8, 1, 41); + let mut casefolded = root; + casefolded.inode_flags = EXT4_CASEFOLD_FLAG; + let mut read_only = root; + read_only.mount_flags = StatVfsMountFlags::RDONLY; + let mut foreign_format = root; + foreign_format.filesystem_type = NFS_SUPER_MAGIC; + + assert!(admit_linux_child_properties(root, root).is_ok()); + assert_unsupported(admit_linux_child_properties(root, properties(8, 2, 41))); + assert_unsupported(admit_linux_child_properties(root, properties(8, 1, 42))); + assert_unsupported(admit_linux_child_properties(root, casefolded)); + assert_unsupported(admit_linux_child_properties(root, read_only)); + assert_unsupported(admit_linux_child_properties(root, foreign_format)); + } + + fn assert_unsupported(result: std::io::Result<()>) { assert!(matches!( - admit_linux_properties(NFS_SUPER_MAGIC, StatVfsMountFlags::empty(), 0), - Err(ref error) if error.kind() == std::io::ErrorKind::Unsupported + result, + Err(ref error) + if error.kind() == std::io::ErrorKind::Unsupported + && error.to_string() + == "store namespace does not satisfy one local writable case-sensitive ext4 profile" )); } + + const fn properties( + device_major: u32, + device_minor: u32, + mount_id: u64, + ) -> LinuxDirectoryProperties { + LinuxDirectoryProperties { + filesystem_type: EXT4_SUPER_MAGIC, + mount_flags: StatVfsMountFlags::empty(), + inode_flags: 0, + device_major, + device_minor, + mount_id, + } + } } From 4c26dc5554b9202d738808f1ae93ef0225870d4f Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 16:10:03 -0700 Subject: [PATCH 42/49] Fix: Revalidate bytes before writable recovery handoff --- CHANGELOG.md | 6 ++-- README.md | 4 +-- docs/formats/segment-store-v1/recovery.md | 7 +++-- docs/formats/segment-store-v1/requirements.md | 2 +- ...esystem_recovery_segment_resume_storage.rs | 16 ++++++++++ .../refusal_laws.rs | 31 +++++++++++++++++++ src/adapters/filesystem_recovery_stage.rs | 17 ++++++++++ ...lesystem_recovery_stage_materialization.rs | 2 +- 8 files changed, 76 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b2f5ecb..a265f42 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -56,9 +56,9 @@ after its public API and format compatibility policies are established. - Filesystem reusable-segment recovery now retains pinned root, namespace, and writer-lock authority in the returned stage; reopens `current.seg` read-write without following links or truncation; bounds, materializes, and re-admits - its exact prefix; verifies the final entry and append position; and refuses - missing, changed, linked, replaced, or namespace-drifted evidence before - writing. + its exact prefix; recomputes exact stage evidence immediately before handoff; + revalidates the final entry and append position; and refuses missing, + changed, linked, replaced, or namespace-drifted evidence before writing. - Complete caller-supplied catalog and candidate-head stages now distinguish exact fixed-header, declared-body, or fixed-width truncation from canonical bytes. Complete-looking corruption and oversized stages remain typed diff --git a/README.md b/README.md index 786d137..4cb77eb 100644 --- a/README.md +++ b/README.md @@ -87,8 +87,8 @@ bounded prefix, rebuilds digest and duplicate-identity state, and returns the ordinary append-only stage without rewriting admitted bytes. `FilesystemRecoverySegmentResumer` implements that contract with pinned namespaces and writer authority, no-follow read-write reopening, exact bounded -materialization, final entry and namespace revalidation, and an append -position equal to the admitted prefix length. +materialization, final streamed fingerprint plus entry and namespace +revalidation, and an append position equal to the admitted prefix length. Exact truncation assessments can authorize durable, evidence-bound discard. Complete segment and catalog assessments can authorize verified immutable-pool diff --git a/docs/formats/segment-store-v1/recovery.md b/docs/formats/segment-store-v1/recovery.md index edd8b22..d80f8c1 100644 --- a/docs/formats/segment-store-v1/recovery.md +++ b/docs/formats/segment-store-v1/recovery.md @@ -186,8 +186,11 @@ filesystem profile. It pins the root and all protocol directories, acquires regular read-write file opened without following links or truncation. Its complete protocol-bounded bytes are fingerprinted, materialized, and re-admitted; the handle and canonical entry retain one file identity and exact -length, the pinned namespaces are reverified, and the handle is positioned at -the admitted append boundary before handoff. +length, the pinned namespaces are reverified, and the bytes are fingerprinted +again through the retained writable handle immediately before handoff. The +second fingerprint must equal the requested evidence, entry identity is +reverified after reading, and the handle remains positioned at the admitted +append boundary. The returned `FilesystemRecoverySegmentStage` owns the pinned authority and writer lock. Zero-record and nonempty prefixes both enter the same append and diff --git a/docs/formats/segment-store-v1/requirements.md b/docs/formats/segment-store-v1/requirements.md index ab41a7a..f6d729d 100644 --- a/docs/formats/segment-store-v1/requirements.md +++ b/docs/formats/segment-store-v1/requirements.md @@ -128,7 +128,7 @@ collection, and host-power-loss simulation remain outside issue #17. | `KEEP-RECOVERY-017` | Next-head finalization plans only from an exact complete `head.next` assessment and its matching complete transitive catalog snapshot, admits only generation one over an uninitialized root or the exact successor of an expected current snapshot, synchronizes a ready candidate before atomic replacement, accepts an already-finalized retry, and returns a receipt only after root synchronization | Snapshot-coordinate, transition, candidate-sync, operation-order, fault-stop, and post-replacement retry matrix | `tests/recovery_next_head_finalization.rs`, `tests/recovery_next_head_finalization/*.rs` | Implemented in #17 | | `KEEP-RECOVERY-018` | Filesystem next-head finalization retains root and `writer.lock` authority, pins every protocol directory, revalidates namespace identity and exact stage evidence around bounded complete current and candidate loads, synchronizes and reverifies the exact candidate before atomic replacement, refuses current drift, missing or reappeared candidates, links, corrupt transitive views, and namespace replacement, and returns only after root synchronization | Initial and successor finalization, exact retry, candidate-sync, evidence-drift, missing, link, corruption, namespace-replacement, current-drift, and writer-exclusion matrix | `src/adapters/filesystem_recovery_next_head_finalization_tests.rs`, `src/adapters/filesystem_recovery_next_head_finalization_tests/*.rs` | Implemented in #17 | | `KEEP-RECOVERY-019` | Reusable-segment continuation plans only from an exact reusable `current.seg` assessment within the selected record policy, consumes the storage authority that reopens the stage, re-admits the complete materialized prefix against prior evidence, rebuilds digest and duplicate-identity state, and returns the ordinary append-only stage without rewriting admitted bytes | Reusable-only planning, policy refusal, changed-evidence, storage-failure, duplicate-identity, append, seal, and independent decode matrix | `tests/recovery_segment_resume.rs`, `tests/recovery_segment_resume/*.rs` | Implemented in #17 | -| `KEEP-RECOVERY-020` | Filesystem reusable-segment continuation retains root and `writer.lock` authority in the returned stage, pins every protocol directory, opens `current.seg` read-write without following links or truncation, bounds and re-admits its complete bytes, positions the handle at the exact validated append boundary, refuses missing, changed, linked, replaced, or namespace-drifted evidence, and preserves the prefix on append or empty seal | Empty and nonempty continuation, writer exclusion, missing, changed evidence, link, namespace replacement, writable-handoff replacement, append, seal, and independent decode matrix | `src/adapters/filesystem_recovery_segment_resume_tests.rs`, `src/adapters/filesystem_recovery_segment_resume_tests/*.rs` | Implemented in #17 | +| `KEEP-RECOVERY-020` | Filesystem reusable-segment continuation retains root and `writer.lock` authority in the returned stage, pins every protocol directory, opens `current.seg` read-write without following links or truncation, bounds and re-admits its complete bytes, recomputes exact evidence immediately before handoff, revalidates entry identity after reading, positions the handle at the exact validated append boundary, refuses missing, changed, linked, replaced, or namespace-drifted evidence, and preserves the prefix on append or empty seal | Empty and nonempty continuation, writer exclusion, missing, changed evidence, link, namespace replacement, writable-handoff entry or byte replacement, append, seal, and independent decode matrix | `src/adapters/filesystem_recovery_segment_resume_tests.rs`, `src/adapters/filesystem_recovery_segment_resume_tests/*.rs` | Implemented in #17 | | `KEEP-RECOVERY-021` | Every crash point has exact before, during, and after coordinates; each child executes the production initialization, segment-writing, catalog-publication, or recovery-discard protocol through a fault-injecting port decorator; a deadline-bounded parent receives readiness over a Unix socket, retains that socket, terminates the isolated child process group, and independently verifies the exact Golden File Worldline namespace, bytes, hard-link identity, released writer lock, recovery-stage class, immutable-artifact admission, generation, and visible chunk after restart | Production protocol driver, ordered 105-case model, independent expected-state model, production recovery classifiers, production restart loader, and explicit debug/optimized CI commands | `xtask/tests/durability_crash_production_contract.rs`, `xtask/tests/durability_crash_case_contract.rs`, `xtask/tests/durability_crash_process_contract.rs`, `xtask/src/durability_crash_matrix/`, `.github/workflows/ci.yml` | Implemented in #17 | diff --git a/src/adapters/filesystem_recovery_segment_resume_storage.rs b/src/adapters/filesystem_recovery_segment_resume_storage.rs index 1b8ba76..ec33b3e 100644 --- a/src/adapters/filesystem_recovery_segment_resume_storage.rs +++ b/src/adapters/filesystem_recovery_segment_resume_storage.rs @@ -73,6 +73,22 @@ where RecoveryStageNamespacePhase::AfterObservation, ) .map_err(stage_error)?; + observed + .verify( + directory, + RecoveryStage::Segment.file_name(), + RecoveryStage::Segment, + ) + .map_err(stage_error)?; + let final_evidence = observed + .refingerprint_and_position(RecoveryStage::Segment) + .map_err(stage_error)?; + if final_evidence != request.evidence() { + return Err(RecoverySegmentResumeStorageError::EvidenceMismatch { + expected: request.evidence(), + observed: final_evidence, + }); + } observed .verify( directory, diff --git a/src/adapters/filesystem_recovery_segment_resume_tests/refusal_laws.rs b/src/adapters/filesystem_recovery_segment_resume_tests/refusal_laws.rs index d49f972..467c99f 100644 --- a/src/adapters/filesystem_recovery_segment_resume_tests/refusal_laws.rs +++ b/src/adapters/filesystem_recovery_segment_resume_tests/refusal_laws.rs @@ -112,6 +112,37 @@ fn replacement_at_the_writable_handoff_is_preserved_and_refused() -> Result<(), Ok(()) } +#[test] +fn byte_replacement_at_the_writable_handoff_is_preserved_and_refused() -> Result<(), Box> +{ + let fixture = ResumeFixture::new("filesystem-segment-resume-byte-replaced")?; + let prefix = reusable_prefix()?; + fs::write(fixture.stage_path(), &prefix)?; + let stage_path = fixture.stage_path(); + let mut replacement = prefix.clone(); + let tail = replacement.last_mut().ok_or("missing segment tail")?; + *tail ^= 1; + let replacement_for_hook = replacement.clone(); + let stage_for_hook = stage_path.clone(); + let resumer = fixture.resumer_before_handoff(move || { + let _written = fs::write(&stage_for_hook, replacement_for_hook); + })?; + + let error = execute_recovery_segment_resume(resumer, resume_request(&prefix)?) + .err() + .ok_or("byte-replaced stage unexpectedly resumed")?; + + assert!(matches!( + error, + RecoverySegmentResumeError::Open { + source: RecoverySegmentResumeStorageError::EvidenceMismatch { .. } + } + )); + assert_eq!(fs::read(stage_path)?, replacement); + fixture.remove()?; + Ok(()) +} + #[test] fn replaced_staging_namespace_is_refused_before_stage_open() -> Result<(), Box> { let fixture = ResumeFixture::new("filesystem-segment-resume-namespace")?; diff --git a/src/adapters/filesystem_recovery_stage.rs b/src/adapters/filesystem_recovery_stage.rs index 982a5cb..3359dab 100644 --- a/src/adapters/filesystem_recovery_stage.rs +++ b/src/adapters/filesystem_recovery_stage.rs @@ -1,5 +1,7 @@ //! This module owns pinned filesystem recovery-stage observation. +use std::io::{Seek, SeekFrom}; + use cap_fs_ext::{FollowSymlinks, MetadataExt, OpenOptionsFollowExt, OpenOptionsSyncExt}; use cap_std::fs::{Dir, File, Metadata, OpenOptions}; @@ -70,6 +72,21 @@ impl ObservedRecoveryStage { verify_current_entry(directory, name, stage, &self.admitted) } + pub(super) fn refingerprint_and_position( + &mut self, + stage: RecoveryStage, + ) -> Result { + let length = self.admitted.metadata.length(); + self.file + .seek(SeekFrom::Start(0)) + .map_err(|source| FilesystemRecoveryStageError::Position { stage, source })?; + let evidence = fingerprint_recovery_stage(self.admitted.metadata, &mut self.file) + .map_err(|source| FilesystemRecoveryStageError::Fingerprint { stage, source })?; + verify_length(stage, length, evidence.length().get())?; + filesystem_recovery_stage_materialization::verify_position(&mut self.file, stage, length)?; + Ok(evidence) + } + pub(super) fn into_file(self) -> File { self.file } diff --git a/src/adapters/filesystem_recovery_stage_materialization.rs b/src/adapters/filesystem_recovery_stage_materialization.rs index 124bb20..0f769a2 100644 --- a/src/adapters/filesystem_recovery_stage_materialization.rs +++ b/src/adapters/filesystem_recovery_stage_materialization.rs @@ -46,7 +46,7 @@ fn allocate( Ok(encoded) } -fn verify_position( +pub(super) fn verify_position( file: &mut File, stage: RecoveryStage, expected: RecoveryStageLength, From 430b4fb4d0dedfe8123a354f296991d2911aa3ed Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 16:25:14 -0700 Subject: [PATCH 43/49] Fix: Refuse corrupt partial recovery framing --- CHANGELOG.md | 11 +- README.md | 6 +- docs/formats/segment-store-v1/recovery.md | 11 +- docs/formats/segment-store-v1/requirements.md | 4 +- src/adapters/mod.rs | 3 + src/adapters/recovery_fixed_field_prefix.rs | 16 ++ .../recovery_next_head_stage_error.rs | 6 +- .../recovery_publication_fixed_framing.rs | 167 ++++++++++++++++++ .../recovery_publication_stage_classifier.rs | 15 +- src/adapters/recovery_segment_classifier.rs | 20 ++- .../recovery_segment_fixed_framing.rs | 121 +++++++++++++ .../catalog_laws.rs | 35 ++++ .../next_head_laws.rs | 32 ++++ .../refusal_laws.rs | 92 +++++++++- 14 files changed, 515 insertions(+), 24 deletions(-) create mode 100644 src/adapters/recovery_fixed_field_prefix.rs create mode 100644 src/adapters/recovery_publication_fixed_framing.rs create mode 100644 src/adapters/recovery_segment_fixed_framing.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index a265f42..156e40f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -46,8 +46,9 @@ after its public API and format compatibility policies are established. replacement or length drift without mutating protocol state. - Complete caller-supplied segment-stage bytes now classify as a validated reusable prefix, a complete admitted immutable segment, or an exact - truncation. Complete-looking corruption, duplicate identities, and - caller-policy excess remain typed refusals. + truncation only while every available fixed-framing byte remains canonical. + Proven partial-framing corruption, complete-looking corruption, duplicate + identities, and caller-policy excess remain typed refusals. - Storage-independent reusable-segment recovery now plans only from an exact reusable assessment within the selected resource policy, consumes reopening authority, re-admits the materialized prefix against saved evidence, rebuilds @@ -61,8 +62,10 @@ after its public API and format compatibility policies are established. changed, linked, replaced, or namespace-drifted evidence before writing. - Complete caller-supplied catalog and candidate-head stages now distinguish exact fixed-header, declared-body, or fixed-width truncation from canonical - bytes. Complete-looking corruption and oversized stages remain typed - refusals without claiming transitive catalog reachability. + bytes only while every available fixed-framing byte remains canonical. + Proven partial-framing corruption, complete-looking corruption, and + oversized stages remain typed refusals without claiming transitive catalog + reachability. - Read-only recovery assessment now admits materialized stage bytes only when their canonical-name stage, exact length, and recomputed versioned fingerprint equal prior observation evidence, then dispatches through the diff --git a/README.md b/README.md index 4cb77eb..3fc5302 100644 --- a/README.md +++ b/README.md @@ -77,8 +77,10 @@ admitted Linux ext4 profile. Its bounded stage-fingerprint operation opens fixed stages relative to those capabilities, refuses links and nonregular files, and verifies entry identity and length after reading. Complete caller-supplied segment-stage bytes can be classified as a reusable prefix, -complete admitted segment, or exact truncation. Catalog and next-head stages -likewise distinguish exact truncation from complete canonical bytes. +complete admitted segment, or exact truncation only while every available +fixed-framing byte remains canonical. Catalog and next-head stages apply the +same prefix rule before distinguishing truncation from complete canonical +bytes. Materialized bytes enter read-only semantic assessment only after their stage, length, and recomputed fingerprint match prior observation evidence. An exact reusable segment assessment can authorize storage-independent diff --git a/docs/formats/segment-store-v1/recovery.md b/docs/formats/segment-store-v1/recovery.md index d80f8c1..20c3402 100644 --- a/docs/formats/segment-store-v1/recovery.md +++ b/docs/formats/segment-store-v1/recovery.md @@ -51,10 +51,13 @@ the pinned root or staging capability, opens without following links, admits only regular files, and refuses entry replacement or length drift after reading. `classify_recovery_segment_stage` classifies complete caller-supplied stage bytes as a validated reusable prefix, a complete admitted segment, or an -exact truncation and preserves complete-looking corruption as a typed refusal. -Catalog- and next-head-stage classifiers likewise distinguish exact truncation -from complete canonical bytes. Transitive publication-view admission and -filesystem-streaming semantic classification remain unimplemented. +exact truncation only when every available segment- or record-header framing +byte remains canonical. It preserves proven partial-framing and +complete-looking corruption as typed refusals. Catalog- and +next-head-stage classifiers apply the same available-fixed-framing rule before +distinguishing exact truncation from complete canonical bytes. Transitive +publication-view admission and filesystem-streaming semantic classification +remain unimplemented. `admit_recovery_stage_bytes` first requires the canonical-name stage, exact length, and recomputed stage fingerprint to match prior observation evidence; only `assess_recovery_stage` may dispatch those admitted bytes to a semantic diff --git a/docs/formats/segment-store-v1/requirements.md b/docs/formats/segment-store-v1/requirements.md index f6d729d..fe544a2 100644 --- a/docs/formats/segment-store-v1/requirements.md +++ b/docs/formats/segment-store-v1/requirements.md @@ -118,8 +118,8 @@ collection, and host-power-loss simulation remain outside issue #17. | `KEEP-RECOVERY-007` | Name classification requires the four initialized root entries, admits only fixed protocol names and canonical pool coordinates in their owning namespaces, refuses simultaneous fixed recovery stages before artifact reads, and moves a refused raw name without duplicating its allocation | Canonical-name matrix and allocation counter | `tests/recovery_name_classification.rs`, `tests/recovery_name_classification_memory.rs` | Implemented in #17 | | `KEEP-RECOVERY-008` | Stage evidence is fingerprinted through a zero-allocation bounded streaming reader under the named recovery domain; metadata and observed bytes cannot exceed the name-selected protocol maximum, and failures retain exact stage and offset | Independent framing oracle, adversarial reader matrix, and allocation counter | `tests/recovery_stage_fingerprint.rs`, `tests/recovery_stage_fingerprint_memory.rs` | Implemented in #17 | | `KEEP-RECOVERY-009` | Filesystem stage observation uses the pinned inventory capability, never follows a fixed-stage link, admits only regular files, and refuses entry replacement or length drift after bounded fingerprinting | Capability-relative replacement fixtures | `src/adapters/filesystem_recovery_stage_tests.rs` | Implemented in #17 | -| `KEEP-RECOVERY-010` | Whole-byte segment-stage classification distinguishes a validated reusable prefix, a complete admitted immutable segment, and exact header, record, or seal truncation; complete-looking corruption, duplicates, and resource-limit excess remain typed refusals | Canonical prefix and corruption matrix | `tests/recovery_segment_classification.rs`, `tests/recovery_segment_classification/*.rs` | Implemented in #17 | -| `KEEP-RECOVERY-011` | Whole-byte catalog and next-head stage classification distinguishes exact fixed-header, declared-body, and fixed-width truncation from complete canonical bytes; complete-looking corruption and oversize remain typed format or metadata refusals | Canonical publication-artifact truncation and corruption matrix | `tests/recovery_publication_stage_classification.rs`, `tests/recovery_publication_stage_classification/*.rs` | Implemented in #17 | +| `KEEP-RECOVERY-010` | Whole-byte segment-stage classification distinguishes a validated reusable prefix, a complete admitted immutable segment, and exact header, record, or seal truncation only while every available fixed-framing byte remains canonical; proven partial-framing corruption, complete-looking corruption, duplicates, and resource-limit excess remain typed refusals | Exhaustive available-framing-byte, canonical prefix, and corruption matrix | `tests/recovery_segment_classification.rs`, `tests/recovery_segment_classification/*.rs` | Implemented in #17 | +| `KEEP-RECOVERY-011` | Whole-byte catalog and next-head stage classification distinguishes exact fixed-header, declared-body, and fixed-width truncation from complete canonical bytes only while every available fixed-framing byte remains canonical; proven partial-framing corruption, complete-looking corruption, and oversize remain typed format or metadata refusals | Exhaustive available-framing-byte, canonical publication-artifact truncation, and corruption matrix | `tests/recovery_publication_stage_classification.rs`, `tests/recovery_publication_stage_classification/*.rs` | Implemented in #17 | | `KEEP-RECOVERY-012` | Read-only semantic assessment admits materialized stage bytes only when the canonical-name stage, exact observed length, and `KEEP:RECOVERY:STAGE\0` fingerprint equal prior evidence, then dispatches through the name-selected segment, catalog, or next-head classifier | Evidence-binding mutation matrix and canonical stage assessments | `tests/recovery_stage_assessment.rs`, `tests/recovery_stage_assessment/*.rs` | Implemented in #17 | | `KEEP-RECOVERY-013` | Explicit discard plans only from an exact truncation assessment, retains the observation evidence and typed truncation reason, refuses changed evidence without mutation, synchronizes the name-selected parent after exact removal or admitted absence, and returns a receipt only after synchronization | Truncation-planning, evidence-drift, operation-order, and retry matrix | `tests/recovery_stage_discard.rs`, `tests/recovery_stage_discard/*.rs` | Implemented in #17 | | `KEEP-RECOVERY-014` | Filesystem discard retains root and `writer.lock` authority, pins every protocol directory, never follows a fixed-stage link, revalidates bounded fingerprint and entry identity before unlink, refuses drift without mutation, and synchronizes the typed parent after removal or admitted absence | Exact removal, absent retry, mismatch, symlink, replacement, and writer-exclusion matrix | `src/adapters/filesystem_recovery_stage_discard_tests.rs`, `src/adapters/filesystem_recovery_stage_discard_tests/fixture.rs` | Implemented in #17 | diff --git a/src/adapters/mod.rs b/src/adapters/mod.rs index 8244a14..ead1ffd 100644 --- a/src/adapters/mod.rs +++ b/src/adapters/mod.rs @@ -153,6 +153,7 @@ mod recovery_catalog_stage; mod recovery_catalog_stage_error; mod recovery_entry_name; mod recovery_entry_role; +mod recovery_fixed_field_prefix; mod recovery_inventory; mod recovery_inventory_error; mod recovery_inventory_limit; @@ -177,9 +178,11 @@ mod recovery_next_head_stage; mod recovery_next_head_stage_error; mod recovery_pool_name; mod recovery_pool_name_error; +mod recovery_publication_fixed_framing; mod recovery_publication_stage_classifier; mod recovery_required_entry; mod recovery_segment_classifier; +mod recovery_segment_fixed_framing; mod recovery_segment_resume_error; mod recovery_segment_resume_executor; mod recovery_segment_resume_plan_error; diff --git a/src/adapters/recovery_fixed_field_prefix.rs b/src/adapters/recovery_fixed_field_prefix.rs new file mode 100644 index 0000000..d7b5ad9 --- /dev/null +++ b/src/adapters/recovery_fixed_field_prefix.rs @@ -0,0 +1,16 @@ +//! This module owns completion of available fixed-field recovery prefixes. + +pub(super) fn observed_field( + encoded: &[u8], + offset: usize, + canonical: [u8; LENGTH], +) -> [u8; LENGTH] { + let Some(available) = encoded.get(offset..) else { + return canonical; + }; + let mut observed = canonical; + for (target, source) in observed.iter_mut().zip(available) { + *target = *source; + } + observed +} diff --git a/src/adapters/recovery_next_head_stage_error.rs b/src/adapters/recovery_next_head_stage_error.rs index 3faa555..dbfdf5d 100644 --- a/src/adapters/recovery_next_head_stage_error.rs +++ b/src/adapters/recovery_next_head_stage_error.rs @@ -5,7 +5,7 @@ use std::fmt; use super::{PublicationHeadDecodeError, RecoveryStageMetadataError}; -/// Why complete supplied `head.next` bytes could not be classified lawfully. +/// Why supplied `head.next` bytes could not be classified lawfully. #[derive(Debug)] pub enum RecoveryNextHeadStageError { /// The caller-supplied slice length cannot fit the protocol coordinate. @@ -18,7 +18,7 @@ pub enum RecoveryNextHeadStageError { /// Exact metadata-admission refusal. source: RecoveryStageMetadataError, }, - /// Complete-looking candidate-head bytes were refused. + /// Available fixed framing or complete candidate-head bytes were refused. Complete { /// Exact canonical publication-head refusal. source: PublicationHeadDecodeError, @@ -36,7 +36,7 @@ impl fmt::Display for RecoveryNextHeadStageError { write!(formatter, "next-head metadata was refused: {source}") } Self::Complete { source } => { - write!(formatter, "complete next-head stage was refused: {source}") + write!(formatter, "next-head stage was refused: {source}") } } } diff --git a/src/adapters/recovery_publication_fixed_framing.rs b/src/adapters/recovery_publication_fixed_framing.rs new file mode 100644 index 0000000..e26375f --- /dev/null +++ b/src/adapters/recovery_publication_fixed_framing.rs @@ -0,0 +1,167 @@ +//! This module owns fixed framing for incomplete recovery publication stages. + +use super::recovery_fixed_field_prefix::observed_field; +use super::{ + CatalogDecodeError, PublicationHeadDecodeError, catalog_decoder, catalog_header_decoder, + publication_head_decoder, +}; + +pub(super) fn catalog_header(encoded: &[u8]) -> Result<(), CatalogDecodeError> { + let magic = observed_field(encoded, 0, catalog_decoder::MAGIC); + if magic != catalog_decoder::MAGIC { + return Err(CatalogDecodeError::InvalidMagic { observed: magic }); + } + let version = u16::from_be_bytes(observed_field( + encoded, + 16, + catalog_decoder::VERSION.to_be_bytes(), + )); + if version != catalog_decoder::VERSION { + return Err(CatalogDecodeError::UnsupportedVersion { + expected: catalog_decoder::VERSION, + observed: version, + }); + } + validate_catalog_widths(encoded)?; + validate_catalog_coordinates(encoded) +} + +fn validate_catalog_widths(encoded: &[u8]) -> Result<(), CatalogDecodeError> { + let flags = u16::from_be_bytes(observed_field( + encoded, + 18, + catalog_decoder::FLAGS.to_be_bytes(), + )); + if flags != catalog_decoder::FLAGS { + return Err(CatalogDecodeError::Flags { + expected: catalog_decoder::FLAGS, + observed: flags, + }); + } + let header = u16::from_be_bytes(observed_field( + encoded, + 20, + catalog_header_decoder::HEADER_LENGTH.to_be_bytes(), + )); + if header != catalog_header_decoder::HEADER_LENGTH { + return Err(CatalogDecodeError::HeaderLength { + expected: catalog_header_decoder::HEADER_LENGTH, + observed: header, + }); + } + let entry = u16::from_be_bytes(observed_field( + encoded, + 22, + catalog_header_decoder::ENTRY_LENGTH.to_be_bytes(), + )); + if entry != catalog_header_decoder::ENTRY_LENGTH { + return Err(CatalogDecodeError::EntryLength { + expected: catalog_header_decoder::ENTRY_LENGTH, + observed: entry, + }); + } + Ok(()) +} + +fn validate_catalog_coordinates(encoded: &[u8]) -> Result<(), CatalogDecodeError> { + let checksum = u8::from_be_bytes(observed_field(encoded, 80, [catalog_decoder::ALGORITHM])); + if checksum != catalog_decoder::ALGORITHM { + return Err(CatalogDecodeError::ChecksumAlgorithm { + expected: catalog_decoder::ALGORITHM, + observed: checksum, + }); + } + let digest = u8::from_be_bytes(observed_field(encoded, 81, [catalog_decoder::ALGORITHM])); + if digest != catalog_decoder::ALGORITHM { + return Err(CatalogDecodeError::DigestAlgorithm { + expected: catalog_decoder::ALGORITHM, + observed: digest, + }); + } + let expected = [0_u8; 46]; + let observed = observed_field(encoded, 82, expected); + if observed == expected { + Ok(()) + } else { + Err(CatalogDecodeError::Reserved { expected, observed }) + } +} + +pub(super) fn next_head(encoded: &[u8]) -> Result<(), PublicationHeadDecodeError> { + let magic = observed_field(encoded, 0, publication_head_decoder::MAGIC); + if magic != publication_head_decoder::MAGIC { + return Err(PublicationHeadDecodeError::InvalidMagic { observed: magic }); + } + let version = u16::from_be_bytes(observed_field( + encoded, + 16, + publication_head_decoder::VERSION.to_be_bytes(), + )); + if version != publication_head_decoder::VERSION { + return Err(PublicationHeadDecodeError::UnsupportedVersion { + expected: publication_head_decoder::VERSION, + observed: version, + }); + } + validate_next_head_coordinates(encoded) +} + +fn validate_next_head_coordinates(encoded: &[u8]) -> Result<(), PublicationHeadDecodeError> { + let flags = u16::from_be_bytes(observed_field( + encoded, + 18, + publication_head_decoder::FLAGS.to_be_bytes(), + )); + if flags != publication_head_decoder::FLAGS { + return Err(PublicationHeadDecodeError::Flags { + expected: publication_head_decoder::FLAGS, + observed: flags, + }); + } + let length = u16::from_be_bytes(observed_field( + encoded, + 20, + publication_head_decoder::HEAD_LENGTH.to_be_bytes(), + )); + if length != publication_head_decoder::HEAD_LENGTH { + return Err(PublicationHeadDecodeError::HeadLength { + expected: publication_head_decoder::HEAD_LENGTH, + observed: length, + }); + } + validate_next_head_algorithms(encoded)?; + let expected = [0_u8; 24]; + let observed = observed_field(encoded, 72, expected); + if observed == expected { + Ok(()) + } else { + Err(PublicationHeadDecodeError::Reserved { expected, observed }) + } +} + +fn validate_next_head_algorithms(encoded: &[u8]) -> Result<(), PublicationHeadDecodeError> { + let checksum = u8::from_be_bytes(observed_field( + encoded, + 22, + [publication_head_decoder::ALGORITHM], + )); + if checksum != publication_head_decoder::ALGORITHM { + return Err(PublicationHeadDecodeError::ChecksumAlgorithm { + expected: publication_head_decoder::ALGORITHM, + observed: checksum, + }); + } + let digest = u8::from_be_bytes(observed_field( + encoded, + 23, + [publication_head_decoder::ALGORITHM], + )); + if digest == publication_head_decoder::ALGORITHM { + Ok(()) + } else { + Err(PublicationHeadDecodeError::DigestAlgorithm { + expected: publication_head_decoder::ALGORITHM, + observed: digest, + }) + } +} diff --git a/src/adapters/recovery_publication_stage_classifier.rs b/src/adapters/recovery_publication_stage_classifier.rs index 60371c1..d2ae2cd 100644 --- a/src/adapters/recovery_publication_stage_classifier.rs +++ b/src/adapters/recovery_publication_stage_classifier.rs @@ -4,6 +4,7 @@ use super::{ ChecksummedCatalog, ChecksummedPublicationHead, RecoveryCatalogStage, RecoveryCatalogStageError, RecoveryNextHeadStage, RecoveryNextHeadStageError, RecoveryStage, RecoveryStageMetadata, catalog_decoder, catalog_header_decoder, publication_head_decoder, + recovery_publication_fixed_framing, }; /// Classifies one complete caller-supplied catalog-stage byte sequence. @@ -13,14 +14,17 @@ use super::{ /// /// # Errors /// -/// Returns [`RecoveryCatalogStageError`] for oversized input, a complete -/// invalid header, or complete-looking canonical catalog refusal. Known -/// incomplete boundaries are returned as truncation states. +/// Returns [`RecoveryCatalogStageError`] for oversized input, proven +/// fixed-framing corruption, or complete-looking canonical catalog refusal. +/// Known incomplete boundaries are returned as truncation states only while +/// every available fixed-framing byte remains canonical. pub fn classify_recovery_catalog_stage( encoded: &[u8], ) -> Result, RecoveryCatalogStageError> { let observed = catalog_metadata_length(encoded)?; if encoded.len() < catalog_header_decoder::HEADER_LENGTH_BYTES { + recovery_publication_fixed_framing::catalog_header(encoded) + .map_err(|source| RecoveryCatalogStageError::Header { source })?; return Ok(RecoveryCatalogStage::HeaderTruncated { required: catalog_header_decoder::HEADER_LENGTH_BYTES, observed: encoded.len(), @@ -50,12 +54,15 @@ pub fn classify_recovery_catalog_stage( /// /// Returns [`RecoveryNextHeadStageError`] for oversized input or a /// complete-looking canonical publication-head refusal. Short input is -/// returned as a truncation state. +/// returned as a truncation state only while every available fixed-framing +/// byte remains canonical. pub fn classify_recovery_next_head_stage( encoded: &[u8], ) -> Result, RecoveryNextHeadStageError> { admit_next_head_metadata(encoded)?; if encoded.len() < publication_head_decoder::ENCODED_LENGTH { + recovery_publication_fixed_framing::next_head(encoded) + .map_err(|source| RecoveryNextHeadStageError::Complete { source })?; return Ok(RecoveryNextHeadStage::Truncated { required: publication_head_decoder::ENCODED_LENGTH, observed: encoded.len(), diff --git a/src/adapters/recovery_segment_classifier.rs b/src/adapters/recovery_segment_classifier.rs index 0e679f5..51cbccf 100644 --- a/src/adapters/recovery_segment_classifier.rs +++ b/src/adapters/recovery_segment_classifier.rs @@ -3,8 +3,8 @@ use super::{ AdmittedSegment, RecoverySegmentStage, RecoverySegmentStageError, RecoverySegmentTruncation, RecoveryStage, RecoveryStageMetadata, ReusableRecoverySegment, SegmentHeader, SegmentReadError, - SegmentReadPolicy, SegmentSeal, segment_identity_index, segment_record_cursor_decode, - segment_record_header, segment_seal, + SegmentReadPolicy, SegmentSeal, recovery_segment_fixed_framing, segment_identity_index, + segment_record_cursor_decode, segment_record_header, segment_seal, }; /// Classifies one complete caller-supplied segment-stage byte sequence. @@ -19,7 +19,8 @@ use super::{ /// Returns [`RecoverySegmentStageError`] for oversized input, complete-looking /// corruption, duplicate identities, resource refusal, arithmetic failure, or /// an unsupported format coordinate. Known incomplete boundaries are returned -/// as [`RecoverySegmentStage::Truncated`]. +/// as [`RecoverySegmentStage::Truncated`] only while every available +/// fixed-framing byte remains canonical. pub fn classify_recovery_segment_stage( encoded: &[u8], policy: SegmentReadPolicy, @@ -31,6 +32,8 @@ pub fn classify_recovery_segment_stage( let metadata = RecoveryStageMetadata::new(RecoveryStage::Segment, observed) .map_err(|source| RecoverySegmentStageError::Metadata { source })?; let Some(header_bytes) = encoded.get(..SegmentHeader::ENCODED_LENGTH) else { + recovery_segment_fixed_framing::segment_header(encoded) + .map_err(|source| RecoverySegmentStageError::Header { source })?; return Ok(RecoverySegmentStage::Truncated( RecoverySegmentTruncation::Header { required: SegmentHeader::ENCODED_LENGTH, @@ -77,6 +80,17 @@ fn classify_tail( .map(RecoverySegmentStage::Complete) .map_err(|source| RecoverySegmentStageError::Complete { source }); } + if cursor.remaining.len() < segment_record_header::ENCODED_LENGTH { + recovery_segment_fixed_framing::segment_tail(cursor.remaining).map_err(|source| { + RecoverySegmentStageError::Record { + source: SegmentReadError::RecordHeader { + record_index: cursor.record_index, + offset: cursor.offset, + source, + }, + } + })?; + } match cursor.advance(policy) { Ok(()) => {} Err(source) => return classify_cursor_error(source), diff --git a/src/adapters/recovery_segment_fixed_framing.rs b/src/adapters/recovery_segment_fixed_framing.rs new file mode 100644 index 0000000..c748245 --- /dev/null +++ b/src/adapters/recovery_segment_fixed_framing.rs @@ -0,0 +1,121 @@ +//! This module owns fixed-framing validation for incomplete recovery segments. + +use super::recovery_fixed_field_prefix::observed_field; +use super::segment_record_header::{ + CHECKSUM_ALGORITHM, FLAGS, HEADER_LENGTH, IDENTITY_ALGORITHM, IDENTITY_VERSION, + MAGIC as RECORD_MAGIC, RECORD_VERSION, +}; +use super::{ + SegmentHeader, SegmentHeaderError, SegmentRecordHeaderError, + segment_record_kind::SegmentRecordKind, segment_seal, +}; + +pub(super) fn segment_header(encoded: &[u8]) -> Result<(), SegmentHeaderError> { + let completed = observed_field(encoded, 0, SegmentHeader::admitted().encode()); + SegmentHeader::decode(&completed).map(|_header| ()) +} + +pub(super) fn segment_tail(encoded: &[u8]) -> Result<(), SegmentRecordHeaderError> { + if segment_seal::MAGIC.starts_with(encoded) { + return Ok(()); + } + record_header(encoded) +} + +fn record_header(encoded: &[u8]) -> Result<(), SegmentRecordHeaderError> { + let magic = observed_field(encoded, 0, RECORD_MAGIC); + if magic != RECORD_MAGIC { + return Err(SegmentRecordHeaderError::InvalidMagic { + expected: RECORD_MAGIC, + observed: magic, + }); + } + let version = u16::from_be_bytes(observed_field(encoded, 16, RECORD_VERSION.to_be_bytes())); + if version != RECORD_VERSION { + return Err(SegmentRecordHeaderError::UnsupportedVersion { + expected: RECORD_VERSION, + observed: version, + }); + } + let kind = encoded + .get(18) + .copied() + .map(SegmentRecordKind::admit) + .transpose()?; + let flags = u8::from_be_bytes(observed_field(encoded, 19, [FLAGS])); + if flags != FLAGS { + return Err(SegmentRecordHeaderError::UnknownFlags { + expected: FLAGS, + observed: flags, + }); + } + let header_length = + u16::from_be_bytes(observed_field(encoded, 20, HEADER_LENGTH.to_be_bytes())); + if header_length != HEADER_LENGTH { + return Err(SegmentRecordHeaderError::HeaderLength { + expected: HEADER_LENGTH, + observed: header_length, + }); + } + if let Some(kind) = kind { + validate_identity_length(encoded, kind)?; + } + validate_coordinates(encoded) +} + +fn validate_identity_length( + encoded: &[u8], + kind: SegmentRecordKind, +) -> Result<(), SegmentRecordHeaderError> { + let expected = kind.identity_length(); + let observed = u16::from_be_bytes(observed_field(encoded, 22, expected.to_be_bytes())); + if observed == expected { + Ok(()) + } else { + Err(SegmentRecordHeaderError::IdentityLength { + record_kind: kind.code(), + expected, + observed, + }) + } +} + +fn validate_coordinates(encoded: &[u8]) -> Result<(), SegmentRecordHeaderError> { + let checksum = u8::from_be_bytes(observed_field(encoded, 40, [CHECKSUM_ALGORITHM])); + if checksum != CHECKSUM_ALGORITHM { + return Err(SegmentRecordHeaderError::RecordChecksumAlgorithm { + expected: CHECKSUM_ALGORITHM, + observed: checksum, + }); + } + let version = u16::from_be_bytes(observed_field(encoded, 41, IDENTITY_VERSION.to_be_bytes())); + if version != IDENTITY_VERSION { + return Err(SegmentRecordHeaderError::IdentityVersion { + expected: IDENTITY_VERSION, + observed: version, + }); + } + let algorithm = u8::from_be_bytes(observed_field(encoded, 43, [IDENTITY_ALGORITHM])); + if algorithm != IDENTITY_ALGORITHM { + return Err(SegmentRecordHeaderError::IdentityAlgorithm { + expected: IDENTITY_ALGORITHM, + observed: algorithm, + }); + } + validate_reserved(encoded, 44)?; + validate_reserved(encoded, 108) +} + +fn validate_reserved(encoded: &[u8], offset: u16) -> Result<(), SegmentRecordHeaderError> { + let expected = [0_u8; 4]; + let observed = observed_field(encoded, usize::from(offset), expected); + if observed == expected { + Ok(()) + } else { + Err(SegmentRecordHeaderError::ReservedBytes { + offset, + expected, + observed, + }) + } +} diff --git a/tests/recovery_publication_stage_classification/catalog_laws.rs b/tests/recovery_publication_stage_classification/catalog_laws.rs index 5dbe5ef..e6a1a66 100644 --- a/tests/recovery_publication_stage_classification/catalog_laws.rs +++ b/tests/recovery_publication_stage_classification/catalog_laws.rs @@ -44,6 +44,41 @@ fn partial_catalog_header_is_exactly_truncated() -> Result<(), Box> { Ok(()) } +#[test] +fn every_corrupt_available_catalog_framing_byte_is_refused() -> Result<(), Box> { + let complete = fixture(CATALOG_HEX)?; + for offset in (0_usize..24).chain(80..128) { + let end = offset.checked_add(1).ok_or("catalog-header end overflow")?; + let mut encoded = complete + .get(..end) + .ok_or("missing catalog-header prefix")? + .to_vec(); + let byte = encoded + .get_mut(offset) + .ok_or("missing catalog-header byte")?; + *byte ^= 1; + + let error = classify_recovery_catalog_stage(&encoded) + .err() + .ok_or("corrupt partial catalog header was classified as truncation")?; + + assert!(matches!( + error, + RecoveryCatalogStageError::Header { + source: CatalogDecodeError::InvalidMagic { .. } + | CatalogDecodeError::UnsupportedVersion { .. } + | CatalogDecodeError::Flags { .. } + | CatalogDecodeError::HeaderLength { .. } + | CatalogDecodeError::EntryLength { .. } + | CatalogDecodeError::ChecksumAlgorithm { .. } + | CatalogDecodeError::DigestAlgorithm { .. } + | CatalogDecodeError::Reserved { .. }, + } + )); + } + Ok(()) +} + #[test] fn partial_declared_catalog_body_is_exactly_truncated() -> Result<(), Box> { let complete = fixture(CATALOG_HEX)?; diff --git a/tests/recovery_publication_stage_classification/next_head_laws.rs b/tests/recovery_publication_stage_classification/next_head_laws.rs index b8692af..3e004f6 100644 --- a/tests/recovery_publication_stage_classification/next_head_laws.rs +++ b/tests/recovery_publication_stage_classification/next_head_laws.rs @@ -40,6 +40,38 @@ fn partial_next_head_is_exactly_truncated() -> Result<(), Box> { Ok(()) } +#[test] +fn every_corrupt_available_next_head_framing_byte_is_refused() -> Result<(), Box> { + let complete = fixture(HEAD_HEX)?; + for offset in (0_usize..24).chain(72..96) { + let end = offset.checked_add(1).ok_or("next-head end overflow")?; + let mut encoded = complete + .get(..end) + .ok_or("missing next-head prefix")? + .to_vec(); + let byte = encoded.get_mut(offset).ok_or("missing next-head byte")?; + *byte ^= 1; + + let error = classify_recovery_next_head_stage(&encoded) + .err() + .ok_or("corrupt partial next head was classified as truncation")?; + + assert!(matches!( + error, + RecoveryNextHeadStageError::Complete { + source: PublicationHeadDecodeError::InvalidMagic { .. } + | PublicationHeadDecodeError::UnsupportedVersion { .. } + | PublicationHeadDecodeError::Flags { .. } + | PublicationHeadDecodeError::HeadLength { .. } + | PublicationHeadDecodeError::ChecksumAlgorithm { .. } + | PublicationHeadDecodeError::DigestAlgorithm { .. } + | PublicationHeadDecodeError::Reserved { .. }, + } + )); + } + Ok(()) +} + #[test] fn oversized_next_head_is_refused_before_decoding() -> Result<(), Box> { let mut encoded = fixture(HEAD_HEX)?; diff --git a/tests/recovery_segment_classification/refusal_laws.rs b/tests/recovery_segment_classification/refusal_laws.rs index ac4dad8..1b48925 100644 --- a/tests/recovery_segment_classification/refusal_laws.rs +++ b/tests/recovery_segment_classification/refusal_laws.rs @@ -3,8 +3,9 @@ use std::error::Error; use keep::{ - LayoutEntryLimit, RecoverySegmentStageError, SegmentReadError, SegmentReadPolicy, - SegmentRecordLimit, classify_recovery_segment_stage, + LayoutEntryLimit, RecoverySegmentStageError, SegmentHeaderError, SegmentReadError, + SegmentReadPolicy, SegmentRecordHeaderError, SegmentRecordLimit, + classify_recovery_segment_stage, }; use super::{HEADER_LENGTH, ONE_ZERO_SEGMENT_HEX, RECORD_END, maximum_policy, segment_bytes}; @@ -23,6 +24,93 @@ fn complete_invalid_header_is_a_typed_refusal() -> Result<(), Box> { Ok(()) } +#[test] +fn every_corrupt_available_segment_header_byte_is_refused() -> Result<(), Box> { + let complete = segment_bytes(ONE_ZERO_SEGMENT_HEX)?; + for offset in 0..HEADER_LENGTH { + let end = offset.checked_add(1).ok_or("segment-header end overflow")?; + let mut encoded = complete + .get(..end) + .ok_or("missing segment-header prefix")? + .to_vec(); + let byte = encoded + .get_mut(offset) + .ok_or("missing segment-header byte")?; + *byte ^= 1; + + let error = classify_recovery_segment_stage(&encoded, maximum_policy()) + .err() + .ok_or("corrupt partial segment header was classified as truncation")?; + + assert!(matches!( + error, + RecoverySegmentStageError::Header { + source: SegmentHeaderError::InvalidMagic { .. } + | SegmentHeaderError::UnsupportedVersion { .. } + | SegmentHeaderError::UnknownFlags { .. } + | SegmentHeaderError::HeaderLength { .. } + | SegmentHeaderError::RecordHeaderLength { .. } + | SegmentHeaderError::SealLength { .. } + | SegmentHeaderError::ReservedU16 { .. } + | SegmentHeaderError::MaximumRecordPayloadLength { .. } + | SegmentHeaderError::MaximumSegmentLength { .. } + | SegmentHeaderError::MaximumRecordCount { .. } + | SegmentHeaderError::RecordChecksumAlgorithm { .. } + | SegmentHeaderError::SegmentDigestAlgorithm { .. } + | SegmentHeaderError::ReservedBytes { .. }, + } + )); + } + Ok(()) +} + +#[test] +fn every_corrupt_available_record_framing_byte_is_refused() -> Result<(), Box> { + let complete = segment_bytes(ONE_ZERO_SEGMENT_HEX)?; + for offset in (0..24).chain(40..48).chain(108..112) { + let end = HEADER_LENGTH + .checked_add(offset) + .and_then(|value| value.checked_add(1)) + .ok_or("record-header end overflow")?; + let mut encoded = complete + .get(..end) + .ok_or("missing record-header prefix")? + .to_vec(); + let absolute = HEADER_LENGTH + .checked_add(offset) + .ok_or("record-header offset overflow")?; + let byte = encoded + .get_mut(absolute) + .ok_or("missing record-header byte")?; + *byte ^= 1; + + let error = classify_recovery_segment_stage(&encoded, maximum_policy()) + .err() + .ok_or("corrupt partial record header was classified as truncation")?; + + assert!(matches!( + error, + RecoverySegmentStageError::Record { + source: SegmentReadError::RecordHeader { + record_index: 0, + offset: 64, + source: SegmentRecordHeaderError::InvalidMagic { .. } + | SegmentRecordHeaderError::UnsupportedVersion { .. } + | SegmentRecordHeaderError::UnknownRecordKind { .. } + | SegmentRecordHeaderError::UnknownFlags { .. } + | SegmentRecordHeaderError::HeaderLength { .. } + | SegmentRecordHeaderError::IdentityLength { .. } + | SegmentRecordHeaderError::RecordChecksumAlgorithm { .. } + | SegmentRecordHeaderError::IdentityVersion { .. } + | SegmentRecordHeaderError::IdentityAlgorithm { .. } + | SegmentRecordHeaderError::ReservedBytes { .. }, + }, + } + )); + } + Ok(()) +} + #[test] fn complete_invalid_record_is_a_typed_refusal() -> Result<(), Box> { let mut encoded = segment_bytes(ONE_ZERO_SEGMENT_HEX)?; From 069d7d18d76c364af831b6e6476895cc392de9b2 Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 16:37:23 -0700 Subject: [PATCH 44/49] Fix: Use canonical truncation in discard fixtures --- ...filesystem_recovery_stage_discard_tests.rs | 63 ++++++++++--------- .../fixture.rs | 19 +++++- 2 files changed, 51 insertions(+), 31 deletions(-) diff --git a/src/adapters/filesystem_recovery_stage_discard_tests.rs b/src/adapters/filesystem_recovery_stage_discard_tests.rs index 78097dd..f5eda4d 100644 --- a/src/adapters/filesystem_recovery_stage_discard_tests.rs +++ b/src/adapters/filesystem_recovery_stage_discard_tests.rs @@ -11,24 +11,25 @@ use super::{ mod fixture; -use fixture::{DiscardFixture, evidence, request}; +use fixture::{DiscardFixture, evidence, request, truncated_bytes}; #[test] fn exact_stage_discard_and_absent_retry_are_durable_for_every_parent() -> Result<(), Box> { let fixture = DiscardFixture::new("filesystem-stage-discard")?; - let cases: [(RecoveryStage, &[u8]); 3] = [ - (RecoveryStage::Segment, b"s"), - (RecoveryStage::Catalog, b"c"), - (RecoveryStage::NextHead, b"h"), + let stages = [ + RecoveryStage::Segment, + RecoveryStage::Catalog, + RecoveryStage::NextHead, ]; - for (stage, bytes) in cases { - fs::write(fixture.stage_path(stage), bytes)?; + for stage in stages { + fs::write(fixture.stage_path(stage), truncated_bytes(stage, 1)?)?; } let mut discarder = fixture.discarder()?; - for (stage, bytes) in cases { - let request = request(stage, bytes)?; + for stage in stages { + let bytes = truncated_bytes(stage, 1)?; + let request = request(stage, &bytes)?; let removed = execute_recovery_stage_discard(&mut discarder, request)?; let retried = execute_recovery_stage_discard(&mut discarder, request)?; @@ -47,9 +48,11 @@ fn exact_stage_discard_and_absent_retry_are_durable_for_every_parent() -> Result #[test] fn changed_stage_evidence_is_preserved_and_refused_before_unlink() -> Result<(), Box> { let fixture = DiscardFixture::new("filesystem-stage-discard-mismatch")?; - let expected = request(RecoveryStage::Segment, b"old")?; - fs::write(fixture.stage_path(RecoveryStage::Segment), b"new")?; - let observed = evidence(RecoveryStage::Segment, b"new")?; + let old = truncated_bytes(RecoveryStage::Segment, 1)?; + let new = truncated_bytes(RecoveryStage::Segment, 2)?; + let expected = request(RecoveryStage::Segment, &old)?; + fs::write(fixture.stage_path(RecoveryStage::Segment), &new)?; + let observed = evidence(RecoveryStage::Segment, &new)?; let mut discarder = fixture.discarder()?; let error = execute_recovery_stage_discard(&mut discarder, expected) @@ -65,10 +68,7 @@ fn changed_stage_evidence_is_preserved_and_refused_before_unlink() -> Result<(), }, } if actual_expected == expected.evidence() && actual_observed == observed )); - assert_eq!( - fs::read(fixture.stage_path(RecoveryStage::Segment))?, - b"new" - ); + assert_eq!(fs::read(fixture.stage_path(RecoveryStage::Segment))?, new); drop(discarder); fixture.remove()?; Ok(()) @@ -79,9 +79,10 @@ fn symbolic_stage_is_never_followed_or_removed() -> Result<(), Box> { use std::os::unix::fs::symlink; let fixture = DiscardFixture::new("filesystem-stage-discard-symlink")?; - let request = request(RecoveryStage::Segment, b"outside")?; + let bytes = truncated_bytes(RecoveryStage::Segment, 1)?; + let request = request(RecoveryStage::Segment, &bytes)?; let target = fixture.root().join("outside"); - fs::write(&target, b"outside")?; + fs::write(&target, &bytes)?; symlink(&target, fixture.stage_path(RecoveryStage::Segment))?; let mut discarder = fixture.discarder()?; @@ -99,7 +100,7 @@ fn symbolic_stage_is_never_followed_or_removed() -> Result<(), Box> { .. } )); - assert_eq!(fs::read(&target)?, b"outside"); + assert_eq!(fs::read(&target)?, bytes); assert!(fixture.stage_path(RecoveryStage::Segment).is_symlink()); drop(discarder); fixture.remove()?; @@ -111,14 +112,16 @@ fn replacement_after_open_refuses_without_removing_the_new_entry() -> Result<(), let fixture = DiscardFixture::new("filesystem-stage-discard-replaced")?; let stage_path = fixture.stage_path(RecoveryStage::Segment); let retained_path = fixture.root().join("retained-stage"); - fs::write(&stage_path, b"old")?; - let request = request(RecoveryStage::Segment, b"old")?; + let old = truncated_bytes(RecoveryStage::Segment, 1)?; + let new = truncated_bytes(RecoveryStage::Segment, 2)?; + fs::write(&stage_path, &old)?; + let request = request(RecoveryStage::Segment, &old)?; let discarder = fixture.discarder()?; let mut hook_result = Ok(()); let result = discarder.remove_if_matching_with(request.evidence(), || { hook_result = - fs::rename(&stage_path, &retained_path).and_then(|()| fs::write(&stage_path, b"new")); + fs::rename(&stage_path, &retained_path).and_then(|()| fs::write(&stage_path, &new)); }); hook_result?; @@ -129,8 +132,8 @@ fn replacement_after_open_refuses_without_removing_the_new_entry() -> Result<(), stage: RecoveryStage::Segment, } )); - assert_eq!(fs::read(&stage_path)?, b"new"); - assert_eq!(fs::read(&retained_path)?, b"old"); + assert_eq!(fs::read(&stage_path)?, new); + assert_eq!(fs::read(&retained_path)?, old); drop(discarder); fixture.remove()?; Ok(()) @@ -142,14 +145,16 @@ fn replacement_after_observation_refuses_without_removing_the_new_entry() let fixture = DiscardFixture::new("filesystem-stage-discard-final-handoff")?; let stage_path = fixture.stage_path(RecoveryStage::Segment); let retained_path = fixture.root().join("retained-observed-stage"); - fs::write(&stage_path, b"old")?; - let request = request(RecoveryStage::Segment, b"old")?; + let old = truncated_bytes(RecoveryStage::Segment, 1)?; + let new = truncated_bytes(RecoveryStage::Segment, 2)?; + fs::write(&stage_path, &old)?; + let request = request(RecoveryStage::Segment, &old)?; let discarder = fixture.discarder()?; let mut hook_result = Ok(()); let result = discarder.remove_if_matching_after_observation_with(request.evidence(), || { hook_result = - fs::rename(&stage_path, &retained_path).and_then(|()| fs::write(&stage_path, b"new")); + fs::rename(&stage_path, &retained_path).and_then(|()| fs::write(&stage_path, &new)); }); hook_result?; @@ -160,8 +165,8 @@ fn replacement_after_observation_refuses_without_removing_the_new_entry() stage: RecoveryStage::Segment, } )); - assert_eq!(fs::read(&stage_path)?, b"new"); - assert_eq!(fs::read(&retained_path)?, b"old"); + assert_eq!(fs::read(&stage_path)?, new); + assert_eq!(fs::read(&retained_path)?, old); drop(discarder); fixture.remove()?; Ok(()) diff --git a/src/adapters/filesystem_recovery_stage_discard_tests/fixture.rs b/src/adapters/filesystem_recovery_stage_discard_tests/fixture.rs index 5b6346a..6199f7f 100644 --- a/src/adapters/filesystem_recovery_stage_discard_tests/fixture.rs +++ b/src/adapters/filesystem_recovery_stage_discard_tests/fixture.rs @@ -8,9 +8,9 @@ use crate::LayoutEntryLimit; use super::super::{ FilesystemRecoveryStageDiscardOpenError, FilesystemRecoveryStageDiscarder, RecoveryStage, RecoveryStageDiscardRequest, RecoveryStageEvidence, RecoveryStageMetadata, SegmentReadPolicy, - SegmentRecordLimit, admit_recovery_stage_bytes, assess_recovery_stage, + SegmentRecordLimit, admit_recovery_stage_bytes, assess_recovery_stage, catalog_decoder, filesystem_test_sandbox::TestDirectory, fingerprint_recovery_stage, - plan_recovery_stage_discard, + plan_recovery_stage_discard, publication_head_decoder, segment_header, }; pub(super) fn request( @@ -34,6 +34,21 @@ pub(super) fn evidence( )?) } +pub(super) fn truncated_bytes( + stage: RecoveryStage, + length: usize, +) -> Result, Box> { + let framing: &[u8] = match stage { + RecoveryStage::Segment => &segment_header::MAGIC, + RecoveryStage::Catalog => &catalog_decoder::MAGIC, + RecoveryStage::NextHead => &publication_head_decoder::MAGIC, + }; + Ok(framing + .get(..length) + .ok_or("truncated fixture length exceeds fixed framing")? + .to_vec()) +} + const fn maximum_policy() -> SegmentReadPolicy { SegmentReadPolicy::new(SegmentRecordLimit::MAXIMUM, LayoutEntryLimit::MAXIMUM) } From 4015d8db7a057e5c4c27e1540347ab63988d7189 Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 16:40:24 -0700 Subject: [PATCH 45/49] Fix: Use canonical finalization truncation fixture --- tests/recovery_next_head_finalization/planning_laws.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/recovery_next_head_finalization/planning_laws.rs b/tests/recovery_next_head_finalization/planning_laws.rs index d86c217..5ece69c 100644 --- a/tests/recovery_next_head_finalization/planning_laws.rs +++ b/tests/recovery_next_head_finalization/planning_laws.rs @@ -137,8 +137,11 @@ fn only_a_complete_next_head_can_enter_finalization() -> Result<(), Box Date: Wed, 29 Jul 2026 16:41:17 -0700 Subject: [PATCH 46/49] Fix: Use canonical completion truncation fixture --- tests/recovery_stage_completion/planning_laws.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/recovery_stage_completion/planning_laws.rs b/tests/recovery_stage_completion/planning_laws.rs index 7b5a0d8..64f3a35 100644 --- a/tests/recovery_stage_completion/planning_laws.rs +++ b/tests/recovery_stage_completion/planning_laws.rs @@ -62,7 +62,11 @@ fn reusable_and_truncated_pool_stages_are_not_completion_requests() -> Result<() .get(..reusable_length) .ok_or("reusable segment prefix is outside the fixture")?; let reusable = assessment(RecoveryStage::Segment, reusable_bytes)?; - let truncated_catalog = assessment(RecoveryStage::Catalog, &[0_u8])?; + let catalog = fixture(CATALOG_HEX)?; + let truncated_catalog_bytes = catalog + .get(..1) + .ok_or("canonical catalog fixture is unexpectedly empty")?; + let truncated_catalog = assessment(RecoveryStage::Catalog, truncated_catalog_bytes)?; for assessed in [&reusable, &truncated_catalog] { let error = plan_recovery_stage_completion(assessed) From 8a26b6c894bf47e4102fdfb8f5d996ade8ce13bd Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 16:42:40 -0700 Subject: [PATCH 47/49] Fix: Use canonical discard truncation fixtures --- tests/recovery_stage_discard.rs | 12 ++++++++++++ tests/recovery_stage_discard/execution_laws.rs | 10 +++++----- tests/recovery_stage_discard/planning_laws.rs | 7 ++++--- 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/tests/recovery_stage_discard.rs b/tests/recovery_stage_discard.rs index e8f4ff3..1759c12 100644 --- a/tests/recovery_stage_discard.rs +++ b/tests/recovery_stage_discard.rs @@ -31,6 +31,18 @@ fn fixture(hex: &str) -> Result, Box> { .map_err(Into::into) } +fn truncated_fixture(stage: RecoveryStage) -> Result, Box> { + let complete = fixture(match stage { + RecoveryStage::Segment => SEGMENT_HEX, + RecoveryStage::Catalog => CATALOG_HEX, + RecoveryStage::NextHead => HEAD_HEX, + })?; + complete + .get(..1) + .map(<[u8]>::to_vec) + .ok_or_else(|| "canonical recovery fixture is unexpectedly empty".into()) +} + fn evidence(stage: RecoveryStage, encoded: &[u8]) -> Result> { let length = u64::try_from(encoded.len())?; Ok(fingerprint_recovery_stage( diff --git a/tests/recovery_stage_discard/execution_laws.rs b/tests/recovery_stage_discard/execution_laws.rs index ffd4522..e825a4d 100644 --- a/tests/recovery_stage_discard/execution_laws.rs +++ b/tests/recovery_stage_discard/execution_laws.rs @@ -9,11 +9,11 @@ use keep::{ }; use super::storage_double::{Operation, StageDiscardDouble}; -use super::{discard_request, evidence}; +use super::{discard_request, evidence, truncated_fixture}; #[test] fn exact_evidence_is_removed_before_its_parent_is_synchronized() -> Result<(), Box> { - let bytes = [0_u8]; + let bytes = truncated_fixture(RecoveryStage::Segment)?; let request = discard_request(RecoveryStage::Segment, &bytes)?; let mut storage = StageDiscardDouble::new(Some(request.evidence())); @@ -40,7 +40,7 @@ fn absent_exact_retry_still_synchronizes_the_selected_parent() -> Result<(), Box RecoveryStage::Catalog, RecoveryStage::NextHead, ] { - let bytes = [0_u8]; + let bytes = truncated_fixture(stage)?; let request = discard_request(stage, &bytes)?; let mut storage = StageDiscardDouble::new(None); @@ -63,7 +63,7 @@ fn absent_exact_retry_still_synchronizes_the_selected_parent() -> Result<(), Box #[test] fn changed_evidence_refuses_without_removal_or_parent_sync() -> Result<(), Box> { - let bytes = [0_u8]; + let bytes = truncated_fixture(RecoveryStage::Segment)?; let changed = [1_u8]; let request = discard_request(RecoveryStage::Segment, &bytes)?; let observed = evidence(RecoveryStage::Segment, &changed)?; @@ -92,7 +92,7 @@ fn changed_evidence_refuses_without_removal_or_parent_sync() -> Result<(), Box Result<(), Box> { - let bytes = [0_u8]; + let bytes = truncated_fixture(RecoveryStage::NextHead)?; let request = discard_request(RecoveryStage::NextHead, &bytes)?; let mut storage = StageDiscardDouble::new(Some(request.evidence())).fail_next_synchronization(); diff --git a/tests/recovery_stage_discard/planning_laws.rs b/tests/recovery_stage_discard/planning_laws.rs index 583d5b1..85b9f0e 100644 --- a/tests/recovery_stage_discard/planning_laws.rs +++ b/tests/recovery_stage_discard/planning_laws.rs @@ -9,13 +9,14 @@ use keep::{ use super::{ CATALOG_HEX, HEAD_HEX, SEGMENT_HEX, SEGMENT_SEAL_LENGTH, assessment, evidence, fixture, + truncated_fixture, }; #[test] fn every_exact_truncation_retains_its_reason_and_evidence() -> Result<(), Box> { - let segment = [0_u8]; - let catalog = [0_u8]; - let head = [0_u8]; + let segment = truncated_fixture(RecoveryStage::Segment)?; + let catalog = truncated_fixture(RecoveryStage::Catalog)?; + let head = truncated_fixture(RecoveryStage::NextHead)?; let segment_evidence = evidence(RecoveryStage::Segment, &segment)?; let catalog_evidence = evidence(RecoveryStage::Catalog, &catalog)?; let head_evidence = evidence(RecoveryStage::NextHead, &head)?; From f88ab38fb35443ad265354af348665263daa639c Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 16:46:39 -0700 Subject: [PATCH 48/49] Fix: Synchronize fuzz dependency lock --- fuzz/Cargo.lock | 1 + 1 file changed, 1 insertion(+) diff --git a/fuzz/Cargo.lock b/fuzz/Cargo.lock index 7a86f0f..7d421ee 100644 --- a/fuzz/Cargo.lock +++ b/fuzz/Cargo.lock @@ -210,6 +210,7 @@ dependencies = [ "blake3", "cap-fs-ext", "cap-std", + "rustix", ] [[package]] From 1c8937d3fb14a362fc4dada5afed12389dc16957 Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 16:59:56 -0700 Subject: [PATCH 49/49] Fix: Borrow Linux profile test results --- src/adapters/filesystem_platform_profile.rs | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/adapters/filesystem_platform_profile.rs b/src/adapters/filesystem_platform_profile.rs index 1781a55..62e561d 100644 --- a/src/adapters/filesystem_platform_profile.rs +++ b/src/adapters/filesystem_platform_profile.rs @@ -144,17 +144,17 @@ mod tests { #[test] fn only_writable_case_sensitive_ext4_is_admitted() { assert!(admit_linux_properties(EXT4_SUPER_MAGIC, StatVfsMountFlags::empty(), 0).is_ok()); - assert_unsupported(admit_linux_properties( + assert_unsupported(&admit_linux_properties( EXT4_SUPER_MAGIC, StatVfsMountFlags::empty(), EXT4_CASEFOLD_FLAG, )); - assert_unsupported(admit_linux_properties( + assert_unsupported(&admit_linux_properties( EXT4_SUPER_MAGIC, StatVfsMountFlags::RDONLY, 0, )); - assert_unsupported(admit_linux_properties( + assert_unsupported(&admit_linux_properties( NFS_SUPER_MAGIC, StatVfsMountFlags::empty(), 0, @@ -173,17 +173,17 @@ mod tests { foreign_format.filesystem_type = NFS_SUPER_MAGIC; assert!(admit_linux_child_properties(root, root).is_ok()); - assert_unsupported(admit_linux_child_properties(root, properties(8, 2, 41))); - assert_unsupported(admit_linux_child_properties(root, properties(8, 1, 42))); - assert_unsupported(admit_linux_child_properties(root, casefolded)); - assert_unsupported(admit_linux_child_properties(root, read_only)); - assert_unsupported(admit_linux_child_properties(root, foreign_format)); + assert_unsupported(&admit_linux_child_properties(root, properties(8, 2, 41))); + assert_unsupported(&admit_linux_child_properties(root, properties(8, 1, 42))); + assert_unsupported(&admit_linux_child_properties(root, casefolded)); + assert_unsupported(&admit_linux_child_properties(root, read_only)); + assert_unsupported(&admit_linux_child_properties(root, foreign_format)); } - fn assert_unsupported(result: std::io::Result<()>) { + fn assert_unsupported(result: &std::io::Result<()>) { assert!(matches!( result, - Err(ref error) + Err(error) if error.kind() == std::io::ErrorKind::Unsupported && error.to_string() == "store namespace does not satisfy one local writable case-sensitive ext4 profile"