diff --git a/src/api/router/snapshot_content.rs b/src/api/router/snapshot_content.rs index 6a1f191f..275f7dd7 100644 --- a/src/api/router/snapshot_content.rs +++ b/src/api/router/snapshot_content.rs @@ -50,7 +50,7 @@ pub(super) async fn fixed_root_tree( +async fn resolve_legacy_file_metadata( handler: &T, root_tree: &git_internal::internal::object::tree::Tree, scope: &str, @@ -89,6 +89,132 @@ pub(super) async fn resolve_file_metadata( + handler: &T, + ctx: &crate::ceres::snapshot::runtime::SnapshotContext, +) -> Result, Response> { + use crate::jupiter::storage::qualified_metadata_family::SnapshotMetadataFamily; + if !handler.get_context().config().mst2.publication_enabled { + return fixed_root_tree(handler, &ctx.root_tree_oid).await.map(Some); + } + match handler + .get_context() + .snapshot_metadata_family(&ctx.lease_id, true) + .await + .map_err(mst2_error_response)? + { + Some(SnapshotMetadataFamily::Rooted) => Ok(None), + Some(SnapshotMetadataFamily::Generic) => { + fixed_root_tree(handler, &ctx.root_tree_oid).await.map(Some) + } + None => Err(mst2_error_response(SnapshotError::new( + SnapshotErrorCode::SnapshotGone, + "fixed content lease has no permanent storage route", + ))), + } +} + +#[allow(clippy::result_large_err)] +async fn resolve_file_metadata( + handler: &T, + root_tree: Option<&git_internal::internal::object::tree::Tree>, + ctx: &crate::ceres::snapshot::runtime::SnapshotContext, + path: &str, + expected_digest: Option<&str>, +) -> Result { + if let Some(root_tree) = root_tree { + return resolve_legacy_file_metadata( + handler, + root_tree, + &ctx.built.descriptor.scope, + path, + expected_digest, + ) + .await; + } + use crate::jupiter::storage::qualified_metadata_family::RootedLookupStatus; + let storage = handler.get_context(); + let repository = storage + .rooted_qualified_metadata_writer() + .await + .map_err(internal)?; + let (entry, git_oid) = match repository + .fixed_path_metadata(ctx, path) + .await + .map_err(mst2_error_response)? + { + RootedLookupStatus::File { entry, git_oid } => (entry, git_oid), + RootedLookupStatus::Directory(_) => { + return Err(mst2_error_response(SnapshotError::new( + SnapshotErrorCode::NotDirectory, + format!("{path} is a directory"), + ))); + } + RootedLookupStatus::Absent => { + return Err(mst2_error_response(SnapshotError::new( + SnapshotErrorCode::PathNotFound, + format!("{path} absent in the fixed view"), + ))); + } + RootedLookupStatus::NotDirectory { symlink } => { + return Err(mst2_error_response(SnapshotError::new( + if symlink { + SnapshotErrorCode::SymlinkTraversal + } else { + SnapshotErrorCode::NotDirectory + }, + format!("{path}: intermediate component is not a directory"), + ))); + } + }; + let fs_kind = match entry.kind { + mst2_codec::metapage::EntryKind::Regular => FsKind::Regular, + mst2_codec::metapage::EntryKind::Executable => FsKind::Executable, + mst2_codec::metapage::EntryKind::Symlink => FsKind::Symlink, + mst2_codec::metapage::EntryKind::Directory => { + return Err(mst2_error_response(internal( + "fixed source file has a directory kind", + ))); + } + }; + let oid = git_oid + .split_once(':') + .ok_or_else(|| mst2_error_response(internal("fixed source OID has no hash kind")))? + .1 + .to_owned(); + let file = verified_file_metadata(handler, fs_kind, oid, path, expected_digest).await?; + if file.size != entry.size || file.digest != entry.content_id { + return Err(mst2_error_response(SnapshotError::new( + SnapshotErrorCode::IntegrityError, + "fixed content metadata changed from its certified current source occurrence", + ))); + } + Ok(file) +} + +#[allow(clippy::result_large_err)] +pub(super) async fn resolve_snapshot_file_metadata( + state: &crate::api::MonoApiServiceState, + ctx: &crate::ceres::snapshot::runtime::SnapshotContext, + path: &str, + expected_digest: Option<&str>, +) -> Result { + let handler = state + .api_handler(std::path::Path::new("/")) + .await + .map_err(internal)?; + let root_tree = fixed_content_root(handler.as_ref(), ctx).await?; + resolve_file_metadata( + handler.as_ref(), + root_tree.as_ref(), + ctx, + path, + expected_digest, + ) + .await +} + #[allow(clippy::result_large_err)] pub(super) async fn verified_file_metadata( handler: &T, @@ -264,11 +390,11 @@ pub(super) async fn blob_head( .api_handler(std::path::Path::new("/")) .await .map_err(internal)?; - let root_tree = fixed_root_tree(handler.as_ref(), &ctx.root_tree_oid).await?; + let root_tree = fixed_content_root(handler.as_ref(), &ctx).await?; let f = resolve_file_metadata( handler.as_ref(), - &root_tree, - &ctx.built.descriptor.scope, + root_tree.as_ref(), + &ctx, &q.path, q.expected_digest.as_deref(), ) @@ -331,20 +457,19 @@ pub(super) async fn objects( .api_handler(std::path::Path::new("/")) .await .map_err(internal)?; - let root_tree = fixed_root_tree(handler.as_ref(), &ctx.root_tree_oid).await?; - let scope = ctx.built.descriptor.scope.clone(); + let root_tree = fixed_content_root(handler.as_ref(), &ctx).await?; // Copied references: each per-item future borrows the shared walk state // without moving it (the stream is an FnMut over owned items). let handler_ref = handler.as_ref(); - let root_ref = &root_tree; - let scope_ref = &scope; + let root_ref = root_tree.as_ref(); + let ctx_ref = &ctx; let resolved: Vec> = futures::stream::iter(req.items.clone()) .map(move |item| async move { validate_scope_relative_path(&item.path).map_err(mst2_error_response)?; let f = resolve_file_metadata( handler_ref, root_ref, - scope_ref, + ctx_ref, &item.path, Some(&item.expected_digest), ) @@ -468,13 +593,13 @@ pub(super) struct ChunkMapQuery { #[allow(clippy::result_large_err)] async fn project_for( handler: &T, - root_tree: &git_internal::internal::object::tree::Tree, - scope: &str, + root_tree: Option<&git_internal::internal::object::tree::Tree>, + ctx: &crate::ceres::snapshot::runtime::SnapshotContext, path: &str, expected_digest: Option<&str>, ) -> Result, Response> { - let f = resolve_file_metadata(handler, root_tree, scope, path, expected_digest).await?; + let f = resolve_file_metadata(handler, root_tree, ctx, path, expected_digest).await?; project_resolved(handler, &f).await } @@ -552,12 +677,11 @@ pub(super) async fn chunk_map( .api_handler(std::path::Path::new("/")) .await .map_err(internal)?; - let root_tree = fixed_root_tree(handler.as_ref(), &ctx.root_tree_oid).await?; - let scope = ctx.built.descriptor.scope.clone(); + let root_tree = fixed_content_root(handler.as_ref(), &ctx).await?; let proj = project_for( handler.as_ref(), - &root_tree, - &scope, + root_tree.as_ref(), + &ctx, &q.path, q.expected_digest.as_deref(), ) @@ -619,12 +743,11 @@ pub(super) async fn chunk_map_pages( .api_handler(std::path::Path::new("/")) .await .map_err(internal)?; - let root_tree = fixed_root_tree(handler.as_ref(), &ctx.root_tree_oid).await?; - let scope = ctx.built.descriptor.scope.clone(); + let root_tree = fixed_content_root(handler.as_ref(), &ctx).await?; let proj = project_for( handler.as_ref(), - &root_tree, - &scope, + root_tree.as_ref(), + &ctx, &q.path, q.expected_digest.as_deref(), ) @@ -894,8 +1017,7 @@ pub(super) async fn chunks( .api_handler(std::path::Path::new("/")) .await .map_err(internal)?; - let root_tree = fixed_root_tree(handler.as_ref(), &ctx.root_tree_oid).await?; - let scope = ctx.built.descriptor.scope.clone(); + let root_tree = fixed_content_root(handler.as_ref(), &ctx).await?; let mut planned: Vec = Vec::new(); let mut resolved: Vec = Vec::new(); let mut units: Vec<(String, u64)> = Vec::new(); @@ -907,8 +1029,8 @@ pub(super) async fn chunks( parse_decimal_count(&item.chunk_index, "chunk_index").map_err(mst2_error_response)?; let file = resolve_file_metadata( handler.as_ref(), - &root_tree, - &scope, + root_tree.as_ref(), + &ctx, &item.path, Some(&item.expected_digest), ) diff --git a/src/api/router/snapshot_content_tests.rs b/src/api/router/snapshot_content_tests.rs index 617cf2ea..65234d8b 100644 --- a/src/api/router/snapshot_content_tests.rs +++ b/src/api/router/snapshot_content_tests.rs @@ -107,6 +107,8 @@ mod generation_qualified_fixture; mod install_capability_fixture; type ReceiptWriteHold = (Arc, Arc); +#[path = "snapshot_rooted_metadata_tests.rs"] +mod rooted_metadata; #[derive(Default)] struct ReadCounts { @@ -393,6 +395,7 @@ impl LogStorage for CountingStorage { struct Fixture { app: Router, state: MonoApiServiceState, + generic_history: bool, snapshot: String, lease: String, oid: String, @@ -503,12 +506,47 @@ impl Fixture { rebuildable: bool, directory_count: usize, objects: &[(String, Vec)], + ) -> Self { + Self::new_in_metadata_family(rebuildable, directory_count, objects, false).await + } + + async fn new_generic_history_with_pg_config(rebuildable: bool) -> Self { + Self::new_generic_history_with_pg_config_and_directories(rebuildable, 0).await + } + + async fn new_generic_history_with_pg_config_and_directories( + rebuildable: bool, + directory_count: usize, + ) -> Self { + Self::new_in_metadata_family(rebuildable, directory_count, &[], true).await + } + + async fn new_in_metadata_family( + rebuildable: bool, + directory_count: usize, + objects: &[(String, Vec)], + generic_history: bool, + ) -> Self { + Self::new_in_publication_mode(rebuildable, directory_count, objects, generic_history, true) + .await + } + + async fn new_without_publication() -> Self { + Self::new_in_publication_mode(true, 0, &[], false, false).await + } + + async fn new_in_publication_mode( + rebuildable: bool, + directory_count: usize, + objects: &[(String, Vec)], + generic_history: bool, + publication_enabled: bool, ) -> Self { let temp = tempfile::tempdir().unwrap(); let mut config = isolated_config(temp.path().join("config")); config.monorepo.push_policy = PushPolicy::Trunk; config.mst2.enabled = true; - config.mst2.publication_enabled = true; + config.mst2.publication_enabled = publication_enabled; config.mst2.instance_uuid = Some(uuid::Uuid::new_v4().to_string()); config.mst2.auth_token = Some(TOKEN.to_string()); let backend = build_object_storage(&config.object_storage).await.unwrap(); @@ -516,22 +554,37 @@ impl Fixture { let (mut storage, schema) = if rebuildable { let (database, schema) = test_db_config(temp.path()).await; config.database = database; - let connection = crate::jupiter::storage::init::database_connection(&config.database) - .await - .unwrap(); - ( + let assembly = async { + let connection = + crate::jupiter::storage::init::database_connection(&config.database) + .await + .unwrap(); crate::jupiter::storage::Storage::new_with_connection( Arc::new(config), Arc::new(connection), backend.clone(), ) .await - .unwrap(), - Some(schema), - ) + .unwrap() + }; + let storage = if generic_history { + crate::jupiter::storage::init::with_generic_history_bootstrap(assembly).await + } else { + assembly.await + }; + (storage, Some(schema)) } else { (test_storage_with_config(temp.path(), config).await, None) }; + if generic_history { + let q_rows:i64=storage.mono_storage().get_connection().query_one_raw(sea_orm::Statement::from_string( + sea_orm::DbBackend::Postgres,"SELECT count(*) FROM mst2_metadata_namespace WHERE graph_domain='qualified-v1'")) + .await.unwrap().unwrap().try_get_by_index(0).unwrap(); + assert_eq!( + q_rows, 0, + "G history fixtures must not create or erase actual Q ownership" + ); + } storage.git_service = GitService { obj_storage: MegaObjectStorageWrapper::new(Arc::new(CountingStorage { inner: backend, @@ -664,18 +717,24 @@ impl Fixture { ) .await .unwrap(); - mono.initialize_native_publication(storage.config().mst2.instance_uuid.as_deref().unwrap()) - .await - .unwrap(); - publish_native_push(&storage, "/project", old_tip.id, &new_tip).await; - let head = mono - .read_native_publication_head(storage.config().mst2.instance_uuid.as_deref().unwrap()) + if publication_enabled { + mono.initialize_native_publication( + storage.config().mst2.instance_uuid.as_deref().unwrap(), + ) .await .unwrap(); - assert_eq!(head.token.sequence, 1); - assert!(head.token.certificate.is_some()); - assert_eq!(head.root.commit, commit.id.to_string()); - assert_eq!(head.root.tree, root.id.to_string()); + publish_native_push(&storage, "/project", old_tip.id, &new_tip).await; + let head = mono + .read_native_publication_head( + storage.config().mst2.instance_uuid.as_deref().unwrap(), + ) + .await + .unwrap(); + assert_eq!(head.token.sequence, 1); + assert!(head.token.certificate.is_some()); + assert_eq!(head.root.commit, commit.id.to_string()); + assert_eq!(head.root.tree, root.id.to_string()); + } let state = MonoApiServiceState { entity_store: storage.entity_store.clone(), storage, @@ -686,7 +745,12 @@ impl Fixture { }), listen_addr: "127.0.0.1:0".to_string(), }; - let app = Router::new().nest("/api/v2", routers(state.clone()).with_state(state.clone())); + let routes = if generic_history { + crate::api::router::snapshot_router::generic_history_routers(state.clone()) + } else { + routers(state.clone()) + }; + let app = Router::new().nest("/api/v2", routes.with_state(state.clone())); let response = app .clone() .oneshot( @@ -724,6 +788,7 @@ impl Fixture { digest: Sha256::digest(&raw).into(), raw, counts, + generic_history, _temp: temp, _schema: schema, } @@ -915,6 +980,123 @@ async fn mst2_fixed_head_uses_verified_facts_without_body_reads_and_preserves_ra assert_eq!(fixture.counts.receipt_writes.load(Ordering::SeqCst), 1); } +#[tokio::test] +async fn mst2_publication_disabled_resolve_preserves_head_body_object_map_page_and_chunk_content() { + let fixture = Fixture::new_without_publication().await; + assert!(!fixture.state.storage.config().mst2.publication_enabled); + assert_eq!( + fixture + .state + .storage + .snapshot_metadata_family(&fixture.lease, true) + .await + .unwrap(), + None + ); + let routes: i64 = fixture + .state + .storage + .mono_storage() + .get_connection() + .query_one_raw(sea_orm::Statement::from_string( + sea_orm::DbBackend::Postgres, + "SELECT count(*) FROM mst2_snapshot_storage_route", + )) + .await + .unwrap() + .unwrap() + .try_get_by_index(0) + .unwrap(); + assert_eq!( + routes, 0, + "runtime resolve does not create a permanent SID route" + ); + let head = fixture.send("HEAD", "blob?path=/file", Body::empty()).await; + assert_eq!(head.status(), 200); + assert_eq!( + head.headers()["content-length"], + fixture.raw.len().to_string() + ); + assert_eq!( + head.headers()["etag"], + format!("\"{}\"", fixture.digest_string()) + ); + assert!(to_bytes(head.into_body(), 1024).await.unwrap().is_empty()); + fixture.counts.assert(0, 0); + let body = fixture.send("GET", "blob?path=/file", Body::empty()).await; + assert_eq!(body.status(), 200); + assert_eq!( + to_bytes(body.into_body(), 2 * 1024 * 1024) + .await + .unwrap() + .as_ref(), + fixture.raw + ); + let link_digest = digest(b"file"); + let request=json!({"items":[{"path":"/link","expected_digest":format!("sha256:{}",hex_of(&link_digest))}],"encoding":"identity"}).to_string(); + let objects = fixture + .send("POST", "objects", Body::from(request.clone())) + .await; + assert_eq!(objects.status(), 200); + let wire = to_bytes(objects.into_body(), 2 * 1024 * 1024) + .await + .unwrap(); + let frames = parse_stream(&wire).unwrap(); + let [Frame::Object(object), Frame::End(end)] = frames.as_slice() else { + panic!("expected OBJECT and terminal END"); + }; + assert_eq!(object.objects, vec![(link_digest, b"file".to_vec())]); + assert_eq!(end.request_item_count, 1); + assert_eq!(end.logical_bytes, 4); + assert_eq!( + end.request_body_sha256, + <[u8; 32]>::from(Sha256::digest(request.as_bytes())) + ); + let map = fixture.map("/file").await; + let map_id = map["map"]["map_id"].as_str().unwrap(); + assert_eq!(map["map"]["file_content_id"], fixture.digest_string()); + let page = success_json( + fixture + .send( + "GET", + &format!("chunk-map/pages?path=/file&map_id={map_id}&page_index=0"), + Body::empty(), + ) + .await, + ) + .await; + let leaf = ChunkLeaf::decode( + &STANDARD + .decode(page["leaf_base64"].as_str().unwrap()) + .unwrap(), + ) + .unwrap(); + let hashes: Vec<[u8; 32]> = fixture + .raw + .chunks(CHUNK_SIZE as usize) + .map(|bytes| Sha256::digest(bytes).into()) + .collect(); + assert_eq!(leaf.chunk_sha256, hashes); + let chunks = fixture + .send( + "POST", + "chunks", + Body::from(fixture.chunk_body("/file", map_id, "0").to_string()), + ) + .await; + assert_eq!(chunks.status(), 200); + let wire = to_bytes(chunks.into_body(), 2 * 1024 * 1024).await.unwrap(); + let frames = parse_stream(&wire).unwrap(); + let [Frame::Chunk(chunk), Frame::End(end)] = frames.as_slice() else { + panic!("expected CHUNK and terminal END"); + }; + assert_eq!(chunk.chunk_bytes, &fixture.raw[..CHUNK_SIZE as usize]); + assert_eq!(chunk.file_content_id, fixture.digest); + assert_eq!(chunk.chunk_index, 0); + assert_eq!(end.request_item_count, 1); + assert_eq!(end.logical_bytes, CHUNK_SIZE); +} + #[tokio::test] async fn mst2_fixed_warm_map_and_leaf_aliases_skip_body_reads_and_chunks_use_current_ranges() { let fixture = Fixture::new().await; diff --git a/src/api/router/snapshot_generation_upgrade_tests.rs b/src/api/router/snapshot_generation_upgrade_tests.rs index 08684866..28f4a203 100644 --- a/src/api/router/snapshot_generation_upgrade_tests.rs +++ b/src/api/router/snapshot_generation_upgrade_tests.rs @@ -17,7 +17,7 @@ async fn scalar(db: &sea_orm::DatabaseConnection, sql: &str) -> i64 { #[tokio::test] async fn mst2_generation_additive_upgrade_preserves_legacy_v3_sid_lease_and_new_resolve() { - let fixture = Fixture::new_with_pg_config(true).await; + let fixture = Fixture::new_generic_history_with_pg_config(true).await; let mono = fixture.state.storage.mono_storage(); let db = mono.get_connection(); assert_eq!( @@ -102,10 +102,12 @@ async fn mst2_generation_additive_upgrade_preserves_legacy_v3_sid_lease_and_new_ let connection = crate::jupiter::storage::init::postgres_connection(&config.database) .await .unwrap(); - let storage = crate::jupiter::storage::Storage::new_with_connection( - config, - Arc::new(connection), - fixture.state.storage.git_service.obj_storage.clone(), + let storage = crate::jupiter::storage::init::with_generic_history_bootstrap( + crate::jupiter::storage::Storage::new_with_connection( + config, + Arc::new(connection), + fixture.state.storage.git_service.obj_storage.clone(), + ), ) .await .unwrap(); @@ -113,7 +115,11 @@ async fn mst2_generation_additive_upgrade_preserves_legacy_v3_sid_lease_and_new_ storage, ..fixture.state.clone() }; - let app = Router::new().nest("/api/v2", routers(state.clone()).with_state(state)); + let app = Router::new().nest( + "/api/v2", + crate::api::router::snapshot_router::generic_history_routers(state.clone()) + .with_state(state), + ); let restored = success_json( app.clone() .oneshot(fixture.request("GET", "descriptor", Body::empty())) diff --git a/src/api/router/snapshot_persisted_metadata_tests.rs b/src/api/router/snapshot_persisted_metadata_tests.rs index 057d9d26..ba95602d 100644 --- a/src/api/router/snapshot_persisted_metadata_tests.rs +++ b/src/api/router/snapshot_persisted_metadata_tests.rs @@ -65,7 +65,7 @@ fn assert_metadata(bytes: &[u8], body: &[u8], count: u32, expected: &[([u8; 32], #[tokio::test] async fn mst2_persisted_meta_matches_canonical_routes_after_rebuild_and_advance_without_git_reads() { - let fixture = Fixture::new_with_pg_config_and_directories(true, 140).await; + let fixture = Fixture::new_generic_history_with_pg_config_and_directories(true, 140).await; let context = fixture .state .storage @@ -184,7 +184,7 @@ async fn mst2_persisted_meta_matches_canonical_routes_after_rebuild_and_advance_ #[tokio::test] async fn mst2_persisted_meta_absence_scope_digest_limits_and_release_oracles() { - let fixture = Fixture::new_with_pg_config(true).await; + let fixture = Fixture::new_generic_history_with_pg_config(true).await; for (items, status, code) in [ ( json!([{"directory_path":"/missing"}]), @@ -336,7 +336,7 @@ async fn damage_page(fixture: &Fixture, id: [u8; 32], remove: bool) { #[tokio::test] async fn mst2_persisted_meta_warm_missing_and_corrupt_pages_never_reproject_from_git() { for remove in [true, false] { - let fixture = Fixture::new_with_pg_config(true).await; + let fixture = Fixture::new_generic_history_with_pg_config(true).await; let (id, _) = page_row(&fixture, "/nested").await; damage_page(&fixture, id, remove).await; let body = metadata_body(json!([{"directory_path":"/nested"}]), "identity"); @@ -398,7 +398,7 @@ async fn wait_retention_waiter(txn: &sea_orm::DatabaseTransaction) { #[tokio::test] async fn mst2_persisted_meta_rechecks_deadline_and_release_after_waiting_for_retention_lock() { for expire in [true, false] { - let fixture = Fixture::new_with_pg_config(true).await; + let fixture = Fixture::new_generic_history_with_pg_config(true).await; let mono = fixture.state.storage.mono_storage(); let held = mono.get_connection().begin().await.unwrap(); held.execute_unprepared( @@ -441,7 +441,7 @@ async fn bound_fixture() -> ( Fixture, crate::ceres::snapshot::retention_dag::MetadataPagePayload, ) { - let mut fixture = Fixture::new_with_pg_config(true).await; + let mut fixture = Fixture::new_generic_history_with_pg_config(true).await; advance(&fixture).await; let mono = fixture.state.storage.mono_storage(); let head = mono @@ -546,7 +546,7 @@ async fn mst2_persisted_meta_serves_bound_generic_pages_with_null_members_and_ex #[tokio::test] async fn mst2_persisted_meta_rejects_touched_graph_damage_and_prepare_membership_loss() { for case in 0..5 { - let fixture = Fixture::new_with_pg_config(true).await; + let fixture = Fixture::new_generic_history_with_pg_config(true).await; let (id, _) = page_row(&fixture, "/nested").await; let mono = fixture.state.storage.mono_storage(); let db = mono.get_connection(); @@ -618,7 +618,7 @@ async fn mst2_persisted_meta_reader_holds_protection_until_all_route_bytes_are_o use crate::jupiter::storage::native_snapshot_session::with_metadata_read_barriers; for release_lease in [false, true] { - let fixture = Fixture::new_with_pg_config(true).await; + let fixture = Fixture::new_generic_history_with_pg_config(true).await; let expected = vec![ page_row(&fixture, "/nested").await, page_row(&fixture, "/directory").await, @@ -691,10 +691,11 @@ async fn mst2_persisted_meta_reader_holds_protection_until_all_route_bytes_are_o assert_eq!(observer_config.max_connection, 2); observer_config.max_connection = 1; observer_config.min_connection = 0; - let observer_connection = - crate::jupiter::storage::init::database_connection(&observer_config) - .await - .unwrap(); + let observer_connection = crate::jupiter::storage::init::with_generic_history_bootstrap( + crate::jupiter::storage::init::database_connection(&observer_config), + ) + .await + .unwrap(); let observer = observer_connection.begin().await.unwrap(); tokio::select! { () = wait_retention_waiter(&observer) => {}, diff --git a/src/api/router/snapshot_raw_blob.rs b/src/api/router/snapshot_raw_blob.rs index 6aee3b5c..ea438ec1 100644 --- a/src/api/router/snapshot_raw_blob.rs +++ b/src/api/router/snapshot_raw_blob.rs @@ -91,11 +91,9 @@ pub(super) async fn blob_with_budgets( .api_handler(std::path::Path::new("/")) .await .map_err(internal)?; - let root = content::fixed_root_tree(handler.as_ref(), &context.root_tree_oid).await?; - let file = content::resolve_file_metadata( - handler.as_ref(), - &root, - &context.built.descriptor.scope, + let file = content::resolve_snapshot_file_metadata( + &state, + &context, &query.path, query.expected_digest.as_deref(), ) diff --git a/src/api/router/snapshot_rooted_metadata.rs b/src/api/router/snapshot_rooted_metadata.rs new file mode 100644 index 00000000..a35c7bd1 --- /dev/null +++ b/src/api/router/snapshot_rooted_metadata.rs @@ -0,0 +1,185 @@ +//! JSON metadata endpoints use the same certified, protected fixed-root reader. + +use mst2_codec::metapage::{Entry, EntryKind}; + +use super::*; +use crate::{ + ceres::snapshot::runtime::SnapshotContext, + jupiter::storage::qualified_metadata_family::RootedLookupStatus, +}; + +fn entry_json(entry: &Entry) -> Result { + let name = std::str::from_utf8(&entry.name).map_err(internal)?; + let kind = match entry.kind { + EntryKind::Regular => "regular", + EntryKind::Executable => "executable", + EntryKind::Symlink => "symlink", + EntryKind::Directory => "directory", + }; + let mut value = json!({"name":name,"fs_kind":kind}); + if entry.is_dir() { + value["directory_root"] = json!(format!("sha256:{}", hex_of(&entry.child_root))); + value["node_class"] = json!("native_tree"); + value["lifecycle"] = json!("mutable"); + } else { + value["size"] = json!(entry.size.to_string()); + value["content_digest"] = json!(format!("sha256:{}", hex_of(&entry.content_id))); + } + Ok(value) +} + +fn cursor_name(sid: &str, path: &str, query: &DirectoryQuery) -> Result, Response> { + let Some(cursor) = query.cursor.as_deref() else { + return Ok(None); + }; + let invalid = || { + mst2_error_response(SnapshotError::new( + SnapshotErrorCode::CursorInvalid, + "cursor bound to different parameters or invalid", + )) + }; + let (payload, signature) = cursor.rsplit_once('.').ok_or_else(invalid)?; + if runtime().sign_cursor(payload) != signature { + return Err(invalid()); + } + let bytes = base64::engine::general_purpose::STANDARD + .decode(payload) + .map_err(|_| invalid())?; + let value: serde_json::Value = serde_json::from_slice(&bytes).map_err(|_| invalid())?; + if value["s"].as_str() != Some(sid) + || value["p"].as_str() != Some(path) + || value["l"].as_u64() != Some(query.limit as u64) + { + return Err(invalid()); + } + let name = value["a"].as_str().ok_or_else(invalid)?; + if name.is_empty() || name.len() > 255 || name.contains('/') || name.contains('\0') { + return Err(invalid()); + } + Ok(Some(name.into())) +} + +#[allow(clippy::result_large_err)] +pub(super) async fn directory_response( + state: &MonoApiServiceState, + ctx: &SnapshotContext, + sid: &str, + query: &DirectoryQuery, +) -> Result { + let absolute = abs_view_path(&ctx.built.descriptor.scope, &query.path); + let last = cursor_name(sid, &absolute, query)?; + let repository = state + .storage + .rooted_qualified_metadata_writer() + .await + .map_err(internal)?; + let window = repository + .directory_window(ctx, &query.path, last.as_deref(), query.limit as usize) + .await + .map_err(mst2_error_response)?; + let entries = window + .entries + .iter() + .map(entry_json) + .collect::, _>>()?; + let next = if window.has_more { + let name = window.entries.last().ok_or_else(|| { + mst2_error_response(internal( + "qualified directory returned an empty continuation", + )) + })?; + let payload = json!({"s":sid,"p":absolute,"l":query.limit,"a":std::str::from_utf8(&name.name).map_err(internal)?}); + let encoded = base64_of(&serde_json::to_vec(&payload).map_err(internal)?); + Some(format!("{encoded}.{}", runtime().sign_cursor(&encoded))) + } else { + None + }; + let ancestors = if query.ancestors.as_deref() == Some("chain") { + window + .ancestors + .iter() + .filter(|(path, _)| path != &query.path) + .map(|(path, root)| { + json!({"path":path, + "directory_root":format!("sha256:{}",hex_of(root)),"node_class":"native_tree"}) + }) + .collect::>() + } else { + Vec::new() + }; + revalidate_request(state, ctx) + .await + .map_err(mst2_error_response)?; + let body = json!({"snapshot_id":sid,"path":query.path,"metadata_root":ctx.built.metadata_root, + "directory_root":format!("sha256:{}",hex_of(&window.directory_root)),"node_class":"native_tree","lifecycle":"mutable", + "range_start_exclusive":last,"entries":entries,"entry_count":window.entry_count.to_string(),"next_cursor":next, + "proof_pages":window.proof_pages.iter().map(|(root,bytes)|json!({"digest":format!("sha256:{}",hex_of(root)), + "data_base64":base64_of(bytes)})).collect::>(),"ancestor_chain":ancestors}); + let mut response = Json(body).into_response(); + let tag = format!( + "\"{}:{}:{}:{}\"", + &sid[..16.min(sid.len())], + hex_of(&window.directory_root), + query.limit, + query.cursor.as_deref().unwrap_or("") + ); + if let Ok(value) = HeaderValue::from_str(&tag) { + response.headers_mut().insert("etag", value); + } + response.headers_mut().insert( + "cache-control", + HeaderValue::from_static("private, no-cache, no-transform"), + ); + Ok(response) +} + +#[allow(clippy::result_large_err)] +pub(super) async fn lookup_response( + state: &MonoApiServiceState, + ctx: &SnapshotContext, + sid: &str, + request: &LookupRequest, +) -> Result { + let repository = state + .storage + .rooted_qualified_metadata_writer() + .await + .map_err(internal)?; + let batch = repository + .lookup_metadata(ctx, &request.paths) + .await + .map_err(mst2_error_response)?; + let mut results = Vec::with_capacity(batch.results.len()); + for (path, status) in request.paths.iter().zip(batch.results) { + let mut item = json!({"path":path}); + match status { + RootedLookupStatus::Directory(root) => { + let mut node = json!({"fs_kind":"directory","directory_root":format!("sha256:{}",hex_of(&root)), + "node_class":"native_tree","lifecycle":"mutable"}); + if path != "/" { + node["name"] = json!(path.rsplit('/').next()); + } + item["status"] = json!("found"); + item["node"] = node; + } + RootedLookupStatus::File { entry, .. } => { + item["status"] = json!("found"); + item["node"] = entry_json(&entry)?; + } + RootedLookupStatus::Absent => item["status"] = json!("absent"), + RootedLookupStatus::NotDirectory { symlink } => { + item["status"] = json!(if symlink { + "symlink_traversal" + } else { + "not_directory" + }) + } + } + results.push(item); + } + revalidate_request(state, ctx) + .await + .map_err(mst2_error_response)?; + Ok(Json(json!({"snapshot_id":sid,"results":results,"proof_pages":batch.proof_pages.iter().map(|(root,bytes)|json!({ + "digest":format!("sha256:{}",hex_of(root)),"data_base64":base64_of(bytes)})).collect::>()})).into_response()) +} diff --git a/src/api/router/snapshot_rooted_metadata_tests.rs b/src/api/router/snapshot_rooted_metadata_tests.rs new file mode 100644 index 00000000..7c944913 --- /dev/null +++ b/src/api/router/snapshot_rooted_metadata_tests.rs @@ -0,0 +1,784 @@ +use sea_orm::{DbBackend, Statement, TransactionTrait}; +use tokio::sync::Barrier; + +use super::*; +use crate::jupiter::storage::qualified_metadata_family::{ + SnapshotMetadataFamily, with_rooted_reader_barriers, with_rooted_source_fact_barriers, + with_rooted_source_temporary_shadow, +}; + +async fn q_schema(fixture: &Fixture) -> String { + fixture + .state + .storage + .mono_storage() + .get_connection() + .query_one_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + "SELECT namespace.metadata_schema FROM mst2_snapshot_storage_route route + JOIN mst2_metadata_namespace namespace USING(namespace_uuid) WHERE route.snapshot_id=$1 + AND namespace.graph_domain='qualified-v1'", + [fixture.snapshot.clone().into()], + )) + .await + .unwrap() + .unwrap() + .try_get_by_index(0) + .unwrap() +} + +async fn q_count(fixture: &Fixture, query: &str) -> i64 { + let schema = q_schema(fixture).await; + let query = query.replace("{q}", &format!("\"{}\"", schema.replace('"', "\"\""))); + fixture + .state + .storage + .mono_storage() + .get_connection() + .query_one_raw(Statement::from_string(DbBackend::Postgres, query)) + .await + .unwrap() + .unwrap() + .try_get_by_index(0) + .unwrap() +} + +async fn resolve(fixture: &Fixture) -> Value { + success_json( + fixture + .app + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri("/api/v2/snapshots/resolve") + .header("authorization", format!("Bearer {TOKEN}")) + .header("content-type", "application/json") + .body(Body::from( + json!({"target":{"kind":"latest"},"scope":"/project"}).to_string(), + )) + .unwrap(), + ) + .await + .unwrap(), + ) + .await +} + +async fn release(fixture: &Fixture, lease: &str) -> Value { + success_json( + fixture + .app + .clone() + .oneshot( + Request::builder() + .method("DELETE") + .uri(format!("/api/v2/snapshots/leases/{lease}")) + .header("authorization", format!("Bearer {TOKEN}")) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(), + ) + .await +} + +async fn collect_unowned(fixture: &Fixture) { + let writer = fixture + .state + .storage + .rooted_qualified_metadata_writer() + .await + .unwrap(); + for _ in 0..8 { + let work = writer.maintenance_tick(64).await.unwrap(); + assert!(work.collector_enabled && work.examined <= 64); + if q_count(fixture, "SELECT count(*) FROM {q}.mst2_metadata_payload").await == 0 { + return; + } + } + panic!("bounded maintenance failed to collect the small unowned fixture"); +} + +#[tokio::test] +async fn default_rooted_http_handoff_renew_release_and_fresh_incarnation_keep_exact_ownership() { + let fixture = Fixture::new_with_pg_config(true).await; + assert_eq!( + fixture + .state + .storage + .snapshot_metadata_family(&fixture.lease, true) + .await + .unwrap(), + Some(SnapshotMetadataFamily::Rooted) + ); + assert_eq!( + q_count( + &fixture, + "SELECT count(*) FROM {q}.mst2_qualified_session_incarnation WHERE state='READY'" + ) + .await, + 1 + ); + let first_generation = q_count( + &fixture, + "SELECT root_generation FROM {q}.mst2_qualified_session_incarnation WHERE state='READY'", + ) + .await; + assert_eq!(q_count(&fixture,"SELECT count(*) FROM {q}.mst2_metadata_root_anchor WHERE anchor_kind IN ('PREPARE','REUSE')").await,0); + assert_eq!(q_count(&fixture,"SELECT count(*) FROM {q}.mst2_metadata_prepare WHERE plan_kind='ROOTED' AND state='COMMITTED' AND coverage_retired_at IS NOT NULL").await,1); + let original_payloads = + q_count(&fixture, "SELECT count(*) FROM {q}.mst2_metadata_payload").await; + let next = resolve(&fixture).await; + assert_eq!(next["descriptor"]["snapshot_id"], fixture.snapshot); + let second = next["lease_id"].as_str().unwrap(); + assert_ne!(second, fixture.lease); + assert_eq!( + q_count(&fixture, "SELECT count(*) FROM {q}.mst2_metadata_prepare").await, + 1 + ); + assert_eq!( + q_count(&fixture, "SELECT count(*) FROM {q}.mst2_metadata_payload").await, + original_payloads + ); + let previous_deadline = fixture + .state + .storage + .snapshot_context(&fixture.snapshot, second) + .await + .unwrap() + .lease_expires_at_unix; + let renewed = fixture + .state + .storage + .snapshot_renew(second, 1) + .await + .unwrap(); + assert_eq!(renewed.snapshot_id, fixture.snapshot); + assert!(renewed.expires_at_unix >= previous_deadline); + assert_eq!(release(&fixture, &fixture.lease).await["released"], true); + assert_eq!(release(&fixture, &fixture.lease).await["released"], false); + assert_eq!(q_count(&fixture,"SELECT count(*) FROM {q}.mst2_qualified_lease_binding WHERE state='RELEASED' AND lease_epoch=2").await,1); + let mut request = fixture.request("GET", "descriptor", Body::empty()); + request + .headers_mut() + .insert("x-mega-snapshot-lease", second.parse().unwrap()); + success_json(fixture.app.clone().oneshot(request).await.unwrap()).await; + assert_eq!(release(&fixture, second).await["released"], true); + assert_eq!( + q_count( + &fixture, + "SELECT count(*) FROM {q}.mst2_metadata_root_anchor" + ) + .await, + 0 + ); + assert_eq!( + q_count( + &fixture, + "SELECT count(*) FROM {q}.mst2_qualified_session_incarnation WHERE state='RETIRED'" + ) + .await, + 1 + ); + collect_unowned(&fixture).await; + assert_eq!( + q_count( + &fixture, + "SELECT count(*) FROM {q}.mst2_metadata_graph_node" + ) + .await, + 0 + ); + assert!( + q_count( + &fixture, + "SELECT count(*) FROM {q}.mst2_metadata_page_certificate" + ) + .await + > 0 + ); + let fresh = resolve(&fixture).await; + assert_eq!(fresh["descriptor"]["snapshot_id"], fixture.snapshot); + assert_ne!(fresh["lease_id"], next["lease_id"]); + assert_eq!( + q_count( + &fixture, + "SELECT count(*) FROM {q}.mst2_qualified_session_incarnation" + ) + .await, + 2 + ); + assert_eq!( + q_count( + &fixture, + "SELECT count(*) FROM {q}.mst2_qualified_session_incarnation WHERE state='READY'" + ) + .await, + 1 + ); + assert_eq!( + q_count(&fixture, "SELECT count(*) FROM {q}.mst2_metadata_payload").await, + original_payloads + ); + assert_eq!( + q_count( + &fixture, + "SELECT root_generation FROM {q}.mst2_qualified_session_incarnation WHERE state='READY'" + ) + .await, + first_generation + 1 + ); + fixture.counts.assert(0, 0); +} + +#[tokio::test] +async fn rooted_wide_directory_windows_and_lookup_survive_rebuild_with_valid_proofs() { + let fixture = Fixture::new_with_pg_config_and_directories(true, 140).await; + let config = fixture.state.storage.config(); + let connection = crate::jupiter::storage::init::postgres_connection(&config.database) + .await + .unwrap(); + let storage = crate::jupiter::storage::Storage::new_with_connection( + config, + Arc::new(connection), + fixture.state.storage.git_service.obj_storage.clone(), + ) + .await + .unwrap(); + let state = MonoApiServiceState { + storage, + ..fixture.state.clone() + }; + let app = Router::new().nest("/api/v2", routers(state.clone()).with_state(state)); + let mut cursor = None; + let mut names = Vec::new(); + for _ in 0..12 { + let mut suffix = "directory?path=/&limit=17".to_string(); + if let Some(current) = &cursor { + suffix.push_str(&format!("&cursor={current}")); + } + let response = app + .clone() + .oneshot(fixture.request("GET", &suffix, Body::empty())) + .await + .unwrap(); + let window = success_json(response).await; + assert_eq!(window["entry_count"], "147"); + let entries = window["entries"].as_array().unwrap(); + assert!(!entries.is_empty() && entries.len() <= 17); + names.extend( + entries + .iter() + .map(|entry| entry["name"].as_str().unwrap().to_owned()), + ); + for proof in window["proof_pages"].as_array().unwrap() { + let bytes = STANDARD + .decode(proof["data_base64"].as_str().unwrap()) + .unwrap(); + mst2_codec::metapage::Page::decode(&bytes).unwrap(); + assert_eq!( + proof["digest"], + format!("sha256:{}", hex_of(&mst2_codec::metapage::page_id(&bytes))) + ); + } + cursor = window["next_cursor"].as_str().map(str::to_owned); + if cursor.is_none() { + break; + } + } + assert!(cursor.is_none()); + assert_eq!(names.len(), 147); + assert!( + names + .windows(2) + .all(|pair| pair[0].as_bytes() < pair[1].as_bytes()) + ); + let lookup=success_json(app.oneshot(fixture.request("POST","lookup", + Body::from(json!({"paths":["/nested/file","/wide-139/file-139","/missing","/file/child","/link/child"]}).to_string()))) + .await.unwrap()).await; + let statuses: Vec<_> = lookup["results"] + .as_array() + .unwrap() + .iter() + .map(|row| row["status"].as_str().unwrap()) + .collect(); + assert_eq!( + statuses, + [ + "found", + "found", + "absent", + "not_directory", + "symlink_traversal" + ] + ); + assert_eq!( + q_count( + &fixture, + "SELECT count(*) FROM {q}.mst2_metadata_reader_operation WHERE state='ACTIVE'" + ) + .await, + 0 + ); + assert_eq!(q_count(&fixture,"SELECT count(*) FROM {q}.mst2_metadata_root_anchor WHERE anchor_kind IN ('REQUEST','READER')").await,0); + fixture.counts.assert(0, 0); +} + +#[tokio::test] +async fn rooted_reader_release_race_keeps_independent_roots_until_owned_buffers_finish() { + let fixture = Fixture::new_with_pg_config(true).await; + let admitted = Arc::new(Barrier::new(2)); + let resume = Arc::new(Barrier::new(2)); + let request = fixture.request( + "POST", + "metadata/pages", + Body::from( + json!({"encoding":"identity","items":[{"directory_path":"/nested"}]}).to_string(), + ), + ); + let app = fixture.app.clone(); + let pending = tokio::spawn(with_rooted_reader_barriers( + admitted.clone(), + resume.clone(), + async move { app.oneshot(request).await.unwrap() }, + )); + tokio::time::timeout(Duration::from_secs(4), admitted.wait()) + .await + .unwrap(); + assert_eq!(q_count(&fixture,"SELECT count(*) FROM {q}.mst2_metadata_root_anchor WHERE anchor_kind IN ('REQUEST','READER')").await,2); + let work = fixture + .state + .storage + .rooted_qualified_metadata_writer() + .await + .unwrap() + .maintenance_tick(64) + .await + .unwrap(); + assert_eq!(work.payload_pages_removed, 0); + assert_eq!(release(&fixture, &fixture.lease).await["released"], true); + assert_eq!(q_count(&fixture,"SELECT count(*) FROM {q}.mst2_metadata_root_anchor WHERE anchor_kind IN ('REQUEST','READER')").await,2); + let work = fixture + .state + .storage + .rooted_qualified_metadata_writer() + .await + .unwrap() + .maintenance_tick(64) + .await + .unwrap(); + assert_eq!(work.payload_pages_removed, 0); + assert_eq!( + q_count( + &fixture, + "SELECT count(*) FROM {q}.mst2_metadata_root_anchor WHERE anchor_kind='LEASE'" + ) + .await, + 1 + ); + resume.wait().await; + error( + tokio::time::timeout(Duration::from_secs(4), pending) + .await + .unwrap() + .unwrap(), + 410, + "LEASE_EXPIRED", + false, + ) + .await; + assert_eq!( + q_count( + &fixture, + "SELECT count(*) FROM {q}.mst2_metadata_reader_operation WHERE state='FINISHED'" + ) + .await, + 1 + ); + assert_eq!( + q_count( + &fixture, + "SELECT count(*) FROM {q}.mst2_metadata_root_anchor" + ) + .await, + 0 + ); + collect_unowned(&fixture).await; + fixture.counts.assert(0, 0); +} + +async fn source_oid(fixture: &Fixture) -> String { + let schema = q_schema(fixture).await; + let quoted = format!("\"{}\"", schema.replace('"', "\"\"")); + fixture.state.storage.mono_storage().get_connection().query_one_raw(Statement::from_string(DbBackend::Postgres, + format!("SELECT split_part(a.tagged_tree_oid,':',2) FROM {quoted}.mst2_qualified_session_incarnation s + JOIN {quoted}.mst2_metadata_source_root_attestation a ON a.attestation_id=s.attestation_id WHERE s.state='READY'"))) + .await.unwrap().unwrap().try_get_by_index(0).unwrap() +} + +async fn source_revision(fixture: &Fixture, oid: &str) -> String { + fixture + .state + .storage + .mono_storage() + .get_connection() + .query_one_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + "SELECT revision::text FROM mst2_rooted_source_tree_revision WHERE tree_id=$1", + [oid.into()], + )) + .await + .unwrap() + .unwrap() + .try_get_by_index(0) + .unwrap() +} + +#[tokio::test] +async fn actual_q_body_callers_preserve_unchanged_source_revision_and_ignore_temp_shadow() { + let fixture = Fixture::new_with_pg_config_and_directories(true, 140).await; + fixture.map("/file").await; + fixture.counts.reset(); + let oid = source_oid(&fixture).await; + let revision = source_revision(&fixture, &oid).await; + fixture + .state + .storage + .mono_storage() + .get_connection() + .execute_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + "UPDATE mega_tree SET sub_trees=sub_trees,pack_offset=pack_offset WHERE tree_id=$1", + [oid.clone().into()], + )) + .await + .unwrap(); + assert_eq!(source_revision(&fixture, &oid).await, revision); + let response = + with_rooted_source_temporary_shadow(fixture.send("HEAD", "blob?path=/file", Body::empty())) + .await; + assert_eq!(response.status(), 200); + assert_eq!( + response.headers()["x-mega-content-size"], + fixture.raw.len().to_string() + ); + fixture.counts.assert(0, 0); + let response = + with_rooted_source_temporary_shadow(fixture.send("GET", "blob?path=/file", Body::empty())) + .await; + assert_eq!(response.status(), 200); + assert_eq!( + to_bytes(response.into_body(), fixture.raw.len() + 1) + .await + .unwrap() + .as_ref(), + fixture.raw + ); + assert_eq!( + q_count( + &fixture, + "SELECT count(*) FROM {q}.mst2_metadata_reader_operation WHERE state='ACTIVE'" + ) + .await, + 0 + ); +} + +#[tokio::test] +async fn actual_q_body_callers_reject_changed_or_deleted_source_before_opening_a_body() { + for delete in [false, true] { + let fixture = Fixture::new_with_pg_config(true).await; + let oid = source_oid(&fixture).await; + let revision = source_revision(&fixture, &oid).await; + fixture + .state + .storage + .mono_storage() + .get_connection() + .execute_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + if delete { + "DELETE FROM mega_tree WHERE tree_id=$1" + } else { + "UPDATE mega_tree SET sub_trees=decode('00','hex') WHERE tree_id=$1" + }, + [oid.clone().into()], + )) + .await + .unwrap(); + assert_ne!(source_revision(&fixture, &oid).await, revision); + let response = fixture.send("GET", "blob?path=/file", Body::empty()).await; + assert!(response.status().is_client_error() || response.status().is_server_error()); + fixture.counts.assert(0, 0); + } +} + +#[tokio::test] +async fn actual_q_body_callers_recheck_only_returned_current_file_facts() { + let fixture = Fixture::new_with_pg_config(true).await; + fixture.state.storage.mono_storage().get_connection().execute_raw(Statement::from_sql_and_values(DbBackend::Postgres, + "UPDATE mst2_verified_object SET verification_version=1 WHERE storage_domain='git' AND object_kind='blob' AND git_oid=$1", + [fixture.oid.clone().into()])).await.unwrap(); + let untouched = fixture + .send("HEAD", "blob?path=/empty", Body::empty()) + .await; + assert_eq!(untouched.status(), 200); + assert_eq!(untouched.headers()["x-mega-content-size"], "0"); + error( + fixture.send("GET", "blob?path=/file", Body::empty()).await, + 503, + "METADATA_NOT_READY", + false, + ) + .await; + fixture.counts.assert(0, 0); +} + +#[tokio::test] +async fn actual_q_body_caller_source_mutation_after_reader_admission_cannot_serve_stale_content() { + let fixture = Fixture::new_with_pg_config(true).await; + let oid = source_oid(&fixture).await; + let admitted = Arc::new(Barrier::new(2)); + let resume = Arc::new(Barrier::new(2)); + let app = fixture.app.clone(); + let request = fixture.request("GET", "blob?path=/file", Body::empty()); + let pending = tokio::spawn(with_rooted_reader_barriers( + admitted.clone(), + resume.clone(), + async move { app.oneshot(request).await.unwrap() }, + )); + tokio::time::timeout(Duration::from_secs(4), admitted.wait()) + .await + .unwrap(); + fixture + .state + .storage + .mono_storage() + .get_connection() + .execute_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + "UPDATE mega_tree SET sub_trees=decode('00','hex') WHERE tree_id=$1", + [oid.into()], + )) + .await + .unwrap(); + resume.wait().await; + let response = tokio::time::timeout(Duration::from_secs(4), pending) + .await + .unwrap() + .unwrap(); + assert!(response.status().is_client_error() || response.status().is_server_error()); + assert_eq!(q_count(&fixture,"SELECT count(*) FROM {q}.mst2_metadata_root_anchor WHERE anchor_kind IN ('REQUEST','READER')").await,0); + assert_eq!( + q_count( + &fixture, + "SELECT count(*) FROM {q}.mst2_metadata_reader_operation WHERE state='FINISHED'" + ) + .await, + 1 + ); + fixture.counts.assert(0, 0); +} + +#[tokio::test] +async fn actual_q_body_caller_retries_a_busy_current_file_fact_without_opening_a_body() { + let fixture = Fixture::new_with_pg_config(true).await; + let writer = fixture + .state + .storage + .mono_storage() + .get_connection() + .begin() + .await + .unwrap(); + writer.execute_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + "UPDATE mst2_verified_object SET raw_sha256=raw_sha256 WHERE storage_domain='git' AND object_kind='blob' AND git_oid=$1", + [fixture.oid.clone().into()], + )).await.unwrap(); + error( + fixture.send("GET", "blob?path=/file", Body::empty()).await, + 503, + "TEMPORARY_UNAVAILABLE", + true, + ) + .await; + assert_eq!(q_count(&fixture,"SELECT count(*) FROM {q}.mst2_metadata_root_anchor WHERE anchor_kind IN ('REQUEST','READER')").await,0); + fixture.counts.assert(0, 0); + writer.rollback().await.unwrap(); + assert_eq!( + fixture + .send("HEAD", "blob?path=/file", Body::empty()) + .await + .status(), + 200 + ); + fixture.counts.assert(0, 0); +} + +#[tokio::test] +async fn actual_q_body_caller_holds_the_selected_current_fact_until_the_read_transaction_finishes() +{ + let fixture = Fixture::new_with_pg_config(true).await; + let admitted = Arc::new(Barrier::new(2)); + let resume = Arc::new(Barrier::new(2)); + let app = fixture.app.clone(); + let request = fixture.request("HEAD", "blob?path=/file", Body::empty()); + let pending = tokio::spawn(with_rooted_source_fact_barriers( + admitted.clone(), + resume.clone(), + async move { app.oneshot(request).await.unwrap() }, + )); + tokio::time::timeout(Duration::from_secs(4), admitted.wait()) + .await + .unwrap(); + let core_writer = fixture + .state + .storage + .mono_storage() + .get_connection() + .clone(); + let oid = fixture.oid.clone(); + let (ready, received) = tokio::sync::oneshot::channel(); + let update = tokio::spawn(async move { + let txn = core_writer.begin().await.unwrap(); + let pid: i32 = txn + .query_one_raw(Statement::from_string( + DbBackend::Postgres, + "SELECT pg_backend_pid()", + )) + .await + .unwrap() + .unwrap() + .try_get_by_index(0) + .unwrap(); + ready.send(pid).unwrap(); + txn.execute_raw(Statement::from_sql_and_values(DbBackend::Postgres, + "UPDATE mst2_verified_object SET raw_sha256=raw_sha256 WHERE storage_domain='git' AND object_kind='blob' AND git_oid=$1", + [oid.into()])).await.unwrap(); + txn.commit().await.unwrap(); + }); + let pid = received.await.unwrap(); + tokio::time::timeout(Duration::from_secs(4),async { + loop { + let waiting:bool=fixture.state.storage.mono_storage().get_connection().query_one_raw(Statement::from_sql_and_values( + DbBackend::Postgres,"SELECT coalesce(wait_event_type='Lock',false) FROM pg_stat_activity WHERE pid=$1",[pid.into()])) + .await.unwrap().unwrap().try_get_by_index(0).unwrap(); + if waiting {break;} tokio::task::yield_now().await; + } + }).await.unwrap(); + assert!(!update.is_finished()); + resume.wait().await; + let response = tokio::time::timeout(Duration::from_secs(4), pending) + .await + .unwrap() + .unwrap(); + assert_eq!(response.status(), 200); + tokio::time::timeout(Duration::from_secs(4), update) + .await + .unwrap() + .unwrap(); + assert_eq!(q_count(&fixture,"SELECT count(*) FROM {q}.mst2_metadata_root_anchor WHERE anchor_kind IN ('REQUEST','READER')").await,0); + fixture.counts.assert(0, 0); +} + +async fn durable_lease_state(fixture: &Fixture) -> Value { + let schema = q_schema(fixture).await; + let quoted = format!("\"{}\"", schema.replace('"', "\"\"")); + fixture.state.storage.mono_storage().get_connection().query_one_raw(Statement::from_string(DbBackend::Postgres, + format!("SELECT jsonb_build_object( + 'sessions',(SELECT jsonb_agg(to_jsonb(s) ORDER BY s.snapshot_id,s.session_incarnation) FROM {quoted}.mst2_qualified_session_incarnation s), + 'leases',(SELECT jsonb_agg(to_jsonb(l) ORDER BY l.lease_id) FROM {quoted}.mst2_qualified_lease_binding l), + 'anchors',(SELECT jsonb_agg(to_jsonb(a) ORDER BY a.anchor_id) FROM {quoted}.mst2_metadata_root_anchor a), + 'roots',(SELECT jsonb_agg(to_jsonb(r) ORDER BY r.prepare_id,r.page_id,r.generation) FROM {quoted}.mst2_metadata_graph_root r), + 'routes',(SELECT jsonb_agg(to_jsonb(r) ORDER BY r.lease_id) FROM mst2_lease_storage_route r))"))) + .await.unwrap().unwrap().try_get_by_index(0).unwrap() +} + +#[tokio::test] +async fn actual_q_lease_http_and_direct_handoff_retry_source_lock_without_partial_durable_mutation() +{ + let fixture = Fixture::new_with_pg_config(true).await; + let storage = &fixture.state.storage; + let context = storage + .snapshot_context(&fixture.snapshot, &fixture.lease) + .await + .unwrap(); + let config = storage.config(); + let instance = config.mst2.instance_uuid.as_deref().unwrap(); + let head = storage + .mono_storage() + .read_native_publication_head(instance) + .await + .unwrap(); + let repository = storage.rooted_qualified_metadata_writer().await.unwrap(); + let before = durable_lease_state(&fixture).await; + let revision = source_revision(&fixture, &source_oid(&fixture).await).await; + let writer = storage + .mono_storage() + .get_connection() + .begin() + .await + .unwrap(); + let locked=writer.query_one_raw(Statement::from_sql_and_values(DbBackend::Postgres, + "SELECT revision::text FROM mst2_rooted_source_tree_revision WHERE revision=$1::uuid FOR UPDATE", + [revision.clone().into()])).await.unwrap().unwrap(); + assert_eq!(locked.try_get_by_index::(0).unwrap(), revision); + let direct_errors = tokio::time::timeout(Duration::from_secs(4), async { + [ + repository + .open_session(&head, &context.built, None, 600) + .await + .unwrap_err(), + repository + .renew(&fixture.lease, 600, instance) + .await + .unwrap_err(), + repository.release(&fixture.lease).await.unwrap_err(), + ] + }) + .await + .unwrap(); + for error in direct_errors { + assert_eq!(error.code, SnapshotErrorCode::TemporaryUnavailable); + } + assert_eq!(durable_lease_state(&fixture).await, before); + for (method, uri) in [ + ("POST", "/api/v2/snapshots/resolve".to_owned()), + ( + "POST", + format!("/api/v2/snapshots/leases/{}/renew", fixture.lease), + ), + ( + "DELETE", + format!("/api/v2/snapshots/leases/{}", fixture.lease), + ), + ] { + let body = if uri.ends_with("/resolve") { + Body::from(json!({"target":{"kind":"latest"},"scope":"/project"}).to_string()) + } else { + Body::empty() + }; + let request = Request::builder() + .method(method) + .uri(uri) + .header("authorization", format!("Bearer {TOKEN}")) + .header("content-type", "application/json") + .body(body) + .unwrap(); + let response = + tokio::time::timeout(Duration::from_secs(4), fixture.app.clone().oneshot(request)) + .await + .unwrap() + .unwrap(); + error(response, 503, "TEMPORARY_UNAVAILABLE", true).await; + assert_eq!(durable_lease_state(&fixture).await, before); + } + writer.rollback().await.unwrap(); + assert_eq!(durable_lease_state(&fixture).await, before); + let next = resolve(&fixture).await; + assert_ne!(next["lease_id"], fixture.lease); + assert_eq!(release(&fixture, &fixture.lease).await["released"], true); +} diff --git a/src/api/router/snapshot_router.rs b/src/api/router/snapshot_router.rs index a6838aea..4b568aeb 100644 --- a/src/api/router/snapshot_router.rs +++ b/src/api/router/snapshot_router.rs @@ -76,6 +76,19 @@ pub fn routers(api_state: MonoApiServiceState) -> Router { )) } +/// Explicit fixture for the historical G authority and corruption contracts. +/// Production resolve always installs the rooted family for a new SID. +#[cfg(test)] +pub(crate) fn generic_history_routers( + api_state: MonoApiServiceState, +) -> Router { + routers(api_state).layer(axum::middleware::from_fn( + |request: axum::extract::Request, next: axum::middleware::Next| async move { + GENERIC_HISTORY_RESOLVE.scope(true, next.run(request)).await + }, + )) +} + #[path = "snapshot_content.rs"] mod content; @@ -212,6 +225,7 @@ tokio::task_local! { static NATIVE_RESOLVE_BARRIERS: (std::sync::Arc, std::sync::Arc); static NATIVE_HANDOFF_BARRIERS: (std::sync::Arc, std::sync::Arc); static REJECT_NATIVE_OBSERVATION_SOURCE: bool; + static GENERIC_HISTORY_RESOLVE: bool; } #[cfg(test)] @@ -770,48 +784,141 @@ async fn resolve( // The scope root doubles as metadata_root; building it also validates // that the scope exists and is a directory in this view. let projection_started = std::time::Instant::now(); - let (scope_page, projection_work) = - build_directory_page_with_work(handler.as_ref(), &root_tree, &req.scope) + #[cfg(test)] + let prepared_generic = if selected_native_head.is_some() + && GENERIC_HISTORY_RESOLVE + .try_with(|value| *value) + .unwrap_or(false) + { + let (page, work) = build_directory_page_with_work(handler.as_ref(), &root_tree, &req.scope) .await .map_err(mst2_error_response)?; - let projection_elapsed = projection_started.elapsed(); - - let built = build_descriptor(&config.mst2, &view, &req.scope, scope_page.page_id) + let prepared = crate::ceres::snapshot::pages::prepare_native_metadata_retention( + handler.as_ref(), + &root_tree, + &req.scope, + crate::ceres::snapshot::retention_dag::MetadataDagLimits::default(), + ) + .await .map_err(mst2_error_response)?; - let ctx = if let Some(head) = selected_native_head.as_ref() { - let sessions = state.storage.snapshot_sessions().await; - if let Some(context) = sessions - .open(head, &built, None, req.lease_seconds) + Some((page.page_id, work, prepared)) + } else { + None + }; + #[cfg(not(test))] + let prepared_generic: Option<( + [u8; 32], + crate::ceres::snapshot::pages::ProjectionWork, + crate::ceres::snapshot::pages::PreparedNativeMetadataRetention, + )> = None; + let (metadata_root, projection_work, prepared_rooted) = if let Some((root, work, _)) = + prepared_generic.as_ref() + { + (*root, Some(work.clone()), None) + } else if selected_native_head.is_some() { + let repository = state + .storage + .rooted_qualified_metadata_writer() .await - .map_err(mst2_error_response)? - { - context - } else { - let prepared = crate::ceres::snapshot::pages::prepare_native_metadata_retention( + .map_err(internal)?; + let prepared = + crate::ceres::snapshot::rooted_metadata_projection::prepare_rooted_native_metadata( handler.as_ref(), &root_tree, &req.scope, - crate::ceres::snapshot::retention_dag::MetadataDagLimits::default(), + repository, ) .await .map_err(mst2_error_response)?; - let receipt = sessions - .install(&built, &prepared) + (prepared.plan.root, None, Some(prepared)) + } else { + let (page, work) = build_directory_page_with_work(handler.as_ref(), &root_tree, &req.scope) + .await + .map_err(mst2_error_response)?; + (page.page_id, Some(work), None) + }; + let projection_elapsed = projection_started.elapsed(); + + let built = build_descriptor(&config.mst2, &view, &req.scope, metadata_root) + .map_err(mst2_error_response)?; + let ctx = if let Some(head) = selected_native_head.as_ref() { + use crate::jupiter::storage::qualified_metadata_family::SnapshotMetadataFamily; + let family = state + .storage + .snapshot_metadata_family(&built.snapshot_id, false) + .await + .map_err(mst2_error_response)?; + if family == Some(SnapshotMetadataFamily::Generic) || prepared_generic.is_some() { + // A permanent SID route retains its original physical family. + let existing = state + .storage + .snapshot_sessions() + .await + .open(head, &built, None, req.lease_seconds) .await .map_err(mst2_error_response)?; - #[cfg(test)] - if let Ok((prepared, release)) = NATIVE_HANDOFF_BARRIERS.try_with(|value| value.clone()) - { - prepared.wait().await; - release.wait().await; + if let Some(existing) = existing { + existing + } else { + let (_, _, prepared) = prepared_generic.as_ref().ok_or_else(|| { + mst2_error_response(internal("existing generic route has no durable session")) + })?; + let sessions = state.storage.snapshot_sessions().await; + let receipt = sessions + .install(&built, prepared) + .await + .map_err(mst2_error_response)?; + #[cfg(test)] + if let Ok((prepared, release)) = NATIVE_HANDOFF_BARRIERS.try_with(Clone::clone) { + prepared.wait().await; + release.wait().await; + } + sessions + .open(head, &built, Some(&receipt), req.lease_seconds) + .await + .map_err(mst2_error_response)? + .ok_or_else(|| { + mst2_error_response(internal( + "generic history fixture handoff returned no context", + )) + })? } - sessions - .open(head, &built, Some(&receipt), req.lease_seconds) + } else { + let repository = state + .storage + .rooted_qualified_metadata_writer() + .await + .map_err(internal)?; + if let Some(context) = repository + .open_session(head, &built, None, req.lease_seconds) .await .map_err(mst2_error_response)? - .ok_or_else(|| { - mst2_error_response(internal("durable session handoff returned no context")) - })? + { + context + } else { + let prepared = prepared_rooted.as_ref().ok_or_else(|| { + mst2_error_response(internal("rooted resolve has no source projection")) + })?; + let receipt = repository + .install(&built, prepared) + .await + .map_err(crate::jupiter::storage::native_snapshot_session::install_error) + .map_err(mst2_error_response)?; + #[cfg(test)] + if let Ok((prepared, release)) = + NATIVE_HANDOFF_BARRIERS.try_with(|value| value.clone()) + { + prepared.wait().await; + release.wait().await; + } + repository + .open_session(head, &built, Some(&receipt), req.lease_seconds) + .await + .map_err(mst2_error_response)? + .ok_or_else(|| { + mst2_error_response(internal("durable session handoff returned no context")) + })? + } } } else { runtime() @@ -821,20 +928,26 @@ async fn resolve( if let Some(source) = native_source { let request_id = current_request_id(); - match source.observe( - ResolvedProjection { - descriptor: &ctx.built.descriptor, - snapshot_id: &ctx.built.snapshot_id, - metadata_root: &ctx.built.metadata_root, - context_commit: &ctx.commit_oid, - context_root_tree: &ctx.root_tree_oid, - fixed_root_tree: root_tree.id, - requested_scope: &req.scope, - request_id: &request_id, - }, - projection_work, - projection_elapsed, - ) { + let resolved = ResolvedProjection { + descriptor: &ctx.built.descriptor, + snapshot_id: &ctx.built.snapshot_id, + metadata_root: &ctx.built.metadata_root, + context_commit: &ctx.commit_oid, + context_root_tree: &ctx.root_tree_oid, + fixed_root_tree: root_tree.id, + requested_scope: &req.scope, + request_id: &request_id, + }; + let observation = if let Some(prepared) = prepared_rooted { + source.observe_rooted(resolved, prepared.work, projection_elapsed) + } else { + source.observe( + resolved, + projection_work.unwrap_or_default(), + projection_elapsed, + ) + }; + match observation { Ok(observation) => { if let Some(sink) = &state.storage.projection_observation_sink { let _ = sink.enqueue(&observation); @@ -963,6 +1076,9 @@ struct DirectoryQuery { ancestors: Option, } +#[path = "snapshot_rooted_metadata.rs"] +mod rooted_metadata; + fn default_limit() -> u32 { 128 } @@ -986,6 +1102,18 @@ async fn directory( "limit must be 1..256", ))); } + if state.storage.config().mst2.publication_enabled + && state + .storage + .snapshot_metadata_family(&ctx.lease_id, true) + .await + .map_err(mst2_error_response)? + == Some( + crate::jupiter::storage::qualified_metadata_family::SnapshotMetadataFamily::Rooted, + ) + { + return rooted_metadata::directory_response(&state, &ctx, &snapshot_id, &q).await; + } let handler = state .api_handler(std::path::Path::new("/")) @@ -1201,11 +1329,24 @@ async fn lookup( ))); } + let ctx = request_context(&state, &snapshot_id).map_err(mst2_error_response)?; + if state.storage.config().mst2.publication_enabled + && state + .storage + .snapshot_metadata_family(&ctx.lease_id, true) + .await + .map_err(mst2_error_response)? + == Some( + crate::jupiter::storage::qualified_metadata_family::SnapshotMetadataFamily::Rooted, + ) + { + return rooted_metadata::lookup_response(&state, &ctx, &snapshot_id, &req).await; + } + let handler = state .api_handler(std::path::Path::new("/")) .await .map_err(internal)?; - let ctx = request_context(&state, &snapshot_id).map_err(mst2_error_response)?; let root_tree = handler .get_tree_by_hash(&ctx.root_tree_oid) .await @@ -1371,9 +1512,7 @@ async fn metadata_pages( .collect(); let batch = state .storage - .snapshot_sessions() - .await - .metadata_routes(&ctx, &items) + .snapshot_metadata_routes(&ctx, &items) .await .map_err(mst2_error_response)?; tracing::debug!( diff --git a/src/api/router/snapshot_session_tests.rs b/src/api/router/snapshot_session_tests.rs index ae84d3c6..36101a05 100644 --- a/src/api/router/snapshot_session_tests.rs +++ b/src/api/router/snapshot_session_tests.rs @@ -84,12 +84,16 @@ async fn rebuilt(fixture: &Fixture) -> MonoApiServiceState { let connection = crate::jupiter::storage::init::postgres_connection(&config.database) .await .unwrap(); - let storage = crate::jupiter::storage::Storage::new_with_connection( + let assembly = crate::jupiter::storage::Storage::new_with_connection( config, Arc::new(connection), fixture.state.storage.git_service.obj_storage.clone(), - ) - .await + ); + let storage = if fixture.generic_history { + crate::jupiter::storage::init::with_generic_history_bootstrap(assembly).await + } else { + assembly.await + } .unwrap(); MonoApiServiceState { storage, @@ -102,7 +106,11 @@ async fn rebuilt(fixture: &Fixture) -> MonoApiServiceState { } fn app(state: &MonoApiServiceState) -> Router { - Router::new().nest("/api/v2", routers(state.clone()).with_state(state.clone())) + Router::new().nest( + "/api/v2", + crate::api::router::snapshot_router::generic_history_routers(state.clone()) + .with_state(state.clone()), + ) } async fn lease_control(fixture: &Fixture, lease: &str, method: &str, renew: bool) -> Value { @@ -150,7 +158,7 @@ async fn advance(fixture: &Fixture) { #[tokio::test] async fn mst2_durable_http_resolve_installs_complete_dag_and_warm_leases_share_it() { - let fixture = Fixture::new_with_pg_config(true).await; + let fixture = Fixture::new_generic_history_with_pg_config(true).await; let mono = fixture.state.storage.mono_storage(); let db = mono.get_connection(); let pages = scalar(db, "SELECT count(*) FROM mst2_metadata_payload").await; @@ -306,7 +314,7 @@ async fn mst2_durable_http_resolve_installs_complete_dag_and_warm_leases_share_i #[tokio::test] async fn mst2_durable_http_old_sid_and_original_lease_survive_real_publication_and_fresh_service() { - let fixture = Fixture::new_with_pg_config(true).await; + let fixture = Fixture::new_generic_history_with_pg_config(true).await; let old = success_json(fixture.send("GET", "descriptor", Body::empty()).await).await; advance(&fixture).await; let next = success_json( @@ -376,7 +384,7 @@ async fn mst2_durable_http_old_sid_and_original_lease_survive_real_publication_a #[tokio::test] async fn mst2_durable_http_renew_release_expiry_and_last_lease_retire_protection() { - let fixture = Fixture::new_with_pg_config(true).await; + let fixture = Fixture::new_generic_history_with_pg_config(true).await; let another = success_json( fixture .app @@ -491,7 +499,7 @@ async fn mst2_durable_http_renew_release_expiry_and_last_lease_retire_protection #[tokio::test] async fn mst2_durable_http_live_to_deleting_winner_rejects_paused_resolve_without_leak() { - let fixture = Fixture::new_with_pg_config(true).await; + let fixture = Fixture::new_generic_history_with_pg_config(true).await; let captured = Arc::new(Barrier::new(2)); let release = Arc::new(Barrier::new(2)); let resolving = { @@ -542,7 +550,7 @@ async fn mst2_durable_http_live_to_deleting_winner_rejects_paused_resolve_withou #[tokio::test] async fn mst2_durable_http_lease_winner_blocks_gc_until_the_last_release() { - let fixture = Fixture::new_with_pg_config(true).await; + let fixture = Fixture::new_generic_history_with_pg_config(true).await; let another = success_json( fixture .app @@ -595,7 +603,7 @@ async fn mst2_durable_http_lease_winner_blocks_gc_until_the_last_release() { #[tokio::test] async fn mst2_durable_http_renew_waits_for_lock_before_checking_database_deadline() { - let fixture = Fixture::new_with_pg_config(true).await; + let fixture = Fixture::new_generic_history_with_pg_config(true).await; let mono = fixture.state.storage.mono_storage(); let db = mono.get_connection(); let held = db.begin().await.unwrap(); @@ -671,7 +679,7 @@ async fn mst2_durable_http_renew_waits_for_lock_before_checking_database_deadlin #[tokio::test] async fn mst2_durable_http_publication_advance_rejects_mixed_resolve_then_retries_current() { - let fixture = Fixture::new_with_pg_config(true).await; + let fixture = Fixture::new_generic_history_with_pg_config(true).await; let captured = Arc::new(Barrier::new(2)); let release = Arc::new(Barrier::new(2)); let resolving = { @@ -725,7 +733,7 @@ async fn mst2_durable_http_publication_advance_rejects_mixed_resolve_then_retrie #[tokio::test] async fn mst2_durable_http_install_fault_never_hands_off_or_returns_success() { - let fixture = Fixture::new_with_pg_config(true).await; + let fixture = Fixture::new_generic_history_with_pg_config(true).await; let mono = fixture.state.storage.mono_storage(); let db = mono.get_connection(); let prepared = root_metadata(&fixture).await; @@ -828,7 +836,7 @@ async fn mst2_durable_http_install_fault_never_hands_off_or_returns_success() { #[tokio::test] async fn mst2_durable_http_frame_and_raw_delivery_recheck_after_release() { for raw in [false, true] { - let fixture = Fixture::new_with_pg_config(true).await; + let fixture = Fixture::new_generic_history_with_pg_config(true).await; let response = if raw { fixture.send("GET", "blob?path=/file", Body::empty()).await } else { @@ -862,7 +870,7 @@ async fn mst2_durable_http_frame_and_raw_delivery_recheck_after_release() { #[tokio::test] async fn mst2_durable_http_current_state_wrong_lease_and_corrupt_source_reject_warm_reads() { - let fixture = Fixture::new_with_pg_config(true).await; + let fixture = Fixture::new_generic_history_with_pg_config(true).await; let warm = success_json(fixture.send("GET", "descriptor", Body::empty()).await).await; let mono = fixture.state.storage.mono_storage(); let db = mono.get_connection(); @@ -935,7 +943,7 @@ async fn mst2_durable_http_current_state_wrong_lease_and_corrupt_source_reject_w #[tokio::test] async fn mst2_durable_http_large_dag_batches_handoff_without_scanning_pages_or_edges() { - let fixture = Fixture::new_with_pg_config_and_directories(true, 80).await; + let fixture = Fixture::new_generic_history_with_pg_config_and_directories(true, 80).await; let mono = fixture.state.storage.mono_storage(); let db = mono.get_connection(); let projected = root_metadata(&fixture).await; @@ -1083,7 +1091,7 @@ async fn mst2_durable_http_handoff_rejects_changed_plan_or_receipt_summary() { "projection_revision=projection_revision+1", "total_bytes=total_bytes+1", ] { - let fixture = Fixture::new_with_pg_config_and_directories(true, 80).await; + let fixture = Fixture::new_generic_history_with_pg_config_and_directories(true, 80).await; let prepared = Arc::new(Barrier::new(2)); let release = Arc::new(Barrier::new(2)); let resolving = { @@ -1254,7 +1262,7 @@ async fn overwrite_primary_scope_for_test(db: &DatabaseConnection, storage_uuid: #[tokio::test] async fn mst2_durable_http_warm_reads_renew_and_frame_delivery_reject_primary_scope_drift() { - let fixture = Fixture::new_with_pg_config(true).await; + let fixture = Fixture::new_generic_history_with_pg_config(true).await; success_json(fixture.send("GET", "descriptor", Body::empty()).await).await; let response = fixture .send( diff --git a/src/api/router/snapshot_storage_route_fixture.rs b/src/api/router/snapshot_storage_route_fixture.rs index abc3aa48..71fb3204 100644 --- a/src/api/router/snapshot_storage_route_fixture.rs +++ b/src/api/router/snapshot_storage_route_fixture.rs @@ -30,12 +30,19 @@ pub(super) async fn restore_pre_route_schema(db: &DatabaseConnection) { // The original contexts, leases, plans, payloads and protection stay intact. txn.execute_unprepared( "DO $$ DECLARE r record; functions text; BEGIN + IF EXISTS(SELECT 1 FROM mst2_metadata_namespace WHERE graph_domain='qualified-v1') THEN + RAISE EXCEPTION 'route upgrade fixture cannot tear down a provisioned qualified family'; + END IF; + IF EXISTS(SELECT 1 FROM mst2_rooted_source_tree_revision) THEN + RAISE EXCEPTION 'route upgrade fixture cannot erase actual qualified source history'; + END IF; + DROP FUNCTION mst2_metadata_has_generic_overlap(bytea); FOR r IN SELECT c.relname,t.tgname FROM pg_trigger t JOIN pg_class c ON c.oid=t.tgrelid JOIN pg_namespace n ON n.oid=c.relnamespace JOIN pg_proc p ON p.oid=t.tgfoid WHERE n.nspname=current_schema() AND NOT t.tgisinternal AND left(p.proname,11)='mst2_route_' LOOP EXECUTE format('DROP TRIGGER %I ON %I.%I',r.tgname,current_schema(),r.relname); END LOOP; - DROP TABLE mst2_lease_storage_route,mst2_generic_session_storage_binding,mst2_snapshot_storage_route,mst2_metadata_namespace; + DROP TABLE mst2_lease_storage_route,mst2_generic_session_storage_binding,mst2_snapshot_storage_route,mst2_metadata_namespace,mst2_qualified_family_policy,mst2_rooted_source_tree_revision; FOR r IN SELECT conname FROM pg_constraint WHERE conrelid='mst2_snapshot_context'::regclass AND contype='u' AND pg_get_constraintdef(oid)='UNIQUE (snapshot_id, prepare_id, metadata_root)' @@ -46,7 +53,8 @@ pub(super) async fn restore_pre_route_schema(db: &DatabaseConnection) { IF functions IS NULL THEN RAISE EXCEPTION 'route upgrade fixture has no route functions'; END IF; EXECUTE 'DROP FUNCTION '||functions; END $$; - DELETE FROM seaql_migrations WHERE version='m20261007_000600_add_mst2_storage_routes'", + DELETE FROM seaql_migrations WHERE version IN ( + 'm20261007_000600_add_mst2_storage_routes','m20261008_000200_add_mst2_rooted_qualified_family')", ) .await .unwrap(); diff --git a/src/api/router/snapshot_storage_route_tests.rs b/src/api/router/snapshot_storage_route_tests.rs index 25d40ec3..63b59fc6 100644 --- a/src/api/router/snapshot_storage_route_tests.rs +++ b/src/api/router/snapshot_storage_route_tests.rs @@ -176,7 +176,11 @@ async fn lease_control(fixture: &Fixture, lease: &str, renew: bool) -> Response async fn rebuilt(fixture: &Fixture) -> Router { let state = rebuilt_state(fixture).await; - Router::new().nest("/api/v2", routers(state.clone()).with_state(state)) + Router::new().nest( + "/api/v2", + crate::api::router::snapshot_router::generic_history_routers(state.clone()) + .with_state(state), + ) } async fn rebuilt_state(fixture: &Fixture) -> MonoApiServiceState { @@ -187,10 +191,12 @@ async fn rebuilt_state(fixture: &Fixture) -> MonoApiServiceState { let connection = crate::jupiter::storage::init::postgres_connection(&database) .await .unwrap(); - let storage = crate::jupiter::storage::Storage::new_with_connection( - config, - Arc::new(connection), - fixture.state.storage.git_service.obj_storage.clone(), + let storage = crate::jupiter::storage::init::with_generic_history_bootstrap( + crate::jupiter::storage::Storage::new_with_connection( + config, + Arc::new(connection), + fixture.state.storage.git_service.obj_storage.clone(), + ), ) .await .unwrap(); @@ -326,7 +332,7 @@ async fn rejected(db: &DatabaseConnection, sql: &str) { #[tokio::test] async fn mst2_generic_storage_routes_actual_resolve_warm_and_fresh_service_keep_exact_tuple() { - let fixture = Fixture::new_with_pg_config(true).await; + let fixture = Fixture::new_generic_history_with_pg_config(true).await; let mono = fixture.state.storage.mono_storage(); let db = mono.get_connection(); assert_exact_bindings(db, 1).await; @@ -374,7 +380,7 @@ async fn mst2_generic_storage_routes_actual_resolve_warm_and_fresh_service_keep_ #[tokio::test] async fn mst2_generic_storage_routes_every_member_delete_and_truncate_are_immutable() { - let fixture = Fixture::new_with_pg_config(true).await; + let fixture = Fixture::new_generic_history_with_pg_config(true).await; let mono = fixture.state.storage.mono_storage(); let db = mono.get_connection(); let original = routes(db).await; @@ -431,7 +437,7 @@ fn lease_insert(lease: &str) -> String { #[tokio::test] async fn mst2_generic_storage_routes_raw_lease_derives_atomically_and_half_rows_roll_back() { - let fixture = Fixture::new_with_pg_config(true).await; + let fixture = Fixture::new_generic_history_with_pg_config(true).await; let mono = fixture.state.storage.mono_storage(); let db = mono.get_connection(); let original = routes(db).await; @@ -488,7 +494,7 @@ async fn mst2_generic_storage_routes_raw_lease_derives_atomically_and_half_rows_ #[tokio::test] async fn mst2_generic_storage_routes_renew_terminal_and_unknown_release_preserve_route_and_roots() { - let fixture = Fixture::new_with_pg_config(true).await; + let fixture = Fixture::new_generic_history_with_pg_config(true).await; let mono = fixture.state.storage.mono_storage(); let db = mono.get_connection(); let original = routes(db).await; @@ -592,7 +598,7 @@ async fn lock_count(held: &DatabaseTransaction, pid: i64, key: i32) -> i64 { #[tokio::test] async fn mst2_generic_storage_routes_raw_statement_waits_mono_then_route_then_retention() { - let fixture = Fixture::new_with_pg_config(true).await; + let fixture = Fixture::new_generic_history_with_pg_config(true).await; let mono = fixture.state.storage.mono_storage(); let db = mono.get_connection(); for held_key in [MONO_WRITE_LOCK_KEY1, ROUTE_LOCK_KEY, RETENTION_LOCK_KEY] { @@ -692,8 +698,8 @@ async fn mst2_generic_storage_routes_raw_statement_waits_mono_then_route_then_re #[tokio::test] async fn mst2_generic_storage_routes_schema_isolation_wrong_caller_rr_and_temp_shadow_fail_closed() { - let fixture = Fixture::new_with_pg_config(true).await; - let alien = Fixture::new_with_pg_config(true).await; + let fixture = Fixture::new_generic_history_with_pg_config(true).await; + let alien = Fixture::new_generic_history_with_pg_config(true).await; let mono = fixture.state.storage.mono_storage(); let db = mono.get_connection(); let original = routes(db).await; @@ -801,7 +807,7 @@ async fn mst2_generic_storage_routes_schema_isolation_wrong_caller_rr_and_temp_s #[tokio::test] async fn mst2_generic_storage_routes_additive_backfill_keeps_active_terminal_sources_bytes_and_roots() { - let fixture = Fixture::new_with_pg_config(true).await; + let fixture = Fixture::new_generic_history_with_pg_config(true).await; let mono = fixture.state.storage.mono_storage(); let db = mono.get_connection(); let warm = success_json( @@ -862,9 +868,13 @@ async fn mst2_generic_storage_routes_additive_backfill_keeps_active_terminal_sou #[tokio::test] async fn mst2_generic_storage_routes_actual_http_ignores_temp_source_and_ledger_shadows() { - let fixture = Fixture::new_with_pg_config(true).await; + let fixture = Fixture::new_generic_history_with_pg_config(true).await; let state = rebuilt_state(&fixture).await; - let app = Router::new().nest("/api/v2", routers(state.clone()).with_state(state.clone())); + let app = Router::new().nest( + "/api/v2", + crate::api::router::snapshot_router::generic_history_routers(state.clone()) + .with_state(state.clone()), + ); let mono = state.storage.mono_storage(); let db = mono.get_connection(); let original = routes(db).await; @@ -936,7 +946,7 @@ async fn mst2_generic_storage_routes_actual_http_ignores_temp_source_and_ledger_ #[tokio::test] async fn mst2_generic_storage_routes_temp_prepare_shadow_rejects_qualified_context_and_registered_generic_rebind() { - let fixture = Fixture::new_with_pg_config(true).await; + let fixture = Fixture::new_generic_history_with_pg_config(true).await; let mono = fixture.state.storage.mono_storage(); let db = mono.get_connection(); let original_routes = routes(db).await; @@ -1118,7 +1128,7 @@ async fn mst2_generic_storage_routes_temp_prepare_shadow_rejects_qualified_conte #[tokio::test] async fn mst2_generic_storage_routes_corrupt_original_incarnation_rejects_reads_renew_release_without_repair() { - let fixture = Fixture::new_with_pg_config(true).await; + let fixture = Fixture::new_generic_history_with_pg_config(true).await; let mono = fixture.state.storage.mono_storage(); let db = mono.get_connection(); let source = sources(db).await; diff --git a/src/ceres/snapshot/mod.rs b/src/ceres/snapshot/mod.rs index 3fae7f32..267a78bb 100644 --- a/src/ceres/snapshot/mod.rs +++ b/src/ceres/snapshot/mod.rs @@ -19,5 +19,7 @@ pub mod publication; pub mod resolver; pub mod retention; pub mod retention_dag; +pub(crate) mod rooted_metadata_install; +pub(crate) mod rooted_metadata_projection; pub mod runtime; pub mod view; diff --git a/src/ceres/snapshot/projection_observation.rs b/src/ceres/snapshot/projection_observation.rs index 819e0e86..532b3f18 100644 --- a/src/ceres/snapshot/projection_observation.rs +++ b/src/ceres/snapshot/projection_observation.rs @@ -74,6 +74,7 @@ pub(crate) struct NativeProjectionObservation { request_id: String, projection_elapsed_micros: u64, work: ProjectionWork, + rooted_work: Option, } /// Closed wire fields, borrowed only from the already validated observation. @@ -106,7 +107,9 @@ pub(crate) struct ProjectionWireRecord<'a> { page_counter_scope: &'static str, codec_radix_work: &'static str, #[serde(flatten)] - work: &'a ProjectionWork, + work: Option<&'a ProjectionWork>, + #[serde(skip_serializing_if = "Option::is_none")] + rooted_work: Option<&'a super::rooted_metadata_projection::RootedProjectionWork>, message: &'static str, } @@ -190,15 +193,32 @@ impl NativeResolveSource { projection_elapsed_micros: u64::try_from(projection_elapsed.as_micros()) .map_err(|_| invalid())?, work, + rooted_work: None, }) } + + pub(crate) fn observe_rooted( + self, + resolved: ResolvedProjection<'_>, + work: super::rooted_metadata_projection::RootedProjectionWork, + projection_elapsed: Duration, + ) -> Result { + let mut observation = + self.observe(resolved, ProjectionWork::default(), projection_elapsed)?; + observation.rooted_work = Some(work); + Ok(observation) + } } impl NativeProjectionObservation { pub(crate) fn wire_record(&self) -> ProjectionWireRecord<'_> { ProjectionWireRecord { - observation_revision: 1, - phase: "resolve_directory_projection", + observation_revision: if self.rooted_work.is_some() { 2 } else { 1 }, + phase: if self.rooted_work.is_some() { + "resolve_rooted_projection" + } else { + "resolve_directory_projection" + }, source_domain: "native-git", request_id: &self.request_id, instance_id: self.source.instance.to_string(), @@ -219,14 +239,37 @@ impl NativeProjectionObservation { snapshot_id: &self.snapshot_id, metadata_root: &self.metadata_root, projection_elapsed_micros: self.projection_elapsed_micros, - page_counter_scope: "returned-directory-root-pages", - codec_radix_work: "NOT_EXPOSED", - work: &self.work, + page_counter_scope: if self.rooted_work.is_some() { + "physical-delta-and-reuse-boundaries" + } else { + "returned-directory-root-pages" + }, + codec_radix_work: if self.rooted_work.is_some() { + "OPERATION_CALLS_ONLY_NOT_TOTAL_SQL" + } else { + "NOT_EXPOSED" + }, + work: self.rooted_work.is_none().then_some(&self.work), + rooted_work: self.rooted_work.as_ref(), message: "native resolve directory projection succeeded", } } pub(crate) fn emit(self) { + if let Some(work) = &self.rooted_work { + tracing::debug!(target:"mst2::native_projection_observation", + observation_revision=2u16,phase="resolve_rooted_projection",source_domain="native-git", + request_id=%self.request_id,instance_id=%self.source.instance, + root_commit_oid=%self.source.commit.to_tagged_string(),root_tree_oid=%self.source.tree.to_tagged_string(), + native_certificate_receipt_id=self.source.certificate_receipt_id, + native_writer_epoch=self.source.writer_epoch,native_publication_sequence=self.source.publication_sequence, + scope=?self.scope,snapshot_id=%self.snapshot_id,metadata_root=%self.metadata_root, + projection_elapsed_micros=self.projection_elapsed_micros,rooted_work=?work, + "native rooted projection succeeded; counters exclude total SQL body and catalog work"); + #[cfg(test)] + let _ = OBSERVATIONS.try_with(|observations| observations.lock().unwrap().push(self)); + return; + } tracing::debug!( target: "mst2::native_projection_observation", observation_revision = 1u16, @@ -406,6 +449,45 @@ mod tests { assert_eq!(observation.snapshot_id, fixture.snapshot_id); } + #[test] + fn rooted_observation_exposes_delta_boundaries_without_legacy_or_total_sql_counters() { + let fixture = Fixture::new(); + let commit = fixture.commit.to_string(); + let tree = fixture.tree.to_string(); + let work = super::super::rooted_metadata_projection::RootedProjectionWork { + tree_fetches: 1, + delta_pages: 2, + reused_roots: 7, + ..Default::default() + }; + let observation = fixture + .source() + .observe_rooted( + fixture.resolved(&commit, &tree), + work, + Duration::from_micros(456), + ) + .unwrap(); + let wire = serde_json::to_value(observation.wire_record()).unwrap(); + assert_eq!(wire["observation_revision"], 2); + assert_eq!(wire["phase"], "resolve_rooted_projection"); + assert_eq!( + wire["page_counter_scope"], + "physical-delta-and-reuse-boundaries" + ); + assert_eq!( + wire["codec_radix_work"], + "OPERATION_CALLS_ONLY_NOT_TOTAL_SQL" + ); + assert_eq!(wire["rooted_work"]["tree_fetches"], 1); + assert_eq!(wire["rooted_work"]["delta_pages"], 2); + assert_eq!(wire["rooted_work"]["reused_roots"], 7); + assert!(wire.get("directories_rebuilt").is_none()); + assert!(wire.get("directory_root_pages_returned").is_none()); + assert_eq!(wire["snapshot_id"], fixture.snapshot_id); + assert_eq!(wire["projection_elapsed_micros"], 456); + } + #[test] fn observation_rejects_each_changed_binding_or_unbounded_trace_token() { let fixture = Fixture::new(); diff --git a/src/ceres/snapshot/rooted_metadata_install.rs b/src/ceres/snapshot/rooted_metadata_install.rs new file mode 100644 index 00000000..5086bfa5 --- /dev/null +++ b/src/ceres/snapshot/rooted_metadata_install.rs @@ -0,0 +1,848 @@ +//! Bounded native metadata delta and certified reused-root installation identity. +//! Reused roots are opaque boundaries; their database proofs grant no authority here. + +use std::collections::{BTreeMap, BTreeSet}; + +use mst2_codec::metapage::{HEADER_LEN, PAGE_MAX_BYTES}; +use sha2::{Digest, Sha256}; +use uuid::Uuid; + +use super::{ + error::{SnapshotError, SnapshotErrorCode}, + metadata_install::{MAX_PLAN_BYTES, MetadataInstallIdentity}, + projection_observation::NATIVE_PROJECTION_REVISION, + retention_dag::{MetadataDagLimits, MetadataPageId}, +}; + +const DOMAIN: &[u8] = b"mega.mst2.rooted-install.v1\0"; +const ENCODING_VERSION: u16 = 1; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct RootedReuseRoot { + pub(crate) generation: i64, + pub(crate) attestation_id: Uuid, + pub(crate) attestation_digest: [u8; 32], + pub(crate) certificate_digest: [u8; 32], +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct RootedMetadataInstallPlan { + pub(crate) identity: MetadataInstallIdentity, + pub(crate) root: MetadataPageId, + pub(crate) delta: BTreeMap, + pub(crate) edges: BTreeSet<(MetadataPageId, MetadataPageId)>, + pub(crate) reused: BTreeMap, + pub(crate) source_roots: BTreeMap, +} + +impl RootedMetadataInstallPlan { + pub(crate) fn new( + identity: MetadataInstallIdentity, + root: MetadataPageId, + delta: BTreeMap, + edges: BTreeSet<(MetadataPageId, MetadataPageId)>, + reused: BTreeMap, + source_roots: BTreeMap, + ) -> Result { + let plan = Self { + identity, + root, + delta, + edges, + reused, + source_roots, + }; + plan.validate()?; + Ok(plan) + } + + pub(crate) fn digest(&self) -> Result<[u8; 32], SnapshotError> { + Ok(Sha256::digest(self.encode()?).into()) + } + + pub(crate) fn delta_bytes(&self) -> Result { + self.delta + .values() + .try_fold(0u64, |total, size| total.checked_add(*size)) + .ok_or_else(|| limit("rooted metadata delta byte count overflow")) + } + + pub(crate) fn encode(&self) -> Result, SnapshotError> { + self.validate()?; + let mut bytes = Vec::with_capacity(self.encoded_len()); + bytes.extend_from_slice(DOMAIN); + bytes.extend_from_slice(&ENCODING_VERSION.to_be_bytes()); + for value in [ + &self.identity.source_domain, + &self.identity.tagged_root_tree_oid, + &self.identity.scope, + ] { + write_string(&mut bytes, value); + } + for value in [ + self.identity.schema_version, + self.identity.metadata_codec, + self.identity.materialization_policy, + self.identity.fs_semantics, + self.identity.access_projection, + ] { + bytes.extend_from_slice(&value.to_be_bytes()); + } + bytes.extend_from_slice(&self.identity.verification_revision.to_be_bytes()); + bytes.extend_from_slice(&self.identity.projection_revision.to_be_bytes()); + bytes.extend_from_slice(&self.root); + bytes.extend_from_slice(&(self.delta.len() as u32).to_be_bytes()); + for (page, size) in &self.delta { + bytes.extend_from_slice(page); + bytes.extend_from_slice(&size.to_be_bytes()); + } + bytes.extend_from_slice(&(self.edges.len() as u32).to_be_bytes()); + for (parent, child) in &self.edges { + bytes.extend_from_slice(parent); + bytes.extend_from_slice(child); + } + bytes.extend_from_slice(&(self.reused.len() as u32).to_be_bytes()); + for (page, reused) in &self.reused { + bytes.extend_from_slice(page); + bytes.extend_from_slice(&reused.generation.to_be_bytes()); + bytes.extend_from_slice(reused.attestation_id.as_bytes()); + bytes.extend_from_slice(&reused.attestation_digest); + bytes.extend_from_slice(&reused.certificate_digest); + } + bytes.extend_from_slice(&(self.source_roots.len() as u32).to_be_bytes()); + for (tree_oid, page) in &self.source_roots { + write_string(&mut bytes, tree_oid); + bytes.extend_from_slice(page); + } + Ok(bytes) + } + + pub(crate) fn decode(bytes: &[u8], expected_digest: &[u8; 32]) -> Result { + if bytes.len() > MAX_PLAN_BYTES { + return Err(limit("stored rooted metadata plan exceeds its byte budget")); + } + let digest: [u8; 32] = Sha256::digest(bytes).into(); + if &digest != expected_digest { + return Err(integrity("stored rooted metadata plan digest mismatch")); + } + let mut reader = PlanReader(bytes); + if reader.take(DOMAIN.len())? != DOMAIN || reader.u16()? != ENCODING_VERSION { + return Err(integrity("unsupported rooted metadata plan encoding")); + } + let identity = MetadataInstallIdentity { + source_domain: reader.string(64)?, + tagged_root_tree_oid: reader.string(128)?, + scope: reader.string(4096)?, + schema_version: reader.u16()?, + metadata_codec: reader.u16()?, + materialization_policy: reader.u16()?, + fs_semantics: reader.u16()?, + access_projection: reader.u16()?, + verification_revision: i32::from_be_bytes(reader.array()?), + projection_revision: reader.u16()?, + }; + let root = reader.array()?; + let limits = MetadataDagLimits::default(); + let count = reader.count(limits.nodes)?; + let mut delta = BTreeMap::new(); + let mut last = None; + for _ in 0..count { + let page = reader.array()?; + let size = reader.u64()?; + if last.is_some_and(|previous| previous >= page) { + return Err(integrity("rooted metadata delta is not uniquely ordered")); + } + last = Some(page); + delta.insert(page, size); + } + let count = reader.count(limits.edges)?; + let mut edges = BTreeSet::new(); + let mut last = None; + for _ in 0..count { + let edge = (reader.array()?, reader.array()?); + if last.is_some_and(|previous| previous >= edge) { + return Err(integrity("rooted metadata edges are not uniquely ordered")); + } + last = Some(edge); + edges.insert(edge); + } + let count = reader.count(limits.nodes - delta.len())?; + let mut reused = BTreeMap::new(); + let mut last = None; + for _ in 0..count { + let page = reader.array()?; + let proof = RootedReuseRoot { + generation: i64::from_be_bytes(reader.array()?), + attestation_id: Uuid::from_bytes(reader.array()?), + attestation_digest: reader.array()?, + certificate_digest: reader.array()?, + }; + if last.is_some_and(|previous| previous >= page) { + return Err(integrity( + "rooted metadata reused roots are not uniquely ordered", + )); + } + last = Some(page); + reused.insert(page, proof); + } + let count = reader.count(limits.nodes)?; + let mut source_roots: BTreeMap = BTreeMap::new(); + for _ in 0..count { + let tree_oid = reader.string(128)?; + let page = reader.array()?; + if source_roots + .last_key_value() + .is_some_and(|(previous, _)| previous >= &tree_oid) + { + return Err(integrity( + "rooted metadata source roots are not uniquely ordered", + )); + } + source_roots.insert(tree_oid, page); + } + if !reader.0.is_empty() { + return Err(integrity("rooted metadata plan has trailing bytes")); + } + Self::new(identity, root, delta, edges, reused, source_roots) + } + + pub(crate) fn validate(&self) -> Result<(), SnapshotError> { + self.validate_and_order().map(|_| ()) + } + + /// Child-first delta order. Reused roots have no descendants in this plan. + pub(crate) fn child_first_delta(&self) -> Result, SnapshotError> { + self.validate_and_order() + } + + fn validate_and_order(&self) -> Result, SnapshotError> { + let identity = &self.identity; + if identity.source_domain != "native-git" + || identity.schema_version != mst2_codec::descriptor::SCHEMA_VERSION + || identity.metadata_codec != mst2_codec::descriptor::METADATA_CODEC + || identity.materialization_policy + != mst2_codec::descriptor::MATERIALIZATION_POLICY_GIT_RAW_V1 + || identity.fs_semantics != mst2_codec::descriptor::FS_SEMANTICS_LINUX_CODE_V1 + || identity.access_projection != mst2_codec::descriptor::ACCESS_PROJECTION_EXACT_FULL + || identity.verification_revision + != crate::jupiter::storage::mono_storage::MST2_VERIFICATION_VERSION + || identity.projection_revision != NATIVE_PROJECTION_REVISION + { + return Err(integrity("unsupported rooted native metadata profile")); + } + let hash_kind = tagged_hash_kind(&identity.tagged_root_tree_oid)?; + super::view::validate_scope_relative_path(&identity.scope) + .map_err(|_| integrity("noncanonical rooted metadata scope"))?; + let limits = MetadataDagLimits::default(); + if self.delta.len() > limits.nodes + || self.reused.len() > limits.nodes - self.delta.len() + || self.delta.is_empty() && self.reused.is_empty() + || self.edges.len() > limits.edges + || self.source_roots.is_empty() + || self.source_roots.len() > limits.nodes + { + return Err(limit("rooted metadata plan exceeds its group budget")); + } + if self.delta_bytes()? > limits.payload_bytes { + return Err(limit("rooted metadata delta exceeds its payload budget")); + } + if !self.contains(&self.root) + || self.delta.iter().any(|(page, size)| { + self.reused.contains_key(page) + || !(HEADER_LEN as u64..=PAGE_MAX_BYTES as u64).contains(size) + }) + || self.reused.values().any(|proof| proof.generation <= 0) + || self.delta.is_empty() + && (!self.reused.contains_key(&self.root) || !self.edges.is_empty()) + { + return Err(integrity( + "rooted metadata root, delta or reuse boundary is invalid", + )); + } + for (tree_oid, page) in &self.source_roots { + if tagged_hash_kind(tree_oid)? != hash_kind || !self.contains(page) { + return Err(integrity( + "rooted metadata source binding crossed its profile or boundary", + )); + } + } + if !self.source_roots.values().any(|page| page == &self.root) { + return Err(integrity( + "rooted metadata root lacks a source tree binding", + )); + } + if self.encoded_len() > MAX_PLAN_BYTES { + return Err(limit("rooted metadata plan exceeds its byte budget")); + } + + let mut remaining: BTreeMap<_, usize> = self.delta.keys().map(|page| (*page, 0)).collect(); + let mut parents: BTreeMap<_, Vec<_>> = BTreeMap::new(); + let mut children: BTreeMap<_, Vec<_>> = BTreeMap::new(); + for &(parent, child) in &self.edges { + if parent == child || !self.delta.contains_key(&parent) || !self.contains(&child) { + return Err(integrity( + "rooted metadata edge crossed its delta or reused boundary", + )); + } + children.entry(parent).or_default().push(child); + if self.delta.contains_key(&child) { + *remaining + .get_mut(&parent) + .ok_or_else(|| integrity("rooted metadata delta parent is missing"))? += 1; + parents.entry(child).or_default().push(parent); + } + } + let mut ready: BTreeSet<_> = remaining + .iter() + .filter_map(|(page, count)| (*count == 0).then_some(*page)) + .collect(); + let mut ordered = Vec::with_capacity(self.delta.len()); + while let Some(page) = ready.pop_first() { + ordered.push(page); + for parent in parents.get(&page).into_iter().flatten() { + let count = remaining + .get_mut(parent) + .ok_or_else(|| integrity("rooted metadata delta ancestor is missing"))?; + *count = count + .checked_sub(1) + .ok_or_else(|| integrity("rooted metadata delta edge accounting underflow"))?; + if *count == 0 { + ready.insert(*parent); + } + } + } + if ordered.len() != self.delta.len() { + return Err(integrity("rooted metadata delta contains a cycle")); + } + let mut reachable = BTreeSet::new(); + let mut pending = vec![self.root]; + while let Some(page) = pending.pop() { + if reachable.insert(page) { + pending.extend(children.get(&page).into_iter().flatten()); + } + } + if reachable.len() != self.delta.len() + self.reused.len() { + return Err(integrity( + "rooted metadata plan contains unreachable boundaries", + )); + } + Ok(ordered) + } + + fn contains(&self, page: &MetadataPageId) -> bool { + self.delta.contains_key(page) || self.reused.contains_key(page) + } + + fn encoded_len(&self) -> usize { + DOMAIN.len() + + 2 + + 3 * 4 + + self.identity.source_domain.len() + + self.identity.tagged_root_tree_oid.len() + + self.identity.scope.len() + + 5 * 2 + + 4 + + 2 + + 32 + + 4 * 4 + + self.delta.len() * 40 + + self.edges.len() * 64 + + self.reused.len() * 120 + + self + .source_roots + .keys() + .map(|tree_oid| 4 + tree_oid.len() + 32) + .sum::() + } +} + +fn tagged_hash_kind(oid: &str) -> Result<&str, SnapshotError> { + let (kind, hex) = oid + .split_once(':') + .ok_or_else(|| integrity("untagged rooted metadata source tree"))?; + let length = match kind { + "sha1" => 40, + "sha256" | "blake3" => 64, + _ => return Err(integrity("unsupported rooted metadata source hash kind")), + }; + if hex.len() != length + || !hex + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + { + return Err(integrity( + "noncanonical rooted metadata source tree identity", + )); + } + Ok(kind) +} + +fn write_string(bytes: &mut Vec, value: &str) { + bytes.extend_from_slice(&(value.len() as u32).to_be_bytes()); + bytes.extend_from_slice(value.as_bytes()); +} + +struct PlanReader<'a>(&'a [u8]); + +impl<'a> PlanReader<'a> { + fn take(&mut self, count: usize) -> Result<&'a [u8], SnapshotError> { + if count > self.0.len() { + return Err(integrity("truncated rooted metadata plan")); + } + let (bytes, rest) = self.0.split_at(count); + self.0 = rest; + Ok(bytes) + } + + fn array(&mut self) -> Result<[u8; N], SnapshotError> { + self.take(N)? + .try_into() + .map_err(|_| integrity("invalid rooted metadata plan field")) + } + + fn u16(&mut self) -> Result { + Ok(u16::from_be_bytes(self.array()?)) + } + + fn u32(&mut self) -> Result { + Ok(u32::from_be_bytes(self.array()?)) + } + + fn u64(&mut self) -> Result { + Ok(u64::from_be_bytes(self.array()?)) + } + + fn count(&mut self, maximum: usize) -> Result { + let count = self.u32()? as usize; + if count > maximum { + return Err(limit("rooted metadata plan count exceeds its budget")); + } + Ok(count) + } + + fn string(&mut self, maximum: usize) -> Result { + let count = self.count(maximum)?; + String::from_utf8(self.take(count)?.to_vec()) + .map_err(|_| integrity("non-UTF8 rooted metadata plan identity")) + } +} + +fn integrity(message: &str) -> SnapshotError { + SnapshotError::new(SnapshotErrorCode::IntegrityError, message) +} + +fn limit(message: &str) -> SnapshotError { + SnapshotError::new(SnapshotErrorCode::LimitExceeded, message) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn page(value: u32) -> MetadataPageId { + let mut id = [0; 32]; + id[28..].copy_from_slice(&value.to_be_bytes()); + id + } + + fn identity() -> MetadataInstallIdentity { + MetadataInstallIdentity { + source_domain: "native-git".into(), + tagged_root_tree_oid: format!("sha1:{}", "a".repeat(40)), + scope: "/".into(), + schema_version: mst2_codec::descriptor::SCHEMA_VERSION, + metadata_codec: mst2_codec::descriptor::METADATA_CODEC, + materialization_policy: mst2_codec::descriptor::MATERIALIZATION_POLICY_GIT_RAW_V1, + fs_semantics: mst2_codec::descriptor::FS_SEMANTICS_LINUX_CODE_V1, + access_projection: mst2_codec::descriptor::ACCESS_PROJECTION_EXACT_FULL, + verification_revision: crate::jupiter::storage::mono_storage::MST2_VERIFICATION_VERSION, + projection_revision: NATIVE_PROJECTION_REVISION, + } + } + + fn reuse(value: u128) -> RootedReuseRoot { + RootedReuseRoot { + generation: 7, + attestation_id: Uuid::from_u128(value), + attestation_digest: [11; 32], + certificate_digest: [12; 32], + } + } + + fn plan() -> RootedMetadataInstallPlan { + let identity = identity(); + RootedMetadataInstallPlan::new( + identity.clone(), + page(1), + BTreeMap::from([(page(1), 20), (page(2), 57), (page(3), 16384)]), + BTreeSet::from([ + (page(1), page(2)), + (page(1), page(3)), + (page(1), page(5)), + (page(2), page(4)), + (page(3), page(4)), + ]), + BTreeMap::from([(page(4), reuse(1)), (page(5), reuse(2))]), + BTreeMap::from([ + (identity.tagged_root_tree_oid, page(1)), + (format!("sha1:{}", "b".repeat(40)), page(4)), + ]), + ) + .unwrap() + } + + fn decode(bytes: &[u8]) -> Result { + RootedMetadataInstallPlan::decode(bytes, &Sha256::digest(bytes).into()) + } + + fn records(bytes: &[u8]) -> [Vec>; 4] { + let mut reader = PlanReader(bytes); + reader.take(DOMAIN.len() + 2).unwrap(); + for _ in 0..3 { + reader.string(4096).unwrap(); + } + reader.take(16 + 32).unwrap(); + let mut sections: [Vec>; 4] = std::array::from_fn(|_| Vec::new()); + for (section, width) in sections[..3].iter_mut().zip([40, 64, 120]) { + let count = reader.u32().unwrap(); + for _ in 0..count { + let start = bytes.len() - reader.0.len(); + reader.take(width).unwrap(); + section.push(start..start + width); + } + } + let count = reader.u32().unwrap(); + for _ in 0..count { + let start = bytes.len() - reader.0.len(); + reader.string(128).unwrap(); + reader.take(32).unwrap(); + sections[3].push(start..bytes.len() - reader.0.len()); + } + assert!(reader.0.is_empty()); + sections + } + + #[test] + fn rooted_plan_roundtrip_and_shared_reuse_need_only_delta_order() { + let plan = plan(); + let bytes = plan.encode().unwrap(); + assert_eq!(bytes.len(), plan.encoded_len()); + assert_eq!(bytes.len(), 1004); + assert_eq!( + hex::encode(plan.digest().unwrap()), + "e74e98e8737cfcbdfb264d9f12e15ba2126e96b1e5f35cb0cfd89fb24dde808b" + ); + assert_eq!(decode(&bytes).unwrap(), plan); + assert_eq!( + plan.child_first_delta().unwrap(), + [page(2), page(3), page(1)] + ); + assert_eq!(plan.delta_bytes().unwrap(), 20 + 57 + 16384); + for range in &records(&bytes)[2] { + assert_eq!(range.len(), 120); + } + let original = plan.digest().unwrap(); + for field in 0..4 { + let mut changed = plan.clone(); + let proof = changed.reused.get_mut(&page(4)).unwrap(); + match field { + 0 => proof.generation += 1, + 1 => proof.attestation_id = Uuid::from_u128(9), + 2 => proof.attestation_digest[0] ^= 1, + _ => proof.certificate_digest[0] ^= 1, + } + assert_ne!(changed.digest().unwrap(), original); + } + let mut changed = plan.clone(); + changed.identity.scope = "/目录/a\\b".into(); + assert_ne!(changed.digest().unwrap(), original); + assert_eq!(decode(&changed.encode().unwrap()).unwrap(), changed); + changed = plan; + changed + .source_roots + .insert(format!("sha1:{}", "c".repeat(40)), page(3)); + assert_ne!(changed.digest().unwrap(), original); + } + + #[test] + fn rooted_plan_stored_order_duplicates_digest_domain_version_truncation_and_eof_reject() { + let plan = plan(); + let bytes = plan.encode().unwrap(); + let mut wrong_digest = plan.digest().unwrap(); + wrong_digest[0] ^= 1; + assert!(RootedMetadataInstallPlan::decode(&bytes, &wrong_digest).is_err()); + for section in records(&bytes) { + let mut changed = bytes.clone(); + let a = §ion[0]; + let b = §ion[1]; + assert_eq!(a.len(), b.len()); + changed[a.clone()].copy_from_slice(&bytes[b.clone()]); + changed[b.clone()].copy_from_slice(&bytes[a.clone()]); + assert!(decode(&changed).is_err()); + changed[b.clone()].copy_from_slice(&bytes[b.clone()]); + assert!(decode(&changed).is_err()); + } + for length in [0, DOMAIN.len(), bytes.len() - 1] { + assert!(decode(&bytes[..length]).is_err()); + } + let mut changed = bytes.clone(); + changed.push(0); + assert!(decode(&changed).is_err()); + changed = bytes.clone(); + changed[0] ^= 1; + assert!(decode(&changed).is_err()); + changed = bytes; + changed[DOMAIN.len() + 1] = 2; + assert!(decode(&changed).is_err()); + let oversized = vec![0; MAX_PLAN_BYTES + 1]; + assert_eq!( + decode(&oversized).unwrap_err().code, + SnapshotErrorCode::LimitExceeded + ); + } + + #[test] + fn rooted_plan_decode_bounds_declared_counts_strings_and_reused_generations() { + let bytes = plan().encode().unwrap(); + let sections = records(&bytes); + for section in §ions { + let mut changed = bytes.clone(); + let count_at = section[0].start - 4; + changed[count_at..count_at + 4].copy_from_slice(&u32::MAX.to_be_bytes()); + assert_eq!( + decode(&changed).unwrap_err().code, + SnapshotErrorCode::LimitExceeded + ); + } + let mut changed = bytes.clone(); + let source_at = sections[3][0].start; + changed[source_at..source_at + 4].copy_from_slice(&u32::MAX.to_be_bytes()); + assert_eq!( + decode(&changed).unwrap_err().code, + SnapshotErrorCode::LimitExceeded + ); + changed = bytes.clone(); + changed[DOMAIN.len() + 2 + 4] = 0xff; + assert_eq!( + decode(&changed).unwrap_err().code, + SnapshotErrorCode::IntegrityError + ); + changed = bytes; + let generation_at = sections[2][0].start + 32; + changed[generation_at..generation_at + 8].copy_from_slice(&i64::MIN.to_be_bytes()); + assert_eq!( + decode(&changed).unwrap_err().code, + SnapshotErrorCode::IntegrityError + ); + } + + #[test] + fn rooted_plan_rejects_cycles_unknown_endpoints_reused_parents_and_unreachable_members() { + let original = plan(); + for invalid in 0..7 { + let mut changed = original.clone(); + match invalid { + 0 => { + changed.edges.insert((page(2), page(1))); + } + 1 => { + changed.edges.insert((page(4), page(2))); + } + 2 => { + changed.edges.insert((page(2), page(99))); + } + 3 => { + changed.edges.insert((page(99), page(2))); + } + 4 => { + changed.delta.insert(page(6), 20); + } + 5 => { + changed.reused.insert(page(6), reuse(6)); + } + _ => { + changed.reused.insert(page(2), reuse(2)); + } + } + assert_eq!( + changed.encode().unwrap_err().code, + SnapshotErrorCode::IntegrityError + ); + } + } + + #[test] + fn rooted_plan_zero_delta_requires_its_single_reused_root_and_source_binding() { + let identity = identity(); + let plan = RootedMetadataInstallPlan::new( + identity.clone(), + page(4), + BTreeMap::new(), + BTreeSet::new(), + BTreeMap::from([(page(4), reuse(4))]), + BTreeMap::from([(identity.tagged_root_tree_oid, page(4))]), + ) + .unwrap(); + assert!(plan.child_first_delta().unwrap().is_empty()); + assert_eq!(plan.delta_bytes().unwrap(), 0); + assert_eq!(plan.encode().unwrap().len(), 363); + assert_eq!( + hex::encode(plan.digest().unwrap()), + "81b4cbdaaacd7efe8687477d05bd4db190b974f5aa99ce7e8101326857d0e7b1" + ); + assert_eq!(decode(&plan.encode().unwrap()).unwrap(), plan); + for invalid in 0..5 { + let mut changed = plan.clone(); + match invalid { + 0 => { + changed.root = page(9); + } + 1 => { + changed.edges.insert((page(4), page(4))); + } + 2 => { + changed.reused.insert(page(5), reuse(5)); + } + 3 => { + changed.delta.insert(page(5), 20); + } + _ => { + changed.source_roots.clear(); + } + } + assert!(changed.encode().is_err()); + } + } + + #[test] + fn rooted_plan_rejects_source_profile_scope_size_and_generation_mismatches() { + let original = plan(); + for invalid in 0..18 { + let mut changed = original.clone(); + match invalid { + 0 => changed.identity.metadata_codec += 1, + 1 => changed.identity.projection_revision += 1, + 2 => changed.identity.verification_revision += 1, + 3 => changed.identity.scope = "/a/..".into(), + 4 => changed.identity.scope = format!("/{}", "a".repeat(256)), + 5 => changed.identity.tagged_root_tree_oid = format!("sha1:{}", "A".repeat(40)), + 6 => { + changed.delta.insert(page(2), HEADER_LEN as u64 - 1); + } + 7 => { + changed.delta.insert(page(2), PAGE_MAX_BYTES as u64 + 1); + } + 8 => { + changed.reused.get_mut(&page(4)).unwrap().generation = 0; + } + 9 => { + changed + .source_roots + .insert(format!("sha256:{}", "a".repeat(64)), page(3)); + } + 10 => { + changed + .source_roots + .insert(format!("sha1:{}", "d".repeat(40)), page(99)); + } + 11 => { + changed + .source_roots + .retain(|_, page_id| *page_id != page(1)); + } + 12 => { + changed.source_roots.insert("sha1:bad".into(), page(3)); + } + 13 => changed.identity.source_domain = "other".into(), + 14 => changed.identity.schema_version += 1, + 15 => changed.identity.materialization_policy += 1, + 16 => changed.identity.fs_semantics += 1, + _ => changed.identity.access_projection += 1, + } + assert!(changed.encode().is_err()); + } + for kind in ["sha256", "blake3"] { + let mut changed = original.clone(); + changed.identity.tagged_root_tree_oid = format!("{kind}:{}", "a".repeat(64)); + changed.source_roots = + BTreeMap::from([(changed.identity.tagged_root_tree_oid.clone(), page(1))]); + assert_eq!(decode(&changed.encode().unwrap()).unwrap(), changed); + } + } + + #[test] + fn rooted_plan_admits_exact_group_payload_and_source_count_boundaries() { + let identity = identity(); + let limits = MetadataDagLimits::default(); + let delta: BTreeMap<_, _> = (1..=limits.nodes as u32) + .map(|value| (page(value), PAGE_MAX_BYTES as u64)) + .collect(); + let edges = (2..=limits.nodes as u32) + .map(|value| (page(1), page(value))) + .collect(); + let full_plan = RootedMetadataInstallPlan::new( + identity.clone(), + page(1), + delta, + edges, + BTreeMap::new(), + BTreeMap::from([(identity.tagged_root_tree_oid, page(1))]), + ) + .unwrap(); + assert_eq!(full_plan.delta_bytes().unwrap(), limits.payload_bytes); + assert_eq!(decode(&full_plan.encode().unwrap()).unwrap(), full_plan); + let mut changed = full_plan.clone(); + changed + .delta + .insert(page(limits.nodes as u32 + 1), HEADER_LEN as u64); + assert_eq!( + changed.encode().unwrap_err().code, + SnapshotErrorCode::LimitExceeded + ); + changed = full_plan.clone(); + changed + .reused + .insert(page(limits.nodes as u32 + 1), reuse(9)); + assert_eq!( + changed.encode().unwrap_err().code, + SnapshotErrorCode::LimitExceeded + ); + + let mut source_plan = plan(); + source_plan.source_roots = (0..limits.nodes) + .map(|value| (format!("sha1:{value:040x}"), page(1))) + .collect(); + assert_eq!(decode(&source_plan.encode().unwrap()).unwrap(), source_plan); + source_plan + .source_roots + .insert(format!("sha1:{:040x}", limits.nodes), page(1)); + assert_eq!( + source_plan.encode().unwrap_err().code, + SnapshotErrorCode::LimitExceeded + ); + } + + #[test] + fn rooted_plan_admits_exact_edge_boundary_then_rejects_one_more_edge() { + let mut value = plan(); + value.delta = (1..=4096).map(|id| (page(id), HEADER_LEN as u64)).collect(); + value.reused.clear(); + value.source_roots.retain(|_, root| *root == page(1)); + value.edges = (2..=4096).map(|id| (page(1), page(id))).collect(); + 'fill: for parent in 2..=4096 { + for child in parent + 1..=4096 { + if value.edges.len() == MetadataDagLimits::default().edges { + break 'fill; + } + value.edges.insert((page(parent), page(child))); + } + } + assert_eq!(value.edges.len(), MetadataDagLimits::default().edges); + assert_eq!(decode(&value.encode().unwrap()).unwrap(), value); + assert!(value.edges.insert((page(4095), page(4096)))); + assert_eq!( + value.encode().unwrap_err().code, + SnapshotErrorCode::LimitExceeded + ); + } +} diff --git a/src/ceres/snapshot/rooted_metadata_projection.rs b/src/ceres/snapshot/rooted_metadata_projection.rs new file mode 100644 index 00000000..086db637 --- /dev/null +++ b/src/ceres/snapshot/rooted_metadata_projection.rs @@ -0,0 +1,1108 @@ +//! Native source projection that stops at certified reused-directory boundaries. +//! Hints and operation-local memoization are not database installation authority. + +use std::{ + borrow::Cow, + collections::{BTreeMap, BTreeSet, VecDeque}, +}; + +use async_trait::async_trait; +use git_internal::{hash::ObjectHash, internal::object::tree::Tree}; +use mst2_codec::{ + descriptor::{ + ACCESS_PROJECTION_EXACT_FULL, FS_SEMANTICS_LINUX_CODE_V1, + MATERIALIZATION_POLICY_GIT_RAW_V1, METADATA_CODEC, SCHEMA_VERSION, + }, + metapage::{Entry, EntryKind, HEADER_LEN, PAGE_MAX_BYTES, Page, page_id}, +}; +use sea_orm::ActiveValue::Set; +use sha2::{Digest, Sha256}; + +use super::{ + error::{SnapshotError, SnapshotErrorCode}, + metadata_install::MetadataInstallIdentity, + projection_observation::NATIVE_PROJECTION_REVISION, + resolver::{FsKind, direct_entries}, + retention_dag::{MetadataDagLimits, MetadataPageId, MetadataPagePayload}, + rooted_metadata_install::{RootedMetadataInstallPlan, RootedReuseRoot}, + view::validate_scope_relative_path, +}; +use crate::{ + ceres::api_service::ApiHandler, + common::errors::MegaError, + jupiter::storage::{Storage, mono_storage::MST2_VERIFICATION_VERSION}, +}; + +#[async_trait] +pub(crate) trait RootedReuseLookup: Send + Sync { + async fn lookup_reuse( + &self, + tree_oid: &str, + identity: &MetadataInstallIdentity, + ) -> Result, SnapshotError>; +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct CertifiedReusableDirectory { + pub(crate) page_id: MetadataPageId, + pub(crate) proof: RootedReuseRoot, + pub(crate) relative_path_bytes: usize, + pub(crate) relative_components: usize, + pub(crate) closure_nodes_upper: usize, + pub(crate) closure_edges_upper: usize, + pub(crate) closure_bytes_upper: u64, + pub(crate) closure_entries_upper: usize, +} + +/// API calls and materialized outputs only, not total database or codec work. +#[derive(Debug, Clone, Default, PartialEq, Eq, serde::Serialize)] +pub(crate) struct RootedProjectionWork { + pub(crate) input_tree_entries: u64, + pub(crate) scope_path_entries_examined: u64, + pub(crate) tree_fetches: u64, + pub(crate) reuse_lookups: u64, + pub(crate) directory_memo_hits: u64, + pub(crate) directories_scanned: u64, + pub(crate) direct_entries_scanned: u64, + pub(crate) verified_blob_queries: u64, + pub(crate) verified_blob_facts_loaded: u64, + pub(crate) verified_blob_persistence_batches: u64, + pub(crate) blob_fetches: u64, + pub(crate) raw_bytes_fetched: u64, + pub(crate) raw_bytes_hashed: u64, + pub(crate) directory_root_builds: u64, + pub(crate) radix_route_builds: u64, + pub(crate) codec_input_entries: u64, + pub(crate) delta_pages: u64, + pub(crate) delta_bytes: u64, + pub(crate) reused_roots: u64, + pub(crate) reused_closure_nodes_upper: u64, + pub(crate) reused_closure_edges_upper: u64, + pub(crate) reused_closure_bytes_upper: u64, + pub(crate) reused_closure_entries_upper: u64, + pub(crate) plan_bytes: u64, +} + +#[derive(Debug)] +pub(crate) struct PreparedRootedNativeMetadata { + pub(crate) plan: RootedMetadataInstallPlan, + pub(crate) payloads: Vec, + pub(crate) work: RootedProjectionWork, +} + +pub(crate) async fn prepare_rooted_native_metadata< + T: ApiHandler + ?Sized, + R: RootedReuseLookup + ?Sized, +>( + handler: &T, + root_tree: &Tree, + scope: &str, + reuse: &R, +) -> Result { + validate_scope_relative_path(scope)?; + if !handler.native_snapshot_projection() { + return Err(SnapshotError::new( + SnapshotErrorCode::ScopeInvalid, + "rooted metadata projection requires native Git source semantics", + )); + } + let identity = MetadataInstallIdentity { + source_domain: "native-git".into(), + tagged_root_tree_oid: root_tree.id.to_tagged_string(), + scope: scope.to_owned(), + schema_version: SCHEMA_VERSION, + metadata_codec: METADATA_CODEC, + materialization_policy: MATERIALIZATION_POLICY_GIT_RAW_V1, + fs_semantics: FS_SEMANTICS_LINUX_CODE_V1, + access_projection: ACCESS_PROJECTION_EXACT_FULL, + verification_revision: MST2_VERIFICATION_VERSION, + projection_revision: NATIVE_PROJECTION_REVISION, + }; + let storage = handler.get_context(); + let mut state = ProjectionState::new(identity); + state.work.input_tree_entries = root_tree.tree_items.len() as u64; + let scoped_oid = resolve_scope_oid(handler, root_tree, scope, &mut state.work).await?; + let loaded = (scope == "/").then_some(root_tree); + let root = project_directory( + handler, reuse, &storage, scoped_oid, loaded, scope, &mut state, + ) + .await?; + state.finish(root.page_id) +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +struct PathBounds { + bytes: usize, + components: usize, +} + +impl PathBounds { + fn include(&mut self, name: &str, child: Self) -> Result<(), SnapshotError> { + let bytes = name + .len() + .checked_add(1) + .and_then(|value| value.checked_add(child.bytes)) + .filter(|value| *value <= 4096) + .ok_or_else(path_limit)?; + let components = child + .components + .checked_add(1) + .filter(|value| *value <= 256) + .ok_or_else(path_limit)?; + self.bytes = self.bytes.max(bytes); + self.components = self.components.max(components); + Ok(()) + } + + fn validate_at(self, prefix: &str) -> Result<(), SnapshotError> { + validate_scope_relative_path(prefix)?; + let (bytes, components) = if prefix == "/" { + (0, 0) + } else { + (prefix.len(), prefix[1..].split('/').count()) + }; + bytes + .checked_add(self.bytes) + .filter(|value| *value <= 4096) + .ok_or_else(path_limit)?; + components + .checked_add(self.components) + .filter(|value| *value <= 256) + .ok_or_else(path_limit)?; + Ok(()) + } +} + +#[derive(Debug, Clone, Copy)] +struct DirectorySummary { + page_id: MetadataPageId, + bounds: PathBounds, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct BlobFact { + size: u64, + digest: [u8; 32], +} + +#[derive(Default)] +struct ClosureUpper { + nodes: usize, + edges: usize, + bytes: u64, + entries: usize, +} + +impl ClosureUpper { + fn add(&mut self, directory: &CertifiedReusableDirectory) -> Result<(), SnapshotError> { + self.nodes = self + .nodes + .checked_add(directory.closure_nodes_upper) + .ok_or_else(budget_limit)?; + self.edges = self + .edges + .checked_add(directory.closure_edges_upper) + .ok_or_else(budget_limit)?; + self.bytes = self + .bytes + .checked_add(directory.closure_bytes_upper) + .ok_or_else(budget_limit)?; + self.entries = self + .entries + .checked_add(directory.closure_entries_upper) + .ok_or_else(budget_limit)?; + Ok(()) + } +} + +struct ProjectionState { + identity: MetadataInstallIdentity, + required_trees: BTreeSet, + active_trees: BTreeSet, + directories: BTreeMap, + bounds: BTreeMap, + payloads: BTreeMap, + page_entries: BTreeMap, + edges: BTreeSet<(MetadataPageId, MetadataPageId)>, + reused: BTreeMap, + source_roots: BTreeMap, + blob_facts: BTreeMap, + source_entries: usize, + source_encoded_bytes: u64, + codec_entry_visits: usize, + payload_bytes: u64, + delta_entries: usize, + reuse_upper: ClosureUpper, + work: RootedProjectionWork, +} + +impl ProjectionState { + fn new(identity: MetadataInstallIdentity) -> Self { + Self { + identity, + required_trees: BTreeSet::new(), + active_trees: BTreeSet::new(), + directories: BTreeMap::new(), + bounds: BTreeMap::new(), + payloads: BTreeMap::new(), + page_entries: BTreeMap::new(), + edges: BTreeSet::new(), + reused: BTreeMap::new(), + source_roots: BTreeMap::new(), + blob_facts: BTreeMap::new(), + source_entries: 0, + source_encoded_bytes: 0, + codec_entry_visits: 0, + payload_bytes: 0, + delta_entries: 0, + reuse_upper: ClosureUpper::default(), + work: RootedProjectionWork::default(), + } + } + + fn require_tree(&mut self, tree_oid: &str) -> Result<(), SnapshotError> { + if !self.required_trees.contains(tree_oid) { + if self.required_trees.len() >= MetadataDagLimits::default().nodes { + return Err(budget_limit()); + } + self.required_trees.insert(tree_oid.to_owned()); + } + Ok(()) + } + + fn admit_entries(&mut self, entries: &[(String, FsKind, String)]) -> Result<(), SnapshotError> { + let limits = MetadataDagLimits::default(); + self.source_encoded_bytes = self + .source_encoded_bytes + .checked_add(HEADER_LEN as u64) + .filter(|value| *value <= limits.payload_bytes) + .ok_or_else(budget_limit)?; + let mut previous: Option<&str> = None; + for (name, kind, _) in entries { + if name.is_empty() + || name.len() > 255 + || name == "." + || name == ".." + || name.contains('/') + || name.contains('\0') + || previous.is_some_and(|value| value >= name.as_str()) + { + return Err(integrity( + "source tree has an invalid or duplicate UTF-8 name", + )); + } + previous = Some(name); + let value_bytes = if *kind == FsKind::Directory { 32 } else { 40 }; + self.source_encoded_bytes = self + .source_encoded_bytes + .checked_add((3 + name.len() + value_bytes) as u64) + .filter(|value| *value <= limits.payload_bytes) + .ok_or_else(budget_limit)?; + } + Ok(()) + } + + fn record_bounds( + &mut self, + page: MetadataPageId, + bounds: PathBounds, + ) -> Result<(), SnapshotError> { + if self + .bounds + .get(&page) + .is_some_and(|previous| previous != &bounds) + { + return Err(integrity( + "one canonical directory has inconsistent descendant path bounds", + )); + } + self.bounds.insert(page, bounds); + Ok(()) + } + + fn record_reuse( + &mut self, + tree_oid: &str, + directory: CertifiedReusableDirectory, + path: &str, + ) -> Result { + let limits = MetadataDagLimits::default(); + if directory.page_id == [0; 32] + || directory.proof.generation <= 0 + || directory.closure_nodes_upper == 0 + || directory.closure_nodes_upper > limits.nodes + || directory.closure_edges_upper > limits.edges + || !(HEADER_LEN as u64..=limits.payload_bytes).contains(&directory.closure_bytes_upper) + || directory.closure_entries_upper > limits.entries + || directory.relative_path_bytes > 4096 + || directory.relative_components > 256 + { + return Err(integrity( + "reused directory hint has an invalid certificate summary", + )); + } + let summary = DirectorySummary { + page_id: directory.page_id, + bounds: PathBounds { + bytes: directory.relative_path_bytes, + components: directory.relative_components, + }, + }; + summary.bounds.validate_at(path)?; + self.record_bounds(directory.page_id, summary.bounds)?; + if !self.payloads.contains_key(&directory.page_id) { + if let Some(previous) = self.reused.get(&directory.page_id) { + if previous.proof.generation != directory.proof.generation + || previous.proof.certificate_digest != directory.proof.certificate_digest + || previous.closure_nodes_upper != directory.closure_nodes_upper + || previous.closure_edges_upper != directory.closure_edges_upper + || previous.closure_bytes_upper != directory.closure_bytes_upper + || previous.closure_entries_upper != directory.closure_entries_upper + { + return Err(integrity( + "one reused boundary has inconsistent lifetime or certificate summaries", + )); + } + } else { + self.reuse_upper.add(&directory)?; + self.reused.insert(directory.page_id, directory); + self.check_budget()?; + } + } + self.source_roots + .insert(tree_oid.to_owned(), summary.page_id); + Ok(summary) + } + + fn charge_codec(&mut self, entries: usize, route_length: usize) -> Result<(), SnapshotError> { + let count = entries + .checked_mul(route_length.checked_add(1).ok_or_else(budget_limit)?) + .ok_or_else(budget_limit)?; + self.codec_entry_visits = self + .codec_entry_visits + .checked_add(count) + .filter(|value| *value <= MetadataDagLimits::default().prepare_entry_visits) + .ok_or_else(budget_limit)?; + self.work.codec_input_entries = self + .work + .codec_input_entries + .checked_add(entries as u64) + .ok_or_else(budget_limit)?; + Ok(()) + } + + fn add_edge( + &mut self, + parent: MetadataPageId, + child: MetadataPageId, + ) -> Result<(), SnapshotError> { + if self.edges.insert((parent, child)) { + self.check_budget()?; + } + Ok(()) + } + + fn collect_directory( + &mut self, + entries: &[Entry], + root_bytes: Vec, + ) -> Result { + let root = page_id(&root_bytes); + let mut routes = VecDeque::from([(Vec::::new(), root, Some(root_bytes))]); + while let Some((route, expected, provided)) = routes.pop_front() { + if self.payloads.contains_key(&expected) || self.reused.contains_key(&expected) { + continue; + } + let bytes = if let Some(bytes) = provided { + bytes + } else { + self.charge_codec(entries.len(), route.len())?; + self.work.radix_route_builds += 1; + Page::pages_along_route(entries, &route) + .map_err(codec_error)? + .pop() + .ok_or_else(|| integrity("canonical radix route returned no page"))? + }; + if page_id(&bytes) != expected || !(HEADER_LEN..=PAGE_MAX_BYTES).contains(&bytes.len()) + { + return Err(integrity( + "canonical radix page differs from its parent binding", + )); + } + let (page, _) = Page::decode(&bytes).map_err(codec_error)?; + let page_entries = match page { + Page::Leaf { entries } => { + for entry in entries.iter().filter(|entry| entry.is_dir()) { + self.add_edge(expected, entry.child_root)?; + } + entries.len() + } + Page::Branch { + terminal, children, .. + } => { + if let Some(entry) = terminal.as_ref().filter(|entry| entry.is_dir()) { + self.add_edge(expected, entry.child_root)?; + } + for child in children { + self.add_edge(expected, child.child_page_id)?; + let mut child_route = route.clone(); + child_route.push(child.label); + routes.push_back((child_route, child.child_page_id, None)); + } + usize::from(terminal.is_some()) + } + }; + self.payload_bytes = self + .payload_bytes + .checked_add(bytes.len() as u64) + .ok_or_else(budget_limit)?; + self.delta_entries = self + .delta_entries + .checked_add(page_entries) + .ok_or_else(budget_limit)?; + self.page_entries.insert(expected, page_entries); + self.payloads.insert( + expected, + MetadataPagePayload { + id: expected, + size: bytes.len() as u64, + bytes, + }, + ); + self.check_budget()?; + } + Ok(root) + } + + fn check_budget(&self) -> Result<(), SnapshotError> { + let limits = MetadataDagLimits::default(); + if self + .payloads + .len() + .checked_add(self.reuse_upper.nodes) + .is_none_or(|value| value > limits.nodes) + || self + .edges + .len() + .checked_add(self.reuse_upper.edges) + .is_none_or(|value| value > limits.edges) + || self + .payload_bytes + .checked_add(self.reuse_upper.bytes) + .is_none_or(|value| value > limits.payload_bytes) + || self + .delta_entries + .checked_add(self.reuse_upper.entries) + .is_none_or(|value| value > limits.entries) + { + return Err(budget_limit()); + } + Ok(()) + } + + fn finish( + mut self, + root: MetadataPageId, + ) -> Result { + let mut children: BTreeMap<_, Vec<_>> = BTreeMap::new(); + for &(parent, child) in &self.edges { + children.entry(parent).or_default().push(child); + } + let mut reachable = BTreeSet::new(); + let mut pending = vec![root]; + while let Some(page) = pending.pop() { + if reachable.insert(page) && !self.reused.contains_key(&page) { + pending.extend(children.get(&page).into_iter().flatten()); + } + } + self.payloads.retain(|page, _| reachable.contains(page)); + self.page_entries.retain(|page, _| reachable.contains(page)); + self.reused.retain(|page, _| reachable.contains(page)); + self.source_roots.retain(|_, page| reachable.contains(page)); + self.edges + .retain(|(parent, child)| reachable.contains(parent) && reachable.contains(child)); + self.payload_bytes = self + .payloads + .values() + .try_fold(0u64, |total, page| total.checked_add(page.size)) + .ok_or_else(budget_limit)?; + self.delta_entries = self + .page_entries + .values() + .try_fold(0usize, |total, count| total.checked_add(*count)) + .ok_or_else(budget_limit)?; + self.reuse_upper = ClosureUpper::default(); + for directory in self.reused.values() { + self.reuse_upper.add(directory)?; + } + self.check_budget()?; + let plan = RootedMetadataInstallPlan::new( + self.identity, + root, + self.payloads + .iter() + .map(|(page, payload)| (*page, payload.size)) + .collect(), + self.edges, + self.reused + .into_iter() + .map(|(page, directory)| (page, directory.proof)) + .collect(), + self.source_roots, + )?; + self.work.delta_pages = plan.delta.len() as u64; + self.work.delta_bytes = self.payload_bytes; + self.work.reused_roots = plan.reused.len() as u64; + self.work.reused_closure_nodes_upper = self.reuse_upper.nodes as u64; + self.work.reused_closure_edges_upper = self.reuse_upper.edges as u64; + self.work.reused_closure_bytes_upper = self.reuse_upper.bytes; + self.work.reused_closure_entries_upper = self.reuse_upper.entries as u64; + self.work.plan_bytes = plan.encode()?.len() as u64; + Ok(PreparedRootedNativeMetadata { + plan, + payloads: self.payloads.into_values().collect(), + work: self.work, + }) + } +} + +async fn resolve_scope_oid( + handler: &T, + root: &Tree, + scope: &str, + work: &mut RootedProjectionWork, +) -> Result { + if scope == "/" { + return Ok(root.id); + } + let mut current = Cow::Borrowed(root); + let mut components = scope[1..].split('/').peekable(); + while let Some(component) = components.next() { + let position = current + .tree_items + .iter() + .position(|item| item.name == component); + work.scope_path_entries_examined += + position.map_or(current.tree_items.len(), |index| index + 1) as u64; + let item = position + .map(|index| ¤t.tree_items[index]) + .ok_or_else(|| { + SnapshotError::new( + SnapshotErrorCode::PathNotFound, + "name absent in enumerated parent directory", + ) + })?; + let kind = FsKind::from_git_mode(item.mode).ok_or_else(|| { + SnapshotError::new( + SnapshotErrorCode::UnsupportedEntry, + "gitlink entries are not supported in this profile", + ) + })?; + if kind != FsKind::Directory { + return Err(SnapshotError::new( + SnapshotErrorCode::NotDirectory, + "rooted metadata scope is not a directory", + )); + } + let expected = item.id; + if expected.kind() != root.id.kind() { + return Err(integrity("scope tree crossed its source hash kind")); + } + if components.peek().is_none() { + return Ok(expected); + } + work.tree_fetches += 1; + let fetched = handler + .get_tree_by_hash(&expected.to_string()) + .await + .map_err(source_error)?; + if fetched.id != expected { + return Err(integrity("fetched scope ancestor identity mismatch")); + } + current = Cow::Owned(fetched); + } + Err(integrity("rooted metadata scope walk has no target")) +} + +async fn project_directory( + handler: &T, + reuse: &R, + storage: &Storage, + oid: ObjectHash, + loaded: Option<&Tree>, + path: &str, + state: &mut ProjectionState, +) -> Result { + validate_scope_relative_path(path)?; + let tagged = oid.to_tagged_string(); + if let Some(summary) = state.directories.get(&tagged).copied() { + summary.bounds.validate_at(path)?; + state.work.directory_memo_hits += 1; + return Ok(summary); + } + state.require_tree(&tagged)?; + if !state.active_trees.insert(tagged.clone()) { + return Err(integrity("native source trees contain a cycle")); + } + state.work.reuse_lookups += 1; + if let Some(hint) = reuse.lookup_reuse(&tagged, &state.identity).await? { + let summary = state.record_reuse(&tagged, hint, path)?; + state.active_trees.remove(&tagged); + state.directories.insert(tagged, summary); + return Ok(summary); + } + let fetched; + let tree = if let Some(tree) = loaded { + tree + } else { + state.work.tree_fetches += 1; + fetched = handler + .get_tree_by_hash(&oid.to_string()) + .await + .map_err(source_error)?; + &fetched + }; + if tree.id != oid + || tree + .tree_items + .iter() + .any(|item| item.id.kind() != oid.kind()) + { + return Err(integrity( + "fetched source directory identity or child hash kind mismatch", + )); + } + state.source_entries = state + .source_entries + .checked_add(tree.tree_items.len()) + .filter(|value| *value <= MetadataDagLimits::default().entries) + .ok_or_else(budget_limit)?; + let direct = direct_entries(tree)?; + state.admit_entries(&direct)?; + state.work.directories_scanned += 1; + state.work.direct_entries_scanned += direct.len() as u64; + let requested: Vec<_> = direct + .iter() + .filter(|(_, kind, oid)| *kind != FsKind::Directory && !state.blob_facts.contains_key(oid)) + .map(|(_, _, oid)| oid.clone()) + .collect::>() + .into_iter() + .collect(); + for batch in requested.chunks(64) { + state.work.verified_blob_queries += 1; + let facts = storage + .mono_storage() + .get_verified_blobs(batch.to_vec()) + .await + .map_err(source_error)?; + state.work.verified_blob_facts_loaded += facts.len() as u64; + for (oid, fact) in facts { + state.blob_facts.insert(oid, verified_fact(&fact)?); + } + } + let mut pending_facts = Vec::new(); + let mut bounds = PathBounds::default(); + let mut entries = Vec::with_capacity(direct.len()); + for (name, kind, raw_oid) in direct { + let child_path = if path == "/" { + format!("/{name}") + } else { + format!("{path}/{name}") + }; + validate_scope_relative_path(&child_path)?; + match kind { + FsKind::Directory => { + let child_oid = ObjectHash::from_hex_for_kind(oid.kind(), &raw_oid) + .map_err(|error| integrity(&error.to_string()))?; + let child = Box::pin(project_directory( + handler, + reuse, + storage, + child_oid, + None, + &child_path, + state, + )) + .await?; + bounds.include(&name, child.bounds)?; + entries.push(Entry::dir(name.as_bytes(), child.page_id)); + } + FsKind::Regular | FsKind::Executable | FsKind::Symlink => { + bounds.include(&name, PathBounds::default())?; + let fact = if let Some(fact) = state.blob_facts.get(&raw_oid).copied() { + fact + } else { + state.work.blob_fetches += 1; + let raw = super::pages::fetch_raw_blob(handler, &raw_oid).await?; + state.work.raw_bytes_fetched = state + .work + .raw_bytes_fetched + .checked_add(raw.len() as u64) + .ok_or_else(budget_limit)?; + let size = u64::try_from(raw.len()) + .map_err(|_| integrity("raw file size is not representable"))?; + if size > 8_796_093_022_208 { + return Err(integrity( + "raw file exceeds the native verified-object size profile", + )); + } + let digest: [u8; 32] = Sha256::digest(&raw).into(); + state.work.raw_bytes_hashed = state + .work + .raw_bytes_hashed + .checked_add(size) + .ok_or_else(budget_limit)?; + drop(raw); + let fact = BlobFact { size, digest }; + state.blob_facts.insert(raw_oid.clone(), fact); + pending_facts.push((raw_oid.clone(), fact)); + if pending_facts.len() == 64 { + persist_facts(storage, &pending_facts, &mut state.work).await?; + pending_facts.clear(); + } + fact + }; + let entry_kind = match kind { + FsKind::Regular => EntryKind::Regular, + FsKind::Executable => EntryKind::Executable, + FsKind::Symlink => EntryKind::Symlink, + FsKind::Directory => { + return Err(integrity("file projection received a directory kind")); + } + }; + entries.push(Entry::file( + entry_kind, + name.as_bytes(), + fact.size, + fact.digest, + )); + } + } + } + if !pending_facts.is_empty() { + persist_facts(storage, &pending_facts, &mut state.work).await?; + } + bounds.validate_at(path)?; + state.charge_codec(entries.len(), 0)?; + state.work.directory_root_builds += 1; + let root_bytes = Page::build(&entries).map_err(codec_error)?; + let page_id = state.collect_directory(&entries, root_bytes)?; + state.record_bounds(page_id, bounds)?; + state.source_roots.insert(tagged.clone(), page_id); + let summary = DirectorySummary { page_id, bounds }; + state.active_trees.remove(&tagged); + state.directories.insert(tagged, summary); + Ok(summary) +} + +fn verified_fact( + fact: &crate::callisto::mst2_verified_object::Model, +) -> Result { + if fact.state != "VERIFIED" + || fact.verification_version != MST2_VERIFICATION_VERSION + || !(0..=8_796_093_022_208).contains(&fact.size) + { + return Err(integrity( + "native projection received an invalid current verified blob fact", + )); + } + Ok(BlobFact { + size: u64::try_from(fact.size).map_err(|_| integrity("verified blob size is invalid"))?, + digest: fact + .raw_sha256 + .as_slice() + .try_into() + .map_err(|_| integrity("verified blob digest is invalid"))?, + }) +} + +async fn persist_facts( + storage: &Storage, + facts: &[(String, BlobFact)], + work: &mut RootedProjectionWork, +) -> Result<(), SnapshotError> { + let rows = facts + .iter() + .map( + |(oid, fact)| crate::callisto::mst2_verified_object::ActiveModel { + id: sea_orm::ActiveValue::NotSet, + storage_domain: Set("git".into()), + git_oid: Set(oid.clone()), + object_kind: Set("blob".into()), + raw_sha256: Set(fact.digest.to_vec()), + size: Set(fact.size as i64), + verification_version: Set(MST2_VERIFICATION_VERSION), + state: Set("VERIFIED".into()), + created_at: Set(chrono::Utc::now().fixed_offset()), + }, + ) + .collect(); + work.verified_blob_persistence_batches += 1; + storage + .mono_storage() + .insert_verified_blobs(rows) + .await + .map_err(source_error)?; + work.verified_blob_queries += 1; + let stored = storage + .mono_storage() + .get_verified_blobs(facts.iter().map(|(oid, _)| oid.clone()).collect()) + .await + .map_err(source_error)?; + work.verified_blob_facts_loaded += stored.len() as u64; + for (oid, expected) in facts { + let actual = stored + .get(oid) + .ok_or_else(|| integrity("verified blob persistence lost its current fact"))?; + if verified_fact(actual)? != *expected { + return Err(integrity( + "verified blob persistence conflicts with the hashed raw source", + )); + } + } + Ok(()) +} + +fn source_error(error: MegaError) -> SnapshotError { + let code = match error { + MegaError::ObjStorageNotFound(_) => SnapshotErrorCode::ObjectUnavailable, + MegaError::ObjStorageInconsistent(_) => SnapshotErrorCode::IntegrityError, + _ => SnapshotErrorCode::Internal, + }; + tracing::warn!(error = %error, "rooted native metadata source operation failed"); + SnapshotError::new(code, "rooted native metadata source operation failed") +} + +fn codec_error(error: mst2_codec::CodecError) -> SnapshotError { + integrity(&error.to_string()) +} +fn integrity(message: &str) -> SnapshotError { + SnapshotError::new(SnapshotErrorCode::IntegrityError, message) +} +fn budget_limit() -> SnapshotError { + SnapshotError::new( + SnapshotErrorCode::LimitExceeded, + "rooted metadata projection exceeds its fixed budget", + ) +} +fn path_limit() -> SnapshotError { + SnapshotError::new( + SnapshotErrorCode::ScopeInvalid, + "rooted metadata subtree exceeds its full-path budget", + ) +} + +#[cfg(test)] +mod tests { + use git_internal::hash::HashKind; + use uuid::Uuid; + + use super::*; + + fn state() -> ProjectionState { + ProjectionState::new(MetadataInstallIdentity { + source_domain: "native-git".into(), + tagged_root_tree_oid: ObjectHash::from_hex_for_kind(HashKind::Sha1, &"a".repeat(40)) + .unwrap() + .to_tagged_string(), + scope: "/".into(), + schema_version: SCHEMA_VERSION, + metadata_codec: METADATA_CODEC, + materialization_policy: MATERIALIZATION_POLICY_GIT_RAW_V1, + fs_semantics: FS_SEMANTICS_LINUX_CODE_V1, + access_projection: ACCESS_PROJECTION_EXACT_FULL, + verification_revision: MST2_VERIFICATION_VERSION, + projection_revision: NATIVE_PROJECTION_REVISION, + }) + } + + fn reusable(page: MetadataPageId) -> CertifiedReusableDirectory { + CertifiedReusableDirectory { + page_id: page, + proof: RootedReuseRoot { + generation: 1, + attestation_id: Uuid::from_u128(1), + attestation_digest: [1; 32], + certificate_digest: [2; 32], + }, + relative_path_bytes: 0, + relative_components: 0, + closure_nodes_upper: 1, + closure_edges_upper: 0, + closure_bytes_upper: HEADER_LEN as u64, + closure_entries_upper: 0, + } + } + + #[test] + fn reused_path_bounds_recheck_moved_aliases_and_checked_overflow() { + let bounds = PathBounds { + bytes: 4093, + components: 255, + }; + assert!(bounds.validate_at("/").is_ok()); + assert!(bounds.validate_at("/a").is_ok()); + assert_eq!( + bounds.validate_at("/a/b").unwrap_err().code, + SnapshotErrorCode::ScopeInvalid + ); + assert!( + PathBounds { + bytes: usize::MAX, + components: 0 + } + .validate_at("/a") + .is_err() + ); + let mut parent = PathBounds::default(); + parent + .include( + "a", + PathBounds { + bytes: 3, + components: 1, + }, + ) + .unwrap(); + parent.include("longer", PathBounds::default()).unwrap(); + assert_eq!( + parent, + PathBounds { + bytes: 7, + components: 2 + } + ); + assert!( + parent + .include( + "a", + PathBounds { + bytes: usize::MAX, + components: 0 + } + ) + .is_err() + ); + } + + #[test] + fn canonical_radix_payloads_and_shared_reuse_edges_form_a_delta_only_plan() { + let empty = Page::build(&[]).unwrap(); + let reused_page = page_id(&empty); + let mut state = state(); + state + .record_reuse( + &format!("sha1:{}", "b".repeat(40)), + reusable(reused_page), + "/left", + ) + .unwrap(); + let mut entries: Vec<_> = (0..260) + .map(|index| { + Entry::file( + EntryKind::Regular, + format!("file-{index:03}").as_bytes(), + index, + [3; 32], + ) + }) + .collect(); + entries.push(Entry::dir(b"left", reused_page)); + entries.push(Entry::dir(b"right", reused_page)); + entries.sort_by(|left, right| left.name.cmp(&right.name)); + let bytes = Page::build(&entries).unwrap(); + let root = state.collect_directory(&entries, bytes).unwrap(); + state + .source_roots + .insert(state.identity.tagged_root_tree_oid.clone(), root); + let prepared = state.finish(root).unwrap(); + assert!(prepared.payloads.len() > 1); + assert_eq!(prepared.plan.reused.len(), 1); + assert!(!prepared.plan.delta.contains_key(&reused_page)); + assert_eq!(prepared.work.tree_fetches, 0); + assert_eq!(prepared.work.blob_fetches, 0); + assert_eq!( + prepared.plan.child_first_delta().unwrap().last(), + Some(&root) + ); + let mut decoded_edges = BTreeSet::new(); + for payload in &prepared.payloads { + assert_eq!(page_id(&payload.bytes), payload.id); + match Page::decode(&payload.bytes).unwrap().0 { + Page::Leaf { entries } => { + for entry in entries.iter().filter(|entry| entry.is_dir()) { + decoded_edges.insert((payload.id, entry.child_root)); + } + } + Page::Branch { + terminal, children, .. + } => { + if let Some(entry) = terminal.filter(Entry::is_dir) { + decoded_edges.insert((payload.id, entry.child_root)); + } + for child in children { + decoded_edges.insert((payload.id, child.child_page_id)); + } + } + } + } + assert_eq!(decoded_edges, prepared.plan.edges); + assert_eq!( + RootedMetadataInstallPlan::decode( + &prepared.plan.encode().unwrap(), + &prepared.plan.digest().unwrap() + ) + .unwrap(), + prepared.plan + ); + } + + #[test] + fn reuse_budget_is_deduplicated_conservative_and_never_expands_descendants() { + let empty = Page::build(&[]).unwrap(); + let page = page_id(&empty); + let hint = reusable(page); + let mut state = state(); + state + .record_reuse(&format!("sha1:{}", "b".repeat(40)), hint.clone(), "/one") + .unwrap(); + let mut alias = hint.clone(); + alias.proof.attestation_id = Uuid::from_u128(2); + alias.proof.attestation_digest = [9; 32]; + state + .record_reuse(&format!("sha1:{}", "c".repeat(40)), alias, "/two") + .unwrap(); + assert_eq!(state.reuse_upper.nodes, 1); + assert_eq!(state.reuse_upper.bytes, HEADER_LEN as u64); + assert_eq!(state.reused.get(&page).unwrap().proof, hint.proof); + assert!(state.payloads.is_empty()); + assert!(state.edges.is_empty()); + let mut different_lifetime = hint.clone(); + different_lifetime.proof.generation = 2; + assert!( + state + .record_reuse( + &format!("sha1:{}", "f".repeat(40)), + different_lifetime, + "/lifetime" + ) + .is_err() + ); + let mut bad = hint; + bad.relative_path_bytes = 1; + assert!( + state + .record_reuse(&format!("sha1:{}", "d".repeat(40)), bad, "/three") + .is_err() + ); + let mut huge = reusable([4; 32]); + huge.closure_nodes_upper = MetadataDagLimits::default().nodes; + assert_eq!( + state + .record_reuse(&format!("sha1:{}", "e".repeat(40)), huge, "/four") + .unwrap_err() + .code, + SnapshotErrorCode::LimitExceeded + ); + } +} diff --git a/src/jupiter/migration/m20261008_000200_add_mst2_rooted_qualified_family.rs b/src/jupiter/migration/m20261008_000200_add_mst2_rooted_qualified_family.rs new file mode 100644 index 00000000..fd613dce --- /dev/null +++ b/src/jupiter/migration/m20261008_000200_add_mst2_rooted_qualified_family.rs @@ -0,0 +1,125 @@ +//! A bounded namespace registry; physical Q provisioning occurs at bootstrap. + +use sea_orm::{ConnectionTrait, DbBackend, Statement}; +use sea_orm_migration::prelude::*; + +#[derive(DeriveMigrationName)] +pub struct Migration; + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + let connection = manager.get_connection(); + let row = connection.query_one_raw(Statement::from_string(DbBackend::Postgres, + "SELECT current_schema() AS schema,n.oid::bigint AS oid FROM pg_catalog.pg_namespace n WHERE n.nspname=current_schema()")) + .await?.ok_or_else(|| DbErr::Custom("qualified family core schema is missing".into()))?; + let schema: String = row.try_get("", "schema")?; + let oid: i64 = row.try_get("", "oid")?; + let quoted = format!("\"{}\"", schema.replace('"', "\"\"")); + let literal = format!("'{}'", schema.replace('\'', "''")); + let old = include_str!("m20261007_000600_storage_routes.sql"); + let start = old + .find("CREATE FUNCTION mst2_route_insert_guard()") + .ok_or_else(|| { + DbErr::Custom("generic route insertion guard source is missing".into()) + })?; + let end = old[start..] + .find("CREATE FUNCTION mst2_route_context_insert()") + .ok_or_else(|| { + DbErr::Custom("generic route insertion guard boundary is missing".into()) + })? + + start; + let guard = old[start..end].replace("CREATE FUNCTION", "CREATE OR REPLACE FUNCTION") + .replace("WHERE s.snapshot_id=NEW.snapshot_id AND NEW.canonical_descriptor=s.canonical_descriptor", + "WHERE n.singleton=1 AND n.graph_domain='generic-v1' AND s.snapshot_id=NEW.snapshot_id AND NEW.canonical_descriptor=s.canonical_descriptor"); + let catalog = include_str!("../storage/qualified_family_catalog.sql") + .replace("$CORE_OID$", "c_oid") + .replace("$Q_OID$", "q_oid") + .replace("$EXEMPT_Q_OID$", "exempt_q_oid"); + let shape = include_str!("../storage/qualified_family_shape.sql"); + let implementation = hex::encode( + super::super::storage::qualified_metadata_family::implementation_fingerprint(), + ); + let sql = include_str!("m20261008_000200_rooted_qualified_family.sql") + .replace("$CATALOG_SQL$", &catalog) + .replace("$SHAPE_SQL$", shape) + .replace("$GENERIC_INSERT_GUARD$", &guard) + .replace( + "$SOURCE_REVISION_SQL$", + include_str!("../storage/qualified_source_revision.sql"), + ) + .replace( + "$ROOTED_ROUTES_SQL$", + include_str!("m20261008_000200_rooted_routes.sql"), + ) + .replace("$CORE_SCHEMA$", "ed) + .replace("$CORE_LITERAL$", &literal) + .replace("$IMPLEMENTATION_SHA$", &implementation) + .replace("$CORE_OID$", &oid.to_string()); + connection.execute_unprepared(&sql).await?; + // Build the expected physical shape from trusted source exactly once, + // under this forward migration, without registering a namespace or + // preserving any template relations/history after the transaction. + let template_uuid = uuid::Uuid::new_v4().to_string(); + let template_schema = format!("mst2q_{}", template_uuid.replace('-', "")); + let template_quoted = format!("\"{template_schema}\""); + connection + .execute_unprepared(&format!("CREATE SCHEMA {template_quoted}")) + .await?; + let template_oid: i64 = connection + .query_one_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + "SELECT oid::bigint AS oid FROM pg_catalog.pg_namespace WHERE nspname=$1", + [template_schema.clone().into()], + )) + .await? + .ok_or_else(|| DbErr::Custom("qualified family template schema is missing".into()))? + .try_get("", "oid")?; + let template_storage = uuid::Uuid::new_v4().to_string(); + connection + .execute_unprepared( + &super::super::storage::qualified_metadata_family::render_family( + &schema, + oid, + &template_schema, + template_oid, + &template_uuid, + &template_storage, + ), + ) + .await?; + let expected_shape: Vec = connection.query_one_raw(Statement::from_sql_and_values(DbBackend::Postgres, + format!("SELECT {quoted}.mst2_route_family_shape($1::bigint::oid,$2::uuid,$3) AS fingerprint"), + [template_oid.into(),template_uuid.into(),template_storage.into()])).await? + .ok_or_else(||DbErr::Custom("qualified family template shape is missing".into()))?.try_get("","fingerprint")?; + connection.execute_unprepared(&format!("SET LOCAL search_path={quoted},pg_catalog,pg_temp; DROP SCHEMA {template_quoted} CASCADE")).await?; + let authority_catalog: Vec = connection + .query_one_raw(Statement::from_string( + DbBackend::Postgres, + format!("SELECT {quoted}.mst2_route_family_catalog({oid},0::oid) AS fingerprint"), + )) + .await? + .ok_or_else(|| { + DbErr::Custom("qualified family core authority catalog is missing".into()) + })? + .try_get("", "fingerprint")?; + connection + .execute_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + format!("INSERT INTO {quoted}.mst2_qualified_family_policy VALUES(1,$1,$2,$3)"), + [ + hex::decode(implementation) + .map_err(|e| DbErr::Custom(e.to_string()))? + .into(), + expected_shape.into(), + authority_catalog.into(), + ], + )) + .await?; + Ok(()) + } + + async fn down(&self, _manager: &SchemaManager) -> Result<(), DbErr> { + Ok(()) + } +} diff --git a/src/jupiter/migration/m20261008_000200_rooted_qualified_family.sql b/src/jupiter/migration/m20261008_000200_rooted_qualified_family.sql new file mode 100644 index 00000000..22038ef2 --- /dev/null +++ b/src/jupiter/migration/m20261008_000200_rooted_qualified_family.sql @@ -0,0 +1,161 @@ +SELECT mst2_route_enter(current_schema()); +SET LOCAL search_path=$CORE_SCHEMA$,pg_catalog,pg_temp; +LOCK TABLE mst2_metadata_namespace IN ACCESS EXCLUSIVE MODE; + +ALTER TABLE mst2_metadata_namespace + DROP CONSTRAINT mst2_metadata_namespace_pkey, + ALTER COLUMN singleton DROP NOT NULL, + DROP CONSTRAINT mst2_metadata_namespace_family_identity_check, + DROP CONSTRAINT mst2_metadata_namespace_graph_domain_check, + DROP CONSTRAINT mst2_metadata_namespace_admission_state_check, + DROP CONSTRAINT mst2_metadata_namespace_collector_state_check, + DROP CONSTRAINT mst2_metadata_namespace_check, + ADD COLUMN metadata_storage_uuid text, + ADD COLUMN implementation_fingerprint bytea, + ADD COLUMN catalog_fingerprint bytea, + ADD CONSTRAINT mst2_namespace_family_pair CHECK (( + (singleton=1 AND family_identity='v3-generic-session-1' AND graph_domain='generic-v1' + AND admission_state='G_ADMITTED_Q_CLOSED' AND collector_state='CLOSED' AND core_schema=metadata_schema + AND core_schema_oid=metadata_schema_oid AND metadata_storage_uuid IS NULL + AND implementation_fingerprint IS NULL AND catalog_fingerprint IS NULL) + OR (singleton IS NULL AND family_identity='v3-rooted-qualified-1' AND graph_domain='qualified-v1' + AND admission_state='ROOTED_Q_ADMITTED' AND collector_state='ENABLED' AND core_schema<>metadata_schema + AND core_schema_oid<>metadata_schema_oid AND metadata_storage_uuid IS NOT NULL + AND octet_length(implementation_fingerprint)=32 AND octet_length(catalog_fingerprint)=32) + ) IS TRUE); +CREATE UNIQUE INDEX idx_mst2_namespace_generic_slot ON mst2_metadata_namespace(singleton) + WHERE singleton IS NOT NULL; +CREATE UNIQUE INDEX idx_mst2_namespace_domain ON mst2_metadata_namespace(graph_domain); +CREATE UNIQUE INDEX idx_mst2_namespace_metadata_schema ON mst2_metadata_namespace(metadata_schema_oid); + +CREATE OR REPLACE FUNCTION mst2_route_enter(caller_schema text) RETURNS uuid LANGUAGE plpgsql VOLATILE +SET search_path=$CORE_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE n mst2_metadata_namespace%ROWTYPE; g mst2_metadata_namespace%ROWTYPE; mono_held boolean; route_held boolean; +BEGIN + IF NOT mst2_route_scope_valid(caller_schema) OR (SELECT count(*) FROM mst2_metadata_namespace)>2 THEN + RAISE EXCEPTION 'storage route captured primary scope is unavailable'; + END IF; + SELECT * INTO STRICT g FROM mst2_metadata_namespace WHERE singleton=1; + mono_held:=mst2_route_lock_held(1297043024,g.mono_lock_key2); + route_held:=mst2_route_lock_held(1296718001,pg_catalog.hashtext(g.core_schema)); + IF mst2_route_lock_held(1296718001,pg_catalog.hashtext(g.core_schema),false) AND NOT mono_held THEN + RAISE EXCEPTION 'storage route lock was acquired before core mono'; + END IF; + IF EXISTS(SELECT 1 FROM mst2_metadata_namespace x + WHERE mst2_route_lock_held(1296717362,pg_catalog.hashtext(x.metadata_schema),false)) + AND NOT (mono_held AND route_held) THEN + RAISE EXCEPTION 'storage route cannot acquire core locks after retention'; + END IF; + PERFORM pg_catalog.set_config('lock_timeout','5000ms',true); + PERFORM pg_catalog.pg_advisory_xact_lock(1297043024,g.mono_lock_key2); + PERFORM pg_catalog.pg_advisory_xact_lock(1296718001,pg_catalog.hashtext(g.core_schema)); + FOR n IN SELECT * FROM mst2_metadata_namespace ORDER BY namespace_uuid LOOP + PERFORM pg_catalog.pg_advisory_xact_lock(1296717362,pg_catalog.hashtext(n.metadata_schema)); + END LOOP; + RETURN g.namespace_uuid; +END $$; + +-- The new namespace is included before acquiring any retention lock. +CREATE FUNCTION mst2_route_family_candidate_enter(caller_schema text,candidate uuid,candidate_schema text) +RETURNS void LANGUAGE plpgsql VOLATILE SET search_path=$CORE_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE g mst2_metadata_namespace%ROWTYPE; r record; +BEGIN + IF NOT mst2_route_scope_valid(caller_schema) OR candidate IS NULL + OR substr(candidate::text,15,1)<>'4' OR substr(candidate::text,20,1) NOT IN ('8','9','a','b') + OR candidate_schema IS DISTINCT FROM 'mst2q_'||replace(candidate::text,'-','') THEN + RAISE EXCEPTION 'qualified provisioning candidate is not a fresh server namespace'; + END IF; + IF EXISTS(SELECT 1 FROM pg_catalog.pg_locks WHERE locktype='advisory' AND pid=pg_catalog.pg_backend_pid() + AND database=(SELECT oid FROM pg_catalog.pg_database WHERE datname=pg_catalog.current_database()) + AND classid=1296717362::oid AND objsubid=2 AND granted) THEN + RAISE EXCEPTION 'qualified provisioning cannot extend a previously locked retention set'; + END IF; + SELECT * INTO STRICT g FROM mst2_metadata_namespace WHERE singleton=1; + IF mst2_route_lock_held(1296718001,pg_catalog.hashtext(g.core_schema),false) + AND NOT mst2_route_lock_held(1297043024,g.mono_lock_key2) THEN + RAISE EXCEPTION 'storage route lock was acquired before core mono'; + END IF; + PERFORM pg_catalog.set_config('lock_timeout','5000ms',true); + PERFORM pg_catalog.pg_advisory_xact_lock(1297043024,g.mono_lock_key2); + PERFORM pg_catalog.pg_advisory_xact_lock(1296718001,pg_catalog.hashtext(g.core_schema)); + -- A racing successful bootstrap can register while this caller waits. + IF EXISTS(SELECT 1 FROM mst2_metadata_namespace WHERE graph_domain='qualified-v1') THEN + RAISE EXCEPTION 'qualified provisioning candidate lost bootstrap serialization'; + END IF; + FOR r IN SELECT namespace_uuid,metadata_schema FROM mst2_metadata_namespace + UNION ALL SELECT candidate,candidate_schema ORDER BY namespace_uuid LOOP + PERFORM pg_catalog.pg_advisory_xact_lock(1296717362,pg_catalog.hashtext(r.metadata_schema)); + END LOOP; +END $$; + +CREATE FUNCTION mst2_route_family_catalog(c_oid oid,q_oid oid,exempt_q_oid oid DEFAULT 0::oid) RETURNS bytea LANGUAGE sql VOLATILE +SET search_path=pg_catalog,pg_temp AS $catalog$ $CATALOG_SQL$ $catalog$; +CREATE FUNCTION mst2_route_family_shape(q_oid oid,n_uuid uuid,s_uuid text) RETURNS bytea LANGUAGE sql VOLATILE +SET search_path=pg_catalog,pg_temp AS $shape$ $SHAPE_SQL$ $shape$; +CREATE TABLE mst2_qualified_family_policy ( + singleton smallint PRIMARY KEY CHECK(singleton=1),implementation_fingerprint bytea NOT NULL CHECK(octet_length(implementation_fingerprint)=32), + expected_shape bytea NOT NULL CHECK(octet_length(expected_shape)=32), + authority_catalog bytea NOT NULL CHECK(octet_length(authority_catalog)=32) +); +CREATE TRIGGER mst2_route_family_policy_immutable BEFORE UPDATE OR DELETE ON mst2_qualified_family_policy + FOR EACH ROW EXECUTE FUNCTION mst2_route_immutable(); +CREATE TRIGGER mst2_route_family_policy_truncate_guard BEFORE TRUNCATE ON mst2_qualified_family_policy + FOR EACH STATEMENT EXECUTE FUNCTION mst2_route_immutable(); + +CREATE FUNCTION mst2_route_family_registration_guard() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$CORE_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE g mst2_metadata_namespace%ROWTYPE; stamp record; +BEGIN + SELECT * INTO STRICT g FROM mst2_metadata_namespace WHERE singleton=1; + IF (SELECT count(*) FROM mst2_metadata_namespace)<>1 OR NEW.singleton IS NOT NULL + OR substr(NEW.namespace_uuid::text,15,1)<>'4' OR substr(NEW.namespace_uuid::text,20,1) NOT IN ('8','9','a','b') + OR NEW.graph_domain<>'qualified-v1' OR NEW.family_identity<>'v3-rooted-qualified-1' + OR NEW.admission_state<>'ROOTED_Q_ADMITTED' OR NEW.collector_state<>'ENABLED' + OR ROW(NEW.core_schema,NEW.core_schema_oid,NEW.database_name,NEW.database_oid,NEW.storage_uuid, + NEW.server_address,NEW.server_port,NEW.mono_lock_key2) IS DISTINCT FROM + ROW(g.core_schema,g.core_schema_oid,g.database_name,g.database_oid,g.storage_uuid, + g.server_address,g.server_port,g.mono_lock_key2) + OR NEW.metadata_schema IS DISTINCT FROM 'mst2q_'||replace(NEW.namespace_uuid::text,'-','') + OR NEW.metadata_storage_uuid IS NOT DISTINCT FROM g.storage_uuid + OR NEW.metadata_storage_uuid IS DISTINCT FROM (NEW.metadata_storage_uuid::uuid)::text + OR substr(NEW.metadata_storage_uuid,15,1)<>'4' OR substr(NEW.metadata_storage_uuid,20,1) NOT IN ('8','9','a','b') + OR NOT EXISTS(SELECT 1 FROM pg_catalog.pg_namespace n + WHERE n.oid=NEW.metadata_schema_oid AND n.nspname=NEW.metadata_schema) + OR NEW.implementation_fingerprint IS DISTINCT FROM decode('$IMPLEMENTATION_SHA$','hex') + OR NOT mst2_route_lock_held(1297043024,g.mono_lock_key2) + OR NOT mst2_route_lock_held(1296718001,pg_catalog.hashtext(g.core_schema)) + OR NOT mst2_route_lock_held(1296717362,pg_catalog.hashtext(NEW.metadata_schema)) + OR NOT mst2_route_lock_held(1296717362,pg_catalog.hashtext(g.metadata_schema)) THEN + RAISE EXCEPTION 'qualified namespace registration has no exact admitted rooted family scope and lock set'; + END IF; + EXECUTE pg_catalog.format('SELECT * FROM %I.mst2_metadata_family_identity WHERE singleton=1',NEW.metadata_schema) + INTO STRICT stamp; + IF ROW(stamp.namespace_uuid,stamp.storage_uuid,stamp.core_schema_oid,stamp.metadata_schema_oid, + stamp.family_identity,stamp.implementation_fingerprint) IS DISTINCT FROM + ROW(NEW.namespace_uuid,NEW.metadata_storage_uuid,NEW.core_schema_oid,NEW.metadata_schema_oid, + NEW.family_identity,NEW.implementation_fingerprint) + OR NEW.catalog_fingerprint IS DISTINCT FROM mst2_route_family_catalog(NEW.core_schema_oid,NEW.metadata_schema_oid) THEN + RAISE EXCEPTION 'qualified namespace registration fingerprint disagrees with its physical family'; + END IF; + IF NOT EXISTS(SELECT 1 FROM mst2_qualified_family_policy p WHERE p.singleton=1 + AND p.implementation_fingerprint=NEW.implementation_fingerprint + AND p.authority_catalog=mst2_route_family_catalog(NEW.core_schema_oid,0::oid,NEW.metadata_schema_oid) + AND p.expected_shape=mst2_route_family_shape(NEW.metadata_schema_oid,NEW.namespace_uuid,NEW.metadata_storage_uuid)) THEN + RAISE EXCEPTION 'qualified namespace does not have the trusted complete physical family shape'; + END IF; + RETURN NEW; +END $$; +DROP TRIGGER mst2_route_namespace_registration_closed ON mst2_metadata_namespace; +CREATE TRIGGER mst2_route_namespace_registration_guard BEFORE INSERT ON mst2_metadata_namespace + FOR EACH ROW EXECUTE FUNCTION mst2_route_family_registration_guard(); + +CREATE FUNCTION mst2_metadata_has_generic_overlap(p bytea) RETURNS boolean LANGUAGE sql VOLATILE +SET search_path=$CORE_SCHEMA$,pg_catalog,pg_temp AS $$ + SELECT EXISTS(SELECT 1 FROM mst2_retention_node WHERE node_id='page:sha256:'||encode(p,'hex')) + OR EXISTS(SELECT 1 FROM mst2_retention_gc_op WHERE node_id='page:sha256:'||encode(p,'hex')) +$$; + +-- Existing generic route derivation must never select the newly registered Q row. +$GENERIC_INSERT_GUARD$ +$SOURCE_REVISION_SQL$ +$ROOTED_ROUTES_SQL$ diff --git a/src/jupiter/migration/m20261008_000200_rooted_routes.sql b/src/jupiter/migration/m20261008_000200_rooted_routes.sql new file mode 100644 index 00000000..e4cef146 --- /dev/null +++ b/src/jupiter/migration/m20261008_000200_rooted_routes.sql @@ -0,0 +1,171 @@ +CREATE FUNCTION mst2_route_qualified_namespace_valid(id uuid) RETURNS boolean LANGUAGE sql VOLATILE +SET search_path=$CORE_SCHEMA$,pg_catalog,pg_temp AS $$ + SELECT mst2_route_scope_valid($CORE_LITERAL$) AND EXISTS(SELECT 1 FROM mst2_metadata_namespace q + JOIN mst2_metadata_namespace g ON g.singleton=1 + JOIN mst2_qualified_family_policy policy ON policy.singleton=1 + JOIN pg_catalog.pg_namespace physical ON physical.oid=q.metadata_schema_oid AND physical.nspname=q.metadata_schema + WHERE q.namespace_uuid=id AND q.singleton IS NULL AND q.graph_domain='qualified-v1' + AND q.family_identity='v3-rooted-qualified-1' AND q.admission_state='ROOTED_Q_ADMITTED' AND q.collector_state='ENABLED' + AND q.metadata_schema='mst2q_'||replace(q.namespace_uuid::text,'-','') + AND q.metadata_schema_oid<>q.core_schema_oid + AND ROW(q.core_schema,q.core_schema_oid,q.database_name,q.database_oid,q.storage_uuid, + q.server_address,q.server_port,q.mono_lock_key2) IS NOT DISTINCT FROM + ROW(g.core_schema,g.core_schema_oid,g.database_name,g.database_oid,g.storage_uuid, + g.server_address,g.server_port,g.mono_lock_key2) + AND q.implementation_fingerprint=policy.implementation_fingerprint + AND policy.authority_catalog=mst2_route_family_catalog(q.core_schema_oid,0::oid,q.metadata_schema_oid) + AND q.catalog_fingerprint=mst2_route_family_catalog(q.core_schema_oid,q.metadata_schema_oid) + AND policy.expected_shape=mst2_route_family_shape(q.metadata_schema_oid,q.namespace_uuid,q.metadata_storage_uuid)) +$$; + +CREATE OR REPLACE FUNCTION mst2_route_statement_barrier() RETURNS trigger LANGUAGE plpgsql VOLATILE AS $$ +DECLARE q_id uuid; caller text:=pg_catalog.current_schema(); +BEGIN + IF TG_TABLE_SCHEMA<>$CORE_LITERAL$ OR NOT EXISTS(SELECT 1 FROM pg_catalog.pg_class + WHERE oid=TG_RELID AND relnamespace=$CORE_OID$) THEN + RAISE EXCEPTION 'storage route mutation is outside its captured core schema'; + END IF; + IF caller IS DISTINCT FROM $CORE_LITERAL$ THEN + SELECT n.namespace_uuid INTO q_id FROM $CORE_SCHEMA$.mst2_metadata_namespace n + WHERE n.metadata_schema=caller AND n.graph_domain='qualified-v1'; + IF NOT FOUND OR NOT $CORE_SCHEMA$.mst2_route_qualified_namespace_valid(q_id) THEN + RAISE EXCEPTION 'storage route mutation caller has no exact captured physical family'; END IF; + END IF; + PERFORM $CORE_SCHEMA$.mst2_route_enter($CORE_LITERAL$); + RETURN NULL; +END $$; + +CREATE FUNCTION mst2_route_qualified_snapshot_proof(sid text) RETURNS boolean LANGUAGE plpgsql VOLATILE +SET search_path=$CORE_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE r mst2_snapshot_storage_route%ROWTYPE; n mst2_metadata_namespace%ROWTYPE; valid boolean; +BEGIN + SELECT * INTO r FROM mst2_snapshot_storage_route WHERE snapshot_id=sid; + IF NOT FOUND THEN RETURN false; END IF; + SELECT * INTO n FROM mst2_metadata_namespace WHERE namespace_uuid=r.namespace_uuid; + IF NOT FOUND OR NOT mst2_route_qualified_namespace_valid(n.namespace_uuid) THEN RETURN false; END IF; + EXECUTE pg_catalog.format('SELECT %I.mst2_metadata_snapshot_route_proof($1,$2,$3,$4,$5,$6,$7)',n.metadata_schema) + INTO valid USING r.snapshot_id,r.canonical_descriptor,r.instance_id,r.commit_oid,r.root_tree_oid,r.metadata_root,r.source_profile; + RETURN coalesce(valid,false); +END $$; + +CREATE FUNCTION mst2_route_qualified_lease_proof(lid text) RETURNS boolean LANGUAGE plpgsql VOLATILE +SET search_path=$CORE_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE r mst2_lease_storage_route%ROWTYPE; n mst2_metadata_namespace%ROWTYPE; valid boolean; +BEGIN + SELECT * INTO r FROM mst2_lease_storage_route WHERE lease_id=lid; + IF NOT FOUND THEN RETURN false; END IF; + SELECT * INTO n FROM mst2_metadata_namespace WHERE namespace_uuid=r.namespace_uuid; + IF NOT FOUND OR NOT mst2_route_qualified_namespace_valid(n.namespace_uuid) THEN RETURN false; END IF; + EXECUTE pg_catalog.format('SELECT %I.mst2_metadata_lease_route_proof($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)',n.metadata_schema) + INTO valid USING r.lease_id,r.snapshot_id,r.namespace_uuid,r.session_incarnation,r.prepare_id,r.metadata_root, + r.authorization_epoch,r.publication_sequence,r.writer_epoch,r.certificate_receipt_id; + RETURN coalesce(valid,false); +END $$; + +-- Existing G proof functions remain authoritative for their original family. +CREATE FUNCTION mst2_route_family_for_snapshot(sid text,caller_schema text) +RETURNS TABLE(namespace_uuid uuid,graph_domain text) LANGUAGE plpgsql VOLATILE +SET search_path=$CORE_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE route mst2_snapshot_storage_route%ROWTYPE; namespace mst2_metadata_namespace%ROWTYPE; +BEGIN + IF NOT mst2_route_scope_valid(caller_schema) THEN RAISE EXCEPTION 'storage route captured primary scope is unavailable'; END IF; + SELECT * INTO route FROM mst2_snapshot_storage_route WHERE snapshot_id=sid; + IF NOT FOUND THEN RETURN; END IF; + SELECT * INTO namespace FROM mst2_metadata_namespace n WHERE n.namespace_uuid=route.namespace_uuid; + IF NOT FOUND OR namespace.graph_domain='generic-v1' AND NOT mst2_route_snapshot_proof(sid) + OR namespace.graph_domain='qualified-v1' AND NOT mst2_route_qualified_snapshot_proof(sid) + OR namespace.graph_domain NOT IN ('generic-v1','qualified-v1') THEN + RAISE EXCEPTION 'permanent snapshot route has a corrupt exact family or fixed source binding'; END IF; + RETURN QUERY SELECT namespace.namespace_uuid,namespace.graph_domain; +END $$; + +CREATE FUNCTION mst2_route_family_for_lease(lid text,caller_schema text) +RETURNS TABLE(namespace_uuid uuid,graph_domain text) LANGUAGE plpgsql VOLATILE +SET search_path=$CORE_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE route mst2_lease_storage_route%ROWTYPE; namespace mst2_metadata_namespace%ROWTYPE; +BEGIN + IF NOT mst2_route_scope_valid(caller_schema) THEN RAISE EXCEPTION 'storage route captured primary scope is unavailable'; END IF; + SELECT * INTO route FROM mst2_lease_storage_route WHERE lease_id=lid; + IF NOT FOUND THEN RETURN; END IF; + SELECT * INTO namespace FROM mst2_metadata_namespace n WHERE n.namespace_uuid=route.namespace_uuid; + IF NOT FOUND OR namespace.graph_domain='generic-v1' AND NOT mst2_route_lease_proof(lid) + OR namespace.graph_domain='qualified-v1' AND NOT mst2_route_qualified_lease_proof(lid) + OR namespace.graph_domain NOT IN ('generic-v1','qualified-v1') THEN + RAISE EXCEPTION 'permanent lease route has a corrupt exact family or fixed source binding'; END IF; + RETURN QUERY SELECT namespace.namespace_uuid,namespace.graph_domain; +END $$; + +CREATE OR REPLACE FUNCTION mst2_route_insert_guard() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$CORE_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE namespace mst2_metadata_namespace%ROWTYPE; valid boolean; +BEGIN + SELECT * INTO namespace FROM mst2_metadata_namespace WHERE namespace_uuid=NEW.namespace_uuid; + IF NOT FOUND THEN RAISE EXCEPTION 'storage route target namespace is not registered'; END IF; + IF namespace.graph_domain='qualified-v1' THEN + IF NOT mst2_route_qualified_namespace_valid(namespace.namespace_uuid) THEN + RAISE EXCEPTION 'qualified route has no exact complete physical family catalog'; END IF; + IF TG_TABLE_NAME='mst2_snapshot_storage_route' THEN + EXECUTE pg_catalog.format('SELECT %I.mst2_metadata_snapshot_candidate($1,$2,$3,$4,$5,$6,$7)',namespace.metadata_schema) + INTO valid USING NEW.snapshot_id,NEW.canonical_descriptor,NEW.instance_id,NEW.commit_oid, + NEW.root_tree_oid,NEW.metadata_root,NEW.source_profile; + ELSIF TG_TABLE_NAME='mst2_lease_storage_route' THEN + EXECUTE pg_catalog.format('SELECT %I.mst2_metadata_lease_route_proof($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)',namespace.metadata_schema) + INTO valid USING NEW.lease_id,NEW.snapshot_id,NEW.namespace_uuid,NEW.session_incarnation,NEW.prepare_id, + NEW.metadata_root,NEW.authorization_epoch,NEW.publication_sequence,NEW.writer_epoch,NEW.certificate_receipt_id; + ELSE RAISE EXCEPTION 'qualified route cannot use a generic binding relation'; END IF; + IF NOT coalesce(valid,false) THEN RAISE EXCEPTION 'qualified route is not independently derived from its exact source'; END IF; + RETURN NEW; + END IF; + IF namespace.singleton IS DISTINCT FROM 1 OR namespace.graph_domain<>'generic-v1' THEN + RAISE EXCEPTION 'storage route target family is unsupported'; END IF; + IF TG_TABLE_NAME='mst2_snapshot_storage_route' THEN + IF NOT EXISTS(SELECT 1 FROM mst2_snapshot_context s JOIN mst2_metadata_prepare p ON p.prepare_id=s.prepare_id + JOIN mst2_metadata_namespace n ON n.namespace_uuid=NEW.namespace_uuid + WHERE n.singleton=1 AND n.graph_domain='generic-v1' AND s.snapshot_id=NEW.snapshot_id AND NEW.canonical_descriptor=s.canonical_descriptor + AND NEW.instance_id=s.instance_id AND NEW.commit_oid=s.commit_oid AND NEW.root_tree_oid=s.root_tree_oid + AND NEW.metadata_root=s.metadata_root AND NEW.source_profile=mst2_route_profile(p.source_domain,p.tagged_root_tree_oid, + p.scope,p.schema_version,p.metadata_codec,p.materialization_policy,p.fs_semantics,p.access_projection, + p.verification_revision,p.projection_revision) + AND p.state='COMMITTED' AND p.metadata_root=s.metadata_root AND p.source_domain='native-git' + AND (p.graph_domain IS NULL OR p.graph_domain='generic-v1') + AND p.tagged_root_tree_oid IN ('sha1:'||s.root_tree_oid,'sha256:'||s.root_tree_oid)) THEN + RAISE EXCEPTION 'storage route is not derived from its actual generic context'; + END IF; + ELSIF TG_TABLE_NAME='mst2_generic_session_storage_binding' THEN + IF NOT EXISTS(SELECT 1 FROM mst2_snapshot_context s JOIN mst2_snapshot_storage_route r USING(snapshot_id) + WHERE s.snapshot_id=NEW.snapshot_id AND r.namespace_uuid=NEW.namespace_uuid + AND s.prepare_id=NEW.prepare_id AND s.metadata_root=NEW.metadata_root) THEN + RAISE EXCEPTION 'storage route generic incarnation is not its actual context'; + END IF; + ELSIF TG_TABLE_NAME='mst2_lease_storage_route' THEN + IF NOT EXISTS(SELECT 1 FROM mst2_snapshot_lease l JOIN mst2_generic_session_storage_binding b USING(snapshot_id) + WHERE l.lease_id=NEW.lease_id AND l.snapshot_id=NEW.snapshot_id AND b.namespace_uuid=NEW.namespace_uuid + AND b.session_incarnation=NEW.session_incarnation AND b.prepare_id=NEW.prepare_id AND b.metadata_root=NEW.metadata_root + AND l.authorization_epoch=NEW.authorization_epoch AND l.publication_sequence=NEW.publication_sequence + AND l.writer_epoch=NEW.writer_epoch AND l.certificate_receipt_id=NEW.certificate_receipt_id + AND mst2_route_snapshot_proof(l.snapshot_id)) THEN + RAISE EXCEPTION 'storage route lease is not its exact generic incarnation and source'; + END IF; + ELSE RAISE EXCEPTION 'storage route insertion target is not registered'; END IF; + RETURN NEW; +END $$; + +CREATE FUNCTION mst2_route_permanent_complete() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$CORE_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE domain text; valid boolean; +BEGIN + SELECT graph_domain INTO domain FROM mst2_metadata_namespace WHERE namespace_uuid=NEW.namespace_uuid; + IF TG_TABLE_NAME='mst2_snapshot_storage_route' THEN + valid:=CASE domain WHEN 'generic-v1' THEN mst2_route_snapshot_proof(NEW.snapshot_id) + WHEN 'qualified-v1' THEN mst2_route_qualified_snapshot_proof(NEW.snapshot_id) ELSE false END; + ELSE + valid:=CASE domain WHEN 'generic-v1' THEN mst2_route_lease_proof(NEW.lease_id) + WHEN 'qualified-v1' THEN mst2_route_qualified_lease_proof(NEW.lease_id) ELSE false END; + END IF; + IF NOT coalesce(valid,false) THEN RAISE EXCEPTION 'permanent storage route cannot commit without its exact actual incarnation'; END IF; + RETURN NULL; +END $$; +CREATE CONSTRAINT TRIGGER mst2_route_permanent_complete AFTER INSERT ON mst2_snapshot_storage_route + DEFERRABLE INITIALLY DEFERRED FOR EACH ROW EXECUTE FUNCTION mst2_route_permanent_complete(); +CREATE CONSTRAINT TRIGGER mst2_route_permanent_complete AFTER INSERT ON mst2_lease_storage_route + DEFERRABLE INITIALLY DEFERRED FOR EACH ROW EXECUTE FUNCTION mst2_route_permanent_complete(); diff --git a/src/jupiter/migration/mod.rs b/src/jupiter/migration/mod.rs index f5981f02..2fbc20ce 100644 --- a/src/jupiter/migration/mod.rs +++ b/src/jupiter/migration/mod.rs @@ -152,6 +152,7 @@ mod m20261007_000400_add_mst2_qualified_metadata_gc; mod m20261007_000500_add_mst2_install_capability; mod m20261007_000600_add_mst2_storage_routes; mod m20261008_000100_add_mst2_chunk_maps; +mod m20261008_000200_add_mst2_rooted_qualified_family; mod runner; pub use m20260905_000100_add_push_queue::ensure_queue_control_seed; pub use runner::apply_migrations; @@ -292,6 +293,7 @@ impl MigratorTrait for Migrator { Box::new(m20261007_000500_add_mst2_install_capability::Migration), Box::new(m20261007_000600_add_mst2_storage_routes::Migration), Box::new(m20261008_000100_add_mst2_chunk_maps::Migration), + Box::new(m20261008_000200_add_mst2_rooted_qualified_family::Migration), ] } } @@ -1202,7 +1204,7 @@ mod tests { async fn import_repo_alias_rows_canonicalized() { let names = migration_names(); assert_eq!( - &names[names.len() - 14..names.len() - 5], + &names[names.len() - 16..names.len() - 7], &[ "m20260923_000200_canonicalize_import_repo_paths".to_string(), "m20260925_000100_media_paging".to_string(), @@ -1217,25 +1219,33 @@ mod tests { "native retention, metadata installation and view tables follow media paging" ); assert_eq!( - &names[names.len() - 5], + &names[names.len() - 7], "m20261007_000200_add_mst2_metadata_generations" ); assert_eq!( - &names[names.len() - 4], + &names[names.len() - 6], "m20261007_000300_add_mst2_metadata_lifetime_history" ); assert_eq!( - &names[names.len() - 3], + &names[names.len() - 5], "m20261007_000400_add_mst2_qualified_metadata_gc" ); assert_eq!( - &names[names.len() - 2], + &names[names.len() - 4], "m20261007_000500_add_mst2_install_capability" ); assert_eq!( - names.last().unwrap(), + &names[names.len() - 3], "m20261007_000600_add_mst2_storage_routes" ); + assert_eq!( + &names[names.len() - 2], + "m20261008_000100_add_mst2_chunk_maps" + ); + assert_eq!( + names.last().unwrap(), + "m20261008_000200_add_mst2_rooted_qualified_family" + ); let db = alias_db().await; insert_repo(&db, 1, "/third-party//a").await; diff --git a/src/jupiter/storage/init.rs b/src/jupiter/storage/init.rs index d8480141..47044027 100644 --- a/src/jupiter/storage/init.rs +++ b/src/jupiter/storage/init.rs @@ -10,6 +10,21 @@ use crate::{ jupiter::migration::{apply_migrations, ensure_queue_control_seed}, }; +#[cfg(test)] +tokio::task_local! { + static GENERIC_HISTORY_BOOTSTRAP: (); +} + +#[cfg(test)] +pub(crate) async fn with_generic_history_bootstrap(future: F) -> F::Output { + GENERIC_HISTORY_BOOTSTRAP.scope((), future).await +} + +#[cfg(test)] +pub(super) fn generic_history_bootstrap_active() -> bool { + GENERIC_HISTORY_BOOTSTRAP.try_with(|_| ()).is_ok() +} + /// Create a PostgreSQL database connection. /// /// After a successful connection, applies any pending database migrations and @@ -25,6 +40,11 @@ pub async fn database_connection(db_config: &DbConfig) -> Result>, pub(crate) native_snapshot_sessions: Arc>, + #[cfg(test)] + pub(crate) shadow_qualified_metadata: + Arc>, + pub(crate) rooted_qualified_metadata: Arc< + tokio::sync::OnceCell>, + >, pub(crate) projection_observation_sink: Option>, pub cl_service: CLService, @@ -194,6 +201,46 @@ pub struct Storage { } impl Storage { + pub(crate) async fn rooted_qualified_metadata_writer( + &self, + ) -> Result<&qualified_metadata_family::RootedQualifiedMetadataRepository, MegaError> { + self.rooted_qualified_metadata + .get_or_try_init(|| async { + let repository = Arc::new( + qualified_metadata_family::RootedQualifiedMetadataRepository::open( + self.mono_storage().get_connection(), + &self.config().database, + ) + .await?, + ); + repository + .maintenance_tick(64) + .await + .map_err(|error| MegaError::Other(error.to_string()))?; + qualified_metadata_family::RootedQualifiedMetadataRepository::start_maintenance( + &repository, + ); + Ok::<_, MegaError>(repository) + }) + .await + .map(Arc::as_ref) + } + + #[cfg(test)] + pub(crate) async fn shadow_qualified_metadata_writer( + &self, + ) -> Result<&qualified_metadata_family::ShadowQualifiedMetadataWriter, MegaError> { + self.shadow_qualified_metadata + .get_or_try_init(|| async { + qualified_metadata_family::ShadowQualifiedMetadataWriter::open( + self.mono_storage().get_connection(), + &self.config().database, + ) + .await + }) + .await + } + pub async fn new( config: Arc, object_store: MegaObjectStorageWrapper, @@ -325,11 +372,14 @@ impl Storage { let storage_event_emitter = crate::jupiter::service::storage_event_emitter::StorageEventEmitter::from_config_disabled(&config); - Ok(Storage { + let storage = Storage { app_service: app_service.into(), native_projection_cache: Arc::default(), native_snapshot_sessions: Arc::default(), native_chunk_maps: Arc::default(), + #[cfg(test)] + shadow_qualified_metadata: Arc::default(), + rooted_qualified_metadata: Arc::default(), projection_observation_sink: None, config_handle, config, @@ -349,7 +399,13 @@ impl Storage { view_runtime, entity_store: Arc::new(SharedEntityStore::default()), vault: None, - }) + }; + #[cfg(test)] + if init::generic_history_bootstrap_active() { + return Ok(storage); + } + storage.rooted_qualified_metadata_writer().await?; + Ok(storage) } pub fn config_handle(&self) -> ConfigHandle { @@ -709,6 +765,9 @@ impl Storage { native_projection_cache: Arc::default(), native_snapshot_sessions: Arc::default(), native_chunk_maps: Arc::default(), + #[cfg(test)] + shadow_qualified_metadata: Arc::default(), + rooted_qualified_metadata: Arc::default(), projection_observation_sink: None, // app_service: AppService::mock(), cl_service: CLService::mock(), diff --git a/src/jupiter/storage/native_metadata_generations.rs b/src/jupiter/storage/native_metadata_generations.rs index a2cad697..dde51fc2 100644 --- a/src/jupiter/storage/native_metadata_generations.rs +++ b/src/jupiter/storage/native_metadata_generations.rs @@ -398,7 +398,11 @@ impl PostgresMetadataGenerationRepository { let fixed = self.require_fixed_plan(txn, intent).await?; check_generation_payloads(txn, &fixed).await?; let legacy = if intent.graph_domain == GraphDomain::Qualified { - qualified::finalize_graph(txn, &fixed, &observation.dag).await? + if self.inner.qualified_family.is_some() { + qualified::finalize_canonical_graph(txn, &fixed, &observation.dag).await? + } else { + qualified::finalize_graph(txn, &fixed, &observation.dag).await? + } } else { self.inner .finalize_stored_plan_in_txn(txn, &intent.legacy, &observation.dag, fixed.stored) diff --git a/src/jupiter/storage/native_metadata_install.rs b/src/jupiter/storage/native_metadata_install.rs index 5b97f663..6ecf72c3 100644 --- a/src/jupiter/storage/native_metadata_install.rs +++ b/src/jupiter/storage/native_metadata_install.rs @@ -139,6 +139,7 @@ pub struct PostgresMetadataInstallRepository { connection: DatabaseConnection, barrier_timeout: Duration, storage_scope: PrimaryStorageScope, + qualified_family: Option, } #[derive(Debug, Clone, PartialEq, Eq)] @@ -160,6 +161,7 @@ impl PostgresMetadataInstallRepository { connection, barrier_timeout: Duration::from_secs(5), storage_scope, + qualified_family: None, }) } @@ -654,6 +656,9 @@ impl PostgresMetadataInstallRepository { if txn.get_database_backend() != DbBackend::Postgres { return Err(internal("metadata installation requires PostgreSQL")); } + if let Some(family) = &self.qualified_family { + family.enter(txn).await?; + } if read_storage_scope(txn).await? != self.storage_scope { return Err(internal( "metadata recovery connection is outside the captured primary storage scope", diff --git a/src/jupiter/storage/native_metadata_qualified.rs b/src/jupiter/storage/native_metadata_qualified.rs index 5a29c9c6..221f9c98 100644 --- a/src/jupiter/storage/native_metadata_qualified.rs +++ b/src/jupiter/storage/native_metadata_qualified.rs @@ -115,6 +115,22 @@ impl GcRecord { } impl PostgresQualifiedMetadataRepository { + #[cfg(test)] + pub(crate) async fn registered_shadow( + connection: DatabaseConnection, + family: crate::jupiter::storage::qualified_metadata_family::VerifiedQualifiedNamespace, + ) -> Result { + let mut inner = PostgresMetadataInstallRepository::new(connection).await?; + inner.qualified_family = Some(family); + Ok(Self { + inner: PostgresMetadataGenerationRepository { + inner, + graph_domain: "qualified-v1", + }, + }) + } + + #[cfg(test)] pub async fn new(connection: DatabaseConnection) -> Result { Ok(Self { inner: PostgresMetadataGenerationRepository { @@ -635,8 +651,7 @@ pub(super) async fn allocate_lifetimes( n.state AS graph_state,n.metadata_codec AS graph_codec,n.bytes AS graph_bytes, b.page_id IS NOT NULL AS payload_present,b.generation AS payload_generation,b.metadata_codec AS payload_codec,b.byte_size AS payload_size, EXISTS(SELECT 1 FROM mst2_metadata_gc_op o WHERE o.page_id=l.page_id AND o.generation=l.generation) AS tombstone, - EXISTS(SELECT 1 FROM mst2_retention_node x WHERE x.node_id=l.node_id) - OR EXISTS(SELECT 1 FROM mst2_retention_gc_op x WHERE x.node_id=l.node_id) AS generic_graph + mst2_metadata_has_generic_overlap(l.page_id) AS generic_graph FROM jsonb_to_recordset($1::jsonb) p(id text,size integer) JOIN mst2_metadata_current c ON c.page_id=decode(p.id,'hex') JOIN mst2_metadata_lifetime l USING(page_id,generation) @@ -740,18 +755,16 @@ pub(super) async fn check_lifetimes( OR ($3='COMMITTED' AND l.state<>'LIVE') OR (l.state='LIVE' AND n.page_id IS NULL) OR n.state<>'LIVE' OR n.metadata_codec<>$2 OR n.bytes<>m.expected_size OR EXISTS(SELECT 1 FROM mst2_metadata_gc_op o WHERE o.page_id=m.page_id AND o.generation=m.generation) - OR EXISTS(SELECT 1 FROM mst2_retention_node x WHERE x.node_id=l.node_id) - OR EXISTS(SELECT 1 FROM mst2_retention_gc_op x WHERE x.node_id=l.node_id)) LIMIT 1", + OR mst2_metadata_has_generic_overlap(l.page_id)) LIMIT 1", [stored.record.prepare_id.clone().into(),stored.record.metadata_codec.into(),stored.record.state.clone().into()], )).await.map_err(internal)?.is_some() { return Err(unavailable("fixed qualified incarnation is no longer installable")); } Ok(()) } -pub(super) async fn finalize_graph( - txn: &DatabaseTransaction, +fn validate_fixed_observation( fixed: &FixedPlan, dag: &ValidatedMetadataDag, -) -> Result { +) -> Result<(), SnapshotError> { let actual_pages: BTreeSet<_> = dag.payloads().iter().map(|p| (p.id, p.size)).collect(); let actual_edges: BTreeSet<_> = dag .edges() @@ -780,6 +793,92 @@ pub(super) async fn finalize_graph( "qualified DAG observation differs from immutable plan", )); } + Ok(()) +} + +pub(super) async fn finalize_canonical_graph( + txn: &DatabaseTransaction, + fixed: &FixedPlan, + dag: &ValidatedMetadataDag, +) -> Result { + validate_fixed_observation(fixed, dag)?; + if fixed.stored.record.state == "COMMITTED" { + verify_graph(txn, fixed).await?; + return fixed.stored.receipt(); + } + let mut pending: BTreeMap<_, usize> = fixed.stored.plan.pages.keys().map(|p| (*p, 0)).collect(); + let mut parents: BTreeMap<_, Vec<_>> = BTreeMap::new(); + for &(parent, child) in &fixed.stored.plan.edges { + *pending + .get_mut(&parent) + .ok_or_else(|| integrity("canonical parent is outside its fixed plan"))? += 1; + parents.entry(child).or_default().push(parent); + } + let mut ready: BTreeSet<_> = pending + .iter() + .filter_map(|(p, count)| (*count == 0).then_some(*p)) + .collect(); + let mut ordered = Vec::with_capacity(pending.len()); + while let Some(page) = ready.pop_first() { + ordered.push(json!({"page":hex::encode(page),"generation":fixed.bindings.0[&page].0})); + for parent in parents.get(&page).into_iter().flatten() { + let count = pending + .get_mut(parent) + .ok_or_else(|| integrity("canonical ancestor is outside its fixed plan"))?; + *count = count + .checked_sub(1) + .ok_or_else(|| integrity("canonical edge accounting underflow"))?; + if *count == 0 { + ready.insert(*parent); + } + } + } + if ordered.len() != pending.len() { + return Err(integrity("canonical certification order contains a cycle")); + } + let certified: i32 = txn + .query_one_raw(statement( + "SELECT mst2_metadata_certify_batch($1,$2::jsonb) AS certified", + [ + fixed.intent.prepare_id().into(), + serde_json::to_string(&ordered).map_err(internal)?.into(), + ], + )) + .await + .map_err(internal)? + .ok_or_else(|| integrity("canonical certification batch is missing"))? + .try_get("", "certified") + .map_err(internal)?; + if certified as usize != ordered.len() { + return Err(integrity( + "canonical certification did not cover its exact cold DAG", + )); + } + retain_existing_roots(txn, &fixed.intent).await?; + verify_graph(txn, fixed).await?; + let changed = txn + .execute_raw(statement( + "UPDATE mst2_metadata_prepare SET state='COMMITTED',committed_at=clock_timestamp() + WHERE prepare_id=$1 AND state='PREPARING' AND storage_seal=$2", + [ + fixed.intent.prepare_id().into(), + fixed.intent.storage_seal.to_vec().into(), + ], + )) + .await + .map_err(internal)?; + if changed.rows_affected() != 1 { + return Err(integrity("canonical finalize lost its prepare CAS")); + } + fixed.stored.receipt() +} + +pub(super) async fn finalize_graph( + txn: &DatabaseTransaction, + fixed: &FixedPlan, + dag: &ValidatedMetadataDag, +) -> Result { + validate_fixed_observation(fixed, dag)?; if fixed.stored.record.state == "COMMITTED" { verify_graph(txn, fixed).await?; return fixed.stored.receipt(); diff --git a/src/jupiter/storage/native_snapshot_session.rs b/src/jupiter/storage/native_snapshot_session.rs index fb8d9134..da15a716 100644 --- a/src/jupiter/storage/native_snapshot_session.rs +++ b/src/jupiter/storage/native_snapshot_session.rs @@ -366,9 +366,11 @@ mod routes; #[path = "native_snapshot_metadata_routes.rs"] mod metadata_routes; -pub(crate) use metadata_routes::MetadataRouteRequest; #[cfg(test)] pub(crate) use metadata_routes::with_metadata_read_barriers; +pub(crate) use metadata_routes::{ + MetadataRouteRequest, PersistedMetadataReadWork, PersistedMetadataRouteBatch, +}; const SESSION_SQL: &str = "SELECT s.snapshot_id,s.canonical_descriptor,s.commit_oid,s.root_tree_oid, (SELECT storage_uuid FROM mst2_metadata_storage_scope WHERE singleton=1) AS authority_storage_uuid, @@ -665,7 +667,7 @@ fn forbidden() -> SnapshotError { "snapshot serving state or authorization epoch changed", ) } -fn install_error(error: MetadataInstallError) -> SnapshotError { +pub(crate) fn install_error(error: MetadataInstallError) -> SnapshotError { match error { MetadataInstallError::Rejected(error) if error.code == SnapshotErrorCode::Internal => { tracing::error!(%error,"native metadata installation unavailable"); @@ -690,6 +692,44 @@ fn install_error(error: MetadataInstallError) -> SnapshotError { } impl super::Storage { + pub(crate) async fn snapshot_metadata_family( + &self, + identity: &str, + lease: bool, + ) -> Result, SnapshotError> + { + use super::base_storage::StorageConnector; + super::qualified_metadata_family::select_snapshot_family( + self.mono_storage().get_connection(), + identity, + lease, + ) + .await + } + + pub(crate) async fn snapshot_metadata_routes( + &self, + context: &SnapshotContext, + requests: &[MetadataRouteRequest<'_>], + ) -> Result { + if self + .snapshot_metadata_family(&context.lease_id, true) + .await? + == Some(super::qualified_metadata_family::SnapshotMetadataFamily::Rooted) + { + return self + .rooted_qualified_metadata_writer() + .await + .map_err(internal)? + .metadata_routes(context, requests) + .await; + } + self.snapshot_sessions() + .await + .metadata_routes(context, requests) + .await + } + pub(crate) async fn snapshot_sessions(&self) -> &PostgresNativeSessionRepository { use super::base_storage::StorageConnector; self.native_snapshot_sessions @@ -714,6 +754,16 @@ impl super::Storage { let instance = uuid::Uuid::parse_str(instance) .map_err(|_| not_ready("invalid native instance"))? .to_string(); + if self.snapshot_metadata_family(lease, true).await? + == Some(super::qualified_metadata_family::SnapshotMetadataFamily::Rooted) + { + return self + .rooted_qualified_metadata_writer() + .await + .map_err(internal)? + .context(sid, lease, &instance) + .await; + } self.snapshot_sessions() .await .context(sid, lease, &instance) @@ -740,6 +790,16 @@ impl super::Storage { let instance = uuid::Uuid::parse_str(instance) .map_err(|_| not_ready("invalid native instance"))? .to_string(); + if self.snapshot_metadata_family(lease, true).await? + == Some(super::qualified_metadata_family::SnapshotMetadataFamily::Rooted) + { + return self + .rooted_qualified_metadata_writer() + .await + .map_err(internal)? + .renew(lease, seconds, &instance) + .await; + } self.snapshot_sessions() .await .renew(lease, seconds, &instance) @@ -751,6 +811,16 @@ impl super::Storage { pub(crate) async fn snapshot_release(&self, lease: &str) -> Result { if self.config().mst2.publication_enabled { + if self.snapshot_metadata_family(lease, true).await? + == Some(super::qualified_metadata_family::SnapshotMetadataFamily::Rooted) + { + return self + .rooted_qualified_metadata_writer() + .await + .map_err(internal)? + .release(lease) + .await; + } self.snapshot_sessions().await.release(lease).await } else { Ok(crate::ceres::snapshot::runtime::runtime().release_lease(lease)) diff --git a/src/jupiter/storage/qualified_family_catalog.sql b/src/jupiter/storage/qualified_family_catalog.sql new file mode 100644 index 00000000..acb8cce9 --- /dev/null +++ b/src/jupiter/storage/qualified_family_catalog.sql @@ -0,0 +1,57 @@ +WITH selected_namespaces AS ( + SELECT oid,nspname,nspowner,nspacl FROM pg_catalog.pg_namespace + WHERE oid IN ($CORE_OID$,$Q_OID$) +), selected_relations AS ( + SELECT c.oid,c.relnamespace,c.relname,c.relkind,c.relpersistence,c.relowner,c.relacl, + c.relrowsecurity,c.relforcerowsecurity,c.reloptions,c.relam,c.relhasrules,c.relispartition + FROM pg_catalog.pg_class c + WHERE c.relnamespace=$Q_OID$ + OR (c.relnamespace=$CORE_OID$ AND (c.relname IN ('mst2_metadata_namespace','mst2_qualified_family_policy', + 'mega_tree','mst2_rooted_source_tree_revision','mst2_verified_object','mst2_retention_node','mst2_retention_gc_op', + 'mega_commit','mega_refs','mst2_native_head','mst2_native_publication','mst2_publication','mst2_publication_outbox', + 'mst2_snapshot_storage_route','mst2_lease_storage_route','mst2_generic_session_storage_binding', + 'mst2_snapshot_context','mst2_snapshot_lease') + OR c.oid IN (SELECT i.indexrelid FROM pg_catalog.pg_index i JOIN pg_catalog.pg_class parent ON parent.oid=i.indrelid + WHERE parent.relnamespace=$CORE_OID$ AND parent.relname IN ('mst2_metadata_namespace','mst2_qualified_family_policy', + 'mega_tree','mst2_rooted_source_tree_revision','mst2_verified_object','mst2_retention_node','mst2_retention_gc_op', + 'mega_commit','mega_refs','mst2_native_head','mst2_native_publication','mst2_publication','mst2_publication_outbox', + 'mst2_snapshot_storage_route','mst2_lease_storage_route','mst2_generic_session_storage_binding', + 'mst2_snapshot_context','mst2_snapshot_lease')))) +), selected_triggers AS ( + SELECT t.* FROM pg_catalog.pg_trigger t + WHERE (t.tgrelid IN (SELECT oid FROM selected_relations) + OR EXISTS(SELECT 1 FROM pg_catalog.pg_constraint x + WHERE x.oid=t.tgconstraint AND x.conrelid IN (SELECT oid FROM pg_catalog.pg_class WHERE relnamespace=$Q_OID$))) + AND NOT ($Q_OID$=0 AND $EXEMPT_Q_OID$<>0 AND t.tgisinternal + AND EXISTS(SELECT 1 FROM pg_catalog.pg_constraint fk + JOIN pg_catalog.pg_class source ON source.oid=fk.conrelid + JOIN pg_catalog.pg_class target ON target.oid=fk.confrelid + WHERE fk.oid=t.tgconstraint AND fk.contype='f' AND fk.connamespace=$EXEMPT_Q_OID$ + AND source.relnamespace=$EXEMPT_Q_OID$ AND target.relnamespace=$CORE_OID$ + AND t.tgrelid=fk.confrelid AND t.tgconstrrelid=fk.conrelid)) +), objects AS ( + SELECT 'namespace' AS kind,oid::text AS key,pg_catalog.to_jsonb(n) AS value FROM selected_namespaces n + UNION ALL SELECT 'relation',oid::text,pg_catalog.to_jsonb(c) FROM selected_relations c + UNION ALL SELECT 'column',a.attrelid||':'||a.attnum,pg_catalog.to_jsonb(a) + FROM pg_catalog.pg_attribute a JOIN selected_relations c ON c.oid=a.attrelid WHERE a.attnum>0 + UNION ALL SELECT 'default',d.oid::text,pg_catalog.to_jsonb(d) + FROM pg_catalog.pg_attrdef d JOIN selected_relations c ON c.oid=d.adrelid + UNION ALL SELECT 'constraint',x.oid::text,pg_catalog.to_jsonb(x) + FROM pg_catalog.pg_constraint x JOIN selected_relations c ON c.oid=x.conrelid + UNION ALL SELECT 'index',i.indexrelid::text,pg_catalog.to_jsonb(i) + FROM pg_catalog.pg_index i JOIN selected_relations c ON c.oid=i.indrelid + UNION ALL SELECT 'trigger',t.oid::text,pg_catalog.to_jsonb(t) + FROM selected_triggers t + UNION ALL SELECT 'function',p.oid::text,pg_catalog.to_jsonb(p) + FROM pg_catalog.pg_proc p WHERE p.pronamespace=$Q_OID$ + OR (p.pronamespace=$CORE_OID$ AND (pg_catalog.left(p.proname,11)='mst2_route_' + OR p.proname='mst2_metadata_has_generic_overlap')) + OR EXISTS(SELECT 1 FROM selected_triggers t WHERE t.tgfoid=p.oid) + UNION ALL SELECT 'policy',p.oid::text,pg_catalog.to_jsonb(p) + FROM pg_catalog.pg_policy p JOIN selected_relations c ON c.oid=p.polrelid + UNION ALL SELECT 'rule',r.oid::text,pg_catalog.to_jsonb(r) + FROM pg_catalog.pg_rewrite r JOIN selected_relations c ON c.oid=r.ev_class +) +SELECT pg_catalog.sha256(pg_catalog.convert_to( + pg_catalog.jsonb_agg(pg_catalog.jsonb_build_array(kind,key,value) ORDER BY kind,key)::text,'UTF8')) AS fingerprint +FROM objects diff --git a/src/jupiter/storage/qualified_family_shape.sql b/src/jupiter/storage/qualified_family_shape.sql new file mode 100644 index 00000000..4269fe74 --- /dev/null +++ b/src/jupiter/storage/qualified_family_shape.sql @@ -0,0 +1,65 @@ +WITH q AS ( + SELECT n.oid,n.nspname FROM pg_catalog.pg_namespace n WHERE n.oid=q_oid +), relations AS ( + SELECT c.* FROM pg_catalog.pg_class c WHERE c.relnamespace=q_oid +), objects AS ( + SELECT 'relation' AS kind,c.relname AS key,jsonb_build_array(c.relname,c.relkind,c.relpersistence,c.relowner,c.relacl, + c.relrowsecurity,c.relforcerowsecurity,c.reloptions,c.relispartition,a.amname) AS value + FROM relations c LEFT JOIN pg_catalog.pg_am a ON a.oid=c.relam + UNION ALL SELECT 'column',c.relname||':'||a.attnum,jsonb_build_array(c.relname,a.attnum,a.attname, + CASE WHEN tn.oid=q_oid THEN '$Q_SCHEMA$' ELSE tn.nspname END,t.typname,a.attnotnull,a.attisdropped,a.attidentity,a.attgenerated,a.attislocal,a.attinhcount, + a.atttypmod,a.attndims,a.attacl,a.attlen,a.attbyval,a.attalign,a.attstorage,a.attcompression, + CASE WHEN cn.oid=q_oid THEN '$Q_SCHEMA$' ELSE cn.nspname END,co.collname, + pg_catalog.replace(pg_catalog.pg_get_expr(d.adbin,d.adrelid),(SELECT nspname FROM q),'$Q_SCHEMA$')) + FROM pg_catalog.pg_attribute a JOIN relations c ON c.oid=a.attrelid + JOIN pg_catalog.pg_type t ON t.oid=a.atttypid JOIN pg_catalog.pg_namespace tn ON tn.oid=t.typnamespace + LEFT JOIN pg_catalog.pg_collation co ON co.oid=a.attcollation LEFT JOIN pg_catalog.pg_namespace cn ON cn.oid=co.collnamespace + LEFT JOIN pg_catalog.pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum WHERE a.attnum>0 + UNION ALL SELECT 'constraint',c.relname||':'||x.conname,jsonb_build_array(c.relname,x.conname,x.contype, + pg_catalog.replace(pg_catalog.pg_get_constraintdef(x.oid,true),(SELECT nspname FROM q),'$Q_SCHEMA$'), + x.condeferrable,x.condeferred,x.convalidated,x.conislocal,x.coninhcount,x.connoinherit) + FROM pg_catalog.pg_constraint x JOIN relations c ON c.oid=x.conrelid + UNION ALL SELECT 'index',c.relname||':'||ic.relname,jsonb_build_array(c.relname,ic.relname, + pg_catalog.replace(pg_catalog.pg_get_indexdef(i.indexrelid),(SELECT nspname FROM q),'$Q_SCHEMA$'), + i.indisunique,i.indisprimary,i.indisexclusion,i.indimmediate,i.indisvalid,i.indisready,i.indislive, + i.indnullsnotdistinct,i.indisclustered,i.indisreplident) + FROM pg_catalog.pg_index i JOIN relations c ON c.oid=i.indrelid JOIN relations ic ON ic.oid=i.indexrelid + UNION ALL SELECT 'function',p.proname||':'||pg_catalog.replace(pg_catalog.pg_get_function_identity_arguments(p.oid),(SELECT nspname FROM q),'$Q_SCHEMA$'),jsonb_build_array( + p.proname,pg_catalog.replace(pg_catalog.pg_get_function_identity_arguments(p.oid),(SELECT nspname FROM q),'$Q_SCHEMA$'), + pg_catalog.replace(pg_catalog.pg_get_function_result(p.oid),(SELECT nspname FROM q),'$Q_SCHEMA$'),l.lanname, + p.proowner,p.proacl,p.prokind,p.provolatile,p.proisstrict,p.prosecdef,p.proleakproof,p.proparallel, + p.procost,p.prorows,p.probin,p.pronargdefaults, + pg_catalog.replace(pg_catalog.pg_get_expr(p.proargdefaults,0),(SELECT nspname FROM q),'$Q_SCHEMA$'), + pg_catalog.replace(pg_catalog.replace(pg_catalog.replace(pg_catalog.replace(p.prosrc, + pg_catalog.quote_literal((SELECT nspname FROM q)),pg_catalog.quote_literal('$Q_SCHEMA$')), + pg_catalog.quote_literal(q_oid::text),pg_catalog.quote_literal('$Q_OID$')), + pg_catalog.quote_literal(n_uuid::text),pg_catalog.quote_literal('$NAMESPACE_UUID$')), + pg_catalog.quote_literal(s_uuid),pg_catalog.quote_literal('$STORAGE_UUID$')), + pg_catalog.replace(pg_catalog.array_to_string(p.proconfig,E'\n'),(SELECT nspname FROM q),'$Q_SCHEMA$')) + FROM pg_catalog.pg_proc p JOIN pg_catalog.pg_language l ON l.oid=p.prolang WHERE p.pronamespace=q_oid + UNION ALL SELECT 'trigger',c.relname||':'||p.proname||':'||t.tgtype||':'||coalesce(x.conname,t.tgname),jsonb_build_array( + CASE WHEN c.relnamespace=q_oid THEN '$Q_SCHEMA$' ELSE cns.nspname END,c.relname, + CASE WHEN t.tgisinternal THEN NULL ELSE t.tgname END,t.tgtype,t.tgenabled,t.tgisinternal, + CASE WHEN pn.oid=q_oid THEN '$Q_SCHEMA$' ELSE pn.nspname END,p.proname, + pg_catalog.replace(pg_catalog.pg_get_function_identity_arguments(p.oid),(SELECT nspname FROM q),'$Q_SCHEMA$'), + t.tgdeferrable,t.tginitdeferred,t.tgnargs, + pg_catalog.replace(encode(t.tgargs,'hex'),encode(pg_catalog.convert_to((SELECT nspname FROM q),'UTF8'),'hex'), + encode(pg_catalog.convert_to('$Q_SCHEMA$','UTF8'),'hex')),t.tgattr::text, + pg_catalog.replace(pg_catalog.pg_get_expr(t.tgqual,t.tgrelid),(SELECT nspname FROM q),'$Q_SCHEMA$'),t.tgoldtable,t.tgnewtable,x.conname, + CASE WHEN rn.oid=q_oid THEN '$Q_SCHEMA$' ELSE rn.nspname END,rc.relname, + CASE WHEN t.tgparentid=0 THEN NULL ELSE 'unexpected parent trigger' END) + FROM pg_catalog.pg_trigger t JOIN pg_catalog.pg_class c ON c.oid=t.tgrelid + JOIN pg_catalog.pg_namespace cns ON cns.oid=c.relnamespace + JOIN pg_catalog.pg_proc p ON p.oid=t.tgfoid JOIN pg_catalog.pg_namespace pn ON pn.oid=p.pronamespace + LEFT JOIN pg_catalog.pg_constraint x ON x.oid=t.tgconstraint + LEFT JOIN pg_catalog.pg_class rc ON rc.oid=t.tgconstrrelid LEFT JOIN pg_catalog.pg_namespace rn ON rn.oid=rc.relnamespace + WHERE c.relnamespace=q_oid OR EXISTS(SELECT 1 FROM pg_catalog.pg_constraint fk + WHERE fk.oid=t.tgconstraint AND fk.conrelid IN (SELECT oid FROM relations)) + UNION ALL SELECT 'rule',c.relname||':'||r.rulename,pg_catalog.to_jsonb(r) + FROM pg_catalog.pg_rewrite r JOIN relations c ON c.oid=r.ev_class + UNION ALL SELECT 'policy',c.relname||':'||p.polname,pg_catalog.to_jsonb(p) + FROM pg_catalog.pg_policy p JOIN relations c ON c.oid=p.polrelid +) +SELECT pg_catalog.sha256(pg_catalog.convert_to( + pg_catalog.jsonb_agg(pg_catalog.jsonb_build_array(kind,key,value) ORDER BY kind,key,value::text)::text,'UTF8')) AS fingerprint +FROM objects diff --git a/src/jupiter/storage/qualified_metadata_anchors.sql b/src/jupiter/storage/qualified_metadata_anchors.sql new file mode 100644 index 00000000..08b3d7ea --- /dev/null +++ b/src/jupiter/storage/qualified_metadata_anchors.sql @@ -0,0 +1,426 @@ +CREATE TABLE mst2_metadata_source_root_attestation ( + attestation_id uuid PRIMARY KEY,namespace_uuid uuid NOT NULL REFERENCES mst2_metadata_family_identity(namespace_uuid), + origin_prepare_id text NOT NULL REFERENCES mst2_metadata_prepare(prepare_id), + tagged_tree_oid text NOT NULL CHECK(tagged_tree_oid ~ '^(sha1:[0-9a-f]{40}|sha256:[0-9a-f]{64}|blake3:[0-9a-f]{64})$'), + source_profile jsonb NOT NULL,profile_digest bytea NOT NULL CHECK(octet_length(profile_digest)=32), + source_body_digest bytea NOT NULL CHECK(octet_length(source_body_digest)=32), + source_revision uuid NOT NULL, + root_page bytea NOT NULL,root_generation bigint NOT NULL, + root_certificate_digest bytea NOT NULL CHECK(octet_length(root_certificate_digest)=32), + source_proof jsonb NOT NULL,attestation_digest bytea NOT NULL CHECK(octet_length(attestation_digest)=32), + FOREIGN KEY(root_page,root_generation,root_certificate_digest) + REFERENCES mst2_metadata_page_certificate(page_id,generation,certificate_digest), + UNIQUE(attestation_id,root_page,root_generation,attestation_digest) +); +CREATE INDEX mst2_metadata_source_root_lookup ON mst2_metadata_source_root_attestation(profile_digest,tagged_tree_oid,root_page,root_generation); +CREATE TABLE mst2_metadata_prepare_reuse_root ( + prepare_id text NOT NULL REFERENCES mst2_metadata_prepare(prepare_id),root_page bytea NOT NULL, + root_generation bigint NOT NULL,attestation_id uuid NOT NULL,attestation_digest bytea NOT NULL, + PRIMARY KEY(prepare_id,root_page), + FOREIGN KEY(attestation_id,root_page,root_generation,attestation_digest) + REFERENCES mst2_metadata_source_root_attestation(attestation_id,root_page,root_generation,attestation_digest) +); +CREATE INDEX mst2_metadata_prepare_reuse_root_page ON mst2_metadata_prepare_reuse_root(root_page,root_generation,prepare_id); +CREATE TABLE mst2_metadata_reuse_index ( + profile_digest bytea NOT NULL,tagged_tree_oid text NOT NULL,attestation_id uuid NOT NULL, + root_page bytea NOT NULL,root_generation bigint NOT NULL,attestation_digest bytea NOT NULL, + PRIMARY KEY(profile_digest,tagged_tree_oid), + FOREIGN KEY(attestation_id,root_page,root_generation,attestation_digest) + REFERENCES mst2_metadata_source_root_attestation(attestation_id,root_page,root_generation,attestation_digest) +); +CREATE INDEX mst2_metadata_reuse_index_page ON mst2_metadata_reuse_index(root_page,root_generation); + +ALTER TABLE mst2_qualified_session_incarnation ADD COLUMN canonical_descriptor bytea NOT NULL, + ADD COLUMN attestation_id uuid NOT NULL,ADD COLUMN attestation_digest bytea NOT NULL, + ADD COLUMN state text NOT NULL CHECK(state IN ('READY','RETIRED')), + ADD FOREIGN KEY(attestation_id,metadata_root,root_generation,attestation_digest) + REFERENCES mst2_metadata_source_root_attestation(attestation_id,root_page,root_generation,attestation_digest); +ALTER TABLE mst2_qualified_lease_binding ADD COLUMN namespace_uuid uuid NOT NULL, + ADD COLUMN authorization_epoch bigint NOT NULL,ADD COLUMN publication_sequence bigint NOT NULL, + ADD COLUMN writer_epoch bigint NOT NULL,ADD COLUMN certificate_receipt_id bigint NOT NULL, + ADD COLUMN expires_at_unix bigint NOT NULL,ADD COLUMN state text NOT NULL CHECK(state IN ('ACTIVE','RELEASED','EXPIRED')), + ADD COLUMN lease_epoch bigint NOT NULL CHECK(lease_epoch>0); +CREATE INDEX mst2_qualified_lease_active_incarnation ON mst2_qualified_lease_binding(snapshot_id,session_incarnation,state,lease_id); +CREATE TABLE mst2_metadata_reader_operation ( + operation_id uuid PRIMARY KEY,lease_id text NOT NULL REFERENCES mst2_qualified_lease_binding(lease_id), + snapshot_id text NOT NULL,session_incarnation uuid NOT NULL,root_page bytea NOT NULL,root_generation bigint NOT NULL, + lease_epoch bigint NOT NULL CHECK(lease_epoch>0),hard_deadline_unix bigint NOT NULL, + state text NOT NULL CHECK(state IN ('ACTIVE','FINISHED','EXPIRED')), + FOREIGN KEY(snapshot_id,session_incarnation) REFERENCES mst2_qualified_session_incarnation(snapshot_id,session_incarnation) +); +CREATE INDEX mst2_metadata_reader_active_lease ON mst2_metadata_reader_operation(lease_id,state,operation_id); +CREATE INDEX mst2_metadata_reader_active_deadline ON mst2_metadata_reader_operation(hard_deadline_unix,operation_id) WHERE state='ACTIVE'; +CREATE TABLE mst2_metadata_root_anchor ( + anchor_id uuid PRIMARY KEY,anchor_kind text NOT NULL CHECK(anchor_kind IN ('PREPARE','REUSE','SESSION','LEASE','REQUEST','READER')), + owner_key text NOT NULL CHECK(octet_length(owner_key) BETWEEN 1 AND 512), + root_page bytea NOT NULL,root_generation bigint NOT NULL,root_certificate_digest bytea NOT NULL, + prepare_id text REFERENCES mst2_metadata_prepare(prepare_id), + snapshot_id text,session_incarnation uuid,lease_id text REFERENCES mst2_qualified_lease_binding(lease_id), + reader_operation_id uuid REFERENCES mst2_metadata_reader_operation(operation_id), + UNIQUE(anchor_kind,owner_key,root_page,root_generation), + FOREIGN KEY(root_page,root_generation) REFERENCES mst2_metadata_graph_node(page_id,generation), + FOREIGN KEY(root_page,root_generation,root_certificate_digest) + REFERENCES mst2_metadata_page_certificate(page_id,generation,certificate_digest), + FOREIGN KEY(snapshot_id,session_incarnation) REFERENCES mst2_qualified_session_incarnation(snapshot_id,session_incarnation) +); +CREATE INDEX mst2_metadata_root_anchor_page ON mst2_metadata_root_anchor(root_page,root_generation,anchor_kind,owner_key); +CREATE INDEX mst2_metadata_root_anchor_prepare ON mst2_metadata_root_anchor(prepare_id,anchor_kind,anchor_id); +CREATE INDEX mst2_metadata_root_anchor_lease ON mst2_metadata_root_anchor(lease_id,anchor_kind,anchor_id); + +CREATE FUNCTION mst2_metadata_session_covers_prepare(pid text) RETURNS boolean LANGUAGE sql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ + SELECT EXISTS(SELECT 1 FROM mst2_metadata_prepare q + JOIN mst2_metadata_current cur ON cur.page_id=q.metadata_root + JOIN mst2_metadata_page_certificate certificate USING(page_id,generation) + JOIN mst2_qualified_session_incarnation session ON session.metadata_root=q.metadata_root + AND session.root_generation=cur.generation + JOIN mst2_metadata_source_root_attestation source ON source.attestation_id=session.attestation_id + AND source.root_page=session.metadata_root AND source.root_generation=session.root_generation + AND source.attestation_digest=session.attestation_digest + AND source.root_certificate_digest=certificate.certificate_digest + JOIN mst2_metadata_root_anchor anchor ON anchor.snapshot_id=session.snapshot_id + AND anchor.session_incarnation=session.session_incarnation AND anchor.anchor_kind='SESSION' + AND anchor.owner_key=session.snapshot_id||':'||session.session_incarnation::text + AND anchor.root_page=session.metadata_root AND anchor.root_generation=session.root_generation + AND anchor.root_certificate_digest=certificate.certificate_digest + WHERE q.prepare_id=pid AND q.plan_kind='ROOTED' AND q.state='COMMITTED' AND session.state='READY' + AND session.namespace_uuid=(SELECT namespace_uuid FROM mst2_metadata_family_identity WHERE singleton=1) + AND convert_from(session.source_profile,'UTF8')::jsonb=$CORE_SCHEMA$.mst2_route_profile( + q.source_domain,q.tagged_root_tree_oid,q.scope,q.schema_version,q.metadata_codec, + q.materialization_policy,q.fs_semantics,q.access_projection,q.verification_revision,q.projection_revision)) +$$; + +CREATE FUNCTION mst2_metadata_native_profile(pid text) RETURNS jsonb LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE q record; +BEGIN + SELECT source_domain,tagged_root_tree_oid,schema_version,metadata_codec,materialization_policy, + fs_semantics,access_projection,verification_revision,projection_revision + INTO q FROM mst2_metadata_prepare WHERE prepare_id=pid AND state IN ('PREPARING','COMMITTED') + AND graph_domain='qualified-v1' AND mst2_metadata_scope_matches(primary_scope); + IF NOT FOUND OR q.source_domain<>'native-git' OR q.schema_version<>2 OR q.metadata_codec<>1 + OR q.materialization_policy<>1 OR q.fs_semantics<>1 OR q.access_projection<>0 + OR q.verification_revision<>2 OR q.projection_revision<>1 + OR q.tagged_root_tree_oid !~ '^(sha1:[0-9a-f]{40}|sha256:[0-9a-f]{64}|blake3:[0-9a-f]{64})$' THEN + RAISE EXCEPTION 'source attestation requires the exact current native metadata profile'; + END IF; + RETURN jsonb_build_object('source_domain',q.source_domain,'hash_kind',split_part(q.tagged_root_tree_oid,':',1), + 'schema_version',q.schema_version,'metadata_codec',q.metadata_codec,'materialization_policy',q.materialization_policy, + 'fs_semantics',q.fs_semantics,'access_projection',q.access_projection, + 'verification_revision',q.verification_revision,'projection_revision',q.projection_revision); +END $$; + +CREATE FUNCTION mst2_metadata_decode_git_tree(b bytea,hash_kind text) RETURNS jsonb +LANGUAGE plpgsql IMMUTABLE STRICT SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE width integer; p integer:=0; start integer; mode text; name bytea; oid bytea; kind integer; + result jsonb[]:=ARRAY[]::jsonb[]; sorted jsonb; count integer:=0; +BEGIN + width:=CASE hash_kind WHEN 'sha1' THEN 20 WHEN 'sha256' THEN 32 WHEN 'blake3' THEN 32 ELSE 0 END; + IF width=0 OR octet_length(b)>67108864 THEN RAISE EXCEPTION 'source Git tree kind or byte budget is invalid'; END IF; + WHILE p32 AND p-start<6 LOOP p:=p+1; END LOOP; + IF p>=octet_length(b) OR get_byte(b,p)<>32 THEN RAISE EXCEPTION 'source Git tree mode is malformed'; END IF; + mode:=convert_from(substring(b FROM start+1 FOR p-start),'UTF8'); p:=p+1; start:=p; + WHILE p0 AND p-start<=255 LOOP p:=p+1; END LOOP; + IF p>=octet_length(b) OR get_byte(b,p)<>0 THEN RAISE EXCEPTION 'source Git tree name is malformed'; END IF; + name:=substring(b FROM start+1 FOR p-start); PERFORM mst2_metadata_valid_name(name); p:=p+1; + IF p>octet_length(b)-width THEN RAISE EXCEPTION 'source Git tree object identity is truncated'; END IF; + oid:=substring(b FROM p+1 FOR width); p:=p+width; + kind:=CASE mode WHEN '40000' THEN 4 WHEN '100644' THEN 1 WHEN '100664' THEN 1 WHEN '100640' THEN 1 + WHEN '100755' THEN 2 WHEN '120000' THEN 3 ELSE 0 END; + IF kind=0 THEN RAISE EXCEPTION 'source Git tree contains an unsupported entry'; END IF; + result:=array_append(result,jsonb_build_object('kind',kind,'name',encode(name,'hex'), + 'oid',hash_kind||':'||encode(oid,'hex'))); count:=count+1; + IF count>131072 THEN RAISE EXCEPTION 'source Git tree entry budget exceeded'; END IF; + END LOOP; + IF EXISTS(SELECT 1 FROM unnest(result) AS rows(value) GROUP BY value->>'name' HAVING count(*)>1) THEN + RAISE EXCEPTION 'source Git tree has duplicate names'; + END IF; + SELECT coalesce(jsonb_agg(value ORDER BY decode(value->>'name','hex')),'[]'::jsonb) INTO sorted + FROM unnest(result) AS rows(value); + RETURN sorted; +END $$; + +CREATE FUNCTION mst2_metadata_compute_source_proof(pid text,tree_oid text,p bytea,g bigint) RETURNS jsonb +LANGUAGE plpgsql VOLATILE SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE profile jsonb:=mst2_metadata_native_profile(pid); profile_hash bytea; body bytea; decoded jsonb; + item jsonb; mapped jsonb; entry_rows jsonb[]:=ARRAY[]::jsonb[]; entries jsonb; child_root bytea; + child_binding record; source_entries jsonb; reference_rows jsonb[]:=ARRAY[]::jsonb[]; + reference jsonb; fact record; built jsonb; proof jsonb; source_revision uuid; body_digest bytea; +BEGIN + IF split_part(tree_oid,':',1) IS DISTINCT FROM profile->>'hash_kind' + OR tree_oid !~ '^(sha1:[0-9a-f]{40}|sha256:[0-9a-f]{64}|blake3:[0-9a-f]{64})$' THEN + RAISE EXCEPTION 'source tree attestation crossed its tagged hash kind'; + END IF; + profile_hash:=sha256(convert_to('mega.mst2.native-profile.v1','UTF8')||decode('00','hex')||convert_to(profile::text,'UTF8')); + SELECT sub_trees INTO body FROM $CORE_SCHEMA$.mega_tree WHERE tree_id=split_part(tree_oid,':',2); + IF NOT FOUND THEN RAISE EXCEPTION 'source Git tree is missing from its captured core'; END IF; + body_digest:=sha256(body); + source_revision:=$CORE_SCHEMA$.mst2_route_capture_source_tree(split_part(tree_oid,':',2),body_digest); + decoded:=mst2_metadata_decode_git_tree(body,profile->>'hash_kind'); + FOR item IN SELECT value FROM jsonb_array_elements(decoded) LOOP + mapped:=jsonb_build_object('kind',(item->>'kind')::integer,'name',item->>'name'); + reference:=jsonb_build_object('kind',(item->>'kind')::integer,'git_oid',item->>'oid'); + IF (item->>'kind')::integer=4 THEN + SELECT a.root_page,a.root_generation,a.root_certificate_digest INTO child_binding FROM mst2_metadata_source_root_attestation a + JOIN mst2_metadata_current cur ON cur.page_id=a.root_page AND cur.generation=a.root_generation + JOIN mst2_metadata_lifetime life USING(page_id,generation) + JOIN mst2_metadata_graph_node node USING(page_id,generation) + JOIN mst2_metadata_prepare origin ON origin.prepare_id=a.origin_prepare_id + JOIN $CORE_SCHEMA$.mega_tree child_tree ON child_tree.tree_id=split_part(a.tagged_tree_oid,':',2) + WHERE a.tagged_tree_oid=item->>'oid' AND a.profile_digest=profile_hash AND a.source_profile=profile + AND a.namespace_uuid=(SELECT namespace_uuid FROM mst2_metadata_family_identity WHERE singleton=1) + AND node.state='LIVE' AND node.certificate_digest=a.root_certificate_digest + AND $CORE_SCHEMA$.mst2_route_source_tree_matches(split_part(a.tagged_tree_oid,':',2),a.source_revision,a.source_body_digest) + AND (life.state='LIVE' AND origin.state='COMMITTED' OR life.state='RESERVED' AND origin.prepare_id=pid AND origin.state='PREPARING') + AND NOT EXISTS(SELECT 1 FROM mst2_metadata_gc_op gc WHERE gc.page_id=a.root_page AND gc.generation=a.root_generation) + ORDER BY a.attestation_id LIMIT 1; + IF NOT FOUND THEN RAISE EXCEPTION 'source directory lacks its exact-profile certified child root'; END IF; + child_root:=child_binding.root_page; + mapped:=mapped||jsonb_build_object('child',encode(child_root,'hex')); + reference:=reference||jsonb_build_object('child_root',encode(child_root,'hex'), + 'child_generation',child_binding.root_generation,'child_certificate',encode(child_binding.root_certificate_digest,'hex')); + ELSE + SELECT size,raw_sha256 INTO fact FROM $CORE_SCHEMA$.mst2_verified_object + WHERE storage_domain='git' AND git_oid=split_part(item->>'oid',':',2) AND object_kind='blob' + AND state='VERIFIED' AND verification_version=2 FOR SHARE NOWAIT; + IF NOT FOUND OR fact.size<0 OR fact.size>8796093022208 OR octet_length(fact.raw_sha256)<>32 + OR (item->>'kind')::integer=3 AND fact.size NOT BETWEEN 1 AND 4095 THEN + RAISE EXCEPTION 'source file lacks its valid current verified-object fact'; + END IF; + mapped:=mapped||jsonb_build_object('size',fact.size,'content_id',encode(fact.raw_sha256,'hex')); + reference:=reference||jsonb_build_object('size',fact.size,'content_digest',encode(fact.raw_sha256,'hex')); + END IF; + reference_rows:=array_append(reference_rows,reference||jsonb_build_object('name',item->>'name')); + entry_rows:=array_append(entry_rows,mapped); + END LOOP; + entries:=to_jsonb(entry_rows); + SELECT coalesce(jsonb_object_agg(value->>'name',value-'name'),'{}'::jsonb) INTO source_entries + FROM unnest(reference_rows) input(value); + built:=mst2_metadata_build_map(entries); + IF decode(built->>'page_id','hex')<>p OR NOT EXISTS(SELECT 1 FROM mst2_metadata_page_certificate c + JOIN mst2_metadata_current cur USING(page_id,generation) JOIN mst2_metadata_graph_node n USING(page_id,generation) + WHERE c.page_id=p AND c.generation=g AND n.state='LIVE' AND n.certificate_digest=c.certificate_digest) THEN + RAISE EXCEPTION 'source projection differs from its independently canonical certified root'; + END IF; + proof:=jsonb_build_object('namespace',(SELECT namespace_uuid::text FROM mst2_metadata_family_identity WHERE singleton=1), + 'source_profile',profile,'profile_digest',encode(profile_hash,'hex'),'tagged_tree_oid',tree_oid, + 'source_body_digest',encode(body_digest,'hex'),'source_revision',source_revision::text,'root_page',encode(p,'hex'),'root_generation',g, + 'root_certificate',(SELECT encode(certificate_digest,'hex') FROM mst2_metadata_page_certificate WHERE page_id=p AND generation=g), + 'source_entry_count',jsonb_array_length(entries),'source_entries',source_entries,'source_work_units',built->'source_work_units', + 'encoded_map_digest',encode(sha256(convert_to(entries::text,'UTF8')),'hex')); + RETURN proof||jsonb_build_object('attestation',encode(sha256(convert_to('mega.mst2.source-root.v1','UTF8') + ||decode('00','hex')||convert_to(proof::text,'UTF8')),'hex')); +END $$; + +CREATE FUNCTION mst2_metadata_source_attestation_guard() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE proof jsonb; +BEGIN + IF TG_OP<>'INSERT' THEN RAISE EXCEPTION 'source root attestation history is immutable'; END IF; + proof:=mst2_metadata_compute_source_proof(NEW.origin_prepare_id,NEW.tagged_tree_oid,NEW.root_page,NEW.root_generation); + IF NEW.source_revision IS NULL THEN NEW.source_revision:=(proof->>'source_revision')::uuid; END IF; + IF NEW.namespace_uuid::text IS DISTINCT FROM proof->>'namespace' OR NEW.source_profile IS DISTINCT FROM proof->'source_profile' + OR NEW.profile_digest IS DISTINCT FROM decode(proof->>'profile_digest','hex') + OR NEW.source_body_digest IS DISTINCT FROM decode(proof->>'source_body_digest','hex') + OR NEW.source_revision IS DISTINCT FROM (proof->>'source_revision')::uuid + OR NEW.root_certificate_digest IS DISTINCT FROM decode(proof->>'root_certificate','hex') + OR NEW.attestation_digest IS DISTINCT FROM decode(proof->>'attestation','hex') OR NEW.source_proof IS DISTINCT FROM proof THEN + RAISE EXCEPTION 'source attestation was not independently derived from the captured core and canonical root'; + END IF; + RETURN NEW; +END $$; +CREATE TRIGGER mst2_metadata_source_attestation_guard BEFORE INSERT OR UPDATE OR DELETE ON mst2_metadata_source_root_attestation + FOR EACH ROW EXECUTE FUNCTION mst2_metadata_source_attestation_guard(); + +CREATE FUNCTION mst2_metadata_source_attestation_committed() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +BEGIN + IF NOT EXISTS(SELECT 1 FROM mst2_metadata_prepare WHERE prepare_id=NEW.origin_prepare_id AND state='COMMITTED') THEN + RAISE EXCEPTION 'source attestation cannot commit without its definitive finalized preparation'; + END IF; + IF NOT $CORE_SCHEMA$.mst2_route_source_tree_matches(split_part(NEW.tagged_tree_oid,':',2),NEW.source_revision,NEW.source_body_digest) + OR NEW.source_profile IS DISTINCT FROM mst2_metadata_native_profile(NEW.origin_prepare_id) THEN + RAISE EXCEPTION 'source attestation cannot commit after its exact captured source body or profile changed'; + END IF; + RETURN NULL; +END $$; +CREATE CONSTRAINT TRIGGER mst2_metadata_source_attestation_committed AFTER INSERT ON mst2_metadata_source_root_attestation + DEFERRABLE INITIALLY DEFERRED FOR EACH ROW EXECUTE FUNCTION mst2_metadata_source_attestation_committed(); + +CREATE FUNCTION mst2_metadata_reuse_root_guard() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE profile jsonb; profile_hash bytea; +BEGIN + IF TG_OP<>'INSERT' THEN RAISE EXCEPTION 'rooted reuse membership history is immutable'; END IF; + profile:=mst2_metadata_native_profile(NEW.prepare_id); + profile_hash:=sha256(convert_to('mega.mst2.native-profile.v1','UTF8')||decode('00','hex')||convert_to(profile::text,'UTF8')); + IF NOT EXISTS(SELECT 1 FROM mst2_metadata_prepare q,mst2_metadata_source_root_attestation a + JOIN mst2_metadata_current cur ON cur.page_id=a.root_page AND cur.generation=a.root_generation + JOIN mst2_metadata_lifetime life USING(page_id,generation) + JOIN mst2_metadata_graph_node node USING(page_id,generation) + JOIN mst2_metadata_prepare origin ON origin.prepare_id=a.origin_prepare_id + JOIN $CORE_SCHEMA$.mega_tree tree ON tree.tree_id=split_part(a.tagged_tree_oid,':',2) + WHERE q.prepare_id=NEW.prepare_id AND q.state='PREPARING' AND a.attestation_id=NEW.attestation_id + AND a.root_page=NEW.root_page AND a.root_generation=NEW.root_generation AND a.attestation_digest=NEW.attestation_digest + AND a.profile_digest=profile_hash AND a.source_profile=profile + AND $CORE_SCHEMA$.mst2_route_source_tree_matches(split_part(a.tagged_tree_oid,':',2),a.source_revision,a.source_body_digest) + AND origin.state='COMMITTED' AND life.state='LIVE' AND life.graph_domain='qualified-v1' + AND node.state='LIVE' AND node.certificate_digest=a.root_certificate_digest + AND NOT EXISTS(SELECT 1 FROM mst2_metadata_gc_op gc WHERE gc.page_id=a.root_page AND gc.generation=a.root_generation)) THEN + RAISE EXCEPTION 'rooted reuse membership lacks its exact finalized source and current lifetime'; + END IF; + IF (SELECT count(*) FROM mst2_metadata_prepare_reuse_root WHERE prepare_id=NEW.prepare_id)>=4096 THEN + RAISE EXCEPTION 'rooted reuse boundary budget exceeded'; + END IF; + RETURN NEW; +END $$; +CREATE TRIGGER mst2_metadata_reuse_root_guard BEFORE INSERT OR UPDATE OR DELETE ON mst2_metadata_prepare_reuse_root + FOR EACH ROW EXECUTE FUNCTION mst2_metadata_reuse_root_guard(); + +CREATE FUNCTION mst2_metadata_reuse_index_guard() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +BEGIN + IF TG_OP='UPDATE' THEN RAISE EXCEPTION 'rooted reuse index cannot retarget an exact lifetime'; END IF; + IF TG_OP='DELETE' THEN + IF NOT EXISTS(SELECT 1 FROM mst2_metadata_gc_op WHERE page_id=OLD.root_page AND generation=OLD.root_generation AND state='PENDING') THEN + RAISE EXCEPTION 'rooted reuse index retirement requires its exact GC claim'; + END IF; + RETURN OLD; + END IF; + IF NOT EXISTS(SELECT 1 FROM mst2_metadata_source_root_attestation a + JOIN mst2_metadata_prepare q ON q.prepare_id=a.origin_prepare_id + JOIN mst2_metadata_current cur ON cur.page_id=a.root_page AND cur.generation=a.root_generation + JOIN mst2_metadata_lifetime life USING(page_id,generation) + JOIN mst2_metadata_graph_node node USING(page_id,generation) + WHERE a.attestation_id=NEW.attestation_id AND a.profile_digest=NEW.profile_digest AND a.tagged_tree_oid=NEW.tagged_tree_oid + AND a.root_page=NEW.root_page AND a.root_generation=NEW.root_generation AND a.attestation_digest=NEW.attestation_digest + AND q.state='COMMITTED' AND life.state='LIVE' AND node.state='LIVE' AND node.certificate_digest=a.root_certificate_digest + AND NOT EXISTS(SELECT 1 FROM mst2_metadata_gc_op gc WHERE gc.page_id=a.root_page AND gc.generation=a.root_generation)) THEN + RAISE EXCEPTION 'rooted reuse index is not bound to its definitive exact current source proof'; + END IF; + RETURN NEW; +END $$; +CREATE TRIGGER mst2_metadata_reuse_index_guard BEFORE INSERT OR UPDATE OR DELETE ON mst2_metadata_reuse_index + FOR EACH ROW EXECUTE FUNCTION mst2_metadata_reuse_index_guard(); + +CREATE FUNCTION mst2_metadata_root_anchor_guard() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +BEGIN + IF TG_OP='UPDATE' THEN RAISE EXCEPTION 'rooted anchors cannot retarget immutable owner or root identities'; END IF; + IF TG_OP='DELETE' THEN + IF OLD.anchor_kind IN ('PREPARE','REUSE') THEN + IF NOT EXISTS(SELECT 1 FROM mst2_metadata_prepare WHERE prepare_id=OLD.prepare_id AND state IN ('PREPARING','COMMITTED','ABORTED')) THEN + RAISE EXCEPTION 'temporary anchor owner history is missing'; + END IF; + ELSIF OLD.anchor_kind='SESSION' THEN + IF NOT EXISTS(SELECT 1 FROM mst2_qualified_session_incarnation s + WHERE s.snapshot_id=OLD.snapshot_id AND s.session_incarnation=OLD.session_incarnation AND s.state='RETIRED') + OR EXISTS(SELECT 1 FROM mst2_qualified_lease_binding l WHERE l.snapshot_id=OLD.snapshot_id + AND l.session_incarnation=OLD.session_incarnation AND l.state='ACTIVE') THEN + RAISE EXCEPTION 'session root still has its active incarnation or leases'; + END IF; + ELSIF OLD.anchor_kind='LEASE' THEN + IF NOT EXISTS(SELECT 1 FROM mst2_qualified_lease_binding WHERE lease_id=OLD.lease_id AND state IN ('RELEASED','EXPIRED')) + OR EXISTS(SELECT 1 FROM mst2_metadata_reader_operation WHERE lease_id=OLD.lease_id AND state='ACTIVE') THEN + RAISE EXCEPTION 'lease root still has its active lease or readers'; + END IF; + ELSE + IF NOT EXISTS(SELECT 1 FROM mst2_metadata_reader_operation r WHERE r.operation_id=OLD.reader_operation_id + AND (r.state='FINISHED' OR r.state='EXPIRED' AND r.hard_deadline_unix<=floor(extract(epoch FROM clock_timestamp()))::bigint)) THEN + RAISE EXCEPTION 'reader root still has an active operation'; + END IF; + END IF; + RETURN OLD; + END IF; + IF NOT EXISTS(SELECT 1 FROM mst2_metadata_graph_node node + JOIN mst2_metadata_current cur USING(page_id,generation) + JOIN mst2_metadata_lifetime life USING(page_id,generation) + JOIN mst2_metadata_page_certificate proof USING(page_id,generation) + WHERE node.page_id=NEW.root_page AND node.generation=NEW.root_generation AND node.state='LIVE' + AND node.certificate_digest=NEW.root_certificate_digest AND proof.certificate_digest=NEW.root_certificate_digest + AND life.state IN ('RESERVED','LIVE') AND life.graph_domain='qualified-v1' + AND NOT EXISTS(SELECT 1 FROM mst2_metadata_gc_op gc WHERE gc.page_id=node.page_id AND gc.generation=node.generation)) THEN + RAISE EXCEPTION 'rooted anchor does not protect its exact canonical current graph'; + END IF; + IF NEW.anchor_kind IN ('PREPARE','REUSE') THEN + IF NEW.owner_key IS DISTINCT FROM NEW.prepare_id OR NEW.snapshot_id IS NOT NULL OR NEW.session_incarnation IS NOT NULL + OR NEW.lease_id IS NOT NULL OR NEW.reader_operation_id IS NOT NULL + OR NOT EXISTS(SELECT 1 FROM mst2_metadata_prepare WHERE prepare_id=NEW.prepare_id + AND state IN ('PREPARING','COMMITTED') AND coverage_retired_at IS NULL) + OR NEW.anchor_kind='PREPARE' AND NOT (EXISTS(SELECT 1 FROM mst2_metadata_prepare_page + WHERE prepare_id=NEW.prepare_id AND page_id=NEW.root_page AND generation=NEW.root_generation) + OR EXISTS(SELECT 1 FROM mst2_metadata_prepare q JOIN mst2_metadata_prepare_reuse_root r USING(prepare_id) + WHERE q.prepare_id=NEW.prepare_id AND q.plan_kind='ROOTED' AND q.metadata_root=NEW.root_page + AND r.root_page=NEW.root_page AND r.root_generation=NEW.root_generation)) + OR NEW.anchor_kind='REUSE' AND NOT EXISTS(SELECT 1 FROM mst2_metadata_prepare_reuse_root + WHERE prepare_id=NEW.prepare_id AND root_page=NEW.root_page AND root_generation=NEW.root_generation) THEN + RAISE EXCEPTION 'temporary anchor differs from its immutable delta or reused-root owner'; + END IF; + ELSIF NEW.anchor_kind='SESSION' THEN + IF NEW.owner_key IS DISTINCT FROM NEW.snapshot_id||':'||NEW.session_incarnation::text OR NEW.prepare_id IS NOT NULL + OR NEW.lease_id IS NOT NULL OR NEW.reader_operation_id IS NOT NULL + OR NOT EXISTS(SELECT 1 FROM mst2_qualified_session_incarnation s WHERE s.snapshot_id=NEW.snapshot_id + AND s.session_incarnation=NEW.session_incarnation AND s.metadata_root=NEW.root_page AND s.root_generation=NEW.root_generation AND s.state='READY') THEN + RAISE EXCEPTION 'session anchor differs from its exact ready incarnation'; + END IF; + ELSIF NEW.anchor_kind='LEASE' THEN + IF NEW.owner_key IS DISTINCT FROM NEW.lease_id OR NEW.prepare_id IS NOT NULL OR NEW.reader_operation_id IS NOT NULL + OR NOT EXISTS(SELECT 1 FROM mst2_qualified_lease_binding l WHERE l.lease_id=NEW.lease_id + AND l.snapshot_id=NEW.snapshot_id AND l.session_incarnation=NEW.session_incarnation + AND l.metadata_root=NEW.root_page AND l.root_generation=NEW.root_generation AND l.state='ACTIVE' + AND l.expires_at_unix>floor(extract(epoch FROM clock_timestamp()))::bigint) THEN + RAISE EXCEPTION 'lease anchor differs from its exact active lease'; + END IF; + ELSE + IF NEW.owner_key IS DISTINCT FROM NEW.reader_operation_id::text OR NEW.prepare_id IS NOT NULL + OR NOT EXISTS(SELECT 1 FROM mst2_metadata_reader_operation r JOIN mst2_qualified_lease_binding l USING(lease_id) + WHERE r.operation_id=NEW.reader_operation_id AND r.lease_id=NEW.lease_id AND r.snapshot_id=NEW.snapshot_id + AND r.session_incarnation=NEW.session_incarnation AND r.root_page=NEW.root_page AND r.root_generation=NEW.root_generation + AND r.state='ACTIVE' AND l.state='ACTIVE' AND l.lease_epoch=r.lease_epoch + AND r.hard_deadline_unix<=l.expires_at_unix AND r.hard_deadline_unix>floor(extract(epoch FROM clock_timestamp()))::bigint) THEN + RAISE EXCEPTION 'reader anchor differs from its exact active lease operation'; + END IF; + END IF; + RETURN NEW; +END $$; +CREATE TRIGGER mst2_metadata_root_anchor_guard BEFORE INSERT OR UPDATE OR DELETE ON mst2_metadata_root_anchor + FOR EACH ROW EXECUTE FUNCTION mst2_metadata_root_anchor_guard(); + +CREATE FUNCTION mst2_metadata_reuse_root_protected() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +BEGIN + IF EXISTS(SELECT 1 FROM mst2_metadata_prepare WHERE prepare_id=NEW.prepare_id + AND state IN ('PREPARING','COMMITTED') AND coverage_retired_at IS NULL) + AND NOT EXISTS(SELECT 1 FROM mst2_metadata_root_anchor WHERE anchor_kind='REUSE' AND prepare_id=NEW.prepare_id + AND root_page=NEW.root_page AND root_generation=NEW.root_generation) THEN + RAISE EXCEPTION 'reused boundary must commit with continuously owned root protection'; + END IF; + RETURN NULL; +END $$; +CREATE CONSTRAINT TRIGGER mst2_metadata_reuse_root_protected AFTER INSERT ON mst2_metadata_prepare_reuse_root + DEFERRABLE INITIALLY DEFERRED FOR EACH ROW EXECUTE FUNCTION mst2_metadata_reuse_root_protected(); + +CREATE FUNCTION mst2_metadata_temporary_anchor_continuity() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE q mst2_metadata_prepare%ROWTYPE; +BEGIN + IF OLD.anchor_kind NOT IN ('PREPARE','REUSE') THEN RETURN NULL; END IF; + SELECT * INTO STRICT q FROM mst2_metadata_prepare WHERE prepare_id=OLD.prepare_id; + IF q.state='ABORTED' THEN RETURN NULL; END IF; + IF q.coverage_retired_at IS NULL THEN + IF NOT EXISTS(SELECT 1 FROM mst2_metadata_root_anchor a WHERE a.anchor_kind=OLD.anchor_kind + AND a.owner_key=OLD.owner_key AND a.prepare_id=OLD.prepare_id + AND a.root_page=OLD.root_page AND a.root_generation=OLD.root_generation + AND a.root_certificate_digest=OLD.root_certificate_digest) THEN + RAISE EXCEPTION 'active preparation cannot lose its continuously owned canonical root'; + END IF; + ELSIF q.state<>'COMMITTED' OR NOT (mst2_metadata_session_covers_prepare(q.prepare_id) + OR mst2_metadata_orphan_prepare_retired(q.prepare_id)) THEN + RAISE EXCEPTION 'retired preparation coverage requires a definitive independent session root'; + END IF; + RETURN NULL; +END $$; +CREATE CONSTRAINT TRIGGER mst2_metadata_temporary_anchor_continuity AFTER DELETE ON mst2_metadata_root_anchor + DEFERRABLE INITIALLY DEFERRED FOR EACH ROW EXECUTE FUNCTION mst2_metadata_temporary_anchor_continuity(); diff --git a/src/jupiter/storage/qualified_metadata_canonical.sql b/src/jupiter/storage/qualified_metadata_canonical.sql new file mode 100644 index 00000000..2ce0c283 --- /dev/null +++ b/src/jupiter/storage/qualified_metadata_canonical.sql @@ -0,0 +1,227 @@ +CREATE FUNCTION mst2_metadata_read_le(b bytea,p integer,w integer) RETURNS numeric +LANGUAGE plpgsql IMMUTABLE STRICT SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE v numeric:=0; power numeric:=1; i integer; +BEGIN + IF w NOT IN (2,4,8) OR p<0 OR p>octet_length(b)-w THEN + RAISE EXCEPTION 'MTP2 integer is out of bounds'; + END IF; + FOR i IN 0..w-1 LOOP + v:=v+get_byte(b,p+i)*power; power:=power*256; + END LOOP; + RETURN v; +END $$; + +CREATE FUNCTION mst2_metadata_write_le(v numeric,w integer) RETURNS bytea +LANGUAGE plpgsql IMMUTABLE STRICT SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE result bytea; i integer; +BEGIN + IF w NOT IN (2,4,8) OR v<0 OR v<>trunc(v) OR v>=power(256::numeric,w) THEN + RAISE EXCEPTION 'MTP2 output integer is out of range'; + END IF; + result:=decode(repeat('00',w),'hex'); + FOR i IN 0..w-1 LOOP result:=set_byte(result,i,mod(v,256)::integer); v:=trunc(v/256); END LOOP; + RETURN result; +END $$; + +CREATE FUNCTION mst2_metadata_encode_entry(e jsonb) RETURNS bytea +LANGUAGE plpgsql IMMUTABLE STRICT SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE kind integer:=(e->>'kind')::integer; name bytea:=decode(e->>'name','hex'); target bytea; result bytea; +BEGIN + IF kind IS NULL OR kind NOT BETWEEN 1 AND 4 OR name IS NULL THEN RAISE EXCEPTION 'MTP2 output entry is incomplete'; END IF; + PERFORM mst2_metadata_valid_name(name); + result:=set_byte(decode('00','hex'),0,kind)||mst2_metadata_write_le(octet_length(name),2)||name; + IF kind=4 THEN + target:=decode(e->>'child','hex'); + IF target IS NULL OR octet_length(target)<>32 OR target=decode(repeat('00',32),'hex') THEN + RAISE EXCEPTION 'MTP2 output directory reference is invalid'; + END IF; + ELSE + target:=decode(e->>'content_id','hex'); + IF target IS NULL OR octet_length(target)<>32 OR e->>'size' IS NULL THEN + RAISE EXCEPTION 'MTP2 output file reference is invalid'; + END IF; + result:=result||mst2_metadata_write_le((e->>'size')::numeric,8); + END IF; + RETURN result||target; +END $$; + +CREATE FUNCTION mst2_metadata_build_map(items jsonb,budget bigint DEFAULT 67108864) RETURNS jsonb +LANGUAGE plpgsql IMMUTABLE STRICT SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE n integer; item jsonb; previous bytea; name bytea; minimum bytea; maximum bytea; + raw bytea:=''::bytea; result bytea; prefix bytea; terminal jsonb; + partition record; grouped jsonb:='[]'::jsonb; child jsonb; + visits bigint:=0; encoded_bytes bigint:=0; pages bigint:=1; total_bytes bigint; payload bytea; children integer:=0; +BEGIN + IF jsonb_typeof(items)<>'array' OR budget NOT BETWEEN 1 AND 67108864 THEN + RAISE EXCEPTION 'MTP2 canonical map input or work budget is invalid'; + END IF; + n:=jsonb_array_length(items); + IF n>131072 THEN RAISE EXCEPTION 'MTP2 canonical map exceeds its entry budget'; END IF; + FOR item IN SELECT value FROM jsonb_array_elements(items) LOOP + name:=decode(item->>'name','hex'); + IF previous IS NOT NULL AND previous>=name THEN RAISE EXCEPTION 'MTP2 build names are not strictly byte ordered'; END IF; + previous:=name; maximum:=name; IF minimum IS NULL THEN minimum:=name; END IF; + payload:=mst2_metadata_encode_entry(item); visits:=visits+1+octet_length(payload); + IF visits>budget THEN RAISE EXCEPTION 'MTP2 canonical source work budget exceeded'; END IF; + encoded_bytes:=encoded_bytes+octet_length(payload); + IF n<=128 AND 20+encoded_bytes<=16384 THEN raw:=raw||payload; END IF; + END LOOP; + IF n<=128 AND 20+encoded_bytes<=16384 THEN + result:=decode('4d5450320000','hex')||mst2_metadata_write_le(n,2)||mst2_metadata_write_le(n,8) + ||mst2_metadata_write_le(octet_length(raw),4)||raw; + RETURN jsonb_build_object('bytes',encode(result,'hex'),'page_id',encode(sha256( + convert_to('mega.mst2.metapage','UTF8')||decode('00','hex')||result),'hex'), + 'source_work_units',visits,'pages',pages,'metadata_bytes',octet_length(result)); + END IF; + prefix:=mst2_metadata_lcp(minimum,maximum); + SELECT value INTO terminal FROM jsonb_array_elements(items) WHERE decode(value->>'name','hex')=prefix; + visits:=visits+2*n; + IF visits>budget THEN RAISE EXCEPTION 'MTP2 canonical source grouping work budget exceeded'; END IF; + payload:=mst2_metadata_write_le(octet_length(prefix),2)||prefix; + IF terminal IS NULL THEN payload:=payload||decode('00','hex'); + ELSE payload:=payload||decode('01','hex')||mst2_metadata_encode_entry(terminal); END IF; + total_bytes:=0; + FOR partition IN SELECT get_byte(decode(value->>'name','hex'),octet_length(prefix)) AS label, + jsonb_agg(value ORDER BY decode(value->>'name','hex')) AS members + FROM jsonb_array_elements(items) WHERE octet_length(decode(value->>'name','hex'))>octet_length(prefix) + GROUP BY get_byte(decode(value->>'name','hex'),octet_length(prefix)) ORDER BY label LOOP + grouped:=partition.members; + IF visits>=budget THEN RAISE EXCEPTION 'MTP2 canonical source work budget exceeded'; END IF; + child:=mst2_metadata_build_map(grouped,budget-visits); visits:=visits+(child->>'source_work_units')::bigint; + pages:=pages+(child->>'pages')::bigint; total_bytes:=total_bytes+(child->>'metadata_bytes')::bigint; + IF pages>4096 OR total_bytes>67108864 THEN RAISE EXCEPTION 'MTP2 canonical source encoding exceeds metadata budget'; END IF; + payload:=payload||set_byte(decode('00','hex'),0,partition.label)||mst2_metadata_write_le(jsonb_array_length(grouped),8) + ||decode(child->>'page_id','hex'); children:=children+1; + END LOOP; + IF children+(terminal IS NOT NULL)::integer<2 OR 20+octet_length(payload)>16384 THEN + RAISE EXCEPTION 'MTP2 canonical branch shape or size is invalid'; + END IF; + result:=decode('4d5450320100','hex')||mst2_metadata_write_le(children,2)||mst2_metadata_write_le(n,8) + ||mst2_metadata_write_le(octet_length(payload),4)||payload; + total_bytes:=total_bytes+octet_length(result); + IF total_bytes>67108864 THEN RAISE EXCEPTION 'MTP2 canonical source encoding exceeds metadata byte budget'; END IF; + RETURN jsonb_build_object('bytes',encode(result,'hex'),'page_id',encode(sha256( + convert_to('mega.mst2.metapage','UTF8')||decode('00','hex')||result),'hex'), + 'source_work_units',visits,'pages',pages,'metadata_bytes',total_bytes); +END $$; + +CREATE FUNCTION mst2_metadata_valid_name(n bytea) RETURNS boolean +LANGUAGE plpgsql IMMUTABLE STRICT SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE i integer; +BEGIN + IF octet_length(n) NOT BETWEEN 1 AND 255 OR n IN (decode('2e','hex'),decode('2e2e','hex')) THEN + RAISE EXCEPTION 'MTP2 name has an invalid length or dot component'; + END IF; + FOR i IN 0..octet_length(n)-1 LOOP + IF get_byte(n,i) IN (0,47) THEN RAISE EXCEPTION 'MTP2 name contains NUL or slash'; END IF; + END LOOP; + PERFORM convert_from(n,'UTF8'); + RETURN true; +END $$; + +CREATE FUNCTION mst2_metadata_decode_entry(b bytea,p integer) RETURNS jsonb +LANGUAGE plpgsql IMMUTABLE STRICT SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE start integer:=p; kind integer; n integer; name bytea; target bytea; size numeric; +BEGIN + IF p<0 OR p>=octet_length(b) THEN RAISE EXCEPTION 'MTP2 entry kind is truncated'; END IF; + kind:=get_byte(b,p); p:=p+1; + IF kind NOT BETWEEN 1 AND 4 THEN RAISE EXCEPTION 'MTP2 entry kind is invalid'; END IF; + n:=mst2_metadata_read_le(b,p,2)::integer; p:=p+2; + IF n>octet_length(b)-p THEN RAISE EXCEPTION 'MTP2 entry name is truncated'; END IF; + name:=substring(b FROM p+1 FOR n); p:=p+n; + PERFORM mst2_metadata_valid_name(name); + IF kind=4 THEN + IF p>octet_length(b)-32 THEN RAISE EXCEPTION 'MTP2 directory reference is truncated'; END IF; + target:=substring(b FROM p+1 FOR 32); p:=p+32; + IF target=decode(repeat('00',32),'hex') THEN RAISE EXCEPTION 'MTP2 empty directory has a zero reference'; END IF; + RETURN jsonb_build_object('end',p,'kind',kind,'name',encode(name,'hex'), + 'encoded_bytes',p-start,'child',encode(target,'hex')); + END IF; + size:=mst2_metadata_read_le(b,p,8); p:=p+8; + IF p>octet_length(b)-32 THEN RAISE EXCEPTION 'MTP2 file content reference is truncated'; END IF; + target:=substring(b FROM p+1 FOR 32); p:=p+32; + RETURN jsonb_build_object('end',p,'kind',kind,'name',encode(name,'hex'), + 'encoded_bytes',p-start,'size',size,'content_id',encode(target,'hex')); +END $$; + +CREATE FUNCTION mst2_metadata_decode_local(b bytea) RETURNS jsonb +LANGUAGE plpgsql IMMUTABLE STRICT SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE kind integer; n integer; count numeric; declared numeric; p integer:=20; plen integer; + i integer; label integer; previous_label integer:=-1; previous_name bytea; name bytea; prefix bytea; + terminal integer:=0; item jsonb; entries jsonb:='[]'::jsonb; children jsonb:='[]'::jsonb; + refs jsonb:='[]'::jsonb; child bytea; child_count numeric; entry_bytes bigint:=0; +BEGIN + IF octet_length(b) NOT BETWEEN 20 AND 16384 THEN RAISE EXCEPTION 'MTP2 page length is invalid'; END IF; + IF substring(b FROM 1 FOR 4)<>decode('4d545032','hex') OR get_byte(b,5)<>0 THEN + RAISE EXCEPTION 'MTP2 magic or flags are invalid'; + END IF; + kind:=get_byte(b,4); n:=mst2_metadata_read_le(b,6,2)::integer; + count:=mst2_metadata_read_le(b,8,8); + IF count>9223372036854775807 OR mst2_metadata_read_le(b,16,4)<>octet_length(b)-20 THEN + RAISE EXCEPTION 'MTP2 header count or payload length is invalid'; + END IF; + IF kind=0 THEN + IF n>128 OR count<>n THEN RAISE EXCEPTION 'MTP2 leaf count is invalid'; END IF; + IF n>0 THEN + FOR i IN 1..n LOOP + item:=mst2_metadata_decode_entry(b,p); p:=(item->>'end')::integer; + name:=decode(item->>'name','hex'); + IF previous_name IS NOT NULL AND previous_name>=name THEN + RAISE EXCEPTION 'MTP2 leaf names are not strictly byte ordered'; + END IF; + previous_name:=name; entries:=entries||jsonb_build_array(item-'end'); + entry_bytes:=entry_bytes+(item->>'encoded_bytes')::bigint; + IF (item->>'kind')::integer=4 THEN + refs:=refs||jsonb_build_array(jsonb_build_object('kind','DIRECTORY','name',item->>'name','child',item->>'child')); + END IF; + END LOOP; + END IF; + ELSIF kind=1 THEN + plen:=mst2_metadata_read_le(b,p,2)::integer; p:=p+2; + IF plen>octet_length(b)-p THEN RAISE EXCEPTION 'MTP2 branch prefix is truncated'; END IF; + prefix:=substring(b FROM p+1 FOR plen); p:=p+plen; + IF p>=octet_length(b) THEN RAISE EXCEPTION 'MTP2 terminal flag is truncated'; END IF; + terminal:=get_byte(b,p); p:=p+1; + IF terminal NOT IN (0,1) THEN RAISE EXCEPTION 'MTP2 terminal flag is invalid'; END IF; + IF terminal=1 THEN + item:=mst2_metadata_decode_entry(b,p); p:=(item->>'end')::integer; + IF decode(item->>'name','hex')<>prefix THEN RAISE EXCEPTION 'MTP2 terminal name differs from prefix'; END IF; + entries:=entries||jsonb_build_array(item-'end'); entry_bytes:=(item->>'encoded_bytes')::bigint; + IF (item->>'kind')::integer=4 THEN + refs:=refs||jsonb_build_array(jsonb_build_object('kind','DIRECTORY','name',item->>'name','child',item->>'child')); + END IF; + END IF; + IF n>256 OR n+terminal<2 THEN RAISE EXCEPTION 'MTP2 branch group count is invalid'; END IF; + declared:=terminal; + IF n>0 THEN + FOR i IN 1..n LOOP + IF p>=octet_length(b) THEN RAISE EXCEPTION 'MTP2 branch child label is truncated'; END IF; + label:=get_byte(b,p); p:=p+1; + IF label<=previous_label THEN RAISE EXCEPTION 'MTP2 branch labels are not strictly ordered'; END IF; + previous_label:=label; child_count:=mst2_metadata_read_le(b,p,8); p:=p+8; + IF child_count NOT BETWEEN 1 AND 9223372036854775807 THEN RAISE EXCEPTION 'MTP2 child count is invalid'; END IF; + IF p>octet_length(b)-32 THEN RAISE EXCEPTION 'MTP2 branch child digest is truncated'; END IF; + child:=substring(b FROM p+1 FOR 32); p:=p+32; + declared:=declared+child_count; + IF declared>9223372036854775807 THEN RAISE EXCEPTION 'MTP2 branch count overflows'; END IF; + item:=jsonb_build_object('kind','RADIX','label',label,'count',child_count,'child',encode(child,'hex')); + children:=children||jsonb_build_array(item); refs:=refs||jsonb_build_array(item); + END LOOP; + END IF; + IF declared<>count THEN RAISE EXCEPTION 'MTP2 branch header count differs from children'; END IF; + ELSE + RAISE EXCEPTION 'MTP2 page kind is invalid'; + END IF; + IF p<>octet_length(b) THEN RAISE EXCEPTION 'MTP2 page has trailing bytes'; END IF; + RETURN jsonb_build_object('kind',kind,'count',count,'prefix',encode(prefix,'hex'), + 'entries',entries,'children',children,'refs',refs,'direct_entry_bytes',entry_bytes, + 'page_id',encode(sha256(convert_to('mega.mst2.metapage','UTF8')||decode('00','hex')||b),'hex')); +END $$; + +CREATE FUNCTION mst2_metadata_lcp(a bytea,b bytea) RETURNS bytea +LANGUAGE plpgsql IMMUTABLE STRICT SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE n integer:=least(octet_length(a),octet_length(b)); i integer:=0; +BEGIN + WHILE i Value { + Value::Array( + entries + .iter() + .map(|entry| { + json!({"kind":1,"name":hex::encode(&entry.name),"size":u64::MAX, + "content_id":hex::encode([17;32])}) + }) + .collect(), + ) +} + +fn files(names: impl IntoIterator) -> Vec { + let mut entries: Vec<_> = names + .into_iter() + .map(|name| Entry::file(EntryKind::Regular, name.as_bytes(), u64::MAX, [17; 32])) + .collect(); + entries.sort_by(|left, right| left.name.cmp(&right.name)); + entries +} + +#[tokio::test] +async fn database_canonical_builder_matches_pinned_codec_at_split_and_name_boundaries() { + let (_config, _core, _namespace, q, _guard) = fixture().await; + let terminal_names = + std::iter::once("a".to_owned()).chain((0..129).map(|index| format!("a{index:03}"))); + let cases = [ + Vec::new(), + files((0..128).map(|index| format!("f{index:03}"))), + files((0..129).map(|index| format!("f{index:03}"))), + files((0..128).map(|index| format!("{}{index:03}", "x".repeat(197)))), + files(terminal_names), + files((0..160).map(|index| format!("目录{index:03}"))), + ]; + for entries in cases { + let bytes = Page::build(&entries).unwrap(); + let proof: Value = q + .query_one_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + "SELECT mst2_metadata_build_map($1::jsonb) AS proof", + [encoded_entries(&entries).to_string().into()], + )) + .await + .unwrap() + .unwrap() + .try_get("", "proof") + .unwrap(); + assert_eq!(proof["bytes"].as_str().unwrap(), hex::encode(&bytes)); + assert_eq!( + proof["page_id"].as_str().unwrap(), + hex::encode(page_id(&bytes)) + ); + let decoded: Value = q + .query_one_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + "SELECT mst2_metadata_decode_local($1) AS decoded", + [bytes.into()], + )) + .await + .unwrap() + .unwrap() + .try_get("", "decoded") + .unwrap(); + assert_eq!(decoded["count"].as_u64().unwrap(), entries.len() as u64); + } +} + +#[tokio::test] +async fn database_parser_rejects_nonexact_bytes_and_invalid_names() { + let (_config, _core, _namespace, q, _guard) = fixture().await; + let bytes = Page::build(&files(["file".to_owned()])).unwrap(); + let mut trailing = bytes.clone(); + trailing.push(0); + let mut wrong_length = bytes.clone(); + wrong_length[16] = wrong_length[16].wrapping_add(1); + let mut flags = bytes.clone(); + flags[5] = 1; + let mut wrong_count = bytes.clone(); + wrong_count[8] = 2; + let mut invalid_utf8 = bytes.clone(); + invalid_utf8[23] = 255; + let mut slash = bytes.clone(); + slash[23] = b'/'; + for malformed in [ + trailing, + wrong_length, + flags, + wrong_count, + invalid_utf8, + slash, + ] { + assert!( + q.query_one_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + "SELECT mst2_metadata_decode_local($1)", + [malformed.into()], + )) + .await + .is_err() + ); + } + let mut entries = encoded_entries(&files(["one".to_owned(), "two".to_owned()])); + entries.as_array_mut().unwrap().reverse(); + assert!( + q.query_one_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + "SELECT mst2_metadata_build_map($1::jsonb)", + [entries.to_string().into()], + )) + .await + .is_err() + ); +} + +#[tokio::test] +async fn exact_typed_certificates_keep_shared_directory_occurrences_and_dedup_physical_edges() { + let (config, core, _namespace, q, _guard) = fixture().await; + let writer = ShadowQualifiedMetadataWriter::open(&core, &config) + .await + .unwrap(); + let pages = prepared(); + let receipt = write(&writer, "certified-shared-directory", &pages).await; + assert_eq!( + count(&q, "SELECT count(*) FROM mst2_metadata_page_certificate").await, + 2 + ); + assert_eq!( + count( + &q, + "SELECT count(*) FROM mst2_metadata_verified_ref WHERE reference_kind='DIRECTORY'" + ) + .await, + 2 + ); + assert_eq!( + count(&q, "SELECT count(*) FROM mst2_metadata_graph_edge").await, + 1 + ); + assert_eq!( + count( + &q, + "SELECT max(rank)::bigint FROM mst2_metadata_page_certificate" + ) + .await, + 1 + ); + assert_eq!(writer.finalize(receipt.intent()).await.unwrap(), receipt); + for table in [ + "mst2_metadata_page_certificate", + "mst2_metadata_verified_ref", + "mst2_metadata_source_root_attestation", + "mst2_metadata_prepare_reuse_root", + "mst2_metadata_reuse_index", + "mst2_metadata_root_anchor", + "mst2_metadata_reader_operation", + ] { + assert!( + q.execute_unprepared(&format!("TRUNCATE {table} CASCADE")) + .await + .is_err() + ); + } + for sql in [ + "UPDATE mst2_metadata_page_certificate SET rank=rank+1", + "DELETE FROM mst2_metadata_verified_ref", + "UPDATE mst2_metadata_verified_ref SET reference_ordinal=reference_ordinal+1", + ] { + assert!(q.execute_unprepared(sql).await.is_err()); + } +} + +#[tokio::test] +async fn raw_sql_cannot_certify_a_leafable_branch_from_valid_certified_children() { + let (config, core, namespace, q, _guard) = fixture().await; + let writer = ShadowQualifiedMetadataWriter::open(&core, &config) + .await + .unwrap(); + let receipt = write(&writer, "raw-canonical-oracle", &prepared()).await; + let child = Page::build(&[Entry::file(EntryKind::Regular, b"file", 3, [42; 32])]).unwrap(); + let mut payload = vec![1, 0, b'f', 1, 1, 1, 0, b'f']; + payload.extend_from_slice(&3_u64.to_le_bytes()); + payload.extend_from_slice(&[17; 32]); + payload.push(b'i'); + payload.extend_from_slice(&1_u64.to_le_bytes()); + payload.extend_from_slice(&page_id(&child)); + let mut branch = b"MTP2\x01\x00".to_vec(); + branch.extend_from_slice(&1_u16.to_le_bytes()); + branch.extend_from_slice(&2_u64.to_le_bytes()); + branch.extend_from_slice(&(payload.len() as u32).to_le_bytes()); + branch.extend_from_slice(&payload); + let root = page_id(&branch); + let pid = uuid::Uuid::new_v4().to_string(); + let txn = q + .begin_with_config(Some(IsolationLevel::ReadCommitted), None) + .await + .unwrap(); + namespace.enter(&txn).await.unwrap(); + txn.execute_raw(Statement::from_sql_and_values(DbBackend::Postgres, + "INSERT INTO mst2_metadata_prepare SELECT (jsonb_populate_record(NULL::mst2_metadata_prepare, + to_jsonb(q)||jsonb_build_object('prepare_id',$1::text,'operation_id','raw-leafable-branch','metadata_root',$2::bytea, + 'state','PREPARING','committed_at',NULL,'node_count',2,'edge_count',1,'total_bytes',$3::bigint))).* + FROM mst2_metadata_prepare q WHERE q.prepare_id=$4", + [pid.clone().into(),root.to_vec().into(),((branch.len()+child.len()) as i64).into(),receipt.intent().prepare_id().into()], + )).await.unwrap(); + txn.execute_raw(Statement::from_sql_and_values(DbBackend::Postgres, + "INSERT INTO mst2_metadata_lifetime(page_id,node_id,generation,state,metadata_codec,expected_size,graph_domain) + VALUES($1,'page:sha256:'||encode($1,'hex'),1,'RESERVED',1,$2,'qualified-v1')", + [root.to_vec().into(),(branch.len() as i32).into()], + )).await.unwrap(); + txn.execute_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + "INSERT INTO mst2_metadata_current VALUES($1,1)", + [root.to_vec().into()], + )) + .await + .unwrap(); + txn.execute_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + "INSERT INTO mst2_metadata_prepare_page SELECT $1,$2,1,$3 + UNION ALL SELECT $1,page_id,generation,expected_size + FROM mst2_metadata_prepare_page WHERE prepare_id=$4 AND page_id=$5", + [ + pid.clone().into(), + root.to_vec().into(), + (branch.len() as i32).into(), + receipt.intent().prepare_id().into(), + page_id(&child).to_vec().into(), + ], + )) + .await + .unwrap(); + txn.execute_raw(Statement::from_sql_and_values(DbBackend::Postgres, + "INSERT INTO mst2_metadata_payload(page_id,generation,metadata_codec,byte_size,payload) VALUES($1,1,1,$2,$3)", + [root.to_vec().into(),(branch.len() as i32).into(),branch.into()], + )).await.unwrap(); + let error = txn + .query_one_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + "SELECT mst2_metadata_certify_page($1,$2,1)", + [pid.into(), root.to_vec().into()], + )) + .await + .unwrap_err(); + assert!(error.to_string().contains("leafable"), "{error}"); + txn.rollback().await.unwrap(); +} + +#[tokio::test] +async fn temporary_root_cannot_disappear_in_a_later_transaction() { + let (config, core, namespace, q, _guard) = fixture().await; + let writer = ShadowQualifiedMetadataWriter::open(&core, &config) + .await + .unwrap(); + let receipt = write(&writer, "continuous-owned-root", &prepared()).await; + let txn = q + .begin_with_config(Some(IsolationLevel::ReadCommitted), None) + .await + .unwrap(); + namespace.enter(&txn).await.unwrap(); + txn.execute_raw(Statement::from_sql_and_values(DbBackend::Postgres, + "INSERT INTO mst2_metadata_root_anchor(anchor_id,anchor_kind,owner_key,root_page,root_generation,root_certificate_digest,prepare_id) + SELECT $1::uuid,'PREPARE',$2,c.page_id,c.generation,c.certificate_digest,$2 + FROM mst2_metadata_page_certificate c JOIN mst2_metadata_prepare q ON q.metadata_root=c.page_id + WHERE q.prepare_id=$2", + [uuid::Uuid::new_v4().to_string().into(),receipt.intent().prepare_id().into()], + )).await.unwrap(); + txn.commit().await.unwrap(); + let error = q + .execute_unprepared("DELETE FROM mst2_metadata_root_anchor WHERE anchor_kind='PREPARE'") + .await + .unwrap_err(); + assert!( + error + .to_string() + .contains("continuously owned canonical root"), + "{error}" + ); + assert_eq!( + count(&q, "SELECT count(*) FROM mst2_metadata_root_anchor").await, + 1 + ); +} + +#[tokio::test] +async fn source_root_is_derived_from_current_core_body_and_verified_blob_facts() { + let (config, core, namespace, q, _guard) = fixture().await; + let entries = [Entry::file(EntryKind::Regular, b"file", 3, [17; 32])]; + let page = Page::build(&entries).unwrap(); + let mut builder = MetadataDagBuilder::new(MetadataDagLimits::default()); + builder.add_directory(&page, &entries).unwrap(); + let prepared = PreparedNativeMetadataRetention::test_installation( + Arc::new(builder.finish(page_id(&page)).unwrap()), + "/", + ); + let mut body = b"100644 file\0".to_vec(); + body.extend_from_slice(&[187; 20]); + core.execute_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + "INSERT INTO mega_tree(id,tree_id,sub_trees,size,created_at,pack_id,pack_offset,commit_id) + VALUES(1,$1,$2,0,now(),'fixture',0,'fixture')", + ["a".repeat(40).into(), body.clone().into()], + )) + .await + .unwrap(); + core.execute_raw(Statement::from_sql_and_values(DbBackend::Postgres, + "INSERT INTO mst2_verified_object(storage_domain,git_oid,object_kind,raw_sha256,size,verification_version,state,created_at) + VALUES('git',$1,'blob',$2,3,2,'VERIFIED',now())", + ["b".repeat(40).into(),vec![17_u8;32].into()], + )).await.unwrap(); + let writer = ShadowQualifiedMetadataWriter::open(&core, &config) + .await + .unwrap(); + let receipt = write(&writer, "bound-real-core-source", &prepared).await; + let aid = uuid::Uuid::new_v4().to_string(); + let txn = q + .begin_with_config(Some(IsolationLevel::ReadCommitted), None) + .await + .unwrap(); + namespace.enter(&txn).await.unwrap(); + txn.execute_raw(Statement::from_sql_and_values(DbBackend::Postgres, + "INSERT INTO mst2_metadata_source_root_attestation(attestation_id,namespace_uuid,origin_prepare_id, + tagged_tree_oid,source_profile,profile_digest,source_body_digest,root_page,root_generation, + root_certificate_digest,source_proof,attestation_digest) + SELECT $1::uuid,(proof->>'namespace')::uuid,$2,$3,proof->'source_profile',decode(proof->>'profile_digest','hex'), + decode(proof->>'source_body_digest','hex'),$4,1,decode(proof->>'root_certificate','hex'),proof, + decode(proof->>'attestation','hex') FROM (SELECT mst2_metadata_compute_source_proof($2,$3,$4,1) AS proof) input", + [aid.into(),receipt.intent().prepare_id().into(),prepared.fixed_root_tree_oid().into(),page_id(&page).to_vec().into()], + )).await.unwrap(); + txn.commit().await.unwrap(); + assert_eq!( + count( + &q, + "SELECT count(*) FROM mst2_metadata_source_root_attestation" + ) + .await, + 1 + ); + body[7] = b'F'; + core.execute_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + "UPDATE mega_tree SET sub_trees=$1 WHERE tree_id=$2", + [body.into(), "a".repeat(40).into()], + )) + .await + .unwrap(); + let error = q + .query_one_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + "SELECT mst2_metadata_compute_source_proof($1,$2,$3,1)", + [ + receipt.intent().prepare_id().into(), + prepared.fixed_root_tree_oid().into(), + page_id(&page).to_vec().into(), + ], + )) + .await + .unwrap_err(); + assert!( + error + .to_string() + .contains("independently canonical certified root"), + "{error}" + ); +} + +pub(super) async fn seeded_rooted_plan( + core: &DatabaseConnection, + tree_char: char, + name: &str, + id: i64, +) -> (RootedMetadataInstallPlan, MetadataPagePayload) { + let entries = [Entry::file( + EntryKind::Regular, + name.as_bytes(), + 3, + [17; 32], + )]; + let bytes = Page::build(&entries).unwrap(); + let page = page_id(&bytes); + let mut builder = MetadataDagBuilder::new(MetadataDagLimits::default()); + builder.add_directory(&bytes, &entries).unwrap(); + let prepared = PreparedNativeMetadataRetention::test_installation( + Arc::new(builder.finish(page).unwrap()), + "/", + ); + let mut identity = prepared.install_plan().unwrap().identity; + identity.tagged_root_tree_oid = format!("sha1:{}", tree_char.to_string().repeat(40)); + let mut body = format!("100644 {name}\0").into_bytes(); + body.extend_from_slice(&[187; 20]); + core.execute_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + "INSERT INTO mega_tree(id,tree_id,sub_trees,size,created_at,pack_id,pack_offset,commit_id) + VALUES($1,$2,$3,0,now(),'fixture',0,'fixture')", + [ + id.into(), + tree_char.to_string().repeat(40).into(), + body.into(), + ], + )) + .await + .unwrap(); + core.execute_raw(Statement::from_sql_and_values(DbBackend::Postgres, + "INSERT INTO mst2_verified_object(storage_domain,git_oid,object_kind,raw_sha256,size,verification_version,state,created_at) + VALUES('git',$1,'blob',$2,3,2,'VERIFIED',now()) ON CONFLICT(storage_domain,git_oid,object_kind) DO NOTHING", + ["b".repeat(40).into(),vec![17_u8;32].into()], + )).await.unwrap(); + let source_roots = BTreeMap::from([(identity.tagged_root_tree_oid.clone(), page)]); + let plan = RootedMetadataInstallPlan::new( + identity, + page, + BTreeMap::from([(page, bytes.len() as u64)]), + BTreeSet::new(), + BTreeMap::new(), + source_roots, + ) + .unwrap(); + ( + plan, + MetadataPagePayload { + id: page, + size: bytes.len() as u64, + bytes, + }, + ) +} + +#[tokio::test] +async fn actual_rooted_cold_and_zero_delta_reuse_keep_one_canonical_graph() { + let (config, core, _namespace, q, _guard) = fixture().await; + let (cold, payload) = seeded_rooted_plan(&core, 'a', "file", 1).await; + let writer = RootedQualifiedMetadataRepository::open(&core, &config) + .await + .unwrap(); + let intent = writer + .begin_intent("actual-rooted-cold", &cold) + .await + .unwrap(); + writer.install_pages(&intent, &[payload]).await.unwrap(); + let receipt = writer.finalize(&intent).await.unwrap(); + assert_eq!(receipt.metadata_root(), cold.root); + assert_eq!( + writer.recover("actual-rooted-cold", &cold).await.unwrap(), + Some(receipt.clone()) + ); + let proof=q.query_one_raw(Statement::from_string(DbBackend::Postgres, + "SELECT a.attestation_id::text,a.attestation_digest,a.root_certificate_digest FROM mst2_metadata_source_root_attestation a" + )).await.unwrap().unwrap(); + let reused = RootedReuseRoot { + generation: intent.root_generation(), + attestation_id: uuid::Uuid::parse_str( + &proof.try_get::("", "attestation_id").unwrap(), + ) + .unwrap(), + attestation_digest: proof + .try_get::>("", "attestation_digest") + .unwrap() + .try_into() + .unwrap(), + certificate_digest: proof + .try_get::>("", "root_certificate_digest") + .unwrap() + .try_into() + .unwrap(), + }; + let warm = RootedMetadataInstallPlan::new( + cold.identity.clone(), + cold.root, + BTreeMap::new(), + BTreeSet::new(), + BTreeMap::from([(cold.root, reused)]), + cold.source_roots.clone(), + ) + .unwrap(); + let warm_intent = writer + .begin_intent("actual-zero-delta-root", &warm) + .await + .unwrap(); + assert_eq!( + writer.finalize(&warm_intent).await.unwrap().metadata_root(), + cold.root + ); + assert_eq!( + count(&q, "SELECT count(*) FROM mst2_metadata_payload").await, + 1 + ); + assert_eq!( + count(&q, "SELECT count(*) FROM mst2_metadata_page_certificate").await, + 1 + ); + assert_eq!( + count( + &q, + "SELECT count(*) FROM mst2_metadata_source_root_attestation" + ) + .await, + 1 + ); + assert_eq!(count(&q,"SELECT count(*) FROM mst2_metadata_prepare WHERE plan_kind='ROOTED' AND state='COMMITTED' AND node_count=0").await,1); + let error = q + .execute_unprepared("DELETE FROM mst2_metadata_root_anchor WHERE anchor_kind='REUSE'") + .await + .unwrap_err(); + assert!( + error + .to_string() + .contains("continuously owned canonical root"), + "{error}" + ); +} + +#[tokio::test] +async fn changed_ancestor_installs_only_delta_and_accepts_independently_attested_source_aliases() { + let (config, core, _namespace, q, _guard) = fixture().await; + let writer = RootedQualifiedMetadataRepository::open(&core, &config) + .await + .unwrap(); + let (first, payload) = seeded_rooted_plan(&core, 'a', "file", 1).await; + let first_intent = writer.begin_intent("alias-first", &first).await.unwrap(); + writer + .install_pages(&first_intent, &[payload]) + .await + .unwrap(); + writer.finalize(&first_intent).await.unwrap(); + let (alias, payload) = seeded_rooted_plan(&core, 'c', "file", 2).await; + assert_eq!(alias.root, first.root); + let alias_intent = writer.begin_intent("alias-second", &alias).await.unwrap(); + writer + .install_pages(&alias_intent, &[payload]) + .await + .unwrap(); + writer.finalize(&alias_intent).await.unwrap(); + let first_hint = writer + .lookup_reuse(&first.identity.tagged_root_tree_oid, &first.identity) + .await + .unwrap() + .unwrap(); + let alias_hint = writer + .lookup_reuse(&alias.identity.tagged_root_tree_oid, &alias.identity) + .await + .unwrap() + .unwrap(); + assert_eq!(first_hint.page_id, alias_hint.page_id); + assert_eq!( + first_hint.proof.certificate_digest, + alias_hint.proof.certificate_digest + ); + assert_ne!( + first_hint.proof.attestation_id, + alias_hint.proof.attestation_id + ); + + let entries = [ + Entry::dir(b"one", first.root), + Entry::dir(b"two", first.root), + ]; + let bytes = Page::build(&entries).unwrap(); + let root = page_id(&bytes); + let mut body = b"40000 one\0".to_vec(); + body.extend_from_slice(&[0xaa; 20]); + body.extend_from_slice(b"40000 two\0"); + body.extend_from_slice(&[0xcc; 20]); + core.execute_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + "INSERT INTO mega_tree(id,tree_id,sub_trees,size,created_at,pack_id,pack_offset,commit_id) + VALUES(3,$1,$2,0,now(),'fixture',0,'fixture')", + ["d".repeat(40).into(), body.into()], + )) + .await + .unwrap(); + let mut identity = first.identity.clone(); + identity.tagged_root_tree_oid = format!("sha1:{}", "d".repeat(40)); + let plan = RootedMetadataInstallPlan::new( + identity.clone(), + root, + BTreeMap::from([(root, bytes.len() as u64)]), + BTreeSet::from([(root, first.root)]), + BTreeMap::from([(first.root, first_hint.proof)]), + BTreeMap::from([ + (identity.tagged_root_tree_oid, root), + (first.identity.tagged_root_tree_oid.clone(), first.root), + (alias.identity.tagged_root_tree_oid.clone(), first.root), + ]), + ) + .unwrap(); + let intent = writer + .begin_intent("changed-ancestor-aliases", &plan) + .await + .unwrap(); + writer + .install_pages( + &intent, + &[MetadataPagePayload { + id: root, + size: bytes.len() as u64, + bytes, + }], + ) + .await + .unwrap(); + let receipt = writer.finalize(&intent).await.unwrap(); + assert_eq!(receipt.metadata_root(), root); + assert_eq!( + count(&q, "SELECT count(*) FROM mst2_metadata_payload").await, + 2 + ); + assert_eq!( + count(&q, "SELECT count(*) FROM mst2_metadata_graph_edge").await, + 1 + ); + assert_eq!( + count( + &q, + "SELECT count(*) FROM mst2_metadata_verified_ref WHERE reference_kind='DIRECTORY'" + ) + .await, + 2 + ); + assert_eq!( + count( + &q, + "SELECT count(*) FROM mst2_metadata_source_root_attestation" + ) + .await, + 3 + ); + let row = q.query_one_raw(Statement::from_sql_and_values(DbBackend::Postgres, + "SELECT node_count,edge_count,(SELECT count(*) FROM mst2_metadata_prepare_reuse_root r + WHERE r.prepare_id=q.prepare_id)::bigint AS reuse_count FROM mst2_metadata_prepare q WHERE prepare_id=$1", + [intent.prepare_id().into()])).await.unwrap().unwrap(); + assert_eq!(row.try_get::("", "node_count").unwrap(), 1); + assert_eq!(row.try_get::("", "edge_count").unwrap(), 1); + assert_eq!(row.try_get::("", "reuse_count").unwrap(), 1); + assert_eq!( + writer + .recover("changed-ancestor-aliases", &plan) + .await + .unwrap(), + Some(receipt) + ); +} + +#[tokio::test] +async fn late_raw_delta_membership_is_rejected_after_intent_commit() { + let (config, core, _namespace, q, _guard) = fixture().await; + let (plan, payload) = seeded_rooted_plan(&core, 'a', "file", 1).await; + let (donor, donor_payload) = seeded_rooted_plan(&core, 'c', "other", 2).await; + let writer = RootedQualifiedMetadataRepository::open(&core, &config) + .await + .unwrap(); + let donor_intent = writer + .begin_intent("rooted-extra-donor", &donor) + .await + .unwrap(); + writer + .install_pages(&donor_intent, &[donor_payload.clone()]) + .await + .unwrap(); + writer.finalize(&donor_intent).await.unwrap(); + let intent = writer + .begin_intent("rooted-bounded-intent", &plan) + .await + .unwrap(); + writer.install_pages(&intent, &[payload]).await.unwrap(); + let error=q.execute_raw(Statement::from_sql_and_values(DbBackend::Postgres, + "INSERT INTO mst2_metadata_prepare_page(prepare_id,page_id,generation,expected_size) VALUES($1,$2,1,$3)", + [intent.prepare_id().into(),donor.root.to_vec().into(),(donor_payload.size as i32).into()], + )).await.unwrap_err(); + assert!( + error + .to_string() + .contains("missing or extra exact delta/reuse"), + "{error}" + ); + assert_eq!( + writer.finalize(&intent).await.unwrap().metadata_root(), + plan.root + ); +} diff --git a/src/jupiter/storage/qualified_metadata_certificates.sql b/src/jupiter/storage/qualified_metadata_certificates.sql new file mode 100644 index 00000000..6ddd7e66 --- /dev/null +++ b/src/jupiter/storage/qualified_metadata_certificates.sql @@ -0,0 +1,293 @@ +CREATE TABLE mst2_metadata_page_certificate ( + page_id bytea NOT NULL,generation bigint NOT NULL,certificate_digest bytea NOT NULL CHECK(octet_length(certificate_digest)=32), + origin_prepare_id text NOT NULL REFERENCES mst2_metadata_prepare(prepare_id), + namespace_uuid uuid NOT NULL REFERENCES mst2_metadata_family_identity(namespace_uuid), + proof_revision integer NOT NULL CHECK(proof_revision=1),metadata_codec smallint NOT NULL CHECK(metadata_codec=1), + byte_size integer NOT NULL CHECK(byte_size BETWEEN 20 AND 16384), + rank integer NOT NULL CHECK(rank BETWEEN 0 AND 4095), + map_entry_count bigint NOT NULL CHECK(map_entry_count BETWEEN 0 AND 131072), + map_encoded_entry_bytes bigint NOT NULL CHECK(map_encoded_entry_bytes BETWEEN 0 AND 67108864), + min_name bytea,max_name bytea, + relative_path_bytes integer NOT NULL CHECK(relative_path_bytes BETWEEN 0 AND 4096), + relative_components integer NOT NULL CHECK(relative_components BETWEEN 0 AND 256), + closure_nodes_upper integer NOT NULL CHECK(closure_nodes_upper BETWEEN 1 AND 4096), + closure_edges_upper integer NOT NULL CHECK(closure_edges_upper BETWEEN 0 AND 16384), + closure_bytes_upper bigint NOT NULL CHECK(closure_bytes_upper BETWEEN 20 AND 67108864), + closure_entries_upper bigint NOT NULL CHECK(closure_entries_upper BETWEEN 0 AND 131072), + canonical_proof jsonb NOT NULL, + PRIMARY KEY(page_id,generation),UNIQUE(page_id,generation,certificate_digest), + FOREIGN KEY(page_id,generation) REFERENCES mst2_metadata_lifetime(page_id,generation), + CHECK((map_entry_count=0)=(min_name IS NULL AND max_name IS NULL)), + CHECK(map_entry_count=0 OR (min_name IS NOT NULL AND max_name IS NOT NULL AND min_name<=max_name)) +); +CREATE TABLE mst2_metadata_verified_ref ( + parent_page bytea NOT NULL,parent_generation bigint NOT NULL, + reference_ordinal integer NOT NULL CHECK(reference_ordinal BETWEEN 0 AND 256), + reference_kind text NOT NULL CHECK(reference_kind IN ('DIRECTORY','RADIX')), + name bytea,label integer,advertised_count bigint, + child_page bytea NOT NULL,child_generation bigint NOT NULL, + child_certificate_digest bytea NOT NULL CHECK(octet_length(child_certificate_digest)=32), + PRIMARY KEY(parent_page,parent_generation,reference_ordinal), + FOREIGN KEY(parent_page,parent_generation) REFERENCES mst2_metadata_page_certificate(page_id,generation), + FOREIGN KEY(child_page,child_generation,child_certificate_digest) + REFERENCES mst2_metadata_page_certificate(page_id,generation,certificate_digest), + CHECK((reference_kind='DIRECTORY' AND name IS NOT NULL AND label IS NULL AND advertised_count IS NULL) + OR (reference_kind='RADIX' AND name IS NULL AND label IS NOT NULL AND advertised_count IS NOT NULL + AND label BETWEEN 0 AND 255 AND advertised_count>0)) +); +CREATE INDEX mst2_metadata_verified_ref_child ON mst2_metadata_verified_ref(child_page,child_generation,parent_page,parent_generation); +ALTER TABLE mst2_metadata_graph_node ADD COLUMN certificate_digest bytea NOT NULL, + ADD FOREIGN KEY(page_id,generation,certificate_digest) + REFERENCES mst2_metadata_page_certificate(page_id,generation,certificate_digest); + +CREATE FUNCTION mst2_metadata_child_certificate(p bytea,g bigint,pid text) RETURNS jsonb +LANGUAGE plpgsql VOLATILE SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE c mst2_metadata_page_certificate%ROWTYPE; +BEGIN + SELECT proof.* INTO c FROM mst2_metadata_page_certificate proof + JOIN mst2_metadata_current cur USING(page_id,generation) + JOIN mst2_metadata_lifetime life USING(page_id,generation) + JOIN mst2_metadata_graph_node node USING(page_id,generation) + JOIN mst2_metadata_payload body USING(page_id,generation) + WHERE proof.page_id=p AND proof.generation=g AND life.graph_domain='qualified-v1' + AND node.state='LIVE' AND node.certificate_digest=proof.certificate_digest + AND node.metadata_codec=proof.metadata_codec AND body.metadata_codec=proof.metadata_codec + AND body.byte_size=proof.byte_size AND node.bytes=proof.byte_size AND life.expected_size=proof.byte_size + AND (life.state='LIVE' OR (life.state='RESERVED' AND proof.origin_prepare_id=pid + AND EXISTS(SELECT 1 FROM mst2_metadata_prepare q WHERE q.prepare_id=pid AND q.state='PREPARING'))) + AND NOT EXISTS(SELECT 1 FROM mst2_metadata_gc_op op WHERE op.page_id=p AND op.generation=g); + IF NOT FOUND THEN RAISE EXCEPTION 'MTP2 child lacks its exact certified current graph'; END IF; + RETURN jsonb_build_object('page',encode(c.page_id,'hex'),'generation',c.generation, + 'certificate',encode(c.certificate_digest,'hex'),'rank',c.rank,'map_entry_count',c.map_entry_count, + 'map_encoded_entry_bytes',c.map_encoded_entry_bytes,'min_name',encode(c.min_name,'hex'), + 'max_name',encode(c.max_name,'hex'),'relative_path_bytes',c.relative_path_bytes, + 'relative_components',c.relative_components,'closure_nodes_upper',c.closure_nodes_upper, + 'closure_edges_upper',c.closure_edges_upper,'closure_bytes_upper',c.closure_bytes_upper, + 'closure_entries_upper',c.closure_entries_upper); +END $$; + +CREATE FUNCTION mst2_metadata_compute_certificate(p bytea,g bigint,pid text) RETURNS jsonb +LANGUAGE plpgsql VOLATILE SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE body mst2_metadata_payload%ROWTYPE; q mst2_metadata_prepare%ROWTYPE; decoded jsonb; + ref jsonb; entry jsonb; child jsonb; bound jsonb; bound_refs jsonb:='[]'::jsonb; + seen jsonb:='[]'::jsonb; children jsonb:='{}'::jsonb; child_page bytea; child_generation bigint; + key text; name bytea; child_min bytea; child_max bytea; prefix bytea; minimum bytea; maximum bytea; + map_count bigint:=0; map_bytes bigint:=0; path_bytes integer:=0; components integer:=0; rank integer:=0; + nodes bigint:=1; edges bigint:=0; bytes bigint; entries bigint; proof jsonb; +BEGIN + SELECT * INTO q FROM mst2_metadata_prepare WHERE prepare_id=pid AND state='PREPARING' + AND graph_domain='qualified-v1' AND mst2_metadata_scope_matches(primary_scope); + IF NOT FOUND THEN RAISE EXCEPTION 'MTP2 certification needs an active exact preparation'; END IF; + SELECT b.* INTO body FROM mst2_metadata_payload b JOIN mst2_metadata_current cur USING(page_id,generation) + JOIN mst2_metadata_lifetime life USING(page_id,generation) + JOIN mst2_metadata_prepare_page member USING(page_id,generation) + WHERE b.page_id=p AND b.generation=g AND member.prepare_id=pid AND life.state='RESERVED' + AND life.graph_domain='qualified-v1' AND life.metadata_codec=q.metadata_codec + AND b.metadata_codec=q.metadata_codec AND b.byte_size=member.expected_size AND b.byte_size=life.expected_size + AND NOT EXISTS(SELECT 1 FROM mst2_metadata_gc_op op WHERE op.page_id=p AND op.generation=g); + IF NOT FOUND THEN RAISE EXCEPTION 'MTP2 certification crossed its durable payload lifetime'; END IF; + decoded:=mst2_metadata_decode_local(body.payload); + IF decode(decoded->>'page_id','hex')<>p THEN RAISE EXCEPTION 'MTP2 durable payload digest differs from its page'; END IF; + map_count:=jsonb_array_length(decoded->'entries'); map_bytes:=(decoded->>'direct_entry_bytes')::bigint; + bytes:=body.byte_size; entries:=map_count; + FOR ref IN SELECT value FROM jsonb_array_elements(decoded->'refs') LOOP + child_page:=decode(ref->>'child','hex'); + SELECT member.generation INTO child_generation FROM mst2_metadata_prepare_page member + WHERE member.prepare_id=pid AND member.page_id=child_page; + IF NOT FOUND THEN + SELECT reused.root_generation INTO child_generation FROM mst2_metadata_prepare_reuse_root reused + JOIN mst2_metadata_root_anchor anchor ON anchor.prepare_id=reused.prepare_id + AND anchor.anchor_kind='REUSE' AND anchor.owner_key=pid + AND anchor.root_page=reused.root_page AND anchor.root_generation=reused.root_generation + WHERE reused.prepare_id=pid AND reused.root_page=child_page; + IF NOT FOUND THEN RAISE EXCEPTION 'MTP2 reference is outside exact delta or reused-root membership'; END IF; + END IF; + child:=mst2_metadata_child_certificate(child_page,child_generation,pid); + bound:=ref||jsonb_build_object('generation',child_generation,'certificate',child->>'certificate'); + bound_refs:=bound_refs||jsonb_build_array(bound); + key:=encode(child_page,'hex')||':'||child_generation; + children:=children||jsonb_build_object(encode(child_page,'hex'),child); + IF NOT seen ? key THEN + seen:=seen||jsonb_build_array(key); rank:=greatest(rank,(child->>'rank')::integer+1); + nodes:=nodes+(child->>'closure_nodes_upper')::bigint; + edges:=edges+1+(child->>'closure_edges_upper')::bigint; + bytes:=bytes+(child->>'closure_bytes_upper')::bigint; + entries:=entries+(child->>'closure_entries_upper')::bigint; + END IF; + IF ref->>'kind'='RADIX' THEN + child_min:=decode(child->>'min_name','hex'); child_max:=decode(child->>'max_name','hex'); + prefix:=decode(decoded->>'prefix','hex'); + IF child_min IS NULL OR child_max IS NULL OR (child->>'map_entry_count')::bigint<>(ref->>'count')::bigint + OR octet_length(child_min)<=octet_length(prefix) OR octet_length(child_max)<=octet_length(prefix) + OR substring(child_min FROM 1 FOR octet_length(prefix))<>prefix + OR substring(child_max FROM 1 FOR octet_length(prefix))<>prefix + OR get_byte(child_min,octet_length(prefix))<>(ref->>'label')::integer + OR get_byte(child_max,octet_length(prefix))<>(ref->>'label')::integer THEN + RAISE EXCEPTION 'MTP2 radix child count or name partition differs from its canonical certificate'; + END IF; + IF minimum IS NULL OR child_minmaximum THEN maximum:=child_max; END IF; + map_count:=map_count+(child->>'map_entry_count')::bigint; + map_bytes:=map_bytes+(child->>'map_encoded_entry_bytes')::bigint; + path_bytes:=greatest(path_bytes,(child->>'relative_path_bytes')::integer); + components:=greatest(components,(child->>'relative_components')::integer); + END IF; + IF rank>4095 OR nodes>4096 OR edges>16384 OR bytes>67108864 OR entries>131072 THEN + RAISE EXCEPTION 'MTP2 certified closure upper bound exceeds its fixed budget'; + END IF; + END LOOP; + FOR entry IN SELECT value FROM jsonb_array_elements(decoded->'entries') LOOP + name:=decode(entry->>'name','hex'); + IF minimum IS NULL OR namemaximum THEN maximum:=name; END IF; + IF (entry->>'kind')::integer=4 THEN + child:=children->(entry->>'child'); + path_bytes:=greatest(path_bytes,1+octet_length(name)+(child->>'relative_path_bytes')::integer); + components:=greatest(components,1+(child->>'relative_components')::integer); + ELSE + path_bytes:=greatest(path_bytes,1+octet_length(name)); components:=greatest(components,1); + END IF; + END LOOP; + IF map_count<>(decoded->>'count')::bigint OR map_count>131072 OR map_bytes>67108864 + OR path_bytes>4096 OR components>256 THEN RAISE EXCEPTION 'MTP2 canonical map or path budget is invalid'; END IF; + IF (decoded->>'kind')::integer=1 AND ( + (map_count<=128 AND 20+map_bytes<=16384) + OR decode(decoded->>'prefix','hex') IS DISTINCT FROM mst2_metadata_lcp(minimum,maximum)) THEN + RAISE EXCEPTION 'MTP2 branch is leafable or its prefix is not the true canonical LCP'; + END IF; + proof:=jsonb_build_object('namespace',(SELECT namespace_uuid::text FROM mst2_metadata_family_identity WHERE singleton=1), + 'page',encode(p,'hex'),'generation',g,'codec',body.metadata_codec,'byte_size',body.byte_size, + 'proof_revision',1,'page_kind',(decoded->>'kind')::integer,'references',bound_refs,'rank',rank, + 'map_entry_count',map_count,'map_encoded_entry_bytes',map_bytes,'min_name',encode(minimum,'hex'), + 'max_name',encode(maximum,'hex'),'relative_path_bytes',path_bytes,'relative_components',components, + 'closure_nodes_upper',nodes,'closure_edges_upper',edges,'closure_bytes_upper',bytes,'closure_entries_upper',entries); + RETURN proof||jsonb_build_object('certificate',encode(sha256(convert_to('mega.mst2.canonical-proof.v1','UTF8') + ||decode('00','hex')||convert_to(proof::text,'UTF8')),'hex')); +END $$; + +CREATE FUNCTION mst2_metadata_certificate_guard() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE proof jsonb; +BEGIN + IF TG_OP<>'INSERT' THEN RAISE EXCEPTION 'MTP2 canonical certificate history is immutable'; END IF; + proof:=mst2_metadata_compute_certificate(NEW.page_id,NEW.generation,NEW.origin_prepare_id); + IF NEW.certificate_digest<>decode(proof->>'certificate','hex') OR NEW.canonical_proof<>proof + OR NEW.namespace_uuid::text<>proof->>'namespace' OR NEW.proof_revision<>1 OR NEW.metadata_codec<>(proof->>'codec')::smallint + OR NEW.byte_size<>(proof->>'byte_size')::integer OR NEW.rank<>(proof->>'rank')::integer + OR NEW.map_entry_count<>(proof->>'map_entry_count')::bigint + OR NEW.map_encoded_entry_bytes<>(proof->>'map_encoded_entry_bytes')::bigint + OR NEW.min_name IS DISTINCT FROM decode(proof->>'min_name','hex') + OR NEW.max_name IS DISTINCT FROM decode(proof->>'max_name','hex') + OR NEW.relative_path_bytes<>(proof->>'relative_path_bytes')::integer + OR NEW.relative_components<>(proof->>'relative_components')::integer + OR NEW.closure_nodes_upper<>(proof->>'closure_nodes_upper')::integer + OR NEW.closure_edges_upper<>(proof->>'closure_edges_upper')::integer + OR NEW.closure_bytes_upper<>(proof->>'closure_bytes_upper')::bigint + OR NEW.closure_entries_upper<>(proof->>'closure_entries_upper')::bigint THEN + RAISE EXCEPTION 'MTP2 canonical certificate was not derived from its durable bytes and exact child proofs'; + END IF; + RETURN NEW; +END $$; +CREATE TRIGGER mst2_metadata_certificate_guard BEFORE INSERT OR UPDATE OR DELETE ON mst2_metadata_page_certificate + FOR EACH ROW EXECUTE FUNCTION mst2_metadata_certificate_guard(); + +CREATE FUNCTION mst2_metadata_verified_ref_guard() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE ref jsonb; +BEGIN + IF TG_OP<>'INSERT' THEN RAISE EXCEPTION 'MTP2 canonical reference history is immutable'; END IF; + SELECT canonical_proof->'references'->NEW.reference_ordinal INTO ref FROM mst2_metadata_page_certificate + WHERE page_id=NEW.parent_page AND generation=NEW.parent_generation; + IF ref IS NULL OR NEW.reference_kind<>ref->>'kind' OR NEW.child_page<>decode(ref->>'child','hex') + OR NEW.child_generation<>(ref->>'generation')::bigint + OR NEW.child_certificate_digest<>decode(ref->>'certificate','hex') + OR NEW.name IS DISTINCT FROM decode(ref->>'name','hex') + OR NEW.label IS DISTINCT FROM (ref->>'label')::integer + OR NEW.advertised_count IS DISTINCT FROM (ref->>'count')::bigint THEN + RAISE EXCEPTION 'MTP2 canonical reference differs from its independently derived occurrence'; + END IF; + RETURN NEW; +END $$; +CREATE TRIGGER mst2_metadata_verified_ref_guard BEFORE INSERT OR UPDATE OR DELETE ON mst2_metadata_verified_ref + FOR EACH ROW EXECUTE FUNCTION mst2_metadata_verified_ref_guard(); + +CREATE FUNCTION mst2_metadata_certificate_complete() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE p bytea; g bigint; proof jsonb; +BEGIN + IF TG_TABLE_NAME IN ('mst2_metadata_verified_ref','mst2_metadata_graph_edge') THEN p:=NEW.parent_page; g:=NEW.parent_generation; + ELSE p:=NEW.page_id; g:=NEW.generation; END IF; + SELECT canonical_proof INTO proof FROM mst2_metadata_page_certificate WHERE page_id=p AND generation=g; + IF proof IS NULL OR (SELECT count(*) FROM mst2_metadata_verified_ref WHERE parent_page=p AND parent_generation=g) + <>jsonb_array_length(proof->'references') + OR NOT EXISTS(SELECT 1 FROM mst2_metadata_graph_node n WHERE n.page_id=p AND n.generation=g + AND n.state='LIVE' AND n.certificate_digest=decode(proof->>'certificate','hex')) + OR EXISTS((SELECT child_page,child_generation FROM mst2_metadata_verified_ref WHERE parent_page=p AND parent_generation=g) + EXCEPT (SELECT child_page,child_generation FROM mst2_metadata_graph_edge WHERE parent_page=p AND parent_generation=g)) + OR EXISTS((SELECT child_page,child_generation FROM mst2_metadata_graph_edge WHERE parent_page=p AND parent_generation=g) + EXCEPT (SELECT child_page,child_generation FROM mst2_metadata_verified_ref WHERE parent_page=p AND parent_generation=g)) THEN + RAISE EXCEPTION 'MTP2 canonical certificate must commit with its complete exact graph and typed references'; + END IF; + RETURN NULL; +END $$; +CREATE CONSTRAINT TRIGGER mst2_metadata_certificate_complete AFTER INSERT ON mst2_metadata_page_certificate + DEFERRABLE INITIALLY DEFERRED FOR EACH ROW EXECUTE FUNCTION mst2_metadata_certificate_complete(); +CREATE CONSTRAINT TRIGGER mst2_metadata_verified_refs_complete AFTER INSERT ON mst2_metadata_verified_ref + DEFERRABLE INITIALLY DEFERRED FOR EACH ROW EXECUTE FUNCTION mst2_metadata_certificate_complete(); +CREATE CONSTRAINT TRIGGER mst2_metadata_graph_refs_complete AFTER INSERT ON mst2_metadata_graph_edge + DEFERRABLE INITIALLY DEFERRED FOR EACH ROW EXECUTE FUNCTION mst2_metadata_certificate_complete(); + +CREATE FUNCTION mst2_metadata_certify_page(pid text,p bytea,g bigint) RETURNS bytea LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE proof jsonb; existing bytea; +BEGIN + SELECT c.certificate_digest INTO existing FROM mst2_metadata_page_certificate c + JOIN mst2_metadata_current cur USING(page_id,generation) JOIN mst2_metadata_lifetime life USING(page_id,generation) + JOIN mst2_metadata_graph_node n USING(page_id,generation) JOIN mst2_metadata_prepare_page member USING(page_id,generation) + JOIN mst2_metadata_prepare q USING(prepare_id) + WHERE c.page_id=p AND c.generation=g AND member.prepare_id=pid AND q.state='PREPARING' + AND mst2_metadata_scope_matches(q.primary_scope) AND life.state='LIVE' AND life.graph_domain='qualified-v1' + AND n.state='LIVE' AND n.certificate_digest=c.certificate_digest AND n.bytes=member.expected_size + AND n.metadata_codec=q.metadata_codec + AND NOT EXISTS(SELECT 1 FROM mst2_metadata_gc_op op WHERE op.page_id=p AND op.generation=g); + IF FOUND THEN RETURN existing; END IF; + proof:=mst2_metadata_compute_certificate(p,g,pid); + INSERT INTO mst2_metadata_page_certificate(page_id,generation,certificate_digest,origin_prepare_id,namespace_uuid, + proof_revision,metadata_codec,byte_size,rank,map_entry_count,map_encoded_entry_bytes,min_name,max_name, + relative_path_bytes,relative_components,closure_nodes_upper,closure_edges_upper,closure_bytes_upper,closure_entries_upper,canonical_proof) + VALUES(p,g,decode(proof->>'certificate','hex'),pid,(proof->>'namespace')::uuid,1,(proof->>'codec')::smallint, + (proof->>'byte_size')::integer,(proof->>'rank')::integer,(proof->>'map_entry_count')::bigint, + (proof->>'map_encoded_entry_bytes')::bigint,decode(proof->>'min_name','hex'),decode(proof->>'max_name','hex'), + (proof->>'relative_path_bytes')::integer,(proof->>'relative_components')::integer, + (proof->>'closure_nodes_upper')::integer,(proof->>'closure_edges_upper')::integer, + (proof->>'closure_bytes_upper')::bigint,(proof->>'closure_entries_upper')::bigint,proof); + INSERT INTO mst2_metadata_verified_ref(parent_page,parent_generation,reference_ordinal,reference_kind, + name,label,advertised_count,child_page,child_generation,child_certificate_digest) + SELECT p,g,ordinality::integer-1,value->>'kind',decode(value->>'name','hex'),(value->>'label')::integer, + (value->>'count')::bigint,decode(value->>'child','hex'),(value->>'generation')::bigint,decode(value->>'certificate','hex') + FROM jsonb_array_elements(proof->'references') WITH ORDINALITY refs(value,ordinality); + INSERT INTO mst2_metadata_graph_node(page_id,generation,state,metadata_codec,bytes,incoming_refs,certificate_digest) + VALUES(p,g,'LIVE',(proof->>'codec')::smallint,(proof->>'byte_size')::integer,0,decode(proof->>'certificate','hex')); + INSERT INTO mst2_metadata_graph_edge(parent_page,parent_generation,child_page,child_generation) + SELECT DISTINCT p,g,child_page,child_generation FROM mst2_metadata_verified_ref WHERE parent_page=p AND parent_generation=g; + RETURN decode(proof->>'certificate','hex'); +END $$; + +CREATE FUNCTION mst2_metadata_certify_batch(pid text,members jsonb) RETURNS integer LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE member jsonb; certified integer:=0; +BEGIN + IF members IS NULL OR jsonb_typeof(members)<>'array' OR jsonb_array_length(members) NOT BETWEEN 1 AND 4096 THEN + RAISE EXCEPTION 'canonical certification batch exceeds its fixed member budget'; + END IF; + -- Array order is supplied by the independently validated cold DAG. Each + -- invocation still proves its durable bytes and already certified children. + FOR member IN SELECT value FROM jsonb_array_elements(members) WITH ORDINALITY AS input(value,ordinal) + ORDER BY ordinal LOOP + IF jsonb_typeof(member)<>'object' OR member->>'page' !~ '^[0-9a-f]{64}$' + OR member->>'generation' IS NULL THEN RAISE EXCEPTION 'canonical certification batch member is malformed'; END IF; + PERFORM mst2_metadata_certify_page(pid,decode(member->>'page','hex'),(member->>'generation')::bigint); + certified:=certified+1; + END LOOP; + RETURN certified; +END $$; diff --git a/src/jupiter/storage/qualified_metadata_family.rs b/src/jupiter/storage/qualified_metadata_family.rs new file mode 100644 index 00000000..52594dd3 --- /dev/null +++ b/src/jupiter/storage/qualified_metadata_family.rs @@ -0,0 +1,624 @@ +//! Captured physical Q provisioning and the sealed rooted production repository. + +use mst2_codec::metapage::{HEADER_LEN, PAGE_MAX_BYTES}; +use sea_orm::{ + ConnectionTrait, DatabaseConnection, DatabaseTransaction, DbBackend, IsolationLevel, Statement, + TransactionTrait, +}; +use sha2::{Digest, Sha256}; +use url::Url; + +#[cfg(test)] +use super::native_metadata_install::generations::{ + GenerationMetadataReceipt, GenerationPrepareIntent, + qualified::PostgresQualifiedMetadataRepository, +}; +use super::{init::postgres_connection, native_metadata_install::MetadataInstallError}; +#[cfg(test)] +use crate::ceres::snapshot::pages::PreparedNativeMetadataRetention; +use crate::{ + ceres::snapshot::{error::SnapshotError, retention_dag::MetadataPagePayload}, + common::errors::MegaError, + config::DbConfig, +}; + +const FAMILY: &str = "v3-rooted-qualified-1"; +const FAMILY_SQL: &str = include_str!("qualified_metadata_family.sql"); +const CANONICAL_SQL: &str = include_str!("qualified_metadata_canonical.sql"); +const CERTIFICATES_SQL: &str = include_str!("qualified_metadata_certificates.sql"); +const ANCHORS_SQL: &str = include_str!("qualified_metadata_anchors.sql"); +const SOURCE_READ_SQL: &str = include_str!("qualified_metadata_source_read.sql"); +const SOURCE_REVISION_SQL: &str = include_str!("qualified_source_revision.sql"); +const ROOTED_SQL: &str = include_str!("qualified_metadata_rooted.sql"); +const SERVING_SQL: &str = include_str!("qualified_metadata_serving.sql"); +const GC_SQL: &str = include_str!("qualified_metadata_gc.sql"); + +#[path = "qualified_metadata_rooted.rs"] +mod rooted; +pub(crate) use rooted::{ + RootedDirectoryWindow, RootedLookupBatch, RootedLookupStatus, RootedQualifiedMetadataRepository, +}; +#[cfg(test)] +pub(crate) use rooted::{ + RootedPrepareIntent, with_rooted_reader_barriers, with_rooted_source_fact_barriers, + with_rooted_source_temporary_shadow, +}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum SnapshotMetadataFamily { + Generic, + Rooted, +} + +pub(crate) async fn select_snapshot_family( + connection: &DatabaseConnection, + identity: &str, + lease: bool, +) -> Result, SnapshotError> { + let result = async { + let txn = connection + .begin_with_config(Some(IsolationLevel::ReadCommitted), None) + .await?; + let (schema, _) = captured_core(&txn).await?; + let function = if lease { + "mst2_route_family_for_lease" + } else { + "mst2_route_family_for_snapshot" + }; + let rows = txn + .query_all_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + format!( + "SELECT namespace_uuid::text,graph_domain FROM {}.{function}($1,$2)", + identifier(&schema) + ), + [identity.into(), schema.into()], + )) + .await?; + let family = match rows.as_slice() { + [] => None, + [row] => Some(match row.try_get::("", "graph_domain")?.as_str() { + "generic-v1" => SnapshotMetadataFamily::Generic, + "qualified-v1" => SnapshotMetadataFamily::Rooted, + _ => { + return Err(rejected( + "snapshot route selected an unsupported physical family", + )); + } + }), + _ => { + return Err(rejected( + "snapshot route selected multiple physical families", + )); + } + }; + txn.commit().await?; + Ok::<_, MegaError>(family) + } + .await; + result.map_err(|error| { + if let MegaError::Db(db_error) = &error + && rooted::is_lock_unavailable(db_error) + { + return SnapshotError::new( + crate::ceres::snapshot::error::SnapshotErrorCode::TemporaryUnavailable, + "fixed source is being updated; retry the operation", + ); + } + SnapshotError::new( + crate::ceres::snapshot::error::SnapshotErrorCode::IntegrityError, + error.to_string(), + ) + }) +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct VerifiedQualifiedNamespace { + core_schema: String, + core_oid: i64, + schema: String, + schema_oid: i64, + namespace_uuid: String, + storage_uuid: String, + catalog_fingerprint: Vec, +} + +fn identifier(value: &str) -> String { + format!("\"{}\"", value.replace('"', "\"\"")) +} +fn literal(value: &str) -> String { + format!("'{}'", value.replace('\'', "''")) +} +pub(crate) fn implementation_fingerprint() -> Vec { + let mut hash = Sha256::new(); + hash.update(b"mega.mst2.rooted-qualified-implementation.v1\0"); + // Length-prefix every source component so source selection and proof + // revisions cannot change while retaining an accepted physical stamp. + for (name, bytes) in [ + ("family", FAMILY.as_bytes()), + ("family-ddl", FAMILY_SQL.as_bytes()), + ("canonical-proof-revision-1", CANONICAL_SQL.as_bytes()), + ("indexed-source-read-revision-1", SOURCE_READ_SQL.as_bytes()), + ("captured-source-revision-1", SOURCE_REVISION_SQL.as_bytes()), + ("rooted-collector-revision-1", GC_SQL.as_bytes()), + ("typed-certificate-revision-1", CERTIFICATES_SQL.as_bytes()), + ( + "source-attestation-and-anchor-revision-1", + ANCHORS_SQL.as_bytes(), + ), + ("rooted-plan-and-bindings-revision-1", ROOTED_SQL.as_bytes()), + ( + "rooted-session-and-reader-revision-1", + SERVING_SQL.as_bytes(), + ), + ( + "rooted-physical-route-revision-1", + include_bytes!("../migration/m20261008_000200_rooted_routes.sql").as_slice(), + ), + ( + "authority-catalog-selector", + include_bytes!("qualified_family_catalog.sql").as_slice(), + ), + ( + "normalized-family-shape", + include_bytes!("qualified_family_shape.sql").as_slice(), + ), + ( + "initial-core-registration", + include_bytes!("../migration/m20261008_000200_rooted_qualified_family.sql").as_slice(), + ), + ] { + hash.update((name.len() as u64).to_le_bytes()); + hash.update(name.as_bytes()); + hash.update((bytes.len() as u64).to_le_bytes()); + hash.update(bytes); + } + hash.finalize().to_vec() +} +pub(crate) fn render_family( + core_schema: &str, + core_oid: i64, + q_schema: &str, + q_oid: i64, + n_uuid: &str, + s_uuid: &str, +) -> String { + FAMILY_SQL + .replace("$CANONICAL_SQL$", CANONICAL_SQL) + .replace("$CERTIFICATES_SQL$", CERTIFICATES_SQL) + .replace("$ANCHORS_SQL$", ANCHORS_SQL) + .replace("$SOURCE_READ_SQL$", SOURCE_READ_SQL) + .replace("$ROOTED_SQL$", ROOTED_SQL) + .replace("$SERVING_SQL$", SERVING_SQL) + .replace("$GC_SQL$", GC_SQL) + .replace("$CORE_SCHEMA$", &identifier(core_schema)) + .replace("$CORE_LITERAL$", &literal(core_schema)) + .replace("$Q_SCHEMA$", &identifier(q_schema)) + .replace("$Q_LITERAL$", &literal(q_schema)) + .replace("$CORE_OID$", &core_oid.to_string()) + .replace("$Q_OID$", &q_oid.to_string()) + .replace("$NAMESPACE_UUID$", n_uuid) + .replace("$STORAGE_UUID$", s_uuid) + .replace( + "$IMPLEMENTATION_SHA$", + &hex::encode(implementation_fingerprint()), + ) + .replace("$HEADER_LEN$", &HEADER_LEN.to_string()) + .replace("$PAGE_MAX_BYTES$", &PAGE_MAX_BYTES.to_string()) +} +fn rejected(message: &str) -> MegaError { + MegaError::Other(message.into()) +} + +async fn captured_core(connection: &C) -> Result<(String, i64), MegaError> { + let row = connection.query_one_raw(Statement::from_string(DbBackend::Postgres, + "SELECT current_schema() AS schema,n.oid::bigint AS oid FROM pg_catalog.pg_namespace n WHERE n.nspname=current_schema()")) + .await?.ok_or_else(|| rejected("qualified family core schema is missing"))?; + let schema: String = row.try_get("", "schema")?; + let oid: i64 = row.try_get("", "oid")?; + let valid = connection + .query_one_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + format!( + "SELECT {}.mst2_route_scope_valid($1) AS valid", + identifier(&schema) + ), + [schema.clone().into()], + )) + .await? + .ok_or_else(|| rejected("qualified family core identity is missing"))? + .try_get::("", "valid")?; + if !valid { + return Err(rejected( + "qualified family requires its registered primary core schema", + )); + } + Ok((schema, oid)) +} + +async fn catalog( + connection: &C, + core_oid: i64, + q_oid: i64, +) -> Result, MegaError> { + catalog_with_exemption(connection, core_oid, q_oid, 0).await +} + +async fn catalog_with_exemption( + connection: &C, + core_oid: i64, + q_oid: i64, + exempt_q_oid: i64, +) -> Result, MegaError> { + // Inspect the actual catalogs directly. A replaced helper function cannot + // turn a bad physical structure into a fresh accepted fingerprint. + let sql = include_str!("qualified_family_catalog.sql") + .replace("$CORE_OID$", "$1::bigint::oid") + .replace("$Q_OID$", "$2::bigint::oid") + .replace("$EXEMPT_Q_OID$", "$3::bigint::oid"); + let row = connection + .query_one_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + sql, + [core_oid.into(), q_oid.into(), exempt_q_oid.into()], + )) + .await? + .ok_or_else(|| rejected("qualified family catalog is missing"))?; + Ok(row.try_get("", "fingerprint")?) +} + +async fn registered( + connection: &C, + core: &(String, i64), +) -> Result, MegaError> { + let policy=connection.query_one_raw(Statement::from_string(DbBackend::Postgres,format!( + "SELECT implementation_fingerprint,expected_shape,authority_catalog FROM {}.mst2_qualified_family_policy WHERE singleton=1",identifier(&core.0)))) + .await?.ok_or_else(||rejected("qualified trusted family policy is missing"))?; + if policy.try_get::>("", "implementation_fingerprint")? != implementation_fingerprint() + { + return Err(rejected("qualified trusted core authority catalog changed")); + } + let rows=connection.query_all_raw(Statement::from_string(DbBackend::Postgres,format!( + "SELECT n.namespace_uuid::text,n.metadata_schema,n.metadata_schema_oid::bigint,n.metadata_storage_uuid, + n.implementation_fingerprint,n.catalog_fingerprint, + n.family_identity,n.admission_state,n.collector_state,n.storage_uuid AS core_storage_uuid, + n.core_schema,n.core_schema_oid::bigint, + (SELECT count(*) FROM {c}.mst2_metadata_namespace)::bigint AS namespace_count, + EXISTS(SELECT 1 FROM pg_catalog.pg_namespace p WHERE p.oid=n.metadata_schema_oid AND p.nspname=n.metadata_schema) AS schema_present, + {c}.mst2_route_scope_valid({core_literal}) AS core_valid + FROM {c}.mst2_metadata_namespace n WHERE n.graph_domain='qualified-v1'",c=identifier(&core.0),core_literal=literal(&core.0)))) + .await?; + if rows.is_empty() { + if policy.try_get::>("", "authority_catalog")? + != catalog(connection, core.1, 0).await? + { + return Err(rejected("qualified trusted core authority catalog changed")); + } + return Ok(None); + } + if rows.len() != 1 { + return Err(rejected( + "qualified family registry exceeds its one-Q hard limit", + )); + } + let row = &rows[0]; + let schema: String = row.try_get("", "metadata_schema")?; + let namespace_uuid: String = row.try_get("", "namespace_uuid")?; + let storage_uuid: String = row.try_get("", "metadata_storage_uuid")?; + let schema_oid: i64 = row.try_get("", "metadata_schema_oid")?; + let expected_schema = format!("mst2q_{}", namespace_uuid.replace('-', "")); + let namespace_identity = uuid::Uuid::parse_str(&namespace_uuid) + .map_err(|_| rejected("qualified namespace UUID is invalid"))?; + let storage_identity = uuid::Uuid::parse_str(&storage_uuid) + .map_err(|_| rejected("qualified storage UUID is invalid"))?; + if schema != expected_schema + || namespace_identity.get_version_num() != 4 + || namespace_identity.get_variant() != uuid::Variant::RFC4122 + || namespace_identity.to_string() != namespace_uuid + || storage_identity.get_version_num() != 4 + || storage_identity.get_variant() != uuid::Variant::RFC4122 + || storage_identity.to_string() != storage_uuid + || storage_uuid == row.try_get::("", "core_storage_uuid")? + || row.try_get::("", "core_schema")? != core.0 + || row.try_get::("", "core_schema_oid")? != core.1 + || row.try_get::("", "family_identity")? != FAMILY + || row.try_get::("", "admission_state")? != "ROOTED_Q_ADMITTED" + || row.try_get::("", "collector_state")? != "ENABLED" + || row.try_get::("", "namespace_count")? != 2 + || !row.try_get::("", "schema_present")? + || !row.try_get::("", "core_valid")? + || row.try_get::>("", "implementation_fingerprint")? != implementation_fingerprint() + { + return Err(rejected( + "qualified family registry and physical identity disagree", + )); + } + // A single registered, physically present Q identity is the only permitted + // source of core-side RI triggers omitted from the core authority stamp. + // The complete catalog and trusted shape below still bind all of them. + if policy.try_get::>("", "authority_catalog")? + != catalog_with_exemption(connection, core.1, 0, schema_oid).await? + { + return Err(rejected("qualified trusted core authority catalog changed")); + } + let fingerprint: Vec = row.try_get("", "catalog_fingerprint")?; + if fingerprint.len() != 32 || catalog(connection, core.1, schema_oid).await? != fingerprint { + return Err(rejected( + "qualified family actual catalog fingerprint changed", + )); + } + let shape: Vec = connection + .query_one_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + format!( + "SELECT {}.mst2_route_family_shape($1::bigint::oid,$2::uuid,$3) AS fingerprint", + identifier(&core.0) + ), + [ + schema_oid.into(), + namespace_uuid.clone().into(), + storage_uuid.clone().into(), + ], + )) + .await? + .ok_or_else(|| rejected("qualified family actual shape is missing"))? + .try_get("", "fingerprint")?; + if shape != policy.try_get::>("", "expected_shape")? { + return Err(rejected( + "qualified namespace does not have the trusted complete physical family shape", + )); + } + let stamp=connection.query_one_raw(Statement::from_sql_and_values(DbBackend::Postgres,format!( + "SELECT EXISTS(SELECT 1 FROM {q}.mst2_metadata_family_identity i JOIN {q}.mst2_metadata_storage_scope s USING(singleton) + WHERE i.singleton=1 AND i.namespace_uuid=$1::uuid AND i.storage_uuid=$2 AND s.storage_uuid=$2 + AND i.core_schema_oid=$3::bigint::oid AND i.metadata_schema_oid=$4::bigint::oid + AND i.family_identity=$5 AND i.implementation_fingerprint=$6) AS valid",q=identifier(&schema)), + [namespace_uuid.clone().into(),storage_uuid.clone().into(),core.1.into(),schema_oid.into(),FAMILY.into(),implementation_fingerprint().into()])) + .await?.ok_or_else(||rejected("qualified family physical stamp is missing"))?; + if !stamp.try_get::("", "valid")? { + return Err(rejected("qualified family physical stamp changed")); + } + Ok(Some(VerifiedQualifiedNamespace { + core_schema: core.0.clone(), + core_oid: core.1, + schema, + schema_oid, + namespace_uuid, + storage_uuid, + catalog_fingerprint: fingerprint, + })) +} + +/// Production bootstrap calls this after core migrations, before returning a +/// writable app connection. Existing registrations are verified, never reset. +pub(crate) async fn provision_or_verify_rooted_qualified_family( + connection: &DatabaseConnection, +) -> Result { + let core = captured_core(connection).await?; + for attempt in 0..2 { + let txn = connection + .begin_with_config(Some(IsolationLevel::ReadCommitted), None) + .await?; + if let Some(namespace) = registered(&txn, &core).await? { + txn.execute_unprepared(&format!( + "SELECT {}.mst2_route_enter({})", + identifier(&core.0), + literal(&core.0) + )) + .await?; + let locked = registered(&txn, &core) + .await? + .ok_or_else(|| rejected("qualified registration disappeared"))?; + if locked != namespace { + return Err(rejected("qualified registration changed during bootstrap")); + } + txn.commit().await?; + return Ok(locked); + } + let candidate = uuid::Uuid::new_v4().to_string(); + let schema = format!("mst2q_{}", candidate.replace('-', "")); + let enter = txn + .execute_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + format!( + "SELECT {}.mst2_route_family_candidate_enter($1,$2::uuid,$3)", + identifier(&core.0) + ), + [ + core.0.clone().into(), + candidate.clone().into(), + schema.clone().into(), + ], + )) + .await; + if let Err(error) = enter { + txn.rollback().await?; + if attempt == 0 && error.to_string().contains("lost bootstrap serialization") { + continue; + } + return Err(error.into()); + } + txn.execute_unprepared(&format!("CREATE SCHEMA {}", identifier(&schema))) + .await?; + let row = txn + .query_one_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + "SELECT oid::bigint AS oid FROM pg_catalog.pg_namespace WHERE nspname=$1", + [schema.clone().into()], + )) + .await? + .ok_or_else(|| rejected("qualified provisioning schema was not created"))?; + let schema_oid: i64 = row.try_get("", "oid")?; + let storage_uuid = uuid::Uuid::new_v4().to_string(); + let sql = render_family( + &core.0, + core.1, + &schema, + schema_oid, + &candidate, + &storage_uuid, + ); + txn.execute_unprepared(&sql).await?; + let fingerprint = catalog(&txn, core.1, schema_oid).await?; + txn.execute_unprepared(&format!( + "SET LOCAL search_path={},pg_catalog,pg_temp", + identifier(&core.0) + )) + .await?; + txn.execute_raw(Statement::from_sql_and_values(DbBackend::Postgres,format!( + "INSERT INTO {c}.mst2_metadata_namespace(singleton,namespace_uuid,core_schema,core_schema_oid, + database_name,database_oid,storage_uuid,server_address,server_port,mono_lock_key2,metadata_schema, + metadata_schema_oid,family_identity,graph_domain,admission_state,collector_state, + metadata_storage_uuid,implementation_fingerprint,catalog_fingerprint) + SELECT NULL,$1::uuid,g.core_schema,g.core_schema_oid,g.database_name,g.database_oid,g.storage_uuid, + g.server_address,g.server_port,g.mono_lock_key2,$2,$3::bigint::oid,$4,'qualified-v1','ROOTED_Q_ADMITTED','ENABLED',$5,$6,$7 + FROM {c}.mst2_metadata_namespace g WHERE singleton=1",c=identifier(&core.0)), + [candidate.into(),schema.into(),schema_oid.into(),FAMILY.into(),storage_uuid.into(),implementation_fingerprint().into(),fingerprint.into()])).await?; + let namespace = registered(&txn, &core) + .await? + .ok_or_else(|| rejected("qualified provisioning did not register its family"))?; + txn.commit().await?; + return Ok(namespace); + } + Err(rejected( + "qualified provisioning could not serialize bootstrap", + )) +} + +impl VerifiedQualifiedNamespace { + pub(crate) async fn enter(&self, txn: &DatabaseTransaction) -> Result<(), SnapshotError> { + let result: Result<(), MegaError> = async { + let actual = txn + .query_one_raw(Statement::from_string( + DbBackend::Postgres, + "SELECT current_schema() AS schema", + )) + .await? + .ok_or_else(|| rejected("qualified writer schema is missing"))? + .try_get::("", "schema")?; + if actual != self.schema { + return Err(rejected( + "qualified writer escaped its physical pool schema", + )); + } + txn.execute_unprepared(&format!( + "SELECT {}.mst2_route_enter({})", + identifier(&self.core_schema), + literal(&self.core_schema) + )) + .await?; + if registered(txn, &(self.core_schema.clone(), self.core_oid)) + .await? + .as_ref() + != Some(self) + { + return Err(rejected("qualified writer physical namespace changed")); + } + Ok(()) + } + .await; + result.map_err(|e| { + if let MegaError::Db(error) = &e + && rooted::is_lock_unavailable(error) + { + return SnapshotError::new( + crate::ceres::snapshot::error::SnapshotErrorCode::TemporaryUnavailable, + "qualified source is being updated; retry the operation", + ); + } + SnapshotError::new( + crate::ceres::snapshot::error::SnapshotErrorCode::SnapshotNotReady, + e.to_string(), + ) + }) + } +} + +fn pool_url(db_url: &str, namespace: &VerifiedQualifiedNamespace) -> Result { + let mut url = Url::parse(db_url) + .map_err(|_| rejected("qualified writer requires a valid PostgreSQL URL"))?; + let mut options = Vec::new(); + let mut others = Vec::new(); + for (key, value) in url.query_pairs() { + if key == "options" { + options.push(value.into_owned()); + } else { + others.push((key.into_owned(), value.into_owned())); + } + } + // Only server-generated lowercase schema names can reach this option. + options.push(format!( + "-csearch_path={},pg_catalog,pg_temp", + namespace.schema + )); + url.set_query(None); + { + let mut pairs = url.query_pairs_mut(); + for (key, value) in others { + pairs.append_pair(&key, &value); + } + pairs.append_pair("options", &options.join(" ")); + } + Ok(url.to_string()) +} + +/// The adapter has no Deref or unsealed connection accessor. It permits only +/// test-only cold preparation writes, without a production factory. +#[cfg(test)] +pub(crate) struct ShadowQualifiedMetadataWriter { + repository: PostgresQualifiedMetadataRepository, +} +#[cfg(test)] +impl ShadowQualifiedMetadataWriter { + pub(crate) async fn open( + core: &DatabaseConnection, + config: &DbConfig, + ) -> Result { + let captured = captured_core(core).await?; + let namespace = registered(core, &captured) + .await? + .ok_or_else(|| rejected("qualified shadow family is not provisioned at bootstrap"))?; + let mut q_config = config.clone(); + q_config.db_url = pool_url(&config.db_url, &namespace)?; + q_config.max_connection = q_config.max_connection.clamp(1, 2); + q_config.min_connection = 1; + let connection = postgres_connection(&q_config).await?; + let txn = connection + .begin_with_config(Some(IsolationLevel::ReadCommitted), None) + .await?; + namespace + .enter(&txn) + .await + .map_err(|e| rejected(&e.to_string()))?; + txn.commit().await?; + let repository = + PostgresQualifiedMetadataRepository::registered_shadow(connection, namespace) + .await + .map_err(|e| rejected(&e.to_string()))?; + Ok(Self { repository }) + } + pub(crate) async fn begin_intent( + &self, + operation_id: &str, + prepared: &PreparedNativeMetadataRetention, + ) -> Result { + self.repository.begin_intent(operation_id, prepared).await + } + pub(crate) async fn install_pages( + &self, + intent: &GenerationPrepareIntent, + payloads: &[MetadataPagePayload], + ) -> Result<(), MetadataInstallError> { + self.repository.install_pages(intent, payloads).await + } + pub(crate) async fn finalize( + &self, + intent: &GenerationPrepareIntent, + ) -> Result { + self.repository.finalize(intent).await + } +} + +#[cfg(test)] +#[path = "qualified_metadata_family_tests.rs"] +mod tests; diff --git a/src/jupiter/storage/qualified_metadata_family.sql b/src/jupiter/storage/qualified_metadata_family.sql new file mode 100644 index 00000000..853834e5 --- /dev/null +++ b/src/jupiter/storage/qualified_metadata_family.sql @@ -0,0 +1,491 @@ +-- Dedicated physical v3 Q family with exact generations and owned roots. +SET LOCAL search_path=$Q_SCHEMA$,pg_catalog,pg_temp; +CREATE TABLE mst2_metadata_storage_scope ( + singleton smallint PRIMARY KEY CHECK(singleton=1),storage_uuid text NOT NULL UNIQUE +); +INSERT INTO mst2_metadata_storage_scope VALUES(1,'$STORAGE_UUID$'); +CREATE TABLE mst2_metadata_family_identity ( + singleton smallint PRIMARY KEY CHECK(singleton=1),namespace_uuid uuid NOT NULL UNIQUE, + storage_uuid text NOT NULL UNIQUE,core_schema_oid oid NOT NULL,metadata_schema_oid oid NOT NULL, + family_identity text NOT NULL CHECK(family_identity='v3-rooted-qualified-1'), + implementation_fingerprint bytea NOT NULL CHECK(octet_length(implementation_fingerprint)=32) +); +INSERT INTO mst2_metadata_family_identity VALUES(1,'$NAMESPACE_UUID$'::uuid,'$STORAGE_UUID$', + $CORE_OID$,$Q_OID$,'v3-rooted-qualified-1',decode('$IMPLEMENTATION_SHA$','hex')); +CREATE TABLE mst2_metadata_lifetime ( + page_id bytea NOT NULL CHECK(octet_length(page_id)=32), + node_id text NOT NULL CHECK(node_id='page:sha256:'||encode(page_id,'hex')), + generation bigint NOT NULL CHECK(generation>0), + state text NOT NULL CHECK(state IN ('RESERVED','LIVE','DELETING','REMOVED')), + metadata_codec smallint NOT NULL CHECK(metadata_codec=1), + expected_size integer NOT NULL CHECK(expected_size BETWEEN $HEADER_LEN$ AND $PAGE_MAX_BYTES$), + graph_domain text NOT NULL CHECK(graph_domain='qualified-v1'),PRIMARY KEY(page_id,generation) +); +CREATE INDEX idx_mst2_metadata_lifetime_state_page ON mst2_metadata_lifetime(state,page_id); +CREATE INDEX idx_mst2_metadata_lifetime_node_generation ON mst2_metadata_lifetime(node_id,generation); +CREATE TABLE mst2_metadata_current ( + page_id bytea PRIMARY KEY CHECK(octet_length(page_id)=32),generation bigint NOT NULL CHECK(generation>0), + FOREIGN KEY(page_id,generation) REFERENCES mst2_metadata_lifetime(page_id,generation) +); +CREATE TABLE mst2_metadata_payload ( + page_id bytea PRIMARY KEY CHECK(octet_length(page_id)=32),generation bigint NOT NULL CHECK(generation>0), + metadata_codec smallint NOT NULL CHECK(metadata_codec=1), + byte_size integer NOT NULL CHECK(byte_size BETWEEN $HEADER_LEN$ AND $PAGE_MAX_BYTES$), + payload bytea NOT NULL CHECK(octet_length(payload)=byte_size),created_at timestamptz NOT NULL DEFAULT now(), + FOREIGN KEY(page_id,generation) REFERENCES mst2_metadata_lifetime(page_id,generation) +); +CREATE TABLE mst2_metadata_prepare ( + prepare_id text PRIMARY KEY CHECK(prepare_id ~ '^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$'), + operation_id text NOT NULL UNIQUE CHECK(octet_length(operation_id) BETWEEN 1 AND 255), + manifest_digest bytea NOT NULL CHECK(octet_length(manifest_digest)=32), + canonical_plan bytea NOT NULL CHECK(octet_length(canonical_plan)<=2097152), + source_domain text NOT NULL CHECK(source_domain='native-git'),tagged_root_tree_oid text NOT NULL, + plan_kind text NOT NULL DEFAULT 'COLD' CHECK(plan_kind IN ('COLD','ROOTED')), + bindings_revision bigint NOT NULL DEFAULT 0 CHECK(bindings_revision>=0), + scope text NOT NULL CHECK(octet_length(scope)<=4096),schema_version smallint NOT NULL, + metadata_codec smallint NOT NULL CHECK(metadata_codec=1),materialization_policy smallint NOT NULL, + fs_semantics smallint NOT NULL,access_projection smallint NOT NULL,verification_revision integer NOT NULL, + projection_revision smallint NOT NULL,metadata_root bytea NOT NULL CHECK(octet_length(metadata_root)=32), + node_count integer NOT NULL CHECK(node_count BETWEEN 0 AND 4096), + edge_count integer NOT NULL CHECK(edge_count BETWEEN 0 AND 16384), + total_bytes bigint NOT NULL CHECK(total_bytes BETWEEN 0 AND 67108864), + state text NOT NULL CHECK(state IN ('PREPARING','COMMITTED','ABORTED')), + created_at timestamptz NOT NULL DEFAULT now(),committed_at timestamptz,aborted_at timestamptz, + coverage_retired_at timestamptz, + canonical_bindings bytea NOT NULL CHECK(octet_length(canonical_bindings) BETWEEN 12 AND 196620), + bindings_digest bytea NOT NULL CHECK(octet_length(bindings_digest)=32), + primary_scope bytea NOT NULL CHECK(octet_length(primary_scope) BETWEEN 1 AND 16384), + storage_seal bytea NOT NULL CHECK(octet_length(storage_seal)=32), + graph_domain text NOT NULL CHECK(graph_domain='qualified-v1'),UNIQUE(prepare_id,storage_seal), + CHECK((state='COMMITTED')=(committed_at IS NOT NULL) AND (state='ABORTED')=(aborted_at IS NOT NULL) + AND (coverage_retired_at IS NULL OR state='COMMITTED')) +); +CREATE TABLE mst2_metadata_prepare_page ( + prepare_id text NOT NULL REFERENCES mst2_metadata_prepare(prepare_id), + page_id bytea NOT NULL CHECK(octet_length(page_id)=32),generation bigint NOT NULL CHECK(generation>0), + expected_size integer NOT NULL CHECK(expected_size BETWEEN $HEADER_LEN$ AND $PAGE_MAX_BYTES$), + PRIMARY KEY(prepare_id,page_id),UNIQUE(prepare_id,page_id,generation), + FOREIGN KEY(page_id,generation) REFERENCES mst2_metadata_lifetime(page_id,generation) +); +CREATE INDEX idx_mst2_metadata_prepare_page_lifetime ON mst2_metadata_prepare_page(page_id,generation,prepare_id); +CREATE TABLE mst2_metadata_graph_node ( + page_id bytea NOT NULL CHECK (octet_length(page_id)=32), + generation bigint NOT NULL CHECK (generation>0), + state text NOT NULL CHECK (state IN ('LIVE','DELETING')), + metadata_codec smallint NOT NULL CHECK (metadata_codec=1), + bytes bigint NOT NULL CHECK (bytes BETWEEN $HEADER_LEN$ AND $PAGE_MAX_BYTES$), + incoming_refs bigint NOT NULL DEFAULT 0 CHECK (incoming_refs>=0), + PRIMARY KEY(page_id,generation), + FOREIGN KEY(page_id,generation) REFERENCES mst2_metadata_lifetime(page_id,generation) +); +CREATE INDEX idx_mst2_metadata_graph_node_gc ON mst2_metadata_graph_node(state,incoming_refs,page_id,generation); +CREATE TABLE mst2_metadata_graph_edge ( + parent_page bytea NOT NULL, + parent_generation bigint NOT NULL, + child_page bytea NOT NULL, + child_generation bigint NOT NULL, + PRIMARY KEY(parent_page,parent_generation,child_page,child_generation), + FOREIGN KEY(parent_page,parent_generation) REFERENCES mst2_metadata_graph_node(page_id,generation), + FOREIGN KEY(child_page,child_generation) REFERENCES mst2_metadata_graph_node(page_id,generation), + CHECK (parent_page<>child_page OR parent_generation<>child_generation) +); +CREATE INDEX idx_mst2_metadata_graph_edge_child ON mst2_metadata_graph_edge(child_page,child_generation,parent_page,parent_generation); +CREATE TABLE mst2_metadata_graph_root ( + prepare_id text NOT NULL, + storage_seal bytea NOT NULL CHECK (octet_length(storage_seal)=32), + page_id bytea NOT NULL, + generation bigint NOT NULL, + PRIMARY KEY(prepare_id,page_id,generation), + FOREIGN KEY(prepare_id,storage_seal) REFERENCES mst2_metadata_prepare(prepare_id,storage_seal), + FOREIGN KEY(prepare_id,page_id,generation) REFERENCES mst2_metadata_prepare_page(prepare_id,page_id,generation), + FOREIGN KEY(page_id,generation) REFERENCES mst2_metadata_graph_node(page_id,generation) +); +CREATE INDEX idx_mst2_metadata_graph_root_page ON mst2_metadata_graph_root(page_id,generation,prepare_id); +CREATE TABLE mst2_metadata_gc_op ( + operation_id uuid PRIMARY KEY, + page_id bytea NOT NULL CHECK (octet_length(page_id)=32), + generation bigint NOT NULL CHECK (generation>0), + primary_scope bytea NOT NULL CHECK (octet_length(primary_scope) BETWEEN 1 AND 16384), + graph_domain text NOT NULL CHECK (graph_domain='qualified-v1'), + metadata_codec smallint NOT NULL CHECK (metadata_codec=1), + expected_size integer NOT NULL CHECK (expected_size BETWEEN $HEADER_LEN$ AND $PAGE_MAX_BYTES$), + graph_present boolean NOT NULL, + had_payload boolean NOT NULL, + payload_delete_xid bigint, + state text NOT NULL CHECK (state IN ('PENDING','APPLIED')), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + completed_at timestamptz, + UNIQUE(page_id,generation), + FOREIGN KEY(page_id,generation) REFERENCES mst2_metadata_lifetime(page_id,generation), + CHECK ((state='APPLIED')=(completed_at IS NOT NULL)) +); +CREATE INDEX idx_mst2_metadata_gc_op_pending ON mst2_metadata_gc_op(created_at,operation_id) WHERE state='PENDING'; + +-- Incarnations are historical; SID alone is never a unique physical binding. +CREATE TABLE mst2_qualified_session_incarnation ( + snapshot_id text NOT NULL,session_incarnation uuid NOT NULL,namespace_uuid uuid NOT NULL, + prepare_id text NOT NULL,storage_seal bytea NOT NULL CHECK(octet_length(storage_seal)=32), + metadata_root bytea NOT NULL CHECK(octet_length(metadata_root)=32),root_generation bigint NOT NULL CHECK(root_generation>0), + source_profile bytea NOT NULL,instance_id text NOT NULL,commit_oid text NOT NULL,root_tree_oid text NOT NULL, + authorization_epoch bigint NOT NULL,publication_sequence bigint NOT NULL,writer_epoch bigint NOT NULL, + certificate_receipt_id bigint NOT NULL,PRIMARY KEY(snapshot_id,session_incarnation), + UNIQUE(snapshot_id,session_incarnation,prepare_id,storage_seal,metadata_root,root_generation), + FOREIGN KEY(snapshot_id,namespace_uuid) REFERENCES $CORE_SCHEMA$.mst2_snapshot_storage_route(snapshot_id,namespace_uuid), + FOREIGN KEY(prepare_id,storage_seal) REFERENCES mst2_metadata_prepare(prepare_id,storage_seal) +); +CREATE TABLE mst2_qualified_lease_binding ( + lease_id text PRIMARY KEY,snapshot_id text NOT NULL,session_incarnation uuid NOT NULL, + prepare_id text NOT NULL,storage_seal bytea NOT NULL,metadata_root bytea NOT NULL,root_generation bigint NOT NULL, + FOREIGN KEY(snapshot_id,session_incarnation,prepare_id,storage_seal,metadata_root,root_generation) + REFERENCES mst2_qualified_session_incarnation(snapshot_id,session_incarnation,prepare_id,storage_seal,metadata_root,root_generation) +); + +$CANONICAL_SQL$ +$CERTIFICATES_SQL$ +$ANCHORS_SQL$ +$SOURCE_READ_SQL$ +$ROOTED_SQL$ + +CREATE FUNCTION mst2_metadata_closed() RETURNS trigger LANGUAGE plpgsql +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +BEGIN RAISE EXCEPTION 'qualified session serving and collector are closed'; END $$; +CREATE FUNCTION mst2_metadata_gc_apply(id uuid) RETURNS void LANGUAGE plpgsql +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +BEGIN RAISE EXCEPTION 'qualified collector is closed'; END $$; +CREATE FUNCTION mst2_metadata_gc_finish(id uuid) RETURNS void LANGUAGE plpgsql +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +BEGIN RAISE EXCEPTION 'qualified collector is closed'; END $$; +CREATE FUNCTION mst2_metadata_immutable() RETURNS trigger LANGUAGE plpgsql +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +BEGIN RAISE EXCEPTION 'qualified family identity and history are immutable'; END $$; + +-- Observe the caller before entering fixed trusted functions. Core fallback +-- never enters the pool search path; all cross-family authority is explicit. +CREATE FUNCTION mst2_metadata_dml_barrier() RETURNS trigger LANGUAGE plpgsql VOLATILE AS $$ +BEGIN + IF pg_catalog.current_schema() IS DISTINCT FROM $Q_LITERAL$ OR TG_TABLE_SCHEMA IS DISTINCT FROM $Q_LITERAL$ + OR NOT EXISTS(SELECT 1 FROM pg_catalog.pg_class WHERE oid=TG_RELID AND relnamespace='$Q_OID$'::oid) + OR NOT EXISTS(SELECT 1 FROM pg_catalog.pg_namespace WHERE oid='$Q_OID$'::oid AND nspname=$Q_LITERAL$) + OR pg_catalog.pg_is_in_recovery() OR pg_catalog.current_setting('transaction_isolation')<>'read committed' THEN + RAISE EXCEPTION 'qualified mutation requires its captured primary family and READ COMMITTED'; + END IF; + PERFORM $CORE_SCHEMA$.mst2_route_enter($CORE_LITERAL$); + IF NOT EXISTS(SELECT 1 FROM $CORE_SCHEMA$.mst2_metadata_namespace n + WHERE n.namespace_uuid='$NAMESPACE_UUID$'::uuid AND n.metadata_schema=$Q_LITERAL$ + AND n.metadata_schema_oid='$Q_OID$'::oid AND n.core_schema_oid=$CORE_OID$ + AND n.metadata_storage_uuid='$STORAGE_UUID$' AND n.admission_state='ROOTED_Q_ADMITTED' + AND n.collector_state='ENABLED' AND n.implementation_fingerprint=pg_catalog.decode('$IMPLEMENTATION_SHA$','hex') + AND n.catalog_fingerprint=$CORE_SCHEMA$.mst2_route_family_catalog($CORE_OID$,'$Q_OID$'::oid)) THEN + RAISE EXCEPTION 'qualified physical family catalog fingerprint is unavailable'; + END IF; + RETURN NULL; +END $$; + + +CREATE FUNCTION mst2_metadata_scope_matches(s bytea) RETURNS boolean LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE actual jsonb; +BEGIN + IF pg_is_in_recovery() OR current_setting('transaction_isolation')<>'read committed' THEN RETURN false; END IF; + SELECT jsonb_build_array(x.storage_uuid,current_database(),d.oid::bigint,$Q_LITERAL$,'$Q_OID$'::bigint, + inet_server_addr()::text,inet_server_port()) INTO actual FROM mst2_metadata_storage_scope x + JOIN pg_database d ON d.datname=current_database() WHERE x.singleton=1; + RETURN actual IS NOT NULL AND convert_from(s,'UTF8')::jsonb=actual; +EXCEPTION WHEN OTHERS THEN RETURN false; +END $$; +CREATE FUNCTION mst2_metadata_has_generic_overlap(p bytea) RETURNS boolean LANGUAGE sql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ SELECT false $$; + +CREATE FUNCTION mst2_metadata_lifetime_guard() RETURNS trigger LANGUAGE plpgsql +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +BEGIN + IF TG_OP='DELETE' THEN RAISE EXCEPTION 'metadata lifetime watermark cannot be deleted'; END IF; + IF TG_OP='INSERT' THEN + IF NEW.generation<>1 OR NEW.state<>'RESERVED' OR EXISTS(SELECT 1 FROM mst2_metadata_current WHERE page_id=NEW.page_id) THEN + RAISE EXCEPTION 'qualified initial lifetime cannot adopt old history; collector is closed'; + END IF; + RETURN NEW; + END IF; + IF (to_jsonb(NEW)-'state') IS DISTINCT FROM (to_jsonb(OLD)-'state') THEN + RAISE EXCEPTION 'metadata lifetime identity is immutable'; + END IF; + IF NEW.state=OLD.state THEN RETURN NEW; END IF; + IF OLD.state='RESERVED' AND NEW.state='LIVE' THEN + IF NOT EXISTS(SELECT 1 FROM mst2_metadata_graph_node n JOIN mst2_metadata_payload b USING(page_id,generation) + WHERE n.page_id=OLD.page_id AND n.generation=OLD.generation AND n.state='LIVE' + AND n.metadata_codec=OLD.metadata_codec AND n.bytes=OLD.expected_size + AND b.metadata_codec=OLD.metadata_codec AND b.byte_size=OLD.expected_size) + OR NOT (EXISTS(SELECT 1 FROM mst2_metadata_graph_root r WHERE r.page_id=OLD.page_id AND r.generation=OLD.generation) + OR EXISTS(SELECT 1 FROM mst2_metadata_prepare_page member JOIN mst2_metadata_prepare q USING(prepare_id) + JOIN mst2_metadata_root_anchor anchor ON anchor.prepare_id=q.prepare_id AND anchor.anchor_kind='PREPARE' + AND anchor.owner_key=q.prepare_id AND anchor.root_page=q.metadata_root + WHERE member.page_id=OLD.page_id AND member.generation=OLD.generation AND q.plan_kind='ROOTED' + AND q.state='COMMITTED' AND q.coverage_retired_at IS NULL)) THEN + RAISE EXCEPTION 'qualified LIVE transition needs its graph payload and prepare root'; + END IF; + RETURN NEW; + END IF; + RAISE EXCEPTION 'qualified lifetime collection is closed'; +END $$; +CREATE TRIGGER mst2_metadata_lifetime_guard BEFORE INSERT OR UPDATE OR DELETE ON mst2_metadata_lifetime + FOR EACH ROW EXECUTE FUNCTION mst2_metadata_lifetime_guard(); +CREATE FUNCTION mst2_metadata_current_guard() RETURNS trigger LANGUAGE plpgsql +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +BEGIN + IF TG_OP<>'INSERT' THEN RAISE EXCEPTION 'qualified current collection is closed'; END IF; + IF NEW.generation<>1 OR EXISTS(SELECT 1 FROM mst2_metadata_lifetime WHERE page_id=NEW.page_id AND generation<>1) + OR EXISTS(SELECT 1 FROM mst2_metadata_payload WHERE page_id=NEW.page_id) + OR EXISTS(SELECT 1 FROM mst2_metadata_graph_node WHERE page_id=NEW.page_id) THEN + RAISE EXCEPTION 'initial qualified current cannot reset or adopt old history'; + END IF; + RETURN NEW; +END $$; +CREATE TRIGGER mst2_metadata_current_guard BEFORE INSERT OR UPDATE OR DELETE ON mst2_metadata_current + FOR EACH ROW EXECUTE FUNCTION mst2_metadata_current_guard(); +CREATE FUNCTION mst2_metadata_current_protected() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +BEGIN + IF EXISTS(SELECT 1 FROM mst2_metadata_lifetime WHERE page_id=NEW.page_id AND generation=NEW.generation AND graph_domain='qualified-v1') + AND NOT EXISTS(SELECT 1 FROM mst2_metadata_prepare_page m JOIN mst2_metadata_prepare q USING(prepare_id) + JOIN mst2_metadata_current c ON c.page_id=m.page_id AND c.generation=m.generation + WHERE m.page_id=NEW.page_id AND m.generation=NEW.generation AND q.graph_domain='qualified-v1' + AND q.storage_seal IS NOT NULL AND (q.state='PREPARING' OR (q.state='COMMITTED' AND + (q.coverage_retired_at IS NULL OR mst2_metadata_session_covers_prepare(q.prepare_id))))) THEN + RAISE EXCEPTION 'qualified current change must commit with fresh prepare protection'; + END IF; + RETURN NULL; +END $$; +CREATE CONSTRAINT TRIGGER mst2_metadata_current_protected AFTER INSERT OR UPDATE ON mst2_metadata_current + DEFERRABLE INITIALLY DEFERRED FOR EACH ROW EXECUTE FUNCTION mst2_metadata_current_protected(); + +CREATE FUNCTION mst2_metadata_prepare_guard() RETURNS trigger LANGUAGE plpgsql +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +BEGIN + IF TG_OP='DELETE' THEN RAISE EXCEPTION 'qualified preparation history is immutable'; END IF; + IF TG_OP='INSERT' THEN + IF NEW.state<>'PREPARING' OR NEW.committed_at IS NOT NULL OR NEW.aborted_at IS NOT NULL + OR NEW.coverage_retired_at IS NOT NULL OR NEW.bindings_revision<>0 OR NOT mst2_metadata_scope_matches(NEW.primary_scope) + OR NEW.manifest_digest IS DISTINCT FROM sha256(NEW.canonical_plan) + OR NEW.bindings_digest IS DISTINCT FROM sha256(NEW.canonical_bindings) THEN + RAISE EXCEPTION 'qualified prepare needs its actual fixed primary plan and bindings'; + END IF; + IF NEW.plan_kind='ROOTED' THEN PERFORM mst2_metadata_rooted_manifest(NEW); + ELSIF NEW.node_count=0 OR octet_length(NEW.canonical_bindings)<60 THEN + RAISE EXCEPTION 'cold preparation requires its nonempty full closure'; + END IF; + RETURN NEW; + END IF; + IF (to_jsonb(NEW)-ARRAY['state','committed_at','aborted_at','coverage_retired_at','bindings_revision']) IS DISTINCT FROM + (to_jsonb(OLD)-ARRAY['state','committed_at','aborted_at','coverage_retired_at','bindings_revision']) THEN + RAISE EXCEPTION 'qualified preparation complete identity is immutable'; + END IF; + IF NEW.bindings_revision<>OLD.bindings_revision AND (OLD.plan_kind<>'ROOTED' OR OLD.state<>'PREPARING' + OR NEW.state<>'PREPARING' OR NEW.bindings_revision<>OLD.bindings_revision+1) THEN + RAISE EXCEPTION 'rooted membership revision must advance its exact active preparation'; + END IF; + IF (OLD.state IN ('COMMITTED','ABORTED') AND NEW.state IS DISTINCT FROM OLD.state) + OR (OLD.committed_at IS NOT NULL AND NEW.committed_at IS DISTINCT FROM OLD.committed_at) + OR (OLD.aborted_at IS NOT NULL AND NEW.aborted_at IS DISTINCT FROM OLD.aborted_at) + OR (OLD.coverage_retired_at IS NOT NULL AND NEW.coverage_retired_at IS DISTINCT FROM OLD.coverage_retired_at) THEN + RAISE EXCEPTION 'qualified terminal receipt cannot be revived or rewritten'; + END IF; + IF OLD.state='PREPARING' AND NEW.state='COMMITTED' THEN + IF NEW.plan_kind='ROOTED' THEN + PERFORM mst2_metadata_rooted_finalize_proof(NEW.prepare_id); + ELSE + IF (SELECT count(*) FROM mst2_metadata_prepare_page WHERE prepare_id=NEW.prepare_id)<>NEW.node_count + OR (SELECT coalesce(sum(expected_size),0) FROM mst2_metadata_prepare_page WHERE prepare_id=NEW.prepare_id)<>NEW.total_bytes + OR NOT EXISTS(SELECT 1 FROM mst2_metadata_prepare_page WHERE prepare_id=NEW.prepare_id AND page_id=NEW.metadata_root) + OR EXISTS(SELECT 1 FROM mst2_metadata_prepare_page m + LEFT JOIN mst2_metadata_current c USING(page_id,generation) + LEFT JOIN mst2_metadata_lifetime l USING(page_id,generation) + LEFT JOIN mst2_metadata_payload b USING(page_id,generation) + LEFT JOIN mst2_metadata_graph_node n USING(page_id,generation) + LEFT JOIN mst2_metadata_graph_root r ON r.prepare_id=m.prepare_id AND r.page_id=m.page_id AND r.generation=m.generation + WHERE m.prepare_id=NEW.prepare_id AND (c.page_id IS NULL OR l.page_id IS NULL OR b.page_id IS NULL OR n.page_id IS NULL + OR l.state NOT IN ('RESERVED','LIVE') OR n.state<>'LIVE' OR l.metadata_codec<>NEW.metadata_codec + OR b.metadata_codec<>NEW.metadata_codec OR n.metadata_codec<>NEW.metadata_codec + OR l.expected_size<>m.expected_size OR b.byte_size<>m.expected_size OR n.bytes<>m.expected_size + OR r.storage_seal IS DISTINCT FROM NEW.storage_seal + OR n.incoming_refs<>(SELECT count(*) FROM mst2_metadata_graph_edge WHERE child_page=m.page_id AND child_generation=m.generation))) + OR (SELECT count(*) FROM mst2_metadata_prepare_page m JOIN mst2_metadata_graph_edge e + ON e.parent_page=m.page_id AND e.parent_generation=m.generation WHERE m.prepare_id=NEW.prepare_id)<>NEW.edge_count THEN + RAISE EXCEPTION 'qualified COMMITTED transition lacks its complete exact graph payload and root coverage'; + END IF; + END IF; + END IF; + IF NEW.state='ABORTED' THEN + IF EXISTS(SELECT 1 FROM mst2_metadata_graph_root WHERE prepare_id=NEW.prepare_id) + OR EXISTS(SELECT 1 FROM mst2_qualified_session_incarnation WHERE prepare_id=NEW.prepare_id) THEN + RAISE EXCEPTION 'qualified terminal transition still has exact coverage'; + END IF; + ELSIF NEW.coverage_retired_at IS NOT NULL AND OLD.coverage_retired_at IS NULL THEN + IF NEW.plan_kind<>'ROOTED' THEN + IF EXISTS(SELECT 1 FROM mst2_metadata_graph_root WHERE prepare_id=NEW.prepare_id) + OR EXISTS(SELECT 1 FROM mst2_qualified_session_incarnation WHERE prepare_id=NEW.prepare_id) THEN + RAISE EXCEPTION 'cold terminal transition still has exact coverage'; + END IF; + ELSIF NEW.state<>'COMMITTED' OR EXISTS(SELECT 1 FROM mst2_metadata_graph_root WHERE prepare_id=NEW.prepare_id) + OR NOT (mst2_metadata_session_covers_prepare(NEW.prepare_id) + OR mst2_metadata_orphan_prepare_eligible(NEW.prepare_id)) THEN + RAISE EXCEPTION 'rooted coverage retirement needs its definitive independent ready session root'; + END IF; + END IF; + RETURN NEW; +END $$; +CREATE TRIGGER mst2_metadata_prepare_guard BEFORE INSERT OR UPDATE OR DELETE ON mst2_metadata_prepare + FOR EACH ROW EXECUTE FUNCTION mst2_metadata_prepare_guard(); +CREATE FUNCTION mst2_metadata_generation_mapping_guard() RETURNS trigger LANGUAGE plpgsql +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +BEGIN + IF TG_OP<>'INSERT' THEN RAISE EXCEPTION 'metadata generation mappings are immutable'; END IF; + IF NOT EXISTS(SELECT 1 FROM mst2_metadata_prepare q JOIN mst2_metadata_current c ON c.page_id=NEW.page_id AND c.generation=NEW.generation + JOIN mst2_metadata_lifetime l USING(page_id,generation) WHERE q.prepare_id=NEW.prepare_id AND q.state='PREPARING' + AND q.graph_domain='qualified-v1' AND q.storage_seal IS NOT NULL AND mst2_metadata_scope_matches(q.primary_scope) + AND l.state IN ('RESERVED','LIVE') AND l.expected_size=NEW.expected_size AND l.metadata_codec=q.metadata_codec) THEN + RAISE EXCEPTION 'qualified mapping cannot cross domain/current/state fence'; + END IF; + RETURN NEW; +END $$; +CREATE TRIGGER mst2_metadata_generation_mapping_guard BEFORE INSERT OR UPDATE OR DELETE ON mst2_metadata_prepare_page + FOR EACH ROW EXECUTE FUNCTION mst2_metadata_generation_mapping_guard(); +CREATE FUNCTION mst2_metadata_payload_fenced() RETURNS trigger LANGUAGE plpgsql +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +BEGIN + IF TG_OP<>'INSERT' THEN RAISE EXCEPTION 'qualified payload mutation and collector are closed'; END IF; + IF NOT EXISTS(SELECT 1 FROM mst2_metadata_current c JOIN mst2_metadata_lifetime l USING(page_id,generation) + JOIN mst2_metadata_prepare_page m USING(page_id,generation) JOIN mst2_metadata_prepare q USING(prepare_id) + WHERE c.page_id=NEW.page_id AND c.generation=NEW.generation AND l.state IN ('RESERVED','LIVE') + AND l.metadata_codec=NEW.metadata_codec AND l.expected_size=NEW.byte_size AND q.state='PREPARING' + AND q.graph_domain='qualified-v1' AND mst2_metadata_scope_matches(q.primary_scope)) THEN + RAISE EXCEPTION 'qualified payload INSERT crossed its exact active generation'; + END IF; + RETURN NEW; +END $$; +CREATE TRIGGER mst2_metadata_payload_fenced BEFORE INSERT OR UPDATE OR DELETE ON mst2_metadata_payload + FOR EACH ROW EXECUTE FUNCTION mst2_metadata_payload_fenced(); +CREATE FUNCTION mst2_metadata_graph_node_guard() RETURNS trigger LANGUAGE plpgsql +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE l mst2_metadata_lifetime%ROWTYPE; +BEGIN + IF TG_OP='DELETE' THEN RAISE EXCEPTION 'qualified graph collection is closed'; END IF; + SELECT l0.* INTO l FROM mst2_metadata_current c JOIN mst2_metadata_lifetime l0 USING(page_id,generation) + WHERE c.page_id=NEW.page_id AND c.generation=NEW.generation; + IF NOT FOUND OR l.graph_domain<>'qualified-v1' OR l.metadata_codec<>NEW.metadata_codec OR l.expected_size<>NEW.bytes + OR NEW.state<>'LIVE' OR l.state NOT IN ('RESERVED','LIVE') + OR NEW.incoming_refs<>(SELECT count(*) FROM mst2_metadata_graph_edge WHERE child_page=NEW.page_id AND child_generation=NEW.generation) THEN + RAISE EXCEPTION 'qualified node does not match exact lifetime and actual counters'; + END IF; + IF TG_OP='INSERT' AND NOT EXISTS(SELECT 1 FROM mst2_metadata_prepare_page m JOIN mst2_metadata_prepare q USING(prepare_id) + WHERE m.page_id=NEW.page_id AND m.generation=NEW.generation AND q.state='PREPARING') THEN + RAISE EXCEPTION 'qualified node creation needs active fixed preparation'; + END IF; + IF NOT EXISTS(SELECT 1 FROM mst2_metadata_page_certificate c WHERE c.page_id=NEW.page_id AND c.generation=NEW.generation + AND c.certificate_digest=NEW.certificate_digest AND c.metadata_codec=NEW.metadata_codec AND c.byte_size=NEW.bytes) THEN + RAISE EXCEPTION 'qualified graph node lacks its exact independently canonical certificate'; + END IF; + IF TG_OP='UPDATE' AND (to_jsonb(NEW)-'incoming_refs') IS DISTINCT FROM (to_jsonb(OLD)-'incoming_refs') THEN + RAISE EXCEPTION 'qualified node identity is immutable'; + END IF; + RETURN NEW; +END $$; +CREATE TRIGGER mst2_metadata_graph_node_guard BEFORE INSERT OR UPDATE OR DELETE ON mst2_metadata_graph_node + FOR EACH ROW EXECUTE FUNCTION mst2_metadata_graph_node_guard(); +CREATE FUNCTION mst2_metadata_graph_root_guard() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +BEGIN + IF TG_OP='UPDATE' THEN RAISE EXCEPTION 'qualified roots cannot be retargeted'; END IF; + IF TG_OP='DELETE' THEN + IF EXISTS(SELECT 1 FROM mst2_qualified_session_incarnation WHERE prepare_id=OLD.prepare_id) THEN + RAISE EXCEPTION 'qualified root still has historical incarnation coverage'; + END IF; + RETURN OLD; + END IF; + IF NOT EXISTS(SELECT 1 FROM mst2_metadata_prepare q JOIN mst2_metadata_current c ON c.page_id=NEW.page_id AND c.generation=NEW.generation + JOIN mst2_metadata_lifetime l USING(page_id,generation) JOIN mst2_metadata_graph_node n USING(page_id,generation) + WHERE q.prepare_id=NEW.prepare_id AND q.storage_seal=NEW.storage_seal AND q.state='PREPARING' + AND q.graph_domain='qualified-v1' AND l.graph_domain='qualified-v1' AND l.state IN ('RESERVED','LIVE') + AND n.state='LIVE' AND mst2_metadata_scope_matches(q.primary_scope)) THEN + RAISE EXCEPTION 'qualified root requires its active exact sealed preparation'; + END IF; + RETURN NEW; +END $$; +CREATE TRIGGER mst2_metadata_graph_root_guard BEFORE INSERT OR UPDATE OR DELETE ON mst2_metadata_graph_root + FOR EACH ROW EXECUTE FUNCTION mst2_metadata_graph_root_guard(); + +CREATE FUNCTION mst2_metadata_graph_edge_guard() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE op mst2_metadata_gc_op%ROWTYPE; +BEGIN + IF TG_OP='UPDATE' THEN RAISE EXCEPTION 'qualified edge identity is immutable'; END IF; + IF TG_OP='DELETE' THEN RAISE EXCEPTION 'qualified edge collection is closed'; END IF; + IF NOT EXISTS(SELECT 1 FROM mst2_metadata_verified_ref ref + JOIN mst2_metadata_page_certificate parent ON parent.page_id=ref.parent_page AND parent.generation=ref.parent_generation + JOIN mst2_metadata_page_certificate child ON child.page_id=ref.child_page AND child.generation=ref.child_generation + WHERE ref.parent_page=NEW.parent_page AND ref.parent_generation=NEW.parent_generation + AND ref.child_page=NEW.child_page AND ref.child_generation=NEW.child_generation + AND ref.child_certificate_digest=child.certificate_digest AND parent.rank>child.rank) THEN + RAISE EXCEPTION 'qualified edge differs from its exact canonical reference or strict certified rank'; + END IF; + IF (SELECT count(*) FROM mst2_metadata_graph_node n JOIN mst2_metadata_current c USING(page_id,generation) + JOIN mst2_metadata_lifetime l USING(page_id,generation) + WHERE ((n.page_id=NEW.parent_page AND n.generation=NEW.parent_generation) + OR (n.page_id=NEW.child_page AND n.generation=NEW.child_generation)) + AND n.state='LIVE' AND l.state IN ('RESERVED','LIVE') AND l.graph_domain='qualified-v1')<>2 + OR NOT EXISTS(SELECT 1 FROM mst2_metadata_prepare_page a JOIN mst2_metadata_prepare q USING(prepare_id) + WHERE a.page_id=NEW.parent_page AND a.generation=NEW.parent_generation AND q.state='PREPARING' + AND q.graph_domain='qualified-v1' AND mst2_metadata_scope_matches(q.primary_scope) + AND (EXISTS(SELECT 1 FROM mst2_metadata_prepare_page b WHERE b.prepare_id=a.prepare_id + AND b.page_id=NEW.child_page AND b.generation=NEW.child_generation) + OR EXISTS(SELECT 1 FROM mst2_metadata_prepare_reuse_root r JOIN mst2_metadata_root_anchor anchor + ON anchor.prepare_id=r.prepare_id AND anchor.anchor_kind='REUSE' AND anchor.owner_key=r.prepare_id + AND anchor.root_page=r.root_page AND anchor.root_generation=r.root_generation + WHERE r.prepare_id=a.prepare_id AND r.root_page=NEW.child_page AND r.root_generation=NEW.child_generation))) THEN + RAISE EXCEPTION 'edge creation requires active exact qualified endpoints'; + END IF; + RETURN NEW; +END $$; +CREATE TRIGGER mst2_metadata_graph_edge_guard BEFORE INSERT OR UPDATE OR DELETE ON mst2_metadata_graph_edge + FOR EACH ROW EXECUTE FUNCTION mst2_metadata_graph_edge_guard(); + +CREATE FUNCTION mst2_metadata_edges_added() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +BEGIN + IF (SELECT count(*) FROM added_edges)>16384 THEN RAISE EXCEPTION 'qualified edge batch exceeds limit'; END IF; + IF NOT EXISTS(SELECT 1 FROM added_edges) THEN RETURN NULL; END IF; + IF EXISTS(SELECT 1 FROM mst2_metadata_graph_node n JOIN ( + SELECT child_page,child_generation,count(*) AS delta FROM added_edges GROUP BY child_page,child_generation + ) d ON d.child_page=n.page_id AND d.child_generation=n.generation + WHERE n.incoming_refs+d.delta<>(SELECT count(*) FROM mst2_metadata_graph_edge x WHERE x.child_page=n.page_id AND x.child_generation=n.generation)) THEN + RAISE EXCEPTION 'qualified insertion found preexisting counter drift'; + END IF; + UPDATE mst2_metadata_graph_node n SET incoming_refs=n.incoming_refs+d.delta FROM ( + SELECT child_page,child_generation,count(*) AS delta FROM added_edges GROUP BY child_page,child_generation + ) d WHERE n.page_id=d.child_page AND n.generation=d.child_generation; + RETURN NULL; +END $$; +CREATE TRIGGER mst2_metadata_edges_added AFTER INSERT ON mst2_metadata_graph_edge + REFERENCING NEW TABLE AS added_edges FOR EACH STATEMENT EXECUTE FUNCTION mst2_metadata_edges_added(); + +DO $$ DECLARE t text; BEGIN + FOREACH t IN ARRAY ARRAY['mst2_metadata_storage_scope','mst2_metadata_family_identity', + 'mst2_metadata_lifetime','mst2_metadata_current','mst2_metadata_payload','mst2_metadata_prepare', + 'mst2_metadata_prepare_page','mst2_metadata_graph_node','mst2_metadata_graph_edge','mst2_metadata_graph_root', + 'mst2_metadata_gc_op','mst2_qualified_session_incarnation','mst2_qualified_lease_binding', + 'mst2_metadata_page_certificate','mst2_metadata_verified_ref','mst2_metadata_source_root_attestation', + 'mst2_metadata_prepare_reuse_root','mst2_metadata_reuse_index','mst2_metadata_reader_operation','mst2_metadata_root_anchor', + 'mst2_metadata_source_entry_reference','mst2_metadata_scope_source_reference'] LOOP + EXECUTE format('CREATE TRIGGER mst2_00_family_barrier BEFORE INSERT OR UPDATE OR DELETE ON %I FOR EACH STATEMENT EXECUTE FUNCTION mst2_metadata_dml_barrier()',t); + EXECUTE format('CREATE TRIGGER mst2_metadata_truncate_guard BEFORE TRUNCATE ON %I FOR EACH STATEMENT EXECUTE FUNCTION mst2_metadata_immutable()',t); + END LOOP; + FOREACH t IN ARRAY ARRAY['mst2_metadata_storage_scope','mst2_metadata_family_identity'] LOOP + EXECUTE format('CREATE TRIGGER mst2_metadata_identity_immutable BEFORE INSERT OR UPDATE OR DELETE ON %I FOR EACH STATEMENT EXECUTE FUNCTION mst2_metadata_immutable()',t); + END LOOP; + FOREACH t IN ARRAY ARRAY['mst2_metadata_gc_op','mst2_qualified_session_incarnation','mst2_qualified_lease_binding', + 'mst2_metadata_reader_operation'] LOOP + EXECUTE format('CREATE TRIGGER mst2_01_family_closed BEFORE INSERT OR UPDATE OR DELETE ON %I FOR EACH STATEMENT EXECUTE FUNCTION mst2_metadata_closed()',t); + END LOOP; +END $$; + +$SERVING_SQL$ +$GC_SQL$ diff --git a/src/jupiter/storage/qualified_metadata_family_tests.rs b/src/jupiter/storage/qualified_metadata_family_tests.rs new file mode 100644 index 00000000..54106a11 --- /dev/null +++ b/src/jupiter/storage/qualified_metadata_family_tests.rs @@ -0,0 +1,855 @@ +use std::{sync::Arc, time::Duration}; + +use mst2_codec::metapage::{Entry, EntryKind, Page, page_id}; +use sea_orm::Database; +use sea_orm_migration::MigratorTrait; + +use super::*; +use crate::{ + ceres::snapshot::retention_dag::{MetadataDagBuilder, MetadataDagLimits}, + jupiter::{ + migration::Migrator, + tests::{TestSchemaGuard, test_db_config}, + }, +}; + +#[path = "qualified_metadata_canonical_tests.rs"] +mod canonical_tests; +#[path = "qualified_metadata_gc_tests.rs"] +mod gc_tests; +#[path = "qualified_metadata_source_revision_tests.rs"] +mod source_revision_tests; + +fn prepared() -> PreparedNativeMetadataRetention { + let entries = [Entry::file(EntryKind::Regular, b"file", 3, [42; 32])]; + let child = Page::build(&entries).unwrap(); + let root_entries = [ + Entry::dir(b"one", page_id(&child)), + Entry::dir(b"two", page_id(&child)), + ]; + let root = Page::build(&root_entries).unwrap(); + let mut builder = MetadataDagBuilder::new(MetadataDagLimits::default()); + builder.add_directory(&child, &entries).unwrap(); + builder.add_directory(&root, &root_entries).unwrap(); + PreparedNativeMetadataRetention::test_installation( + Arc::new(builder.finish(page_id(&root)).unwrap()), + "/", + ) +} + +async fn fixture() -> ( + DbConfig, + DatabaseConnection, + VerifiedQualifiedNamespace, + DatabaseConnection, + TestSchemaGuard, +) { + let temp = tempfile::tempdir().unwrap(); + let (mut config, guard) = test_db_config(temp.path()).await; + config.max_connection = 2; + config.min_connection = 1; + // Exercise the actual production bootstrap, not a test-only DDL shortcut. + let core = super::super::init::database_connection(&config) + .await + .unwrap(); + let namespace = provision_or_verify_rooted_qualified_family(&core) + .await + .unwrap(); + let mut q_config = config.clone(); + q_config.db_url = pool_url(&config.db_url, &namespace).unwrap(); + q_config.max_connection = 1; + let q = postgres_connection(&q_config).await.unwrap(); + (config, core, namespace, q, guard) +} + +async fn count(db: &C, sql: &str) -> i64 { + db.query_one_raw(Statement::from_string(DbBackend::Postgres, sql)) + .await + .unwrap() + .unwrap() + .try_get_by_index(0) + .unwrap() +} +async fn write( + writer: &ShadowQualifiedMetadataWriter, + operation: &str, + pages: &PreparedNativeMetadataRetention, +) -> GenerationMetadataReceipt { + let intent = writer.begin_intent(operation, pages).await.unwrap(); + for batch in pages.dag().payloads().chunks(64) { + writer.install_pages(&intent, batch).await.unwrap(); + } + writer.finalize(&intent).await.unwrap() +} + +#[tokio::test] +async fn physical_q_shadow_writer_is_distinct_and_restart_reuses_its_exact_catalog() { + let (config, core, namespace, q, _guard) = fixture().await; + let pages = prepared(); + let generic=super::super::native_metadata_install::generations::PostgresMetadataGenerationRepository::new(core.clone()).await.unwrap(); + let g = generic.begin_intent("same-pages-g", &pages).await.unwrap(); + for batch in pages.dag().payloads().chunks(64) { + generic.install_pages(&g, batch).await.unwrap(); + } + let g_receipt = generic.finalize(&g).await.unwrap(); + let assembly_dir = tempfile::tempdir().unwrap(); + let mut app_config = crate::config::testing::isolated_config(assembly_dir.path()); + app_config.database = config.clone(); + let storage = super::super::Storage::new_with_connection( + Arc::new(app_config), + Arc::new(core.clone()), + super::super::object_storage::mock_object_storage(), + ) + .await + .unwrap(); + let writer = storage.shadow_qualified_metadata_writer().await.unwrap(); + let clone = storage.clone(); + assert!(std::ptr::eq( + writer, + clone.shadow_qualified_metadata_writer().await.unwrap() + )); + let q_receipt = write(writer, "same-pages-q", &pages).await; + assert_eq!(g_receipt.metadata_root(), q_receipt.metadata_root()); + assert_ne!(g.prepare_id(), q_receipt.intent().prepare_id()); + assert_eq!( + count(&core, "SELECT count(*) FROM mst2_metadata_payload").await, + 2 + ); + assert_eq!( + count( + &q, + "SELECT count(*) FROM mst2_metadata_payload WHERE generation=1" + ) + .await, + 2 + ); + assert_eq!(count(&q,"SELECT count(*) FROM mst2_metadata_prepare WHERE graph_domain='qualified-v1' AND state='COMMITTED'").await,1); + assert_eq!( + count( + &q, + "SELECT count(*) FROM mst2_qualified_session_incarnation" + ) + .await, + 0 + ); + assert_eq!( + count(&q, "SELECT count(*) FROM mst2_qualified_lease_binding").await, + 0 + ); + assert_eq!(count(&q,"SELECT count(*) FROM pg_catalog.pg_class WHERE relnamespace=(SELECT oid FROM pg_catalog.pg_namespace WHERE nspname=current_schema()) AND relkind='r'").await,22); + assert_eq!(count(&q,"SELECT count(*) FROM pg_catalog.pg_class WHERE relnamespace=(SELECT oid FROM pg_catalog.pg_namespace WHERE nspname=current_schema()) AND relname IN ('mst2_retention_node','mst2_retention_edge','mst2_snapshot_context','mst2_snapshot_lease','mega_refs','git_repo','seaql_migrations')").await,0); + let q_scope = q + .query_one_raw(Statement::from_string( + DbBackend::Postgres, + "SELECT primary_scope,storage_seal FROM mst2_metadata_prepare", + )) + .await + .unwrap() + .unwrap(); + let g_scope = core + .query_one_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + "SELECT primary_scope,storage_seal FROM mst2_metadata_prepare WHERE prepare_id=$1", + [g.prepare_id().into()], + )) + .await + .unwrap() + .unwrap(); + assert_ne!( + q_scope.try_get::>("", "primary_scope").unwrap(), + g_scope.try_get::>("", "primary_scope").unwrap() + ); + assert_ne!( + q_scope.try_get::>("", "storage_seal").unwrap(), + g_scope.try_get::>("", "storage_seal").unwrap() + ); + let restarted = super::super::init::database_connection(&config) + .await + .unwrap(); + assert_eq!( + provision_or_verify_rooted_qualified_family(&restarted) + .await + .unwrap(), + namespace + ); + let fresh = ShadowQualifiedMetadataWriter::open(&restarted, &config) + .await + .unwrap(); + assert_eq!(fresh.finalize(q_receipt.intent()).await.unwrap(), q_receipt); + assert_eq!( + catalog(&q, namespace.core_oid, namespace.schema_oid) + .await + .unwrap(), + namespace.catalog_fingerprint + ); +} + +#[tokio::test] +async fn q_catalog_trigger_function_and_index_tamper_are_never_refreshed() { + for tamper in [ + "ALTER TABLE {q}.mst2_metadata_payload DISABLE TRIGGER mst2_metadata_payload_fenced", + "CREATE OR REPLACE FUNCTION {q}.mst2_metadata_has_generic_overlap(p bytea) RETURNS boolean LANGUAGE sql AS 'SELECT true'", + "DROP INDEX {q}.idx_mst2_metadata_graph_edge_child", + "CREATE RULE forged_skip AS ON INSERT TO {q}.mst2_metadata_prepare DO INSTEAD NOTHING", + ] { + let (config, core, namespace, q, _guard) = fixture().await; + let writer = ShadowQualifiedMetadataWriter::open(&core, &config) + .await + .unwrap(); + core.execute_unprepared(&tamper.replace("{q}", &identifier(&namespace.schema))) + .await + .unwrap(); + assert!(writer.begin_intent("tampered", &prepared()).await.is_err()); + let error = provision_or_verify_rooted_qualified_family(&core) + .await + .unwrap_err(); + assert!( + error.to_string().contains("catalog fingerprint changed"), + "{error}" + ); + assert_eq!( + count(&q, "SELECT count(*) FROM mst2_metadata_prepare").await, + 0 + ); + assert_eq!( + count(&core, "SELECT count(*) FROM mst2_metadata_namespace").await, + 2 + ); + let stored:Vec=core.query_one_raw(Statement::from_string(DbBackend::Postgres, + "SELECT catalog_fingerprint FROM mst2_metadata_namespace WHERE graph_domain='qualified-v1'")) + .await.unwrap().unwrap().try_get_by_index(0).unwrap(); + assert_eq!(stored, namespace.catalog_fingerprint); + } +} + +#[tokio::test] +async fn q_schema_rename_fails_closed_without_reprovisioning_or_g_fallback() { + let (config, core, namespace, _q, _guard) = fixture().await; + let writer = ShadowQualifiedMetadataWriter::open(&core, &config) + .await + .unwrap(); + let renamed = format!("{}_renamed", namespace.schema); + core.execute_unprepared(&format!( + "ALTER SCHEMA {} RENAME TO {}", + identifier(&namespace.schema), + identifier(&renamed) + )) + .await + .unwrap(); + assert!(writer.begin_intent("renamed", &prepared()).await.is_err()); + assert!( + provision_or_verify_rooted_qualified_family(&core) + .await + .is_err() + ); + assert_eq!( + count(&core, "SELECT count(*) FROM mst2_metadata_namespace").await, + 2 + ); + assert_eq!( + count(&core, "SELECT count(*) FROM mst2_metadata_prepare").await, + 0 + ); + assert_eq!( + count( + &core, + &format!( + "SELECT count(*) FROM {}.mst2_metadata_prepare", + identifier(&renamed) + ) + ) + .await, + 0 + ); +} + +#[tokio::test] +async fn q_actual_schema_rr_temp_shadow_and_admitted_ledgers_keep_their_guards() { + let (_config, core, namespace, q, _guard) = fixture().await; + let bad = core + .execute_unprepared(&format!( + "INSERT INTO {}.mst2_metadata_gc_op SELECT * FROM {}.mst2_metadata_gc_op WHERE false", + identifier(&namespace.schema), + identifier(&namespace.schema) + )) + .await + .unwrap_err(); + assert!(bad.to_string().contains("captured primary family"), "{bad}"); + let rr = q + .begin_with_config(Some(IsolationLevel::RepeatableRead), None) + .await + .unwrap(); + let error = rr + .execute_unprepared( + "INSERT INTO mst2_metadata_gc_op SELECT * FROM mst2_metadata_gc_op WHERE false", + ) + .await + .unwrap_err(); + assert!(error.to_string().contains("READ COMMITTED"), "{error}"); + rr.rollback().await.unwrap(); + for statement in [ + "SELECT mst2_metadata_gc_apply(gen_random_uuid())", + "SELECT mst2_metadata_gc_finish(gen_random_uuid())", + ] { + let error = q.execute_unprepared(statement).await.unwrap_err(); + assert!( + error.to_string().contains("query returned no rows"), + "{error}" + ); + } + assert_eq!( + count(&q, "SELECT count(*) FROM mst2_metadata_gc_op").await, + 0 + ); + for table in [ + "mst2_qualified_session_incarnation", + "mst2_qualified_lease_binding", + ] { + assert!( + q.execute_unprepared(&format!("INSERT INTO {table} DEFAULT VALUES")) + .await + .is_err() + ); + assert_eq!(count(&q, &format!("SELECT count(*) FROM {table}")).await, 0); + } + q.execute_unprepared("CREATE TEMP TABLE mst2_metadata_prepare (prepare_id text)") + .await + .unwrap(); + let repository = + PostgresQualifiedMetadataRepository::registered_shadow(q.clone(), namespace.clone()) + .await + .unwrap(); + repository + .begin_intent("temp-shadow", &prepared()) + .await + .unwrap(); + assert_eq!( + count( + &q, + &format!( + "SELECT count(*) FROM {}.mst2_metadata_prepare", + identifier(&namespace.schema) + ) + ) + .await, + 1 + ); + assert_eq!( + count(&q, "SELECT count(*) FROM pg_temp.mst2_metadata_prepare").await, + 0 + ); + let duplicate=core.execute_unprepared("INSERT INTO mst2_metadata_namespace SELECT * FROM mst2_metadata_namespace WHERE graph_domain='qualified-v1'").await.unwrap_err(); + assert!( + duplicate + .to_string() + .contains("exact admitted rooted family scope"), + "{duplicate}" + ); + assert_eq!( + count(&core, "SELECT count(*) FROM mst2_metadata_namespace").await, + 2 + ); + let pk:String=q.query_one_raw(Statement::from_string(DbBackend::Postgres, + "SELECT conkey::text FROM pg_catalog.pg_constraint WHERE conrelid='mst2_qualified_session_incarnation'::regclass AND contype='p'")) + .await.unwrap().unwrap().try_get_by_index(0).unwrap(); + assert_eq!( + pk, "{1,2}", + "same SID can have future distinct incarnations" + ); +} + +#[tokio::test] +async fn provisioning_locks_candidate_and_registered_union_in_uuid_order() { + let temp = tempfile::tempdir().unwrap(); + let (config, _guard) = test_db_config(temp.path()).await; + let core = Database::connect(config.db_url.clone()).await.unwrap(); + Migrator::up(&core, None).await.unwrap(); + let other = Database::connect(config.db_url).await.unwrap(); + let captured = captured_core(&core).await.unwrap(); + let candidate = "00000000-0000-4000-8000-000000000001"; + let schema = format!("mst2q_{}", candidate.replace('-', "")); + let held = core.begin().await.unwrap(); + held.execute_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + "SELECT pg_catalog.pg_advisory_xact_lock(1296717362,pg_catalog.hashtext($1))", + [schema.clone().into()], + )) + .await + .unwrap(); + let (core_name, q_name) = (captured.0.clone(), schema.clone()); + let waiter = tokio::spawn(async move { + let txn = other.begin().await.unwrap(); + txn.execute_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + format!( + "SELECT {}.mst2_route_family_candidate_enter($1,$2::uuid,$3)", + identifier(&core_name) + ), + [core_name.into(), candidate.into(), q_name.into()], + )) + .await + .unwrap(); + txn.rollback().await.unwrap(); + }); + let deadline = tokio::time::Instant::now() + Duration::from_secs(4); + loop { + let row=held.query_one_raw(Statement::from_sql_and_values(DbBackend::Postgres, + "SELECT EXISTS(SELECT 1 FROM pg_catalog.pg_locks w WHERE w.locktype='advisory' AND NOT w.granted + AND w.classid=1296717362::oid AND w.objid=pg_catalog.hashtext($1)::oid AND w.objsubid=2 + AND w.database=(SELECT oid FROM pg_catalog.pg_database WHERE datname=current_database()) + AND NOT EXISTS(SELECT 1 FROM pg_catalog.pg_locks g WHERE g.pid=w.pid AND g.locktype='advisory' + AND g.classid=1296717362::oid AND g.objid=pg_catalog.hashtext($2)::oid AND g.objsubid=2 AND g.granted)) AS sorted_wait", + [schema.clone().into(),captured.0.clone().into()])).await.unwrap().unwrap(); + if row.try_get::("", "sorted_wait").unwrap() { + break; + } + assert!( + tokio::time::Instant::now() < deadline, + "candidate must be locked before G when its UUID sorts first" + ); + tokio::task::yield_now().await; + } + held.rollback().await.unwrap(); + waiter.await.unwrap(); + assert_eq!( + count(&core, "SELECT count(*) FROM mst2_metadata_namespace").await, + 1 + ); +} + +#[tokio::test] +async fn q_incomplete_commit_null_binding_and_unsealed_plan_are_rejected() { + let (config, core, namespace, q, _guard) = fixture().await; + let writer = ShadowQualifiedMetadataWriter::open(&core, &config) + .await + .unwrap(); + let intent = writer + .begin_intent("fixed-incomplete", &prepared()) + .await + .unwrap(); + let error=q.execute_raw(Statement::from_sql_and_values(DbBackend::Postgres, + "UPDATE mst2_metadata_prepare SET state='COMMITTED',committed_at=clock_timestamp() WHERE prepare_id=$1",[intent.prepare_id().into()])).await.unwrap_err(); + assert!( + error.to_string().contains("complete exact graph payload"), + "{error}" + ); + for mutation in [ + "graph_domain='generic-v1'", + "canonical_plan=canonical_plan||decode('ff','hex')", + "storage_seal=NULL", + "primary_scope=NULL", + ] { + let error = q + .execute_unprepared(&format!("UPDATE mst2_metadata_prepare SET {mutation}")) + .await + .unwrap_err(); + assert!( + error.to_string().contains("complete identity is immutable"), + "{error}" + ); + } + for batch in prepared().dag().payloads().chunks(64) { + writer.install_pages(&intent, batch).await.unwrap(); + } + writer.finalize(&intent).await.unwrap(); + assert_eq!( + count( + &q, + "SELECT count(*) FROM mst2_metadata_prepare WHERE state='COMMITTED'" + ) + .await, + 1 + ); + assert_eq!( + catalog(&q, namespace.core_oid, namespace.schema_oid) + .await + .unwrap(), + namespace.catalog_fingerprint + ); +} + +#[tokio::test] +async fn first_registration_cannot_self_sign_a_minimal_or_half_installed_family() { + for complete in [false, true] { + let temp = tempfile::tempdir().unwrap(); + let (config, _guard) = test_db_config(temp.path()).await; + let core = Database::connect(config.db_url).await.unwrap(); + Migrator::up(&core, None).await.unwrap(); + let captured = captured_core(&core).await.unwrap(); + let candidate = uuid::Uuid::new_v4().to_string(); + let schema = format!("mst2q_{}", candidate.replace('-', "")); + let storage = uuid::Uuid::new_v4().to_string(); + let txn = core.begin().await.unwrap(); + txn.execute_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + format!( + "SELECT {}.mst2_route_family_candidate_enter($1,$2::uuid,$3)", + identifier(&captured.0) + ), + [ + captured.0.clone().into(), + candidate.clone().into(), + schema.clone().into(), + ], + )) + .await + .unwrap(); + txn.execute_unprepared(&format!("CREATE SCHEMA {}", identifier(&schema))) + .await + .unwrap(); + let oid: i64 = txn + .query_one_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + "SELECT oid::bigint FROM pg_catalog.pg_namespace WHERE nspname=$1", + [schema.clone().into()], + )) + .await + .unwrap() + .unwrap() + .try_get_by_index(0) + .unwrap(); + if complete { + txn.execute_unprepared(&render_family( + &captured.0, + captured.1, + &schema, + oid, + &candidate, + &storage, + )) + .await + .unwrap(); + } else { + txn.execute_unprepared(&format!( + "CREATE TABLE {q}.mst2_metadata_storage_scope(singleton integer,storage_uuid text); + INSERT INTO {q}.mst2_metadata_storage_scope VALUES(1,{storage}); + CREATE TABLE {q}.mst2_metadata_family_identity(singleton integer,namespace_uuid uuid,storage_uuid text, + core_schema_oid oid,metadata_schema_oid oid,family_identity text,implementation_fingerprint bytea); + INSERT INTO {q}.mst2_metadata_family_identity VALUES(1,{candidate}::uuid,{storage},{core_oid},{oid},'{family}',decode('{implementation}','hex'))", + q=identifier(&schema),storage=literal(&storage),candidate=literal(&candidate),core_oid=captured.1, + family=FAMILY,implementation=hex::encode(implementation_fingerprint()))).await.unwrap(); + } + let fingerprint = if complete { + vec![0xff; 32] + } else { + catalog(&txn, captured.1, oid).await.unwrap() + }; + txn.execute_unprepared(&format!( + "SET LOCAL search_path={},pg_catalog,pg_temp", + identifier(&captured.0) + )) + .await + .unwrap(); + let error=txn.execute_raw(Statement::from_sql_and_values(DbBackend::Postgres,format!( + "INSERT INTO {c}.mst2_metadata_namespace(singleton,namespace_uuid,core_schema,core_schema_oid,database_name, + database_oid,storage_uuid,server_address,server_port,mono_lock_key2,metadata_schema,metadata_schema_oid, + family_identity,graph_domain,admission_state,collector_state,metadata_storage_uuid,implementation_fingerprint,catalog_fingerprint) + SELECT NULL,$1::uuid,g.core_schema,g.core_schema_oid,g.database_name,g.database_oid,g.storage_uuid,g.server_address, + g.server_port,g.mono_lock_key2,$2,$3::bigint::oid,$4,'qualified-v1','ROOTED_Q_ADMITTED','ENABLED',$5,$6,$7 + FROM {c}.mst2_metadata_namespace g WHERE singleton=1",c=identifier(&captured.0)), + [candidate.into(),schema.clone().into(),oid.into(),FAMILY.into(),storage.into(),implementation_fingerprint().into(),fingerprint.into()])).await.unwrap_err(); + let expected = if complete { + "fingerprint disagrees" + } else { + "trusted complete physical family shape" + }; + assert!(error.to_string().contains(expected), "{error}"); + txn.rollback().await.unwrap(); + assert_eq!( + count(&core, "SELECT count(*) FROM mst2_metadata_namespace").await, + 1 + ); + assert_eq!( + count( + &core, + &format!( + "SELECT count(*) FROM pg_catalog.pg_namespace WHERE nspname={}", + literal(&schema) + ) + ) + .await, + 0 + ); + assert_eq!( + count(&core, "SELECT count(*) FROM mst2_qualified_family_policy").await, + 1 + ); + } +} + +#[tokio::test] +async fn simultaneous_production_bootstraps_reuse_one_physical_q_namespace() { + let temp = tempfile::tempdir().unwrap(); + let (config, _guard) = test_db_config(temp.path()).await; + let core = postgres_connection(&config).await.unwrap(); + Migrator::up(&core, None).await.unwrap(); + let (left, right) = tokio::join!( + super::super::init::database_connection(&config), + super::super::init::database_connection(&config) + ); + let left = left.unwrap(); + let right = right.unwrap(); + let a = provision_or_verify_rooted_qualified_family(&left) + .await + .unwrap(); + let b = provision_or_verify_rooted_qualified_family(&right) + .await + .unwrap(); + assert_eq!(a, b); + assert_eq!( + count(&core, "SELECT count(*) FROM mst2_metadata_namespace").await, + 2 + ); + assert_eq!( + count( + &core, + &format!( + "SELECT count(*) FROM pg_catalog.pg_namespace WHERE nspname={}", + literal(&a.schema) + ) + ) + .await, + 1 + ); +} + +#[tokio::test] +async fn trusted_shape_policy_and_core_validation_functions_are_immutable_or_fail_closed() { + let (config, core, namespace, _q, _guard) = fixture().await; + for sql in [ + "UPDATE mst2_qualified_family_policy SET expected_shape=decode(repeat('ff',32),'hex')", + "DELETE FROM mst2_qualified_family_policy", + "TRUNCATE mst2_qualified_family_policy", + ] { + let error = core.execute_unprepared(sql).await.unwrap_err(); + assert!(error.to_string().contains("immutable"), "{error}"); + } + let writer = ShadowQualifiedMetadataWriter::open(&core, &config) + .await + .unwrap(); + core.execute_unprepared(&format!( + "CREATE OR REPLACE FUNCTION {}.mst2_route_family_shape(q_oid oid,n_uuid uuid,s_uuid text) + RETURNS bytea LANGUAGE sql AS 'SELECT decode(repeat(''ff'',32),''hex'')'", + identifier(&namespace.core_schema) + )) + .await + .unwrap(); + let error = provision_or_verify_rooted_qualified_family(&core) + .await + .unwrap_err(); + assert!( + error + .to_string() + .contains("trusted core authority catalog changed"), + "{error}" + ); + assert!( + writer + .begin_intent("bad-core-validator", &prepared()) + .await + .is_err() + ); + assert_eq!( + count(&core, "SELECT count(*) FROM mst2_metadata_namespace").await, + 2 + ); +} + +#[tokio::test] +async fn random_q_templates_have_one_trusted_shape_and_changed_composite_signature_is_rejected() { + let temp = tempfile::tempdir().unwrap(); + let (config, _guard) = test_db_config(temp.path()).await; + let core = postgres_connection(&config).await.unwrap(); + Migrator::up(&core, None).await.unwrap(); + let captured = captured_core(&core).await.unwrap(); + let expected: Vec = core + .query_one_raw(Statement::from_string( + DbBackend::Postgres, + "SELECT expected_shape FROM mst2_qualified_family_policy WHERE singleton=1", + )) + .await + .unwrap() + .unwrap() + .try_get_by_index(0) + .unwrap(); + let authority = catalog(&core, captured.1, 0).await.unwrap(); + let mut schemas = Vec::new(); + for _ in 0..2 { + let namespace = uuid::Uuid::new_v4().to_string(); + let schema = format!("mst2q_{}", namespace.replace('-', "")); + let storage = uuid::Uuid::new_v4().to_string(); + let txn = core.begin().await.unwrap(); + txn.execute_unprepared(&format!("CREATE SCHEMA {}", identifier(&schema))) + .await + .unwrap(); + let oid = count( + &txn, + &format!( + "SELECT oid::bigint FROM pg_catalog.pg_namespace WHERE nspname={}", + literal(&schema), + ), + ) + .await; + txn.execute_unprepared(&render_family( + &captured.0, + captured.1, + &schema, + oid, + &namespace, + &storage, + )) + .await + .unwrap(); + let shape = || { + Statement::from_sql_and_values( + DbBackend::Postgres, + format!( + "SELECT {}.mst2_route_family_shape($1::bigint::oid,$2::uuid,$3)", + identifier(&captured.0), + ), + [oid.into(), namespace.clone().into(), storage.clone().into()], + ) + }; + let actual: Vec = txn + .query_one_raw(shape()) + .await + .unwrap() + .unwrap() + .try_get_by_index(0) + .unwrap(); + assert_eq!( + actual, expected, + "random template must retain its trusted structural signature" + ); + assert_eq!( + catalog_with_exemption(&txn, captured.1, 0, oid) + .await + .unwrap(), + authority + ); + assert_ne!( + catalog(&txn, captured.1, 0).await.unwrap(), + authority, + "an unregistered candidate is never implicitly exempted" + ); + txn.execute_unprepared(&format!( + "DROP FUNCTION {q}.mst2_metadata_rooted_manifest({q}.mst2_metadata_prepare); + CREATE FUNCTION {q}.mst2_metadata_rooted_manifest(q text) RETURNS jsonb + LANGUAGE sql IMMUTABLE STRICT AS 'SELECT to_jsonb(q)'", + q = identifier(&schema), + )) + .await + .unwrap(); + let altered: Vec = txn + .query_one_raw(shape()) + .await + .unwrap() + .unwrap() + .try_get_by_index(0) + .unwrap(); + assert_ne!( + altered, expected, + "normalization must preserve the composite argument type" + ); + txn.rollback().await.unwrap(); + schemas.push(schema); + } + assert_ne!(schemas[0], schemas[1]); + assert_eq!(catalog(&core, captured.1, 0).await.unwrap(), authority); + assert!(registered(&core, &captured).await.unwrap().is_none()); +} + +#[tokio::test] +async fn fresh_q_core_authority_exempts_only_exact_q_ri_and_restart_keeps_full_catalog_binding() { + for tamper in ["fk", "ri", "external"] { + let (config, core, namespace, _q, _guard) = fixture().await; + let captured = (namespace.core_schema.clone(), namespace.core_oid); + let authority: Vec = core + .query_one_raw(Statement::from_string( + DbBackend::Postgres, + "SELECT authority_catalog FROM mst2_qualified_family_policy WHERE singleton=1", + )) + .await + .unwrap() + .unwrap() + .try_get_by_index(0) + .unwrap(); + assert_eq!( + catalog_with_exemption(&core, captured.1, 0, namespace.schema_oid) + .await + .unwrap(), + authority + ); + assert_ne!(catalog(&core, captured.1, 0).await.unwrap(), authority); + for _ in 0..2 { + assert_eq!( + registered(&core, &captured).await.unwrap(), + Some(namespace.clone()) + ); + } + let restarted = postgres_connection(&config).await.unwrap(); + assert_eq!( + provision_or_verify_rooted_qualified_family(&restarted) + .await + .unwrap(), + namespace + ); + let txn = core.begin().await.unwrap(); + if tamper == "external" { + let external = format!("hostile_{}", uuid::Uuid::new_v4().simple()); + txn.execute_unprepared(&format!( + "CREATE SCHEMA {external}; CREATE TABLE {external}.forged_route(snapshot_id text,namespace_uuid uuid, + FOREIGN KEY(snapshot_id,namespace_uuid) REFERENCES {c}.mst2_snapshot_storage_route(snapshot_id,namespace_uuid))", + external=identifier(&external),c=identifier(&captured.0), + )).await.unwrap(); + assert_ne!( + catalog_with_exemption(&txn, captured.1, 0, namespace.schema_oid) + .await + .unwrap(), + authority, + "another namespace's internal RI triggers remain part of core authority" + ); + } else { + let row=txn.query_one_raw(Statement::from_sql_and_values(DbBackend::Postgres, + "SELECT fk.conname,t.tgname FROM pg_catalog.pg_constraint fk + JOIN pg_catalog.pg_trigger t ON t.tgconstraint=fk.oid AND t.tgrelid=fk.confrelid AND t.tgisinternal + WHERE fk.connamespace=$1::bigint::oid AND fk.confrelid=$2::regclass AND fk.contype='f' LIMIT 1", + [namespace.schema_oid.into(),format!("{}.mst2_snapshot_storage_route",identifier(&captured.0)).into()])) + .await.unwrap().unwrap(); + let conname: String = row.try_get("", "conname").unwrap(); + let trigger: String = row.try_get("", "tgname").unwrap(); + let sql = if tamper == "fk" { + format!( + "ALTER TABLE {}.mst2_qualified_session_incarnation DROP CONSTRAINT {}", + identifier(&namespace.schema), + identifier(&conname) + ) + } else { + format!( + "ALTER TABLE {}.mst2_snapshot_storage_route DISABLE TRIGGER {}", + identifier(&captured.0), + identifier(&trigger) + ) + }; + txn.execute_unprepared(&sql).await.unwrap(); + assert_eq!( + catalog_with_exemption(&txn, captured.1, 0, namespace.schema_oid) + .await + .unwrap(), + authority + ); + assert_ne!( + catalog(&txn, captured.1, namespace.schema_oid) + .await + .unwrap(), + namespace.catalog_fingerprint + ); + } + assert!( + registered(&txn, &captured).await.is_err(), + "{tamper} tampering cannot refresh admission" + ); + txn.rollback().await.unwrap(); + assert_eq!(registered(&core, &captured).await.unwrap(), Some(namespace)); + } +} diff --git a/src/jupiter/storage/qualified_metadata_gc.rs b/src/jupiter/storage/qualified_metadata_gc.rs new file mode 100644 index 00000000..394d9335 --- /dev/null +++ b/src/jupiter/storage/qualified_metadata_gc.rs @@ -0,0 +1,540 @@ +//! Bounded rooted maintenance. Local retry state is a hint, never GC authority. + +use super::*; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum GcPhase { + Claim, + Apply, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct GcSeal { + operation_id: String, + page_id: [u8; 32], + generation: i64, + primary_scope: Vec, + metadata_codec: i16, + expected_size: i32, + graph_present: bool, + had_payload: bool, + certificate_digest: Option<[u8; 32]>, +} + +#[derive(Debug, Clone)] +struct RetryOperation { + seal: GcSeal, + phase: GcPhase, +} + +#[derive(Debug, Default)] +pub(super) struct RootedMaintenanceState { + retry: Option, + cursor: Option<([u8; 32], i64)>, +} + +#[derive(Debug, Default, Clone, serde::Serialize)] +pub(crate) struct RootedMaintenanceWork { + pub collector_enabled: bool, + pub examined: u16, + pub lifetimes_examined: u16, + pub owners_examined: u16, + pub pending_replayed: u16, + pub uncertain_receipts_observed: u16, + pub absent_claims_observed: u16, + pub gc_claimed: u16, + pub gc_applied: u16, + pub payload_pages_removed: u16, + pub payload_bytes_removed: u64, + pub readers_expired: u16, + pub leases_expired: u16, + pub prepares_aborted: u16, + pub handovers_retired: u16, + pub orphans_retired: u16, +} + +struct GcRecord { + seal: GcSeal, + applied: bool, +} + +fn uncertain(operation: &RetryOperation) -> SnapshotError { + SnapshotError::new( + crate::ceres::snapshot::error::SnapshotErrorCode::TemporaryUnavailable, + format!( + "qualified GC outcome unknown for operation {} generation {} phase {:?}; retry the sealed operation on the same primary", + operation.seal.operation_id, operation.seal.generation, operation.phase + ), + ) +} + +fn gc_record(row: &QueryResult) -> Result { + let operation_id: String = row.try_get("", "operation_id").map_err(internal)?; + let id = uuid::Uuid::parse_str(&operation_id).map_err(internal)?; + let certificate: Option> = row.try_get("", "certificate_digest").map_err(internal)?; + let seal = GcSeal { + operation_id, + page_id: digest_column(row, "page_id")?, + generation: row.try_get("", "generation").map_err(internal)?, + primary_scope: row.try_get("", "primary_scope").map_err(internal)?, + metadata_codec: row.try_get("", "metadata_codec").map_err(internal)?, + expected_size: row.try_get("", "expected_size").map_err(internal)?, + graph_present: row.try_get("", "graph_present").map_err(internal)?, + had_payload: row.try_get("", "had_payload").map_err(internal)?, + certificate_digest: certificate + .map(|bytes| bytes.as_slice().try_into().map_err(internal)) + .transpose()?, + }; + if id.get_version_num() != 4 + || id.to_string() != seal.operation_id + || seal.generation <= 0 + || seal.metadata_codec != 1 + || seal.expected_size <= 0 + || seal.graph_present != seal.certificate_digest.is_some() + || row + .try_get::("", "graph_domain") + .map_err(internal)? + != "qualified-v1" + { + return Err(integrity( + "qualified GC immutable operation profile is invalid", + )); + } + let state: String = row.try_get("", "state").map_err(internal)?; + let applied = match state.as_str() { + "PENDING" => false, + "APPLIED" => true, + _ => return Err(integrity("qualified GC operation has an unknown state")), + }; + Ok(GcRecord { seal, applied }) +} + +async fn load_gc( + db: &C, + operation: &str, +) -> Result, SnapshotError> { + db.query_one_raw(sql( + "SELECT operation_id::text,page_id,generation,primary_scope,graph_domain,metadata_codec, + expected_size,graph_present,had_payload,certificate_digest,state + FROM mst2_metadata_gc_op WHERE operation_id=$1::uuid", + [operation.into()], + )) + .await + .map_err(internal)? + .as_ref() + .map(gc_record) + .transpose() +} + +impl RootedQualifiedMetadataRepository { + pub(crate) fn start_maintenance(repository: &std::sync::Arc) { + let repository = std::sync::Arc::downgrade(repository); + tokio::spawn(async move { + let mut interval = tokio::time::interval(std::time::Duration::from_secs(30)); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + interval.tick().await; + loop { + interval.tick().await; + let Some(repository) = repository.upgrade() else { + break; + }; + match tokio::time::timeout( + std::time::Duration::from_secs(15), + repository.maintenance_tick(64), + ) + .await + { + Ok(Ok(_)) => {} + Ok(Err(error)) => { + tracing::warn!(code=?error.code, "rooted metadata maintenance failed; durable roots and sealed operations remain recoverable") + } + Err(_) => tracing::warn!( + "rooted metadata maintenance reached its deadline; retrying durable state on the next tick" + ), + } + } + }); + } + + async fn collector_enabled(&self) -> Result { + let txn = self.transaction().await?; + let result = async { + let row = txn + .query_one_raw(sql("SELECT mst2_metadata_gc_enabled() AS enabled", [])) + .await + .map_err(internal)? + .ok_or_else(|| integrity("qualified collector policy is missing"))?; + row.try_get("", "enabled").map_err(internal) + } + .await; + let _ = txn.rollback().await; + result + } + + fn require_gc_seal(&self, seal: &GcSeal) -> Result<(), SnapshotError> { + if seal.primary_scope != self.primary_scope { + return Err(integrity( + "qualified GC seal belongs to another captured primary", + )); + } + Ok(()) + } + + async fn prove_gc_record( + &self, + txn: &DatabaseTransaction, + record: &GcRecord, + ) -> Result<(), SnapshotError> { + self.require_gc_seal(&record.seal)?; + let seal = &record.seal; + txn.query_one_raw(sql( + "SELECT mst2_metadata_gc_proof($1,$2,$3,$4,$5,$6)", + [ + seal.page_id.to_vec().into(), + seal.generation.into(), + if record.applied { "APPLIED" } else { "PENDING" }.into(), + (!record.applied && seal.graph_present).into(), + (!record.applied && seal.had_payload).into(), + seal.certificate_digest.map(|value| value.to_vec()).into(), + ], + )) + .await + .map_err(internal)?; + Ok(()) + } + + async fn observe_gc_retry( + &self, + operation: &RetryOperation, + ) -> Result, SnapshotError> { + self.require_gc_seal(&operation.seal)?; + // transaction() acquires the same-primary route completion barrier. + // A failed barrier or read never proves that an uncertain claim is absent. + let txn = self.transaction().await.map_err(|_| uncertain(operation))?; + let result = async { + let record = load_gc(&txn, &operation.seal.operation_id).await?; + if let Some(record) = &record { + if record.seal != operation.seal { + return Err(integrity( + "qualified GC retry retargeted its immutable sealed identity", + )); + } + self.prove_gc_record(&txn, record).await?; + } + Ok(record) + } + .await; + txn.rollback().await.map_err(|_| uncertain(operation))?; + result.map_err(|error: SnapshotError| { + if error.code == crate::ceres::snapshot::error::SnapshotErrorCode::Internal { + uncertain(operation) + } else { + error + } + }) + } + + async fn claim_gc_candidate( + &self, + candidate: GcSeal, + state: &mut RootedMaintenanceState, + ) -> Result { + self.require_gc_seal(&candidate)?; + let txn = self.transaction().await?; + let result = async { + txn.query_one_raw(sql( + "SELECT mst2_metadata_gc_claim($1,$2,$3,$4::uuid)", + [ + candidate.page_id.to_vec().into(), + candidate.generation.into(), + candidate.primary_scope.clone().into(), + candidate.operation_id.clone().into(), + ], + )) + .await + .map_err(internal)?; + let record = load_gc(&txn, &candidate.operation_id) + .await? + .ok_or_else(|| integrity("qualified claimed GC operation disappeared"))?; + if record.seal != candidate || record.applied { + return Err(integrity( + "qualified claim differs from its exact captured candidate", + )); + } + self.prove_gc_record(&txn, &record).await?; + Ok(record.seal) + } + .await; + let seal = match result { + Ok(seal) => seal, + Err(error) => { + let _ = txn.rollback().await; + return Err(error); + } + }; + let retry = RetryOperation { + seal: seal.clone(), + phase: GcPhase::Claim, + }; + // Retain the hint before COMMIT so task cancellation cannot lose an + // already transmitted commit whose durable outcome is still unknown. + state.retry = Some(retry.clone()); + txn.commit().await.map_err(|_| uncertain(&retry))?; + state.retry = None; + Ok(seal) + } + + async fn apply_gc_seal( + &self, + seal: &GcSeal, + state: &mut RootedMaintenanceState, + ) -> Result { + self.require_gc_seal(seal)?; + let txn = self.transaction().await?; + let result = async { + let record = load_gc(&txn, &seal.operation_id) + .await? + .ok_or_else(|| integrity("qualified committed GC operation is missing"))?; + if &record.seal != seal { + return Err(integrity( + "qualified apply retargeted its exact immutable GC operation", + )); + } + self.prove_gc_record(&txn, &record).await?; + if record.applied { + return Ok(false); + } + txn.query_one_raw(sql( + "SELECT mst2_metadata_gc_apply($1::uuid)", + [seal.operation_id.clone().into()], + )) + .await + .map_err(internal)?; + let applied = load_gc(&txn, &seal.operation_id) + .await? + .ok_or_else(|| integrity("qualified GC apply lost its durable receipt"))?; + if &applied.seal != seal || !applied.applied { + return Err(integrity( + "qualified GC apply did not preserve its sealed APPLIED identity", + )); + } + self.prove_gc_record(&txn, &applied).await?; + Ok(true) + } + .await; + let changed = match result { + Ok(changed) => changed, + Err(error) => { + let _ = txn.rollback().await; + return Err(error); + } + }; + let retry = RetryOperation { + seal: seal.clone(), + phase: GcPhase::Apply, + }; + state.retry = Some(retry.clone()); + txn.commit().await.map_err(|_| uncertain(&retry))?; + state.retry = None; + Ok(changed) + } + + async fn pending_gc_seals(&self, limit: u16) -> Result, SnapshotError> { + let txn = self.transaction().await?; + let result = async { + let rows = txn.query_all_raw(sql( + "SELECT operation_id::text,page_id,generation,primary_scope,graph_domain,metadata_codec, + expected_size,graph_present,had_payload,certificate_digest,state + FROM mst2_metadata_gc_op WHERE state='PENDING' ORDER BY created_at,operation_id LIMIT $1", + [i64::from(limit).into()], + )).await.map_err(internal)?; + rows.iter().map(|row| { + let record=gc_record(row)?; + self.require_gc_seal(&record.seal)?; + Ok(record.seal) + }).collect() + }.await; + let _ = txn.rollback().await; + result + } + + async fn cleanup_gc_owners( + &self, + limit: u16, + work: &mut RootedMaintenanceWork, + ) -> Result<(), SnapshotError> { + let txn = self.transaction().await?; + let result = async { + txn.query_one_raw(sql( + "SELECT * FROM mst2_metadata_gc_owner_cleanup($1::integer)", + [i32::from(limit).into()], + )) + .await + .map_err(internal)? + .ok_or_else(|| integrity("qualified owner cleanup lost its bounded counters")) + } + .await; + let row = sessions::finish(txn, result).await?; + let count = |name: &str| -> Result { + u16::try_from(row.try_get::("", name).map_err(internal)?).map_err(internal) + }; + let examined = count("examined")?; + let readers = count("readers_expired")?; + let leases = count("leases_expired")?; + let prepares = count("prepares_aborted")?; + let handovers = count("handovers_retired")?; + let orphans = count("orphans_retired")?; + if examined > limit || readers + leases + prepares + handovers + orphans > examined { + return Err(integrity( + "qualified owner cleanup counters exceed their shared budget", + )); + } + work.examined += examined; + work.owners_examined += examined; + work.readers_expired += readers; + work.leases_expired += leases; + work.prepares_aborted += prepares; + work.handovers_retired += handovers; + work.orphans_retired += orphans; + Ok(()) + } + + async fn gc_candidates( + &self, + after: Option<([u8; 32], i64)>, + limit: u16, + ) -> Result, SnapshotError> { + let (page, generation) = after.map(|(p, g)| (p.to_vec(), g)).unwrap_or_default(); + let txn = self.transaction().await?; + let result = async { + let rows = txn.query_all_raw(sql( + "SELECT l.page_id,l.generation,l.metadata_codec,l.expected_size, + n.page_id IS NOT NULL AS graph_present,n.certificate_digest, + EXISTS(SELECT 1 FROM mst2_metadata_payload b WHERE b.page_id=l.page_id AND b.generation=l.generation) AS had_payload, + NOT EXISTS(SELECT 1 FROM mst2_metadata_root_anchor a WHERE a.root_page=l.page_id AND a.root_generation=l.generation) + AND NOT EXISTS(SELECT 1 FROM mst2_metadata_graph_root r WHERE r.page_id=l.page_id AND r.generation=l.generation) + AND NOT EXISTS(SELECT 1 FROM mst2_metadata_graph_edge e WHERE e.child_page=l.page_id AND e.child_generation=l.generation) + AND NOT EXISTS(SELECT 1 FROM mst2_metadata_prepare_page m JOIN mst2_metadata_prepare q USING(prepare_id) + WHERE m.page_id=l.page_id AND m.generation=l.generation + AND (q.state='PREPARING' OR q.state='COMMITTED' AND q.coverage_retired_at IS NULL)) + AND NOT EXISTS(SELECT 1 FROM mst2_metadata_prepare_reuse_root r JOIN mst2_metadata_prepare q USING(prepare_id) + WHERE r.root_page=l.page_id AND r.root_generation=l.generation + AND (q.state='PREPARING' OR q.state='COMMITTED' AND q.coverage_retired_at IS NULL)) + AND (l.state='LIVE' OR EXISTS(SELECT 1 FROM mst2_metadata_prepare_page m JOIN mst2_metadata_prepare q USING(prepare_id) + WHERE m.page_id=l.page_id AND m.generation=l.generation AND q.state='ABORTED' AND q.storage_seal IS NOT NULL) + AND NOT EXISTS(SELECT 1 FROM mst2_metadata_prepare_page m JOIN mst2_metadata_prepare q USING(prepare_id) + WHERE m.page_id=l.page_id AND m.generation=l.generation AND q.state<>'ABORTED')) AS eligible + FROM mst2_metadata_lifetime l JOIN mst2_metadata_current c USING(page_id,generation) + LEFT JOIN mst2_metadata_graph_node n USING(page_id,generation) + WHERE l.state IN ('RESERVED','LIVE') AND (l.page_id,l.generation)>($1,$2) + ORDER BY l.page_id,l.generation LIMIT $3", + [page.into(),generation.into(),i64::from(limit).into()], + )).await.map_err(internal)?; + rows.iter().map(|row| { + let certificate:Option>=row.try_get("","certificate_digest").map_err(internal)?; + Ok((GcSeal { + operation_id:uuid::Uuid::new_v4().to_string(), + page_id:digest_column(row,"page_id")?, + generation:row.try_get("","generation").map_err(internal)?, + primary_scope:self.primary_scope.clone(), + metadata_codec:row.try_get("","metadata_codec").map_err(internal)?, + expected_size:row.try_get("","expected_size").map_err(internal)?, + graph_present:row.try_get("","graph_present").map_err(internal)?, + had_payload:row.try_get("","had_payload").map_err(internal)?, + certificate_digest:certificate.map(|bytes|bytes.as_slice().try_into().map_err(internal)).transpose()?, + },row.try_get("","eligible").map_err(internal)?)) + }).collect() + }.await; + let _ = txn.rollback().await; + result + } + + /// A shared owner/page work budget; SQL proof and capacity scans are separately bounded. + pub(crate) async fn maintenance_tick( + &self, + limit: u16, + ) -> Result { + if !(1..=64).contains(&limit) { + return Err(integrity("qualified maintenance limit must be 1..=64")); + } + let mut state = self.maintenance_state.lock().await; + let mut work = RootedMaintenanceWork { + collector_enabled: self.collector_enabled().await?, + ..Default::default() + }; + if let Some(retry) = state.retry.clone() { + work.examined += 1; + work.lifetimes_examined += 1; + match self.observe_gc_retry(&retry).await? { + Some(record) if record.applied => { + state.retry = None; + work.uncertain_receipts_observed += 1; + } + Some(record) if work.collector_enabled => { + let changed = self.apply_gc_seal(&record.seal, &mut state).await?; + work.pending_replayed += 1; + work.record_applied(&record.seal, changed)?; + } + Some(_) => {} + None if retry.phase == GcPhase::Claim => { + state.retry = None; + work.absent_claims_observed += 1; + } + None => { + return Err(integrity( + "qualified committed GC apply lost its sealed operation history", + )); + } + } + } + if work.collector_enabled && work.examined < limit { + for seal in self.pending_gc_seals(limit - work.examined).await? { + work.examined += 1; + work.lifetimes_examined += 1; + let changed = self.apply_gc_seal(&seal, &mut state).await?; + work.pending_replayed += 1; + work.record_applied(&seal, changed)?; + } + } + if work.examined < limit { + let remaining = limit - work.examined; + let owner_budget = if work.collector_enabled { + remaining.div_ceil(2) + } else { + remaining + }; + self.cleanup_gc_owners(owner_budget, &mut work).await?; + } + if work.collector_enabled && work.examined < limit { + let remaining = limit - work.examined; + let candidates = self.gc_candidates(state.cursor, remaining).await?; + let reached_end = candidates.len() < usize::from(remaining); + for (candidate, eligible) in candidates { + work.examined += 1; + work.lifetimes_examined += 1; + state.cursor = Some((candidate.page_id, candidate.generation)); + if eligible { + let seal = self.claim_gc_candidate(candidate, &mut state).await?; + work.gc_claimed += 1; + let changed = self.apply_gc_seal(&seal, &mut state).await?; + work.record_applied(&seal, changed)?; + } + } + if reached_end { + state.cursor = None; + } + } + Ok(work) + } +} + +impl RootedMaintenanceWork { + fn record_applied(&mut self, seal: &GcSeal, changed: bool) -> Result<(), SnapshotError> { + self.gc_applied += 1; + if changed && seal.had_payload { + self.payload_pages_removed += 1; + self.payload_bytes_removed += u64::try_from(seal.expected_size).map_err(internal)?; + } + Ok(()) + } +} diff --git a/src/jupiter/storage/qualified_metadata_gc.sql b/src/jupiter/storage/qualified_metadata_gc.sql new file mode 100644 index 00000000..13b41998 --- /dev/null +++ b/src/jupiter/storage/qualified_metadata_gc.sql @@ -0,0 +1,574 @@ +-- Historical certificates and source/session records do not own live graph rows. +ALTER TABLE mst2_metadata_gc_op ADD COLUMN certificate_digest bytea, + ADD CONSTRAINT mst2_gc_certificate_binding CHECK( + (graph_present AND octet_length(certificate_digest)=32 OR NOT graph_present AND certificate_digest IS NULL) IS TRUE); +ALTER TABLE mst2_metadata_prepare ADD COLUMN orphan_expires_at timestamptz NOT NULL; +CREATE INDEX mst2_metadata_prepare_orphan_scan ON mst2_metadata_prepare(orphan_expires_at,prepare_id) + WHERE state='PREPARING' OR state='COMMITTED' AND coverage_retired_at IS NULL; +CREATE INDEX mst2_metadata_current_active_scan ON mst2_metadata_lifetime(page_id,generation) + WHERE state IN ('RESERVED','LIVE'); + +CREATE FUNCTION mst2_metadata_prepare_expiry_guard() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +BEGIN + IF TG_OP='INSERT' THEN NEW.created_at:=clock_timestamp(); NEW.orphan_expires_at:=NEW.created_at+interval '3600 seconds'; + ELSIF NEW.created_at IS DISTINCT FROM OLD.created_at OR NEW.orphan_expires_at IS DISTINCT FROM OLD.orphan_expires_at THEN + RAISE EXCEPTION 'qualified preparation expiry is immutable database evidence'; END IF; + RETURN NEW; +END $$; +CREATE TRIGGER mst2_metadata_prepare_00_expiry_guard BEFORE INSERT OR UPDATE ON mst2_metadata_prepare + FOR EACH ROW EXECUTE FUNCTION mst2_metadata_prepare_expiry_guard(); + +CREATE FUNCTION mst2_metadata_orphan_prepare_eligible(pid text) RETURNS boolean LANGUAGE sql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ + SELECT EXISTS(SELECT 1 FROM mst2_metadata_prepare q WHERE q.prepare_id=$1 AND q.state='COMMITTED' + AND q.plan_kind='ROOTED' AND q.graph_domain='qualified-v1' AND mst2_metadata_scope_matches(q.primary_scope) + AND q.orphan_expires_at=q.created_at+interval '3600 seconds' AND q.orphan_expires_at<=clock_timestamp() + AND NOT EXISTS(SELECT 1 FROM mst2_qualified_session_incarnation s WHERE s.state='READY' + AND (s.prepare_id=q.prepare_id OR s.namespace_uuid='$NAMESPACE_UUID$'::uuid AND s.metadata_root=q.metadata_root + AND s.source_profile=convert_to($CORE_SCHEMA$.mst2_route_profile(q.source_domain,q.tagged_root_tree_oid,q.scope, + q.schema_version,q.metadata_codec,q.materialization_policy,q.fs_semantics,q.access_projection, + q.verification_revision,q.projection_revision)::text,'UTF8') + AND s.root_generation IN (SELECT generation FROM mst2_metadata_prepare_page m + WHERE m.prepare_id=q.prepare_id AND m.page_id=q.metadata_root UNION ALL + SELECT root_generation FROM mst2_metadata_prepare_reuse_root r WHERE r.prepare_id=q.prepare_id AND r.root_page=q.metadata_root))) + AND NOT EXISTS(SELECT 1 FROM mst2_qualified_lease_binding l WHERE l.prepare_id=q.prepare_id AND l.state='ACTIVE') + AND NOT EXISTS(SELECT 1 FROM mst2_metadata_reader_operation r JOIN mst2_qualified_lease_binding l USING(lease_id) + WHERE l.prepare_id=q.prepare_id AND r.state='ACTIVE')) +$$; +CREATE FUNCTION mst2_metadata_orphan_prepare_retired(pid text) RETURNS boolean LANGUAGE sql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ + SELECT mst2_metadata_orphan_prepare_eligible($1) AND EXISTS(SELECT 1 FROM mst2_metadata_prepare q + WHERE q.prepare_id=$1 AND q.coverage_retired_at>=q.orphan_expires_at) + AND NOT EXISTS(SELECT 1 FROM mst2_metadata_root_anchor WHERE prepare_id=$1 AND anchor_kind IN ('PREPARE','REUSE')) +$$; + +CREATE FUNCTION mst2_metadata_gc_enabled() RETURNS boolean LANGUAGE sql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ + SELECT NOT pg_is_in_recovery() AND current_setting('transaction_isolation')='read committed' + AND EXISTS(SELECT 1 FROM $CORE_SCHEMA$.mst2_metadata_namespace n WHERE n.namespace_uuid='$NAMESPACE_UUID$'::uuid + AND n.graph_domain='qualified-v1' AND n.metadata_schema=$Q_LITERAL$ AND n.metadata_schema_oid='$Q_OID$'::oid + AND n.core_schema_oid=$CORE_OID$ AND n.metadata_storage_uuid='$STORAGE_UUID$' + AND n.admission_state='ROOTED_Q_ADMITTED' AND n.collector_state='ENABLED' + AND n.implementation_fingerprint=decode('$IMPLEMENTATION_SHA$','hex') + AND n.catalog_fingerprint=$CORE_SCHEMA$.mst2_route_family_catalog($CORE_OID$,'$Q_OID$'::oid)) +$$; +CREATE FUNCTION mst2_metadata_gc_enter() RETURNS void LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +BEGIN + PERFORM $CORE_SCHEMA$.mst2_route_enter($CORE_LITERAL$); + IF NOT mst2_metadata_gc_enabled() THEN RAISE EXCEPTION 'qualified collector is not independently admitted'; END IF; +END $$; + +CREATE FUNCTION mst2_metadata_assert_uncovered(p bytea,g bigint) RETURNS void LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +BEGIN + IF EXISTS(SELECT 1 FROM mst2_metadata_root_anchor WHERE root_page=p AND root_generation=g) + OR EXISTS(SELECT 1 FROM mst2_metadata_graph_root WHERE page_id=p AND generation=g) + OR EXISTS(SELECT 1 FROM mst2_metadata_graph_edge WHERE child_page=p AND child_generation=g) + OR EXISTS(SELECT 1 FROM mst2_metadata_prepare_page m JOIN mst2_metadata_prepare q USING(prepare_id) + WHERE m.page_id=p AND m.generation=g AND (q.state='PREPARING' OR q.state='COMMITTED' AND q.coverage_retired_at IS NULL)) + OR EXISTS(SELECT 1 FROM mst2_metadata_prepare_reuse_root r JOIN mst2_metadata_prepare q USING(prepare_id) + WHERE r.root_page=p AND r.root_generation=g AND (q.state='PREPARING' OR q.state='COMMITTED' AND q.coverage_retired_at IS NULL)) THEN + RAISE EXCEPTION 'qualified current lifetime still has exact owned coverage or incoming edges'; END IF; +END $$; + +CREATE FUNCTION mst2_metadata_gc_proof(p bytea,g bigint,stage text,graph_present boolean,payload_present boolean,c bytea) +RETURNS void LANGUAGE plpgsql VOLATILE SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE life mst2_metadata_lifetime%ROWTYPE; node mst2_metadata_graph_node%ROWTYPE; + body mst2_metadata_payload%ROWTYPE; certificate mst2_metadata_page_certificate%ROWTYPE; actual boolean; +BEGIN + IF p IS NULL OR octet_length(p)<>32 OR g IS NULL OR g<=0 OR stage IS NULL + OR graph_present IS NULL OR payload_present IS NULL OR graph_present AND c IS NULL THEN + RAISE EXCEPTION 'qualified GC proof has an incomplete exact identity'; END IF; + SELECT * INTO life FROM mst2_metadata_lifetime WHERE page_id=p AND generation=g FOR UPDATE; + IF NOT FOUND OR life.graph_domain<>'qualified-v1' OR life.metadata_codec<>1 OR stage NOT IN ('CLAIM','PENDING','APPLIED') + OR stage='CLAIM' AND life.state NOT IN ('RESERVED','LIVE') OR stage='PENDING' AND life.state<>'DELETING' + OR stage='APPLIED' AND life.state<>'REMOVED' THEN RAISE EXCEPTION 'qualified GC stage is not its exact historical lifetime'; END IF; + IF stage<>'APPLIED' AND NOT EXISTS(SELECT 1 FROM mst2_metadata_current WHERE page_id=p AND generation=g) THEN + RAISE EXCEPTION 'qualified GC does not name its exact current generation'; END IF; + PERFORM mst2_metadata_assert_uncovered(p,g); + SELECT * INTO node FROM mst2_metadata_graph_node WHERE page_id=p AND generation=g FOR UPDATE; + actual:=FOUND; + IF actual IS DISTINCT FROM graph_present OR actual AND (node.incoming_refs<>0 OR node.metadata_codec<>life.metadata_codec + OR node.bytes<>life.expected_size OR node.certificate_digest IS DISTINCT FROM c + OR stage='CLAIM' AND node.state<>'LIVE' OR stage='PENDING' AND node.state<>'DELETING') THEN + RAISE EXCEPTION 'qualified GC graph identity or actual counter changed'; END IF; + SELECT * INTO body FROM mst2_metadata_payload WHERE page_id=p AND generation=g FOR UPDATE; + actual:=FOUND; + IF actual IS DISTINCT FROM payload_present OR actual AND (body.metadata_codec<>life.metadata_codec + OR body.byte_size<>life.expected_size OR octet_length(body.payload)<>body.byte_size + OR sha256(convert_to('mega.mst2.metapage','UTF8')||decode('00','hex')||body.payload)<>p) THEN + RAISE EXCEPTION 'qualified GC durable bytes differ from their exact lifetime and page digest'; END IF; + IF payload_present THEN PERFORM mst2_metadata_decode_local(body.payload); END IF; + IF c IS NOT NULL THEN + SELECT * INTO certificate FROM mst2_metadata_page_certificate WHERE page_id=p AND generation=g AND certificate_digest=c; + IF NOT FOUND OR certificate.namespace_uuid<>'$NAMESPACE_UUID$'::uuid OR certificate.metadata_codec<>life.metadata_codec + OR certificate.byte_size<>life.expected_size OR certificate.proof_revision<>1 + OR (SELECT count(*) FROM (SELECT 1 FROM mst2_metadata_verified_ref WHERE parent_page=p AND parent_generation=g LIMIT 258) refs) + <>jsonb_array_length(certificate.canonical_proof->'references') THEN + RAISE EXCEPTION 'qualified GC lost its immutable canonical certificate and typed reference evidence'; END IF; + ELSIF graph_present OR EXISTS(SELECT 1 FROM mst2_metadata_page_certificate WHERE page_id=p AND generation=g) THEN + RAISE EXCEPTION 'qualified GC cannot omit a certified graph identity'; END IF; + IF graph_present THEN + IF (SELECT count(*) FROM (SELECT 1 FROM mst2_metadata_graph_edge WHERE parent_page=p AND parent_generation=g LIMIT 258) edges)>257 THEN + RAISE EXCEPTION 'qualified canonical node has too many physical outgoing edges'; END IF; + IF payload_present AND (EXISTS((SELECT child_page,child_generation FROM mst2_metadata_verified_ref WHERE parent_page=p AND parent_generation=g) + EXCEPT (SELECT child_page,child_generation FROM mst2_metadata_graph_edge WHERE parent_page=p AND parent_generation=g)) + OR EXISTS((SELECT child_page,child_generation FROM mst2_metadata_graph_edge WHERE parent_page=p AND parent_generation=g) + EXCEPT (SELECT child_page,child_generation FROM mst2_metadata_verified_ref WHERE parent_page=p AND parent_generation=g))) THEN + RAISE EXCEPTION 'qualified GC physical outgoing edges differ from exact typed occurrences'; END IF; + END IF; + IF stage='CLAIM' AND life.state='LIVE' AND NOT (graph_present AND payload_present AND c IS NOT NULL) THEN + RAISE EXCEPTION 'LIVE metadata corruption cannot be collected'; END IF; + IF stage='CLAIM' AND graph_present AND NOT payload_present THEN + RAISE EXCEPTION 'certified RESERVED graph corruption cannot be collected without its durable bytes'; END IF; + IF stage='CLAIM' AND life.state='RESERVED' AND (NOT EXISTS(SELECT 1 FROM mst2_metadata_prepare_page m + JOIN mst2_metadata_prepare q USING(prepare_id) WHERE m.page_id=p AND m.generation=g AND m.expected_size=life.expected_size + AND q.graph_domain='qualified-v1' AND q.state='ABORTED' AND q.storage_seal IS NOT NULL) + OR EXISTS(SELECT 1 FROM mst2_metadata_prepare_page m JOIN mst2_metadata_prepare q USING(prepare_id) + WHERE m.page_id=p AND m.generation=g AND q.state<>'ABORTED')) THEN + RAISE EXCEPTION 'RESERVED metadata requires an explicitly aborted exact origin'; END IF; + IF stage='APPLIED' AND (EXISTS(SELECT 1 FROM mst2_metadata_graph_edge WHERE parent_page=p AND parent_generation=g) + OR EXISTS(SELECT 1 FROM mst2_metadata_reuse_index WHERE root_page=p AND root_generation=g)) THEN + RAISE EXCEPTION 'APPLIED qualified receipt still has old physical edges or reuse hints'; END IF; +END $$; + +CREATE FUNCTION mst2_metadata_gc_op_guard() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE life mst2_metadata_lifetime%ROWTYPE; +BEGIN + IF TG_OP='DELETE' THEN RAISE EXCEPTION 'qualified GC operation history is immutable'; END IF; + PERFORM mst2_metadata_gc_enter(); + IF NOT mst2_metadata_scope_matches(NEW.primary_scope) THEN RAISE EXCEPTION 'qualified GC left its captured primary scope'; END IF; + IF TG_OP='INSERT' THEN + IF NEW.state<>'PENDING' OR NEW.completed_at IS NOT NULL OR NEW.payload_delete_xid IS NOT NULL + OR substr(NEW.operation_id::text,15,1)<>'4' OR substr(NEW.operation_id::text,20,1) NOT IN ('8','9','a','b') THEN + RAISE EXCEPTION 'qualified GC must begin with a fresh UUID and unmodified PENDING proof'; END IF; + PERFORM mst2_metadata_gc_proof(NEW.page_id,NEW.generation,'CLAIM',NEW.graph_present,NEW.had_payload,NEW.certificate_digest); + SELECT * INTO STRICT life FROM mst2_metadata_lifetime WHERE page_id=NEW.page_id AND generation=NEW.generation; + IF ROW(NEW.graph_domain,NEW.metadata_codec,NEW.expected_size) IS DISTINCT FROM + ROW(life.graph_domain,life.metadata_codec,life.expected_size) THEN RAISE EXCEPTION 'qualified GC copied profile is incorrect'; END IF; + NEW.created_at:=clock_timestamp(); + ELSE + IF (to_jsonb(NEW)-ARRAY['state','completed_at','payload_delete_xid']) IS DISTINCT FROM + (to_jsonb(OLD)-ARRAY['state','completed_at','payload_delete_xid']) THEN RAISE EXCEPTION 'qualified GC operation cannot retarget immutable evidence'; END IF; + IF OLD.state='APPLIED' AND NEW IS DISTINCT FROM OLD THEN RAISE EXCEPTION 'qualified GC receipt cannot revive or change'; END IF; + IF NEW.payload_delete_xid IS DISTINCT FROM OLD.payload_delete_xid THEN + IF OLD.state<>'PENDING' OR NEW.state<>'PENDING' OR NOT OLD.had_payload OR OLD.payload_delete_xid IS NOT NULL THEN + RAISE EXCEPTION 'qualified payload deletion has no fresh same-transaction phase'; END IF; + PERFORM mst2_metadata_gc_proof(OLD.page_id,OLD.generation,'PENDING',OLD.graph_present,true,OLD.certificate_digest); + NEW.payload_delete_xid:=txid_current(); + END IF; + IF OLD.state='PENDING' AND NEW.state='APPLIED' THEN + IF OLD.had_payload AND OLD.payload_delete_xid IS DISTINCT FROM txid_current() THEN + RAISE EXCEPTION 'qualified GC completion has no same-transaction payload removal'; END IF; + PERFORM mst2_metadata_gc_proof(OLD.page_id,OLD.generation,'APPLIED',false,false,OLD.certificate_digest); + NEW.completed_at:=clock_timestamp(); + ELSIF NEW.state IS DISTINCT FROM OLD.state OR NEW.completed_at IS DISTINCT FROM OLD.completed_at THEN + RAISE EXCEPTION 'qualified GC state transition is invalid'; END IF; + END IF; + RETURN NEW; +END $$; +CREATE TRIGGER mst2_metadata_gc_op_guard BEFORE INSERT OR UPDATE OR DELETE ON mst2_metadata_gc_op + FOR EACH ROW EXECUTE FUNCTION mst2_metadata_gc_op_guard(); + +CREATE FUNCTION mst2_metadata_gc_complete() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE op mst2_metadata_gc_op%ROWTYPE; +BEGIN + SELECT * INTO STRICT op FROM mst2_metadata_gc_op WHERE operation_id=NEW.operation_id; + IF op.state='PENDING' THEN + IF op.payload_delete_xid IS NOT NULL THEN RAISE EXCEPTION 'qualified payload deletion phase cannot escape its atomic apply transaction'; END IF; + PERFORM mst2_metadata_gc_proof(op.page_id,op.generation,'PENDING',op.graph_present,op.had_payload,op.certificate_digest); + ELSE PERFORM mst2_metadata_gc_proof(op.page_id,op.generation,'APPLIED',false,false,op.certificate_digest); END IF; + RETURN NULL; +END $$; +CREATE CONSTRAINT TRIGGER mst2_metadata_gc_complete AFTER INSERT OR UPDATE ON mst2_metadata_gc_op + DEFERRABLE INITIALLY DEFERRED FOR EACH ROW EXECUTE FUNCTION mst2_metadata_gc_complete(); + +CREATE OR REPLACE FUNCTION mst2_metadata_lifetime_guard() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE current_root mst2_metadata_current%ROWTYPE; op mst2_metadata_gc_op%ROWTYPE; previous mst2_metadata_lifetime%ROWTYPE; +BEGIN + IF TG_OP='DELETE' THEN RAISE EXCEPTION 'qualified lifetime watermarks are immutable'; END IF; + IF TG_OP='INSERT' THEN + IF NEW.state<>'RESERVED' THEN RAISE EXCEPTION 'qualified lifetime must begin RESERVED'; END IF; + SELECT * INTO current_root FROM mst2_metadata_current WHERE page_id=NEW.page_id; + IF FOUND THEN + SELECT * INTO previous FROM mst2_metadata_lifetime WHERE page_id=NEW.page_id AND generation=current_root.generation; + SELECT * INTO op FROM mst2_metadata_gc_op WHERE page_id=NEW.page_id AND generation=current_root.generation AND state='APPLIED'; + IF NOT FOUND OR previous.state<>'REMOVED' OR current_root.generation=9223372036854775807 + OR NEW.generation<>current_root.generation+1 OR NEW.metadata_codec<>op.metadata_codec OR NEW.expected_size<>op.expected_size + OR NOT mst2_metadata_scope_matches(op.primary_scope) THEN RAISE EXCEPTION 'qualified reservation needs its exact next-generation APPLIED predecessor'; END IF; + PERFORM mst2_metadata_gc_proof(NEW.page_id,current_root.generation,'APPLIED',false,false,op.certificate_digest); + ELSIF NEW.generation<>1 OR EXISTS(SELECT 1 FROM mst2_metadata_lifetime WHERE page_id=NEW.page_id) + OR EXISTS(SELECT 1 FROM mst2_metadata_payload WHERE page_id=NEW.page_id) + OR EXISTS(SELECT 1 FROM mst2_metadata_graph_node WHERE page_id=NEW.page_id) THEN + RAISE EXCEPTION 'qualified initial reservation cannot adopt history'; END IF; + RETURN NEW; + END IF; + IF (to_jsonb(NEW)-'state') IS DISTINCT FROM (to_jsonb(OLD)-'state') THEN RAISE EXCEPTION 'qualified lifetime identity is immutable'; END IF; + IF NEW.state=OLD.state THEN RETURN NEW; END IF; + IF OLD.state='RESERVED' AND NEW.state='LIVE' THEN + IF NOT EXISTS(SELECT 1 FROM mst2_metadata_graph_node node JOIN mst2_metadata_payload body USING(page_id,generation) + JOIN mst2_metadata_page_certificate certificate USING(page_id,generation) + WHERE node.page_id=OLD.page_id AND node.generation=OLD.generation AND node.state='LIVE' + AND node.certificate_digest=certificate.certificate_digest AND node.metadata_codec=OLD.metadata_codec + AND body.metadata_codec=OLD.metadata_codec AND body.byte_size=OLD.expected_size AND node.bytes=OLD.expected_size) + OR NOT (EXISTS(SELECT 1 FROM mst2_metadata_graph_root root JOIN mst2_metadata_prepare q USING(prepare_id) + WHERE root.page_id=OLD.page_id AND root.generation=OLD.generation AND q.plan_kind='COLD' + AND q.state='COMMITTED' AND root.storage_seal=q.storage_seal AND q.coverage_retired_at IS NULL) + OR EXISTS(SELECT 1 FROM mst2_metadata_prepare_page m JOIN mst2_metadata_prepare q USING(prepare_id) + JOIN mst2_metadata_root_anchor anchor ON anchor.prepare_id=q.prepare_id AND anchor.anchor_kind='PREPARE' + AND anchor.root_page=q.metadata_root AND anchor.owner_key=q.prepare_id + WHERE m.page_id=OLD.page_id AND m.generation=OLD.generation AND q.plan_kind='ROOTED' + AND q.state='COMMITTED' AND q.coverage_retired_at IS NULL)) THEN + RAISE EXCEPTION 'qualified LIVE transition lacks its certified graph and exact owned preparation'; END IF; + RETURN NEW; + END IF; + PERFORM mst2_metadata_gc_enter(); + SELECT * INTO op FROM mst2_metadata_gc_op WHERE page_id=OLD.page_id AND generation=OLD.generation AND state='PENDING'; + IF NOT FOUND OR NOT mst2_metadata_scope_matches(op.primary_scope) THEN RAISE EXCEPTION 'qualified lifecycle transition lacks its exact pending operation'; END IF; + IF OLD.state IN ('RESERVED','LIVE') AND NEW.state='DELETING' THEN + PERFORM mst2_metadata_gc_proof(OLD.page_id,OLD.generation,'CLAIM',op.graph_present,op.had_payload,op.certificate_digest); + ELSIF OLD.state='DELETING' AND NEW.state='REMOVED' THEN + IF op.had_payload AND op.payload_delete_xid IS DISTINCT FROM txid_current() THEN RAISE EXCEPTION 'qualified removal has no atomic payload deletion'; END IF; + PERFORM mst2_metadata_gc_proof(OLD.page_id,OLD.generation,'PENDING',false,false,op.certificate_digest); + IF EXISTS(SELECT 1 FROM mst2_metadata_graph_edge WHERE parent_page=OLD.page_id AND parent_generation=OLD.generation) + OR EXISTS(SELECT 1 FROM mst2_metadata_reuse_index WHERE root_page=OLD.page_id AND root_generation=OLD.generation) THEN + RAISE EXCEPTION 'qualified removal still has old physical references'; END IF; + ELSE RAISE EXCEPTION 'qualified lifecycle transition is invalid'; END IF; + RETURN NEW; +END $$; + +CREATE OR REPLACE FUNCTION mst2_metadata_current_guard() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE op mst2_metadata_gc_op%ROWTYPE; life mst2_metadata_lifetime%ROWTYPE; +BEGIN + IF TG_OP='DELETE' THEN RAISE EXCEPTION 'qualified current watermark cannot be deleted'; END IF; + IF TG_OP='INSERT' THEN + IF NEW.generation<>1 OR EXISTS(SELECT 1 FROM mst2_metadata_lifetime WHERE page_id=NEW.page_id AND generation<>1) + OR EXISTS(SELECT 1 FROM mst2_metadata_payload WHERE page_id=NEW.page_id) + OR EXISTS(SELECT 1 FROM mst2_metadata_graph_node WHERE page_id=NEW.page_id) THEN + RAISE EXCEPTION 'qualified initial current cannot adopt history'; END IF; + RETURN NEW; + END IF; + IF NEW.page_id<>OLD.page_id OR OLD.generation=9223372036854775807 OR NEW.generation<>OLD.generation+1 THEN + RAISE EXCEPTION 'qualified current requires exact next-generation CAS'; END IF; + SELECT * INTO op FROM mst2_metadata_gc_op WHERE page_id=OLD.page_id AND generation=OLD.generation AND state='APPLIED'; + IF NOT FOUND OR NOT mst2_metadata_scope_matches(op.primary_scope) + OR EXISTS(SELECT 1 FROM mst2_metadata_payload WHERE page_id=OLD.page_id) + OR EXISTS(SELECT 1 FROM mst2_metadata_graph_node WHERE page_id=OLD.page_id AND generation=OLD.generation) THEN + RAISE EXCEPTION 'qualified current has no definitive old-generation removal'; END IF; + PERFORM mst2_metadata_gc_proof(OLD.page_id,OLD.generation,'APPLIED',false,false,op.certificate_digest); + SELECT * INTO life FROM mst2_metadata_lifetime WHERE page_id=NEW.page_id AND generation=NEW.generation; + IF NOT FOUND OR life.state<>'RESERVED' OR life.graph_domain<>'qualified-v1' + OR life.metadata_codec<>op.metadata_codec OR life.expected_size<>op.expected_size THEN + RAISE EXCEPTION 'qualified fresh current differs from its immutable page profile'; END IF; + RETURN NEW; +END $$; + +CREATE FUNCTION mst2_metadata_capacity() RETURNS TABLE(resident_pages bigint,resident_bytes bigint) LANGUAGE sql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ + SELECT count(*)::bigint,coalesce(sum(byte_size),0)::bigint FROM (SELECT byte_size FROM mst2_metadata_payload LIMIT 16385) actual +$$; +CREATE FUNCTION mst2_metadata_capacity_guard() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE pages bigint; bytes bigint; actual bigint; +BEGIN + IF TG_TABLE_NAME='mst2_metadata_payload' THEN + SELECT resident_pages,resident_bytes INTO pages,bytes FROM mst2_metadata_capacity(); + IF pages>16384 OR bytes>268435456 THEN RAISE EXCEPTION 'qualified resident payload capacity is exceeded'; END IF; + ELSIF TG_TABLE_NAME='mst2_metadata_source_entry_reference' THEN + SELECT count(*) INTO actual FROM (SELECT 1 FROM mst2_metadata_source_entry_reference LIMIT 262145) bounded; + IF actual>262144 THEN RAISE EXCEPTION 'qualified retained source dictionary capacity is exceeded'; END IF; + RETURN NULL; + ELSIF TG_TABLE_NAME='mst2_qualified_session_incarnation' THEN + SELECT count(*) INTO actual FROM (SELECT 1 FROM mst2_qualified_session_incarnation WHERE state='READY' LIMIT 4097) bounded; + ELSIF TG_TABLE_NAME='mst2_qualified_lease_binding' THEN + SELECT count(*) INTO actual FROM (SELECT 1 FROM mst2_qualified_lease_binding WHERE state='ACTIVE' LIMIT 4097) bounded; + ELSE SELECT count(*) INTO actual FROM (SELECT 1 FROM mst2_metadata_reader_operation WHERE state='ACTIVE' LIMIT 4097) bounded; END IF; + IF actual>4096 THEN RAISE EXCEPTION 'qualified active serving capacity is exceeded'; END IF; + RETURN NULL; +END $$; +DO $$ DECLARE name text; BEGIN + FOREACH name IN ARRAY ARRAY['mst2_metadata_payload','mst2_qualified_session_incarnation', + 'mst2_qualified_lease_binding','mst2_metadata_reader_operation','mst2_metadata_source_entry_reference'] LOOP + EXECUTE format('CREATE TRIGGER mst2_metadata_capacity_guard AFTER INSERT OR UPDATE ON %I FOR EACH STATEMENT EXECUTE FUNCTION mst2_metadata_capacity_guard()',name); + END LOOP; +END $$; + +-- Append-only identity and replay evidence remains bounded independently of +-- resident payloads. The admission check reads actual stored rows, never hints. +CREATE FUNCTION mst2_metadata_history_capacity_guard() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE actual bigint; bytes bigint; row_limit integer; size_expression text; +BEGIN + IF TG_TABLE_NAME IN ('mst2_metadata_prepare_page','mst2_metadata_prepare_reuse_root','mst2_metadata_verified_ref') THEN + row_limit:=262144; size_expression:='0'; + ELSIF TG_TABLE_NAME='mst2_metadata_prepare' THEN + row_limit:=16384; + size_expression:='pg_column_size(canonical_plan)::bigint+pg_column_size(canonical_bindings)+pg_column_size(primary_scope)'; + ELSIF TG_TABLE_NAME='mst2_metadata_source_root_attestation' THEN + row_limit:=16384; size_expression:='pg_column_size(source_proof)::bigint+pg_column_size(source_profile)'; + ELSIF TG_TABLE_NAME='mst2_metadata_page_certificate' THEN + row_limit:=16384; size_expression:='pg_column_size(canonical_proof)::bigint'; + ELSIF TG_TABLE_NAME='mst2_metadata_scope_source_reference' THEN + row_limit:=16384; size_expression:='pg_column_size(ancestor_revisions)::bigint'; + ELSIF TG_TABLE_NAME='mst2_qualified_session_incarnation' THEN + row_limit:=65536; size_expression:='pg_column_size(canonical_descriptor)::bigint+pg_column_size(source_profile)'; + ELSIF TG_TABLE_NAME IN ('mst2_metadata_lifetime','mst2_metadata_current','mst2_metadata_gc_op', + 'mst2_qualified_lease_binding','mst2_metadata_reader_operation') THEN + row_limit:=65536; size_expression:='0'; + ELSE RAISE EXCEPTION 'qualified history quota has an unknown physical relation'; END IF; + EXECUTE format('SELECT count(*),coalesce(sum(size),0) FROM (SELECT %s AS size FROM %I.%I LIMIT %s) actual', + size_expression,TG_TABLE_SCHEMA,TG_TABLE_NAME,row_limit+1) INTO actual,bytes; + IF actual>row_limit OR bytes>268435456 THEN + RAISE EXCEPTION 'qualified immutable history capacity is exceeded for %',TG_TABLE_NAME; END IF; + RETURN NULL; +END $$; +DO $$ DECLARE name text; BEGIN + FOREACH name IN ARRAY ARRAY['mst2_metadata_prepare','mst2_metadata_prepare_page','mst2_metadata_prepare_reuse_root', + 'mst2_metadata_verified_ref','mst2_metadata_source_root_attestation','mst2_metadata_page_certificate','mst2_metadata_scope_source_reference', + 'mst2_qualified_session_incarnation','mst2_metadata_lifetime','mst2_metadata_current','mst2_metadata_gc_op', + 'mst2_qualified_lease_binding','mst2_metadata_reader_operation'] LOOP + EXECUTE format('CREATE TRIGGER mst2_metadata_history_capacity_guard AFTER INSERT ON %I FOR EACH STATEMENT EXECUTE FUNCTION mst2_metadata_history_capacity_guard()',name); + END LOOP; +END $$; + +CREATE FUNCTION mst2_metadata_gc_claim(p bytea,g bigint,scope bytea,id uuid) RETURNS uuid LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE life mst2_metadata_lifetime%ROWTYPE; node mst2_metadata_graph_node%ROWTYPE; op mst2_metadata_gc_op%ROWTYPE; + has_graph boolean; has_body boolean; +BEGIN + PERFORM mst2_metadata_gc_enter(); + IF NOT mst2_metadata_scope_matches(scope) THEN RAISE EXCEPTION 'qualified claim scope differs from its captured primary'; END IF; + SELECT * INTO op FROM mst2_metadata_gc_op WHERE operation_id=id; + IF FOUND THEN + IF op.page_id IS DISTINCT FROM p OR op.generation<>g OR op.primary_scope IS DISTINCT FROM scope THEN + RAISE EXCEPTION 'qualified GC operation cannot be reused for a different lifetime'; END IF; + RETURN id; + END IF; + SELECT * INTO STRICT life FROM mst2_metadata_lifetime WHERE page_id=p AND generation=g; + SELECT * INTO node FROM mst2_metadata_graph_node WHERE page_id=p AND generation=g; + has_graph:=FOUND; has_body:=EXISTS(SELECT 1 FROM mst2_metadata_payload WHERE page_id=p AND generation=g); + INSERT INTO mst2_metadata_gc_op(operation_id,page_id,generation,primary_scope,graph_domain,metadata_codec, + expected_size,graph_present,had_payload,certificate_digest,state) + VALUES(id,p,g,scope,'qualified-v1',life.metadata_codec,life.expected_size,has_graph,has_body, + CASE WHEN has_graph THEN node.certificate_digest ELSE NULL END,'PENDING'); + UPDATE mst2_metadata_lifetime SET state='DELETING' WHERE page_id=p AND generation=g; + IF has_graph THEN UPDATE mst2_metadata_graph_node SET state='DELETING' WHERE page_id=p AND generation=g; END IF; + RETURN id; +END $$; + +CREATE OR REPLACE FUNCTION mst2_metadata_gc_apply(id uuid) RETURNS void LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE op mst2_metadata_gc_op%ROWTYPE; +BEGIN + PERFORM mst2_metadata_gc_enter(); + SELECT * INTO STRICT op FROM mst2_metadata_gc_op WHERE operation_id=id FOR UPDATE; + IF NOT mst2_metadata_scope_matches(op.primary_scope) THEN RAISE EXCEPTION 'qualified GC replay left its captured primary'; END IF; + IF op.state='APPLIED' THEN + PERFORM mst2_metadata_gc_proof(op.page_id,op.generation,'APPLIED',false,false,op.certificate_digest); RETURN; + END IF; + PERFORM mst2_metadata_gc_proof(op.page_id,op.generation,'PENDING',op.graph_present,op.had_payload,op.certificate_digest); + IF op.had_payload THEN + UPDATE mst2_metadata_gc_op SET payload_delete_xid=txid_current() WHERE operation_id=id; + DELETE FROM mst2_metadata_payload WHERE page_id=op.page_id AND generation=op.generation; + IF NOT FOUND THEN RAISE EXCEPTION 'qualified atomic payload deletion lost its exact row'; END IF; + END IF; + DELETE FROM mst2_metadata_graph_edge WHERE parent_page=op.page_id AND parent_generation=op.generation; + DELETE FROM mst2_metadata_reuse_index WHERE root_page=op.page_id AND root_generation=op.generation; + DELETE FROM mst2_metadata_graph_node WHERE page_id=op.page_id AND generation=op.generation; + UPDATE mst2_metadata_lifetime SET state='REMOVED' WHERE page_id=op.page_id AND generation=op.generation AND state='DELETING'; + IF NOT FOUND THEN RAISE EXCEPTION 'qualified atomic removal lost its exact lifecycle'; END IF; + UPDATE mst2_metadata_gc_op SET state='APPLIED' WHERE operation_id=id; +END $$; +CREATE OR REPLACE FUNCTION mst2_metadata_gc_finish(id uuid) RETURNS void LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ BEGIN PERFORM mst2_metadata_gc_apply(id); END $$; + +-- Additional graph/payload guards follow; no history relation is collected. +CREATE OR REPLACE FUNCTION mst2_metadata_payload_fenced() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE op mst2_metadata_gc_op%ROWTYPE; +BEGIN + IF TG_OP='UPDATE' THEN RAISE EXCEPTION 'qualified immutable payload cannot be updated'; END IF; + IF TG_OP='DELETE' THEN + PERFORM mst2_metadata_gc_enter(); + SELECT * INTO op FROM mst2_metadata_gc_op WHERE page_id=OLD.page_id AND generation=OLD.generation AND state='PENDING'; + IF NOT FOUND OR NOT op.had_payload OR op.payload_delete_xid IS DISTINCT FROM txid_current() + OR NOT mst2_metadata_scope_matches(op.primary_scope) OR op.expected_size<>OLD.byte_size OR op.metadata_codec<>OLD.metadata_codec THEN + RAISE EXCEPTION 'qualified payload removal needs its exact same-transaction pending evidence'; END IF; + PERFORM mst2_metadata_gc_proof(OLD.page_id,OLD.generation,'PENDING',op.graph_present,true,op.certificate_digest); + RETURN OLD; + END IF; + IF NOT EXISTS(SELECT 1 FROM mst2_metadata_current c JOIN mst2_metadata_lifetime l USING(page_id,generation) + JOIN mst2_metadata_prepare_page m USING(page_id,generation) JOIN mst2_metadata_prepare q USING(prepare_id) + WHERE c.page_id=NEW.page_id AND c.generation=NEW.generation AND l.state IN ('RESERVED','LIVE') + AND l.metadata_codec=NEW.metadata_codec AND l.expected_size=NEW.byte_size AND q.state='PREPARING' + AND q.graph_domain='qualified-v1' AND mst2_metadata_scope_matches(q.primary_scope)) THEN + RAISE EXCEPTION 'qualified payload INSERT crossed its exact active generation'; END IF; + RETURN NEW; +END $$; + +CREATE OR REPLACE FUNCTION mst2_metadata_graph_node_guard() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE life mst2_metadata_lifetime%ROWTYPE; op mst2_metadata_gc_op%ROWTYPE; +BEGIN + IF TG_OP='DELETE' THEN + PERFORM mst2_metadata_gc_enter(); + SELECT * INTO op FROM mst2_metadata_gc_op WHERE page_id=OLD.page_id AND generation=OLD.generation AND state='PENDING'; + IF NOT FOUND OR NOT op.graph_present OR op.certificate_digest IS DISTINCT FROM OLD.certificate_digest + OR OLD.state<>'DELETING' OR NOT mst2_metadata_scope_matches(op.primary_scope) + OR op.had_payload AND op.payload_delete_xid IS DISTINCT FROM txid_current() + OR EXISTS(SELECT 1 FROM mst2_metadata_graph_edge WHERE parent_page=OLD.page_id AND parent_generation=OLD.generation) THEN + RAISE EXCEPTION 'qualified graph removal lacks its atomic byte removal and empty outgoing edges'; END IF; + PERFORM mst2_metadata_gc_proof(OLD.page_id,OLD.generation,'PENDING',true,false,op.certificate_digest); + RETURN OLD; + END IF; + SELECT l.* INTO life FROM mst2_metadata_current c JOIN mst2_metadata_lifetime l USING(page_id,generation) + WHERE c.page_id=NEW.page_id AND c.generation=NEW.generation; + IF NOT FOUND OR life.graph_domain<>'qualified-v1' OR life.metadata_codec<>NEW.metadata_codec OR life.expected_size<>NEW.bytes + OR NEW.incoming_refs<>(SELECT count(*) FROM mst2_metadata_graph_edge WHERE child_page=NEW.page_id AND child_generation=NEW.generation) + OR NOT EXISTS(SELECT 1 FROM mst2_metadata_page_certificate certificate WHERE certificate.page_id=NEW.page_id + AND certificate.generation=NEW.generation AND certificate.certificate_digest=NEW.certificate_digest + AND certificate.metadata_codec=NEW.metadata_codec AND certificate.byte_size=NEW.bytes) THEN + RAISE EXCEPTION 'qualified graph identity, certificate or physical counter changed'; END IF; + IF TG_OP='INSERT' THEN + IF NEW.state<>'LIVE' OR life.state NOT IN ('RESERVED','LIVE') + OR NOT EXISTS(SELECT 1 FROM mst2_metadata_prepare_page m JOIN mst2_metadata_prepare q USING(prepare_id) + WHERE m.page_id=NEW.page_id AND m.generation=NEW.generation AND q.state='PREPARING') THEN + RAISE EXCEPTION 'qualified graph creation requires an active exact preparation'; END IF; + ELSE + IF (to_jsonb(NEW)-ARRAY['state','incoming_refs']) IS DISTINCT FROM (to_jsonb(OLD)-ARRAY['state','incoming_refs']) THEN + RAISE EXCEPTION 'qualified graph identity is immutable'; END IF; + IF NEW.state<>OLD.state THEN + PERFORM mst2_metadata_gc_enter(); + SELECT * INTO op FROM mst2_metadata_gc_op WHERE page_id=OLD.page_id AND generation=OLD.generation AND state='PENDING'; + IF NOT FOUND OR OLD.state<>'LIVE' OR NEW.state<>'DELETING' OR life.state<>'DELETING' OR NEW.incoming_refs<>0 + OR op.certificate_digest IS DISTINCT FROM NEW.certificate_digest OR NOT mst2_metadata_scope_matches(op.primary_scope) THEN + RAISE EXCEPTION 'qualified graph state change has no exact pending claim'; END IF; + PERFORM mst2_metadata_assert_uncovered(OLD.page_id,OLD.generation); + ELSIF NEW.state='LIVE' AND life.state NOT IN ('RESERVED','LIVE') OR NEW.state='DELETING' AND life.state<>'DELETING' THEN + RAISE EXCEPTION 'qualified graph state differs from its current lifecycle'; END IF; + END IF; + RETURN NEW; +END $$; + +CREATE OR REPLACE FUNCTION mst2_metadata_graph_edge_guard() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE op mst2_metadata_gc_op%ROWTYPE; +BEGIN + IF TG_OP='UPDATE' THEN RAISE EXCEPTION 'qualified edge identity is immutable'; END IF; + IF TG_OP='DELETE' THEN + PERFORM mst2_metadata_gc_enter(); + SELECT * INTO op FROM mst2_metadata_gc_op WHERE page_id=OLD.parent_page AND generation=OLD.parent_generation AND state='PENDING'; + IF NOT FOUND OR NOT op.graph_present OR NOT mst2_metadata_scope_matches(op.primary_scope) + OR op.had_payload AND op.payload_delete_xid IS DISTINCT FROM txid_current() + OR EXISTS(SELECT 1 FROM mst2_metadata_payload WHERE page_id=OLD.parent_page AND generation=OLD.parent_generation) + OR NOT EXISTS(SELECT 1 FROM mst2_metadata_graph_node node JOIN mst2_metadata_current cur USING(page_id,generation) + JOIN mst2_metadata_lifetime life USING(page_id,generation) + WHERE node.page_id=OLD.parent_page AND node.generation=OLD.parent_generation AND node.state='DELETING' + AND life.state='DELETING' AND node.incoming_refs=0 AND node.certificate_digest=op.certificate_digest) THEN + RAISE EXCEPTION 'qualified edge removal is outside its exact atomic graph deletion phase'; END IF; + PERFORM mst2_metadata_assert_uncovered(OLD.parent_page,OLD.parent_generation); + RETURN OLD; + END IF; + IF NOT EXISTS(SELECT 1 FROM mst2_metadata_verified_ref ref + JOIN mst2_metadata_page_certificate parent ON parent.page_id=ref.parent_page AND parent.generation=ref.parent_generation + JOIN mst2_metadata_page_certificate child ON child.page_id=ref.child_page AND child.generation=ref.child_generation + WHERE ref.parent_page=NEW.parent_page AND ref.parent_generation=NEW.parent_generation + AND ref.child_page=NEW.child_page AND ref.child_generation=NEW.child_generation + AND ref.child_certificate_digest=child.certificate_digest AND parent.rank>child.rank) THEN + RAISE EXCEPTION 'qualified edge differs from its exact canonical reference or strict certified rank'; END IF; + IF (SELECT count(*) FROM mst2_metadata_graph_node n JOIN mst2_metadata_current c USING(page_id,generation) + JOIN mst2_metadata_lifetime l USING(page_id,generation) + WHERE ((n.page_id=NEW.parent_page AND n.generation=NEW.parent_generation) + OR (n.page_id=NEW.child_page AND n.generation=NEW.child_generation)) + AND n.state='LIVE' AND l.state IN ('RESERVED','LIVE') AND l.graph_domain='qualified-v1')<>2 + OR NOT EXISTS(SELECT 1 FROM mst2_metadata_prepare_page a JOIN mst2_metadata_prepare q USING(prepare_id) + WHERE a.page_id=NEW.parent_page AND a.generation=NEW.parent_generation AND q.state='PREPARING' + AND q.graph_domain='qualified-v1' AND mst2_metadata_scope_matches(q.primary_scope) + AND (EXISTS(SELECT 1 FROM mst2_metadata_prepare_page b WHERE b.prepare_id=a.prepare_id + AND b.page_id=NEW.child_page AND b.generation=NEW.child_generation) + OR EXISTS(SELECT 1 FROM mst2_metadata_prepare_reuse_root r JOIN mst2_metadata_root_anchor anchor + ON anchor.prepare_id=r.prepare_id AND anchor.anchor_kind='REUSE' AND anchor.owner_key=r.prepare_id + AND anchor.root_page=r.root_page AND anchor.root_generation=r.root_generation + WHERE r.prepare_id=a.prepare_id AND r.root_page=NEW.child_page AND r.root_generation=NEW.child_generation))) THEN + RAISE EXCEPTION 'edge creation requires active exact qualified endpoints'; END IF; + RETURN NEW; +END $$; + +CREATE FUNCTION mst2_metadata_edges_removed() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +BEGIN + IF (SELECT count(*) FROM removed_edges)>257 THEN RAISE EXCEPTION 'qualified GC edge deletion exceeds one canonical node'; END IF; + IF EXISTS(SELECT 1 FROM mst2_metadata_graph_node node JOIN (SELECT child_page,child_generation,count(*) AS delta + FROM removed_edges GROUP BY child_page,child_generation) removed + ON removed.child_page=node.page_id AND removed.child_generation=node.generation + WHERE node.incoming_refs-removed.delta<>(SELECT count(*) FROM mst2_metadata_graph_edge edge + WHERE edge.child_page=node.page_id AND edge.child_generation=node.generation)) THEN + RAISE EXCEPTION 'qualified removal discovered physical incoming counter drift'; END IF; + UPDATE mst2_metadata_graph_node node SET incoming_refs=node.incoming_refs-removed.delta + FROM (SELECT child_page,child_generation,count(*) AS delta FROM removed_edges GROUP BY child_page,child_generation) removed + WHERE node.page_id=removed.child_page AND node.generation=removed.child_generation; + RETURN NULL; +END $$; +CREATE TRIGGER mst2_metadata_edges_removed AFTER DELETE ON mst2_metadata_graph_edge + REFERENCING OLD TABLE AS removed_edges FOR EACH STATEMENT EXECUTE FUNCTION mst2_metadata_edges_removed(); + +CREATE FUNCTION mst2_metadata_gc_owner_cleanup(maximum integer) +RETURNS TABLE(examined bigint,readers_expired bigint,leases_expired bigint,prepares_aborted bigint, + handovers_retired bigint,orphans_retired bigint) +LANGUAGE plpgsql VOLATILE SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE bound integer:=maximum; item record; q mst2_metadata_prepare%ROWTYPE; now_unix bigint; +BEGIN + PERFORM $CORE_SCHEMA$.mst2_route_enter($CORE_LITERAL$); + IF maximum IS NULL OR maximum NOT BETWEEN 0 AND 64 THEN RAISE EXCEPTION 'qualified owner cleanup budget must be 0..=64'; END IF; + examined:=0; readers_expired:=0; leases_expired:=0; prepares_aborted:=0; handovers_retired:=0; orphans_retired:=0; + now_unix:=floor(extract(epoch FROM clock_timestamp()))::bigint; + FOR item IN SELECT * FROM ( + (SELECT 'READER'::text AS kind,operation_id::text AS owner,lease_id,hard_deadline_unix AS deadline + FROM mst2_metadata_reader_operation WHERE state='ACTIVE' AND hard_deadline_unix<=now_unix + ORDER BY hard_deadline_unix,operation_id LIMIT bound) + UNION ALL + (SELECT 'LEASE'::text,lease_id,lease_id,expires_at_unix FROM mst2_qualified_lease_binding + WHERE state='ACTIVE' AND expires_at_unix<=now_unix ORDER BY expires_at_unix,lease_id LIMIT bound) + UNION ALL + (SELECT 'PREPARE'::text,prepare_id,NULL::text,floor(extract(epoch FROM orphan_expires_at))::bigint + FROM mst2_metadata_prepare WHERE (state='PREPARING' OR state='COMMITTED' AND coverage_retired_at IS NULL) + AND orphan_expires_at<=clock_timestamp() ORDER BY orphan_expires_at,prepare_id LIMIT bound) + ) expired ORDER BY deadline,kind,owner LIMIT bound LOOP + examined:=examined+1; + IF item.kind='READER' THEN + UPDATE mst2_metadata_reader_operation SET state='EXPIRED' WHERE operation_id=item.owner::uuid AND state='ACTIVE'; + IF NOT FOUND THEN RAISE EXCEPTION 'qualified expired reader changed behind its mutation barrier'; END IF; + readers_expired:=readers_expired+1; + DELETE FROM mst2_metadata_root_anchor WHERE reader_operation_id=item.owner::uuid AND anchor_kind IN ('REQUEST','READER'); + PERFORM mst2_metadata_cleanup_lease(item.lease_id); + ELSIF item.kind='LEASE' THEN + UPDATE mst2_qualified_lease_binding SET state='EXPIRED',lease_epoch=lease_epoch+1 WHERE lease_id=item.owner AND state='ACTIVE'; + IF NOT FOUND THEN RAISE EXCEPTION 'qualified expired lease changed behind its mutation barrier'; END IF; + leases_expired:=leases_expired+1; PERFORM mst2_metadata_cleanup_lease(item.lease_id); + ELSE + SELECT * INTO STRICT q FROM mst2_metadata_prepare WHERE prepare_id=item.owner; + IF q.state='PREPARING' THEN + DELETE FROM mst2_metadata_graph_root WHERE prepare_id=q.prepare_id; + UPDATE mst2_metadata_prepare SET state='ABORTED',aborted_at=clock_timestamp() WHERE prepare_id=q.prepare_id; + DELETE FROM mst2_metadata_root_anchor WHERE prepare_id=q.prepare_id AND anchor_kind IN ('PREPARE','REUSE'); + prepares_aborted:=prepares_aborted+1; + ELSIF mst2_metadata_session_covers_prepare(q.prepare_id) THEN + UPDATE mst2_metadata_prepare SET coverage_retired_at=clock_timestamp() WHERE prepare_id=q.prepare_id; + DELETE FROM mst2_metadata_root_anchor WHERE prepare_id=q.prepare_id AND anchor_kind IN ('PREPARE','REUSE'); + handovers_retired:=handovers_retired+1; + ELSIF mst2_metadata_orphan_prepare_eligible(q.prepare_id) THEN + UPDATE mst2_metadata_prepare SET coverage_retired_at=clock_timestamp() WHERE prepare_id=q.prepare_id; + DELETE FROM mst2_metadata_root_anchor WHERE prepare_id=q.prepare_id AND anchor_kind IN ('PREPARE','REUSE'); + orphans_retired:=orphans_retired+1; + END IF; + END IF; + END LOOP; + RETURN NEXT; +END $$; +DROP TRIGGER mst2_01_family_closed ON mst2_metadata_gc_op; diff --git a/src/jupiter/storage/qualified_metadata_gc_tests.rs b/src/jupiter/storage/qualified_metadata_gc_tests.rs new file mode 100644 index 00000000..346254bf --- /dev/null +++ b/src/jupiter/storage/qualified_metadata_gc_tests.rs @@ -0,0 +1,456 @@ +use std::collections::{BTreeMap, BTreeSet}; + +use super::{canonical_tests::seeded_rooted_plan, *}; +use crate::ceres::snapshot::{ + rooted_metadata_install::RootedMetadataInstallPlan, + rooted_metadata_projection::RootedReuseLookup, +}; + +async fn write_rooted( + writer: &RootedQualifiedMetadataRepository, + operation: &str, + plan: &RootedMetadataInstallPlan, + payload: MetadataPagePayload, +) -> RootedPrepareIntent { + let intent = writer.begin_intent(operation, plan).await.unwrap(); + writer.install_pages(&intent, &[payload]).await.unwrap(); + writer.finalize(&intent).await.unwrap(); + intent +} + +// Fault injection only: expire database evidence in the isolated test schema, +// restore the original guards/catalog, then use normal production maintenance. +async fn age_orphan(q: &DatabaseConnection, namespace: &VerifiedQualifiedNamespace, prepare: &str) { + let txn = q.begin().await.unwrap(); + for trigger in [ + "mst2_00_family_barrier", + "mst2_metadata_prepare_guard", + "mst2_metadata_prepare_00_expiry_guard", + ] { + txn.execute_unprepared(&format!( + "ALTER TABLE mst2_metadata_prepare DISABLE TRIGGER {trigger}" + )) + .await + .unwrap(); + } + txn.execute_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + "UPDATE mst2_metadata_prepare SET created_at=clock_timestamp()-interval '3601 seconds', + orphan_expires_at=clock_timestamp()-interval '1 second' WHERE prepare_id=$1", + [prepare.into()], + )) + .await + .unwrap(); + // clock_timestamp() calls differ; preserve the exact database TTL relation. + txn.execute_raw(Statement::from_sql_and_values(DbBackend::Postgres, + "UPDATE mst2_metadata_prepare SET orphan_expires_at=created_at+interval '3600 seconds' WHERE prepare_id=$1", + [prepare.into()])).await.unwrap(); + for trigger in [ + "mst2_00_family_barrier", + "mst2_metadata_prepare_guard", + "mst2_metadata_prepare_00_expiry_guard", + ] { + txn.execute_unprepared(&format!( + "ALTER TABLE mst2_metadata_prepare ENABLE TRIGGER {trigger}" + )) + .await + .unwrap(); + } + txn.commit().await.unwrap(); + assert_eq!( + catalog(q, namespace.core_oid, namespace.schema_oid) + .await + .unwrap(), + namespace.catalog_fingerprint + ); +} + +async fn claim( + q: &DatabaseConnection, + intent: &RootedPrepareIntent, +) -> Result { + let operation = uuid::Uuid::new_v4().to_string(); + q.execute_raw(Statement::from_sql_and_values(DbBackend::Postgres, + "SELECT mst2_metadata_gc_claim($1,$2,(SELECT primary_scope FROM mst2_metadata_prepare WHERE prepare_id=$3),$4::uuid)", + [intent.metadata_root().to_vec().into(),intent.root_generation().into(),intent.prepare_id().into(),operation.clone().into()] + )).await?; + Ok(operation) +} + +#[tokio::test] +async fn admitted_orphan_gc_preserves_replay_identity_and_advances_exact_generation() { + let (config, core, namespace, q, _guard) = fixture().await; + let (plan, payload) = seeded_rooted_plan(&core, 'a', "file", 1).await; + let writer = RootedQualifiedMetadataRepository::open(&core, &config) + .await + .unwrap(); + let first = write_rooted(&writer, "gc-generation-one", &plan, payload.clone()).await; + assert_eq!( + count(&q, "SELECT mst2_metadata_gc_enabled()::bigint").await, + 1 + ); + assert!( + claim(&q, &first) + .await + .unwrap_err() + .to_string() + .contains("owned coverage") + ); + assert_eq!( + count(&q, "SELECT count(*) FROM mst2_metadata_gc_op").await, + 0 + ); + age_orphan(&q, &namespace, first.prepare_id()).await; + let work = writer.maintenance_tick(64).await.unwrap(); + assert!(work.collector_enabled); + assert!(work.examined <= 64); + assert_eq!(work.orphans_retired, 1); + assert_eq!(work.payload_pages_removed, 1); + assert_eq!(work.payload_bytes_removed, payload.size); + assert_eq!( + count(&q, "SELECT count(*) FROM mst2_metadata_payload").await, + 0 + ); + assert_eq!( + count(&q, "SELECT count(*) FROM mst2_metadata_graph_node").await, + 0 + ); + assert_eq!( + count(&q, "SELECT count(*) FROM mst2_metadata_page_certificate").await, + 1 + ); + assert_eq!( + count( + &q, + "SELECT count(*) FROM mst2_metadata_source_root_attestation" + ) + .await, + 1 + ); + assert_eq!( + count( + &q, + "SELECT count(*) FROM mst2_metadata_gc_op WHERE state='APPLIED'" + ) + .await, + 1 + ); + let operation: String = q + .query_one_raw(Statement::from_string( + DbBackend::Postgres, + "SELECT operation_id::text FROM mst2_metadata_gc_op", + )) + .await + .unwrap() + .unwrap() + .try_get_by_index(0) + .unwrap(); + let fresh = write_rooted(&writer, "gc-generation-two", &plan, payload.clone()).await; + assert_eq!(fresh.root_generation(), first.root_generation() + 1); + assert!(writer.recover("gc-generation-one", &plan).await.is_err()); + for _ in 0..2 { + q.execute_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + "SELECT mst2_metadata_gc_apply($1::uuid)", + [operation.clone().into()], + )) + .await + .unwrap(); + } + assert_eq!( + count( + &q, + "SELECT count(*) FROM mst2_metadata_payload WHERE generation=2" + ) + .await, + 1 + ); + assert_eq!( + count( + &q, + "SELECT count(*) FROM mst2_metadata_lifetime WHERE generation=1 AND state='REMOVED'" + ) + .await, + 1 + ); + assert_eq!( + count( + &q, + "SELECT count(*) FROM mst2_metadata_lifetime WHERE generation=2 AND state='LIVE'" + ) + .await, + 1 + ); + assert_eq!( + count(&q, "SELECT count(*) FROM mst2_metadata_page_certificate").await, + 2 + ); + assert_eq!( + count( + &q, + "SELECT count(*) FROM mst2_metadata_current WHERE generation=2" + ) + .await, + 1 + ); +} + +#[tokio::test] +async fn pending_gc_survives_rebuild_and_payload_stamp_cannot_commit_alone() { + let (config, core, namespace, q, _guard) = fixture().await; + let (plan, payload) = seeded_rooted_plan(&core, 'a', "file", 1).await; + let writer = RootedQualifiedMetadataRepository::open(&core, &config) + .await + .unwrap(); + let intent = write_rooted(&writer, "gc-pending", &plan, payload).await; + age_orphan(&q, &namespace, intent.prepare_id()).await; + assert_eq!(writer.maintenance_tick(1).await.unwrap().orphans_retired, 1); + let operation = claim(&q, &intent).await.unwrap(); + assert_eq!( + count( + &q, + "SELECT count(*) FROM mst2_metadata_gc_op WHERE state='PENDING'" + ) + .await, + 1 + ); + let txn = q.begin().await.unwrap(); + txn.execute_raw(Statement::from_sql_and_values(DbBackend::Postgres, + "UPDATE mst2_metadata_gc_op SET payload_delete_xid=txid_current() WHERE operation_id=$1::uuid", + [operation.into()])).await.unwrap(); + let error = txn.commit().await.unwrap_err(); + assert!( + error.to_string().contains("cannot escape its atomic apply"), + "{error}" + ); + assert_eq!( + count(&q, "SELECT count(*) FROM mst2_metadata_payload").await, + 1 + ); + assert_eq!(count(&q,"SELECT count(*) FROM mst2_metadata_gc_op WHERE state='PENDING' AND payload_delete_xid IS NULL").await,1); + drop(writer); + let rebuilt = RootedQualifiedMetadataRepository::open(&core, &config) + .await + .unwrap(); + let work = rebuilt.maintenance_tick(1).await.unwrap(); + assert_eq!(work.examined, 1); + assert_eq!(work.pending_replayed, 1); + assert_eq!(work.gc_applied, 1); + assert_eq!(work.payload_pages_removed, 1); + assert_eq!( + count(&q, "SELECT count(*) FROM mst2_metadata_payload").await, + 0 + ); + assert_eq!( + count( + &q, + "SELECT count(*) FROM mst2_metadata_gc_op WHERE state='APPLIED'" + ) + .await, + 1 + ); +} + +#[tokio::test] +async fn actual_incoming_edge_blocks_child_gc_until_parent_removal() { + let (config, core, namespace, q, _guard) = fixture().await; + let (child, payload) = seeded_rooted_plan(&core, 'a', "file", 1).await; + let writer = RootedQualifiedMetadataRepository::open(&core, &config) + .await + .unwrap(); + let child_intent = write_rooted(&writer, "gc-edge-child", &child, payload).await; + let hint = writer + .lookup_reuse(&child.identity.tagged_root_tree_oid, &child.identity) + .await + .unwrap() + .unwrap(); + let entries = [ + Entry::dir(b"one", child.root), + Entry::dir(b"two", child.root), + ]; + let bytes = Page::build(&entries).unwrap(); + let root = page_id(&bytes); + let mut body = b"40000 one\0".to_vec(); + body.extend_from_slice(&[0xaa; 20]); + body.extend_from_slice(b"40000 two\0"); + body.extend_from_slice(&[0xaa; 20]); + core.execute_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + "INSERT INTO mega_tree(id,tree_id,sub_trees,size,created_at,pack_id,pack_offset,commit_id) + VALUES(2,$1,$2,0,now(),'fixture',0,'fixture')", + ["d".repeat(40).into(), body.into()], + )) + .await + .unwrap(); + let mut identity = child.identity.clone(); + identity.tagged_root_tree_oid = format!("sha1:{}", "d".repeat(40)); + let parent = RootedMetadataInstallPlan::new( + identity.clone(), + root, + BTreeMap::from([(root, bytes.len() as u64)]), + BTreeSet::from([(root, child.root)]), + BTreeMap::from([(child.root, hint.proof)]), + BTreeMap::from([ + (identity.tagged_root_tree_oid, root), + (child.identity.tagged_root_tree_oid.clone(), child.root), + ]), + ) + .unwrap(); + let parent_intent = write_rooted( + &writer, + "gc-edge-parent", + &parent, + MetadataPagePayload { + id: root, + size: bytes.len() as u64, + bytes, + }, + ) + .await; + age_orphan(&q, &namespace, child_intent.prepare_id()).await; + age_orphan(&q, &namespace, parent_intent.prepare_id()).await; + for _ in 0..2 { + assert_eq!(writer.maintenance_tick(1).await.unwrap().orphans_retired, 1); + } + assert_eq!( + count(&q, "SELECT count(*) FROM mst2_metadata_root_anchor").await, + 0 + ); + assert_eq!( + count(&q, "SELECT count(*) FROM mst2_metadata_graph_edge").await, + 1 + ); + assert_eq!( + count( + &q, + "SELECT sum(incoming_refs)::bigint FROM mst2_metadata_graph_node" + ) + .await, + 1 + ); + assert!( + claim(&q, &child_intent) + .await + .unwrap_err() + .to_string() + .contains("incoming edges") + ); + let operation = claim(&q, &parent_intent).await.unwrap(); + q.execute_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + "SELECT mst2_metadata_gc_apply($1::uuid)", + [operation.into()], + )) + .await + .unwrap(); + assert_eq!( + count(&q, "SELECT count(*) FROM mst2_metadata_graph_edge").await, + 0 + ); + assert_eq!( + count( + &q, + "SELECT sum(incoming_refs)::bigint FROM mst2_metadata_graph_node" + ) + .await, + 0 + ); + let operation = claim(&q, &child_intent).await.unwrap(); + q.execute_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + "SELECT mst2_metadata_gc_apply($1::uuid)", + [operation.into()], + )) + .await + .unwrap(); + assert_eq!( + count(&q, "SELECT count(*) FROM mst2_metadata_payload").await, + 0 + ); + assert_eq!( + count(&q, "SELECT count(*) FROM mst2_metadata_verified_ref").await, + 2 + ); +} + +#[tokio::test] +async fn restored_guards_refuse_corrupt_payload_size_counter_and_current_bindings() { + for case in 0..4 { + let (config, core, namespace, q, _guard) = fixture().await; + let (plan, payload) = seeded_rooted_plan(&core, 'a', "file", 1).await; + let writer = RootedQualifiedMetadataRepository::open(&core, &config) + .await + .unwrap(); + let intent = write_rooted(&writer, "gc-corruption", &plan, payload).await; + age_orphan(&q, &namespace, intent.prepare_id()).await; + assert_eq!(writer.maintenance_tick(1).await.unwrap().orphans_retired, 1); + let (table, guard, statement) = match case { + 0 => ( + "mst2_metadata_payload", + "mst2_metadata_payload_fenced", + "UPDATE mst2_metadata_payload SET payload=set_byte(payload,20,255)", + ), + 1 => ( + "mst2_metadata_graph_node", + "mst2_metadata_graph_node_guard", + "UPDATE mst2_metadata_graph_node SET bytes=bytes+1", + ), + 2 => ( + "mst2_metadata_graph_node", + "mst2_metadata_graph_node_guard", + "UPDATE mst2_metadata_graph_node SET incoming_refs=1", + ), + _ => ( + "mst2_metadata_current", + "mst2_metadata_current_guard", + "DELETE FROM mst2_metadata_current", + ), + }; + let txn = q.begin().await.unwrap(); + for trigger in ["mst2_00_family_barrier", guard] { + txn.execute_unprepared(&format!("ALTER TABLE {table} DISABLE TRIGGER {trigger}")) + .await + .unwrap(); + } + txn.execute_unprepared(statement).await.unwrap(); + for trigger in ["mst2_00_family_barrier", guard] { + txn.execute_unprepared(&format!("ALTER TABLE {table} ENABLE TRIGGER {trigger}")) + .await + .unwrap(); + } + txn.commit().await.unwrap(); + assert_eq!( + catalog(&q, namespace.core_oid, namespace.schema_oid) + .await + .unwrap(), + namespace.catalog_fingerprint + ); + let error = claim(&q, &intent).await.unwrap_err(); + assert!( + error.to_string().contains(if case == 0 { + "durable bytes" + } else if case < 3 { + "actual counter" + } else { + "current generation" + }), + "{error}" + ); + assert_eq!( + count(&q, "SELECT count(*) FROM mst2_metadata_gc_op").await, + 0 + ); + assert_eq!( + count(&q, "SELECT count(*) FROM mst2_metadata_payload").await, + 1 + ); + assert_eq!( + count( + &q, + "SELECT count(*) FROM mst2_metadata_lifetime WHERE state='LIVE'" + ) + .await, + 1 + ); + } +} diff --git a/src/jupiter/storage/qualified_metadata_reader.rs b/src/jupiter/storage/qualified_metadata_reader.rs new file mode 100644 index 00000000..63ccbdcd --- /dev/null +++ b/src/jupiter/storage/qualified_metadata_reader.rs @@ -0,0 +1,961 @@ +//! Persisted routes traverse exact certified occurrences, including reused roots. + +use std::collections::{BTreeSet, HashMap}; + +use mst2_codec::metapage::{HEADER_LEN, PAGE_MAX_BYTES}; +use serde::Deserialize; + +use super::*; +use crate::{ + ceres::snapshot::{ + retention_dag::MetadataDagLimits, runtime::SnapshotContext, + view::validate_scope_relative_path, + }, + jupiter::storage::native_snapshot_session::{ + MetadataRouteRequest, PersistedMetadataReadWork, PersistedMetadataRouteBatch, + }, +}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct Binding { + generation: i64, + certificate: [u8; 32], +} + +#[cfg(test)] +tokio::task_local! { + static READER_ADMISSION_BARRIERS: (Arc, Arc); + static SOURCE_FACT_BARRIERS: (Arc, Arc); + pub(super) static READER_TEMP_SOURCE_SHADOW: bool; +} + +#[cfg(test)] +pub(crate) async fn with_rooted_source_fact_barriers( + admitted: Arc, + resume: Arc, + future: F, +) -> F::Output { + SOURCE_FACT_BARRIERS.scope((admitted, resume), future).await +} + +#[cfg(test)] +pub(crate) async fn with_rooted_source_temporary_shadow( + future: F, +) -> F::Output { + READER_TEMP_SOURCE_SHADOW.scope(true, future).await +} + +#[cfg(test)] +pub(crate) async fn with_rooted_reader_barriers( + admitted: Arc, + resume: Arc, + future: F, +) -> F::Output { + READER_ADMISSION_BARRIERS + .scope((admitted, resume), future) + .await +} + +#[derive(Debug, Deserialize)] +struct Reference { + kind: String, + child: String, + generation: i64, + certificate: String, + name: Option, + label: Option, + count: Option, +} + +fn digest_hex(value: &str) -> Result<[u8; 32], SnapshotError> { + hex::decode(value) + .map_err(internal)? + .as_slice() + .try_into() + .map_err(internal) +} +fn limit(message: &str) -> SnapshotError { + SnapshotError::new( + crate::ceres::snapshot::error::SnapshotErrorCode::LimitExceeded, + message, + ) +} +fn absent(message: &str) -> SnapshotError { + SnapshotError::new( + crate::ceres::snapshot::error::SnapshotErrorCode::PathNotFound, + message, + ) +} + +struct StoredPage { + bytes: Vec, + page: Page, + entries: u64, +} + +pub(crate) struct RootedDirectoryWindow { + pub directory_root: [u8; 32], + pub entry_count: u64, + pub entries: Vec, + pub has_more: bool, + pub proof_pages: Vec<([u8; 32], Vec)>, + pub ancestors: Vec<(String, [u8; 32])>, +} + +pub(crate) enum RootedLookupStatus { + Directory([u8; 32]), + File { + entry: mst2_codec::metapage::Entry, + git_oid: String, + }, + Absent, + NotDirectory { + symlink: bool, + }, +} +pub(crate) struct RootedLookupBatch { + pub results: Vec, + pub proof_pages: Vec<([u8; 32], Vec)>, +} +struct Reader<'a> { + txn: &'a DatabaseTransaction, + operation: uuid::Uuid, + bindings: HashMap<[u8; 32], Binding>, + cache: HashMap<[u8; 32], StoredPage>, + work: PersistedMetadataReadWork, + limits: MetadataDagLimits, + directories: BTreeMap, + source_ids: BTreeMap, + file_oids: HashMap<(uuid::Uuid, Vec), String>, +} + +impl RootedQualifiedMetadataRepository { + pub(crate) async fn fixed_path_metadata( + &self, + pinned: &SnapshotContext, + path: &str, + ) -> Result { + validate_scope_relative_path(path)?; + let absolute = if pinned.built.descriptor.scope == "/" { + path.to_owned() + } else if path == "/" { + pinned.built.descriptor.scope.clone() + } else { + format!("{}{}", pinned.built.descriptor.scope, path) + }; + validate_scope_relative_path(&absolute)?; + let (operation, binding, source) = self.admit_reader(pinned).await?; + let result = async { + let txn = self.read_transaction().await?; + let root = pinned.built.descriptor.metadata_root; + let mut reader = Reader::new(&txn, operation, root, binding, source); + let result = reader.lookup(root, path).await; + sessions::finish(txn, result).await + } + .await; + self.finish_reader(operation, result).await + } + + pub(crate) async fn directory_window( + &self, + pinned: &SnapshotContext, + path: &str, + after: Option<&str>, + count: usize, + ) -> Result { + validate_scope_relative_path(path)?; + if !(1..=256).contains(&count) { + return Err(limit("directory limit must be 1..256")); + } + let absolute = if pinned.built.descriptor.scope == "/" { + path.to_owned() + } else if path == "/" { + pinned.built.descriptor.scope.clone() + } else { + format!("{}{}", pinned.built.descriptor.scope, path) + }; + validate_scope_relative_path(&absolute)?; + let (operation, binding, source) = self.admit_reader(pinned).await?; + let result = async { + let txn = self.read_transaction().await?; + let read = async { + let root = pinned.built.descriptor.metadata_root; + let mut reader = Reader::new(&txn, operation, root, binding, source); + let directory = reader.directory(root, path).await?; + reader.load(directory).await?; + let total = reader.cache[&directory].entries; + let mut entries = Vec::with_capacity(count + 1); + reader + .range(directory, after.map(str::as_bytes), count + 1, &mut entries) + .await?; + let has_more = entries.len() > count; + entries.truncate(count); + reader + .source_entries(reader.source_ids[path], directory, &entries) + .await?; + Ok(RootedDirectoryWindow { + directory_root: directory, + entry_count: total, + entries, + has_more, + proof_pages: reader.proofs()?, + ancestors: reader.directories.into_iter().collect(), + }) + } + .await; + sessions::finish(txn, read).await + } + .await; + self.finish_reader(operation, result).await + } + + pub(crate) async fn lookup_metadata( + &self, + pinned: &SnapshotContext, + paths: &[String], + ) -> Result { + if paths.len() > 128 { + return Err(limit("at most 128 paths per lookup")); + } + for path in paths { + validate_scope_relative_path(path)?; + let absolute = if pinned.built.descriptor.scope == "/" { + path.clone() + } else if path == "/" { + pinned.built.descriptor.scope.clone() + } else { + format!("{}{}", pinned.built.descriptor.scope, path) + }; + validate_scope_relative_path(&absolute)?; + } + let (operation, binding, source) = self.admit_reader(pinned).await?; + let result = async { + let txn = self.read_transaction().await?; + let read = async { + let root = pinned.built.descriptor.metadata_root; + let mut reader = Reader::new(&txn, operation, root, binding, source); + let mut results = Vec::with_capacity(paths.len()); + for path in paths { + results.push(reader.lookup(root, path).await?); + } + Ok(RootedLookupBatch { + results, + proof_pages: reader.proofs()?, + }) + } + .await; + sessions::finish(txn, read).await + } + .await; + self.finish_reader(operation, result).await + } + pub(crate) async fn metadata_routes( + &self, + pinned: &SnapshotContext, + requests: &[MetadataRouteRequest<'_>], + ) -> Result { + if requests.is_empty() || requests.len() > 64 { + return Err(limit("items must hold 1..64 entries")); + } + for request in requests { + validate_scope_relative_path(request.directory_path)?; + let scope = &pinned.built.descriptor.scope; + let absolute = if scope == "/" { + request.directory_path.to_owned() + } else if request.directory_path == "/" { + scope.clone() + } else { + format!("{scope}{}", request.directory_path) + }; + validate_scope_relative_path(&absolute)?; + } + // Acquire REQUEST and READER ownership atomically, then release the + // mutation barrier before fetching payloads. Cleanup never precedes + // ownership of the returned byte buffers. + let (operation, binding, source) = self.admit_reader(pinned).await?; + let result = self + .read_routes(pinned, requests, operation, binding, source) + .await; + self.finish_reader(operation, result).await + } + + async fn admit_reader( + &self, + pinned: &SnapshotContext, + ) -> Result<(uuid::Uuid, Binding, uuid::Uuid), SnapshotError> { + let txn = self.transaction().await?; + let admitted = async { + let row = self + .session_row( + &txn, + &pinned.built.snapshot_id, + &pinned.lease_id, + &pinned.built.instance_id, + ) + .await?; + let current = sessions::context( + &row, + &pinned.built.snapshot_id, + &pinned.lease_id, + &pinned.built.instance_id, + )?; + if current.built.descriptor != pinned.built.descriptor + || current.commit_oid != pinned.commit_oid + || current.root_tree_oid != pinned.root_tree_oid + || current.authorization_epoch != pinned.authorization_epoch + { + return Err(integrity( + "qualified fixed session changed during reader admission", + )); + } + let reader = txn + .query_one_raw(sql( + "SELECT operation_id::text,root_generation,certificate_digest + FROM mst2_metadata_begin_reader($1,$2,$3)", + [ + pinned.built.snapshot_id.clone().into(), + pinned.lease_id.clone().into(), + pinned.built.instance_id.clone().into(), + ], + )) + .await + .map_err(database_error)? + .ok_or_else(|| unavailable("qualified reader admission returned no owned root"))?; + Ok(( + uuid::Uuid::parse_str( + &reader + .try_get::("", "operation_id") + .map_err(internal)?, + ) + .map_err(internal)?, + Binding { + generation: reader.try_get("", "root_generation").map_err(internal)?, + certificate: digest_column(&reader, "certificate_digest")?, + }, + row.try_get("", "attestation_id").map_err(internal)?, + )) + } + .await; + let owned = sessions::finish(txn, admitted).await?; + #[cfg(test)] + if let Ok((admitted, resume)) = READER_ADMISSION_BARRIERS.try_with(Clone::clone) { + admitted.wait().await; + resume.wait().await; + } + Ok(owned) + } + + async fn finish_reader( + &self, + operation: uuid::Uuid, + result: Result, + ) -> Result { + let cleanup = self.transaction().await; + let cleanup = match cleanup { + Ok(txn) => { + let finished = txn + .execute_raw(sql( + "SELECT mst2_metadata_finish_reader($1::uuid)", + [operation.to_string().into()], + )) + .await + .map_err(internal) + .map(|_| ()); + sessions::finish(txn, finished).await + } + Err(error) => Err(error), + }; + // A failed cleanup leaves durable roots until the bounded hard deadline. + // Report the failure instead of pretending that a protected operation + // has been definitively finished. + match (result, cleanup) { + (Err(error), _) => Err(error), + (Ok(_), Err(error)) => Err(error), + (Ok(value), Ok(())) => Ok(value), + } + } + + async fn read_routes( + &self, + pinned: &SnapshotContext, + requests: &[MetadataRouteRequest<'_>], + operation: uuid::Uuid, + binding: Binding, + source: uuid::Uuid, + ) -> Result { + let txn = self.read_transaction().await?; + let result = async { + let root = pinned.built.descriptor.metadata_root; + let mut reader = Reader::new(&txn, operation, root, binding, source); + let mut seen = BTreeSet::new(); + let mut pages = Vec::new(); + for request in requests { + let directory = reader.directory(root, request.directory_path).await?; + let route = reader.route(directory, request.route).await?; + let reached = route + .last() + .ok_or_else(|| integrity("qualified metadata route is empty"))?; + if let Some(expected) = request.expected_digest + && expected != format!("sha256:{}", hex::encode(reached)) + { + return Err(SnapshotError::new( + crate::ceres::snapshot::error::SnapshotErrorCode::DigestMismatch, + "qualified metadata route does not reach expected_digest", + )); + } + for page in route { + if seen.insert(page) { + pages.push((page, reader.cache[&page].bytes.clone())); + } + } + } + Ok(PersistedMetadataRouteBatch { + pages, + work: reader.work, + }) + } + .await; + sessions::finish(txn, result).await + } +} + +impl Reader<'_> { + fn new( + txn: &DatabaseTransaction, + operation: uuid::Uuid, + root: [u8; 32], + binding: Binding, + source: uuid::Uuid, + ) -> Reader<'_> { + Reader { + txn, + operation, + bindings: HashMap::from([(root, binding)]), + cache: HashMap::new(), + work: PersistedMetadataReadWork::default(), + limits: MetadataDagLimits::default(), + directories: BTreeMap::from([("/".into(), root)]), + source_ids: BTreeMap::from([("/".into(), source)]), + file_oids: HashMap::new(), + } + } + + async fn source_entries( + &mut self, + source: uuid::Uuid, + directory: [u8; 32], + entries: &[mst2_codec::metapage::Entry], + ) -> Result, uuid::Uuid>, SnapshotError> { + let binding = self + .bindings + .get(&directory) + .ok_or_else(|| integrity("qualified source directory has no certified binding"))?; + let names: Vec<_> = entries + .iter() + .map(|entry| hex::encode(&entry.name)) + .collect(); + let rows = self.txn.query_all_raw(sql("SELECT * FROM mst2_metadata_read_source_entries($1::uuid,$2::uuid,$3,$4,$5,$6::jsonb)", + [self.operation.to_string().into(), source.to_string().into(), directory.to_vec().into(), + binding.generation.into(), binding.certificate.to_vec().into(),json!(names).into()])).await.map_err(database_error)?; + #[cfg(test)] + if entries.iter().any(|entry| !entry.is_dir()) + && let Ok((admitted, resume)) = SOURCE_FACT_BARRIERS.try_with(Clone::clone) + { + admitted.wait().await; + resume.wait().await; + } + if rows.len() != entries.len() { + return Err(integrity( + "qualified source-name index differs from selected actual page entries", + )); + } + let mut children = HashMap::new(); + for row in rows { + let name: Vec = row.try_get("", "name").map_err(internal)?; + let entry = entries + .iter() + .find(|entry| entry.name == name) + .ok_or_else(|| { + integrity("qualified source-name read returned an unrequested occurrence") + })?; + let state: String = row.try_get("", "fact_state").map_err(internal)?; + match state.as_str() { + "READY" => {} + "MISSING" => { + return Err(SnapshotError::new( + crate::ceres::snapshot::error::SnapshotErrorCode::MetadataNotReady, + "selected fixed file has no current verified object metadata", + )); + } + "SOURCE_UNAVAILABLE" => { + return Err(unavailable( + "selected fixed directory has no current exact source attestation", + )); + } + _ => { + return Err(integrity( + "selected fixed file has invalid or different current verified object metadata", + )); + } + } + let kind: i16 = row.try_get("", "kind").map_err(internal)?; + if kind as u8 != entry.kind as u8 { + return Err(integrity( + "qualified selected occurrence changed its fixed filesystem kind", + )); + } + if entry.is_dir() { + let child = self.bindings.get(&entry.child_root).ok_or_else(|| { + integrity("qualified source child has no certified occurrence") + })?; + let root = digest_column(&row, "child_root")?; + if root != entry.child_root + || child.generation + != row + .try_get::("", "child_generation") + .map_err(internal)? + || child.certificate != digest_column(&row, "child_certificate_digest")? + { + return Err(integrity( + "qualified source-name directory differs from its exact certified lifetime", + )); + } + children.insert( + name.clone(), + row.try_get("", "child_attestation_id").map_err(internal)?, + ); + } else if row.try_get::("", "byte_size").map_err(internal)? as u64 != entry.size + || digest_column(&row, "content_digest")? != entry.content_id + { + return Err(integrity( + "qualified source-name file differs from its certified actual page", + )); + } + if !entry.is_dir() { + self.file_oids.insert( + (source, name), + row.try_get("", "git_oid").map_err(internal)?, + ); + } + } + Ok(children) + } + + fn proofs(&self) -> Result)>, SnapshotError> { + let bytes: usize = self.cache.values().map(|page| page.bytes.len()).sum(); + if bytes > 1_048_576 { + return Err(SnapshotError::new( + crate::ceres::snapshot::error::SnapshotErrorCode::ProofBudgetExceeded, + "qualified proof pages exceed the response budget; use metadata/pages", + )); + } + let mut pages: Vec<_> = self + .cache + .iter() + .map(|(id, page)| (*id, page.bytes.clone())) + .collect(); + pages.sort_by_key(|item| item.0); + Ok(pages) + } + + async fn range( + &mut self, + id: [u8; 32], + after: Option<&[u8]>, + count: usize, + out: &mut Vec, + ) -> Result<(), SnapshotError> { + if out.len() >= count { + return Ok(()); + } + self.load(id).await?; + match self.cache[&id].page.clone() { + Page::Leaf { entries } => { + let start = after + .map(|name| entries.partition_point(|entry| entry.name.as_slice() <= name)) + .unwrap_or(0); + out.extend(entries.into_iter().skip(start).take(count - out.len())); + } + Page::Branch { + prefix, + terminal, + children, + } => { + if let Some(entry) = terminal + && after.is_none_or(|name| entry.name.as_slice() > name) + { + out.push(entry); + } + for child in children { + if out.len() >= count { + break; + } + let mut partition = prefix.clone(); + partition.push(child.label); + if after.is_some_and(|name| { + partition.as_slice() < name && !name.starts_with(&partition) + }) { + continue; + } + self.descend(id, child.label).await?; + Box::pin(self.range(child.child_page_id, after, count, out)).await?; + } + } + } + Ok(()) + } + + async fn find_entry( + &mut self, + mut id: [u8; 32], + name: &[u8], + ) -> Result, SnapshotError> { + loop { + self.load(id).await?; + match &self.cache[&id].page { + Page::Leaf { entries } => { + return Ok(entries.iter().find(|entry| entry.name == name).cloned()); + } + Page::Branch { + prefix, + terminal, + children, + } => { + if name == prefix { + return Ok(terminal.clone()); + } + if !name.starts_with(prefix) { + return Ok(None); + } + let Some(label) = name.get(prefix.len()).copied() else { + return Ok(None); + }; + if !children.iter().any(|child| child.label == label) { + return Ok(None); + } + id = self.descend(id, label).await?; + } + } + } + } + + async fn lookup( + &mut self, + mut root: [u8; 32], + path: &str, + ) -> Result { + if path == "/" { + self.load(root).await?; + return Ok(RootedLookupStatus::Directory(root)); + } + let mut prefix = String::new(); + let mut source = self.source_ids["/"]; + let mut parts = path[1..].split('/').peekable(); + while let Some(name) = parts.next() { + let Some(entry) = self.find_entry(root, name.as_bytes()).await? else { + return Ok(RootedLookupStatus::Absent); + }; + if !entry.is_dir() { + if parts.peek().is_some() { + return Ok(RootedLookupStatus::NotDirectory { + symlink: entry.kind == mst2_codec::metapage::EntryKind::Symlink, + }); + } + self.source_entries(source, root, std::slice::from_ref(&entry)) + .await?; + let git_oid = self + .file_oids + .get(&(source, entry.name.clone())) + .cloned() + .ok_or_else(|| { + integrity("selected fixed file has no exact source OID binding") + })?; + return Ok(RootedLookupStatus::File { entry, git_oid }); + } + let children = self + .source_entries(source, root, std::slice::from_ref(&entry)) + .await?; + source = *children.get(&entry.name).ok_or_else(|| { + integrity("qualified named directory has no independently derived source child") + })?; + root = entry.child_root; + prefix.push('/'); + prefix.push_str(name); + self.directories.insert(prefix.clone(), root); + self.source_ids.insert(prefix.clone(), source); + } + self.load(root).await?; + Ok(RootedLookupStatus::Directory(root)) + } + async fn load(&mut self, id: [u8; 32]) -> Result<(), SnapshotError> { + self.work.walk_visits += 1; + if self.work.walk_visits > self.limits.prepare_entry_visits as u64 { + return Err(limit("qualified route work budget exceeded")); + } + if self.cache.contains_key(&id) { + return Ok(()); + } + if self.cache.len() >= self.limits.nodes { + return Err(limit("qualified route page budget exceeded")); + } + let binding = *self.bindings.get(&id).ok_or_else(|| { + integrity("qualified page was not reached through a certified occurrence") + })?; + self.work.page_queries += 1; + let row = self + .txn + .query_one_raw(sql( + PAGE_SQL, + [ + id.to_vec().into(), + binding.generation.into(), + binding.certificate.to_vec().into(), + self.operation.to_string().into(), + ], + )) + .await + .map_err(internal)? + .ok_or_else(|| { + unavailable("qualified certified page or reader ownership is unavailable") + })?; + let bytes: Vec = row.try_get("", "payload").map_err(internal)?; + let size: i32 = row.try_get("", "byte_size").map_err(internal)?; + if !(HEADER_LEN..=PAGE_MAX_BYTES).contains(&bytes.len()) + || bytes.len() != size as usize + || page_id(&bytes) != id + { + return Err(integrity( + "qualified durable metadata bytes differ from their certified page", + )); + } + let (page, entries) = Page::decode(&bytes).map_err(internal)?; + self.work.payload_bytes = self + .work + .payload_bytes + .checked_add(bytes.len() as u64) + .filter(|value| *value <= self.limits.payload_bytes) + .ok_or_else(|| limit("qualified route byte budget exceeded"))?; + let raw_refs: serde_json::Value = row.try_get("", "references").map_err(internal)?; + let refs: Vec = serde_json::from_value(raw_refs).map_err(internal)?; + if refs.len() > 257 { + return Err(integrity("qualified canonical reference count is invalid")); + } + let mut expected = Vec::new(); + let direct = match &page { + Page::Leaf { entries } => entries.as_slice(), + Page::Branch { + terminal, children, .. + } => { + for child in children { + expected.push(( + "RADIX", + child.child_page_id, + None, + Some(child.label), + Some(child.subtree_entries), + )); + } + terminal.as_slice() + } + }; + // Reference ordering is direct DIRECTORY occurrences followed by RADIX, + // as defined by the independent canonical parser. Compare occurrences + // as a set here; SQL validates the certificate's ordinal completeness. + for entry in direct.iter().filter(|entry| entry.is_dir()) { + expected.push(( + "DIRECTORY", + entry.child_root, + Some(hex::encode(&entry.name)), + None, + None, + )); + } + let mut actual = BTreeSet::new(); + for reference in refs { + let child = digest_hex(&reference.child)?; + let binding = Binding { + generation: reference.generation, + certificate: digest_hex(&reference.certificate)?, + }; + if binding.generation <= 0 + || self + .bindings + .get(&child) + .is_some_and(|known| *known != binding) + { + return Err(integrity( + "qualified occurrence retargeted its exact lifetime or certificate", + )); + } + self.bindings.insert(child, binding); + if !actual.insert(( + reference.kind, + child, + reference.name, + reference.label, + reference.count, + )) { + return Err(integrity("qualified typed occurrence has a duplicate")); + } + } + let expected: BTreeSet<_> = expected + .into_iter() + .map(|(kind, child, name, label, count)| (kind.to_owned(), child, name, label, count)) + .collect(); + if actual != expected { + return Err(integrity( + "qualified certificate references differ from its actual durable page", + )); + } + self.work.edge_references_checked += actual.len() as u64; + if self.work.edge_references_checked > self.limits.edges as u64 { + return Err(limit("qualified route reference budget exceeded")); + } + self.work.pages_loaded += 1; + self.cache.insert( + id, + StoredPage { + bytes, + page, + entries, + }, + ); + Ok(()) + } + + async fn descend(&mut self, parent: [u8; 32], label: u8) -> Result<[u8; 32], SnapshotError> { + self.load(parent).await?; + let (prefix, child) = match &self.cache[&parent].page { + Page::Branch { + prefix, children, .. + } => ( + prefix.clone(), + children + .iter() + .find(|child| child.label == label) + .ok_or_else(|| absent("route label is absent in fixed qualified metadata"))? + .clone(), + ), + Page::Leaf { .. } => return Err(absent("route descends past a qualified leaf")), + }; + let id = child.child_page_id; + self.load(id).await?; + let received = &self.cache[&id]; + let mut partition = prefix; + partition.push(label); + let valid = match &received.page { + Page::Leaf { entries } => entries + .iter() + .all(|entry| entry.name.starts_with(&partition)), + Page::Branch { prefix, .. } => prefix.starts_with(&partition), + }; + if !valid || received.entries != child.subtree_entries { + return Err(integrity( + "qualified radix partition differs from its exact child", + )); + } + Ok(id) + } + + async fn directory( + &mut self, + mut root: [u8; 32], + path: &str, + ) -> Result<[u8; 32], SnapshotError> { + if path == "/" { + return Ok(root); + } + let mut directory_path = String::new(); + let mut source = self.source_ids["/"]; + for name in path[1..].split('/') { + let mut id = root; + loop { + self.load(id).await?; + let entry = match &self.cache[&id].page { + Page::Leaf { entries } => entries + .iter() + .find(|entry| entry.name == name.as_bytes()) + .cloned(), + Page::Branch { + prefix, terminal, .. + } => { + if name.as_bytes() == prefix { + terminal.clone() + } else { + if !name.as_bytes().starts_with(prefix) { + return Err(absent("name is absent in qualified directory")); + } + let label = + name.as_bytes().get(prefix.len()).copied().ok_or_else(|| { + absent("name is absent in qualified directory") + })?; + id = self.descend(id, label).await?; + continue; + } + } + } + .ok_or_else(|| absent("name is absent in qualified directory"))?; + if !entry.is_dir() { + return Err(SnapshotError::new( + crate::ceres::snapshot::error::SnapshotErrorCode::NotDirectory, + "fixed qualified path is not a directory", + )); + } + let children = self + .source_entries(source, root, std::slice::from_ref(&entry)) + .await?; + source = *children.get(&entry.name).ok_or_else(|| { + integrity("qualified directory route lost its exact source-name binding") + })?; + root = entry.child_root; + directory_path.push('/'); + directory_path.push_str(name); + self.directories.insert(directory_path.clone(), root); + self.source_ids.insert(directory_path.clone(), source); + break; + } + } + Ok(root) + } + async fn route( + &mut self, + root: [u8; 32], + labels: &[u8], + ) -> Result, SnapshotError> { + self.load(root).await?; + let mut pages = vec![root]; + let mut current = root; + for label in labels { + current = self.descend(current, *label).await?; + pages.push(current); + } + Ok(pages) + } +} + +const PAGE_SQL:&str="SELECT body.payload,body.byte_size,proof.canonical_proof->'references' AS references + FROM mst2_metadata_page_certificate proof JOIN mst2_metadata_current cur USING(page_id,generation) + JOIN mst2_metadata_lifetime life USING(page_id,generation) JOIN mst2_metadata_graph_node node USING(page_id,generation) + JOIN mst2_metadata_payload body USING(page_id,generation) + WHERE proof.page_id=$1 AND proof.generation=$2 AND proof.certificate_digest=$3 + AND proof.namespace_uuid=(SELECT namespace_uuid FROM mst2_metadata_family_identity WHERE singleton=1) + AND life.state='LIVE' AND life.graph_domain='qualified-v1' AND node.state='LIVE' + AND node.certificate_digest=proof.certificate_digest AND node.bytes=proof.byte_size + AND life.expected_size=proof.byte_size AND body.byte_size=proof.byte_size AND body.metadata_codec=1 + AND octet_length(body.payload)=proof.byte_size AND NOT EXISTS(SELECT 1 FROM mst2_metadata_gc_op gc + WHERE gc.page_id=proof.page_id AND gc.generation=proof.generation) + AND EXISTS(SELECT 1 FROM mst2_metadata_reader_operation reader JOIN mst2_metadata_root_anchor anchor + ON anchor.reader_operation_id=reader.operation_id AND anchor.anchor_kind='READER' + AND anchor.root_page=reader.root_page AND anchor.root_generation=reader.root_generation + WHERE reader.operation_id=$4::uuid AND reader.state='ACTIVE' + AND reader.hard_deadline_unix>floor(extract(epoch FROM clock_timestamp()))::bigint) + AND (SELECT count(*) FROM mst2_metadata_verified_ref ref WHERE ref.parent_page=proof.page_id + AND ref.parent_generation=proof.generation)=jsonb_array_length(proof.canonical_proof->'references') + AND NOT EXISTS((SELECT ref.child_page,ref.child_generation FROM mst2_metadata_verified_ref ref + WHERE ref.parent_page=proof.page_id AND ref.parent_generation=proof.generation) EXCEPT + (SELECT edge.child_page,edge.child_generation FROM mst2_metadata_graph_edge edge + WHERE edge.parent_page=proof.page_id AND edge.parent_generation=proof.generation)) + AND NOT EXISTS((SELECT edge.child_page,edge.child_generation FROM mst2_metadata_graph_edge edge + WHERE edge.parent_page=proof.page_id AND edge.parent_generation=proof.generation) EXCEPT + (SELECT ref.child_page,ref.child_generation FROM mst2_metadata_verified_ref ref + WHERE ref.parent_page=proof.page_id AND ref.parent_generation=proof.generation))"; diff --git a/src/jupiter/storage/qualified_metadata_rooted.rs b/src/jupiter/storage/qualified_metadata_rooted.rs new file mode 100644 index 00000000..5dc2f4be --- /dev/null +++ b/src/jupiter/storage/qualified_metadata_rooted.rs @@ -0,0 +1,766 @@ +use std::{collections::BTreeMap, sync::Arc}; + +use mst2_codec::metapage::{Page, page_id}; +use sea_orm::{QueryResult, Value}; +use serde_json::json; + +use super::*; +use crate::ceres::snapshot::{ + metadata_install::MetadataInstallIdentity, + rooted_metadata_install::{RootedMetadataInstallPlan, RootedReuseRoot}, + rooted_metadata_projection::{CertifiedReusableDirectory, RootedReuseLookup}, +}; + +#[path = "qualified_metadata_session.rs"] +mod sessions; + +#[path = "qualified_metadata_gc.rs"] +mod gc; +#[path = "qualified_metadata_reader.rs"] +mod reader; +pub(crate) use reader::{RootedDirectoryWindow, RootedLookupBatch, RootedLookupStatus}; +#[cfg(test)] +pub(crate) use reader::{ + with_rooted_reader_barriers, with_rooted_source_fact_barriers, + with_rooted_source_temporary_shadow, +}; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct RootedPrepareIntent { + prepare_id: String, + operation_id: String, + plan: Arc, + bindings: BTreeMap<[u8; 32], (i64, u64)>, + manifest_digest: [u8; 32], + bindings_digest: [u8; 32], + primary_scope: Vec, + storage_seal: [u8; 32], + root_generation: i64, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct RootedMetadataReceipt { + intent: RootedPrepareIntent, + certificate_digest: [u8; 32], + attestation_id: uuid::Uuid, + attestation_digest: [u8; 32], +} + +impl RootedPrepareIntent { + #[cfg(test)] + pub(crate) fn prepare_id(&self) -> &str { + &self.prepare_id + } + #[cfg(test)] + pub(crate) fn metadata_root(&self) -> [u8; 32] { + self.plan.root + } + #[cfg(test)] + pub(crate) fn root_generation(&self) -> i64 { + self.root_generation + } +} +impl RootedMetadataReceipt { + pub(crate) fn metadata_root(&self) -> [u8; 32] { + self.intent.plan.root + } +} + +pub(crate) struct RootedQualifiedMetadataRepository { + connection: DatabaseConnection, + namespace: VerifiedQualifiedNamespace, + primary_scope: Vec, + maintenance_state: tokio::sync::Mutex, +} + +#[async_trait::async_trait] +impl RootedReuseLookup for RootedQualifiedMetadataRepository { + async fn lookup_reuse( + &self, + tree_oid: &str, + identity: &MetadataInstallIdentity, + ) -> Result, SnapshotError> { + // This is a bounded projection hint. It acquires no writer/retention + // lock and grants no installation authority: begin_intent and finalize + // independently bind the exact attestation and current lifetime. + let kind = identity + .tagged_root_tree_oid + .split_once(':') + .ok_or_else(|| integrity("rooted source identity has no tagged hash kind"))? + .0; + let profile = json!({"source_domain": identity.source_domain,"hash_kind": kind, + "schema_version": identity.schema_version,"metadata_codec": identity.metadata_codec, + "materialization_policy": identity.materialization_policy,"fs_semantics": identity.fs_semantics, + "access_projection": identity.access_projection,"verification_revision": identity.verification_revision, + "projection_revision": identity.projection_revision}); + let q = identifier(&self.namespace.schema); + let c = identifier(&self.namespace.core_schema); + let row = self.connection.query_one_raw(sql(format!( + "SELECT a.root_page,a.root_generation,a.attestation_id::text,a.attestation_digest, + p.certificate_digest,p.relative_path_bytes,p.relative_components,p.closure_nodes_upper, + p.closure_edges_upper,p.closure_bytes_upper,p.closure_entries_upper + FROM {q}.mst2_metadata_reuse_index i JOIN {q}.mst2_metadata_source_root_attestation a + ON a.attestation_id=i.attestation_id AND a.root_page=i.root_page AND a.root_generation=i.root_generation + AND a.attestation_digest=i.attestation_digest + JOIN {q}.mst2_metadata_page_certificate p ON p.page_id=a.root_page AND p.generation=a.root_generation + AND p.certificate_digest=a.root_certificate_digest + JOIN {q}.mst2_metadata_current cur ON cur.page_id=p.page_id AND cur.generation=p.generation + JOIN {q}.mst2_metadata_lifetime life ON life.page_id=p.page_id AND life.generation=p.generation + JOIN {q}.mst2_metadata_graph_node node ON node.page_id=p.page_id AND node.generation=p.generation + JOIN {q}.mst2_metadata_payload body ON body.page_id=p.page_id AND body.generation=p.generation + JOIN {q}.mst2_metadata_prepare origin ON origin.prepare_id=a.origin_prepare_id + JOIN {c}.mega_tree source ON source.tree_id=split_part(a.tagged_tree_oid,':',2) + WHERE i.tagged_tree_oid=$1 AND i.profile_digest=sha256(convert_to('mega.mst2.native-profile.v1','UTF8') + ||decode('00','hex')||convert_to($2::jsonb::text,'UTF8')) AND a.source_profile=$2::jsonb + AND a.namespace_uuid=$3::uuid AND {c}.mst2_route_source_tree_matches(split_part(a.tagged_tree_oid,':',2),a.source_revision,a.source_body_digest) + AND life.state='LIVE' AND life.graph_domain='qualified-v1' AND node.state='LIVE' + AND node.certificate_digest=p.certificate_digest AND node.bytes=p.byte_size + AND body.byte_size=p.byte_size AND body.metadata_codec=p.metadata_codec + AND life.expected_size=p.byte_size AND origin.state='COMMITTED' AND NOT pg_is_in_recovery() + AND NOT EXISTS(SELECT 1 FROM {q}.mst2_metadata_gc_op gc WHERE gc.page_id=p.page_id AND gc.generation=p.generation)" + ),[tree_oid.into(),profile.into(),self.namespace.namespace_uuid.clone().into()])) + .await.map_err(database_error)?; + let Some(row) = row else { + return Ok(None); + }; + let count = |name: &str| -> Result { + usize::try_from(row.try_get::("", name).map_err(internal)?).map_err(internal) + }; + let wide = |name: &str| -> Result { + u64::try_from(row.try_get::("", name).map_err(internal)?).map_err(internal) + }; + Ok(Some(CertifiedReusableDirectory { + page_id: digest_column(&row, "root_page")?, + proof: RootedReuseRoot { + generation: row.try_get("", "root_generation").map_err(internal)?, + attestation_id: uuid::Uuid::parse_str( + &row.try_get::("", "attestation_id") + .map_err(internal)?, + ) + .map_err(internal)?, + attestation_digest: digest_column(&row, "attestation_digest")?, + certificate_digest: digest_column(&row, "certificate_digest")?, + }, + relative_path_bytes: count("relative_path_bytes")?, + relative_components: count("relative_components")?, + closure_nodes_upper: count("closure_nodes_upper")?, + closure_edges_upper: count("closure_edges_upper")?, + closure_bytes_upper: wide("closure_bytes_upper")?, + closure_entries_upper: usize::try_from(wide("closure_entries_upper")?) + .map_err(internal)?, + })) + } +} + +fn sql(text: impl Into, values: impl IntoIterator) -> Statement { + Statement::from_sql_and_values(DbBackend::Postgres, text, values) +} +fn internal(error: impl std::fmt::Display) -> SnapshotError { + SnapshotError::new( + crate::ceres::snapshot::error::SnapshotErrorCode::Internal, + error.to_string(), + ) +} +pub(super) fn is_lock_unavailable(error: &sea_orm::DbErr) -> bool { + let runtime = match error { + sea_orm::DbErr::Exec(runtime) | sea_orm::DbErr::Query(runtime) => runtime, + _ => return false, + }; + let sea_orm::RuntimeErr::SqlxError(sqlx_error) = runtime else { + return false; + }; + let sea_orm::sqlx::Error::Database(database_error) = sqlx_error.as_ref() else { + return false; + }; + database_error.code().as_deref() == Some("55P03") +} +fn database_error(error: sea_orm::DbErr) -> SnapshotError { + if is_lock_unavailable(&error) { + return SnapshotError::new( + crate::ceres::snapshot::error::SnapshotErrorCode::TemporaryUnavailable, + "qualified source is being updated; retry the operation", + ); + } + internal(error) +} +fn integrity(message: &str) -> SnapshotError { + SnapshotError::new( + crate::ceres::snapshot::error::SnapshotErrorCode::IntegrityError, + message, + ) +} +fn unavailable(message: &str) -> SnapshotError { + SnapshotError::new( + crate::ceres::snapshot::error::SnapshotErrorCode::ObjectUnavailable, + message, + ) +} +fn digest_column(row: &QueryResult, name: &str) -> Result<[u8; 32], SnapshotError> { + let bytes: Vec = row.try_get("", name).map_err(internal)?; + bytes.as_slice().try_into().map_err(internal) +} +fn encode_bindings(bindings: &BTreeMap<[u8; 32], (i64, u64)>) -> Vec { + let mut bytes = Vec::with_capacity(12 + 48 * bindings.len()); + bytes.extend_from_slice(b"MST2GEN1"); + bytes.extend_from_slice(&(bindings.len() as u32).to_be_bytes()); + for (page, (generation, size)) in bindings { + bytes.extend_from_slice(page); + bytes.extend_from_slice(&generation.to_be_bytes()); + bytes.extend_from_slice(&size.to_be_bytes()); + } + bytes +} +fn decode_bindings( + bytes: &[u8], + plan: &RootedMetadataInstallPlan, +) -> Result, SnapshotError> { + if bytes.len() != 12 + 48 * plan.delta.len() || bytes.get(..8) != Some(b"MST2GEN1") { + return Err(integrity( + "rooted delta binding encoding differs from its plan", + )); + } + let count = u32::from_be_bytes(bytes[8..12].try_into().map_err(internal)?) as usize; + if count != plan.delta.len() { + return Err(integrity( + "rooted delta binding count differs from its plan", + )); + } + let mut bindings = BTreeMap::new(); + for ((expected, size), record) in plan.delta.iter().zip(bytes[12..].as_chunks::<48>().0) { + let page: [u8; 32] = record[..32].try_into().map_err(internal)?; + let generation = i64::from_be_bytes(record[32..40].try_into().map_err(internal)?); + let recorded_size = u64::from_be_bytes(record[40..48].try_into().map_err(internal)?); + if page != *expected || recorded_size != *size || generation <= 0 { + return Err(integrity("rooted exact delta lifetime binding is invalid")); + } + bindings.insert(page, (generation, recorded_size)); + } + Ok(bindings) +} + +async fn committed( + txn: DatabaseTransaction, + result: Result, + operation: &str, + digest: [u8; 32], + phase: super::super::native_metadata_install::MetadataCommitPhase, +) -> Result { + match result { + Err(error) => { + let _ = txn.rollback().await; + Err(error.into()) + } + Ok(value) => match txn.commit().await { + Ok(()) => Ok(value), + Err(_) => Err(MetadataInstallError::CommitUncertain { + operation_id: operation.into(), + manifest_digest: digest, + phase, + }), + }, + } +} + +impl RootedQualifiedMetadataRepository { + pub(crate) async fn open( + core: &DatabaseConnection, + config: &DbConfig, + ) -> Result { + let captured = captured_core(core).await?; + let namespace = registered(core, &captured) + .await? + .ok_or_else(|| rejected("rooted qualified family is not provisioned"))?; + let mut q_config = config.clone(); + q_config.db_url = pool_url(&config.db_url, &namespace)?; + q_config.max_connection = q_config.max_connection.clamp(1, 4); + q_config.min_connection = 1; + let connection = postgres_connection(&q_config).await?; + let txn = connection + .begin_with_config(Some(IsolationLevel::ReadCommitted), None) + .await?; + namespace + .enter(&txn) + .await + .map_err(|e| rejected(&e.to_string()))?; + let scope:serde_json::Value=txn.query_one_raw(sql( + "SELECT jsonb_build_array(s.storage_uuid,current_database(),d.oid::bigint,current_schema(),n.oid::bigint, + inet_server_addr()::text,inet_server_port()) AS scope FROM mst2_metadata_storage_scope s + JOIN pg_catalog.pg_database d ON d.datname=current_database() + JOIN pg_catalog.pg_namespace n ON n.nspname=current_schema() WHERE s.singleton=1 AND NOT pg_is_in_recovery()",[], + )).await?.ok_or_else(||rejected("rooted captured primary scope is missing"))?.try_get("","scope")?; + let primary_scope = serde_json::to_vec(&scope).map_err(|e| rejected(&e.to_string()))?; + txn.commit().await?; + Ok(Self { + connection, + namespace, + primary_scope, + maintenance_state: tokio::sync::Mutex::default(), + }) + } + + async fn transaction(&self) -> Result { + let txn = self + .connection + .begin_with_config(Some(IsolationLevel::ReadCommitted), None) + .await + .map_err(database_error)?; + self.namespace.enter(&txn).await?; + let valid: bool = txn + .query_one_raw(sql( + "SELECT mst2_metadata_scope_matches($1) AS valid", + [self.primary_scope.clone().into()], + )) + .await + .map_err(database_error)? + .ok_or_else(|| integrity("rooted primary scope is missing"))? + .try_get("", "valid") + .map_err(internal)?; + if !valid { + return Err(integrity("rooted writer left its captured primary scope")); + } + Ok(txn) + } + + async fn load_intent( + &self, + db: &C, + operation: &str, + plan: &RootedMetadataInstallPlan, + ) -> Result, SnapshotError> { + let Some(row)=db.query_one_raw(sql("SELECT prepare_id,plan_kind,state,manifest_digest,canonical_plan,canonical_bindings, + bindings_digest,primary_scope,storage_seal FROM mst2_metadata_prepare WHERE operation_id=$1",[operation.into()])) + .await.map_err(internal)? else {return Ok(None);}; + let digest = plan.digest()?; + if row.try_get::("", "plan_kind").map_err(internal)? != "ROOTED" + || digest_column(&row, "manifest_digest")? != digest + { + return Err(SnapshotError::new( + crate::ceres::snapshot::error::SnapshotErrorCode::Conflict, + "operation is bound to a different rooted manifest", + )); + } + let stored: Vec = row.try_get("", "canonical_plan").map_err(internal)?; + if RootedMetadataInstallPlan::decode(&stored, &digest)? != *plan { + return Err(integrity( + "rooted stored plan differs from its exact source identity", + )); + } + let encoded_bindings: Vec = row.try_get("", "canonical_bindings").map_err(internal)?; + let bindings_digest: [u8; 32] = Sha256::digest(&encoded_bindings).into(); + if bindings_digest != digest_column(&row, "bindings_digest")? { + return Err(integrity( + "rooted stored lifetime binding digest is invalid", + )); + } + let primary_scope: Vec = row.try_get("", "primary_scope").map_err(internal)?; + if primary_scope != self.primary_scope { + return Err(integrity( + "rooted operation belongs to another physical primary", + )); + } + let bindings = decode_bindings(&encoded_bindings, plan)?; + let root_generation = bindings + .get(&plan.root) + .map(|b| b.0) + .or_else(|| plan.reused.get(&plan.root).map(|b| b.generation)) + .ok_or_else(|| integrity("rooted root lifetime is missing"))?; + Ok(Some(( + RootedPrepareIntent { + prepare_id: row.try_get("", "prepare_id").map_err(internal)?, + operation_id: operation.into(), + plan: Arc::new(plan.clone()), + bindings, + manifest_digest: digest, + bindings_digest, + primary_scope, + storage_seal: digest_column(&row, "storage_seal")?, + root_generation, + }, + row.try_get("", "state").map_err(internal)?, + ))) + } + + pub(crate) async fn begin_intent( + &self, + operation: &str, + plan: &RootedMetadataInstallPlan, + ) -> Result { + plan.validate()?; + if operation.is_empty() || operation.len() > 255 || operation.contains('\0') { + return Err(integrity("invalid rooted operation ID").into()); + } + let manifest_digest = plan.digest()?; + let txn = self.transaction().await?; + let result=async { + if let Some((intent,state))=self.load_intent(&txn,operation,plan).await? { + if state=="ABORTED" {return Err(unavailable("rooted preparation was definitively aborted"));} + return Ok(intent); + } + let pages:Vec<_>=plan.delta.iter().map(|(page,size)|json!({"page":hex::encode(page),"size":size})).collect(); + let encoded=serde_json::to_string(&pages).map_err(internal)?; + txn.execute_raw(sql("INSERT INTO mst2_metadata_lifetime(page_id,node_id,generation,state,metadata_codec,expected_size,graph_domain) + SELECT decode(p.page,'hex'),'page:sha256:'||p.page,coalesce(cur.generation+1,1),'RESERVED',1,p.size,'qualified-v1' + FROM jsonb_to_recordset($1::jsonb) p(page text,size integer) + LEFT JOIN mst2_metadata_current cur ON cur.page_id=decode(p.page,'hex') + LEFT JOIN mst2_metadata_lifetime previous ON previous.page_id=cur.page_id AND previous.generation=cur.generation + WHERE (cur.page_id IS NULL AND NOT EXISTS(SELECT 1 FROM mst2_metadata_lifetime life WHERE life.page_id=decode(p.page,'hex'))) + OR (previous.state='REMOVED' AND cur.generation<9223372036854775807 + AND EXISTS(SELECT 1 FROM mst2_metadata_gc_op proof WHERE proof.page_id=cur.page_id + AND proof.generation=cur.generation AND proof.state='APPLIED'))", + [encoded.clone().into()])).await.map_err(internal)?; + txn.execute_raw(sql("INSERT INTO mst2_metadata_current(page_id,generation) + SELECT life.page_id,life.generation FROM jsonb_to_recordset($1::jsonb) p(page text,size integer) + JOIN mst2_metadata_lifetime life ON life.page_id=decode(p.page,'hex') AND life.generation=1 AND life.state='RESERVED' + WHERE NOT EXISTS(SELECT 1 FROM mst2_metadata_current cur WHERE cur.page_id=life.page_id)", + [encoded.clone().into()])).await.map_err(internal)?; + txn.execute_raw(sql("UPDATE mst2_metadata_current cur SET generation=fresh.generation + FROM jsonb_to_recordset($1::jsonb) p(page text,size integer),mst2_metadata_lifetime previous, + mst2_metadata_lifetime fresh + WHERE cur.page_id=decode(p.page,'hex') AND previous.page_id=cur.page_id AND previous.generation=cur.generation + AND previous.state='REMOVED' AND cur.generation<9223372036854775807 + AND fresh.page_id=cur.page_id AND fresh.generation=cur.generation+1 AND fresh.state='RESERVED' + AND fresh.expected_size=p.size AND fresh.metadata_codec=1 AND fresh.graph_domain='qualified-v1' + AND EXISTS(SELECT 1 FROM mst2_metadata_gc_op proof WHERE proof.page_id=cur.page_id + AND proof.generation=cur.generation AND proof.state='APPLIED')",[encoded.clone().into()])) + .await.map_err(internal)?; + let rows=txn.query_all_raw(sql("SELECT cur.page_id,cur.generation,life.expected_size,life.metadata_codec,life.graph_domain,life.state, + n.state AS graph_state,n.certificate_digest,body.byte_size FROM jsonb_to_recordset($1::jsonb) p(page text,size integer) + JOIN mst2_metadata_current cur ON cur.page_id=decode(p.page,'hex') JOIN mst2_metadata_lifetime life USING(page_id,generation) + LEFT JOIN mst2_metadata_graph_node n USING(page_id,generation) LEFT JOIN mst2_metadata_payload body USING(page_id,generation) + WHERE NOT EXISTS(SELECT 1 FROM mst2_metadata_gc_op gc WHERE gc.page_id=cur.page_id AND gc.generation=cur.generation) + ORDER BY cur.page_id",[encoded.into()])).await.map_err(internal)?; + let mut bindings=BTreeMap::new(); + for row in rows { + let page=digest_column(&row,"page_id")?; let size=row.try_get::("","expected_size").map_err(internal)? as u64; + let state:String=row.try_get("","state").map_err(internal)?; + if plan.delta.get(&page)!=Some(&size) || row.try_get::("","metadata_codec").map_err(internal)?!=1 + || row.try_get::("","graph_domain").map_err(internal)?!="qualified-v1" || !matches!(state.as_str(),"RESERVED"|"LIVE") + || state=="LIVE" && (row.try_get::>("","graph_state").map_err(internal)?.as_deref()!=Some("LIVE") + || row.try_get::>("","byte_size").map_err(internal)?!=Some(size as i32)) { + return Err(unavailable("rooted requested delta lifetime is not exactly installable")); + } + bindings.insert(page,(row.try_get("","generation").map_err(internal)?,size)); + } + if bindings.len()!=plan.delta.len() {return Err(unavailable("rooted delta lifetime coverage is incomplete"));} + let canonical_bindings=encode_bindings(&bindings); let bindings_digest:[u8;32]=Sha256::digest(&canonical_bindings).into(); + let prepare_id=uuid::Uuid::new_v4().to_string(); + let root_generation=bindings.get(&plan.root).map(|b|b.0).or_else(||plan.reused.get(&plan.root).map(|b|b.generation)) + .ok_or_else(||integrity("rooted root lifetime is missing"))?; + let mut seal=Sha256::new(); seal.update(b"mega.mst2.rooted-storage-seal.v1\0"); + seal.update(prepare_id.as_bytes()); seal.update(manifest_digest); seal.update(bindings_digest); + seal.update((self.primary_scope.len() as u64).to_be_bytes()); seal.update(&self.primary_scope); + seal.update(plan.root); seal.update(root_generation.to_be_bytes()); + let storage_seal:[u8;32]=seal.finalize().into(); let identity=&plan.identity; + txn.execute_raw(sql("INSERT INTO mst2_metadata_prepare(prepare_id,operation_id,manifest_digest,canonical_plan,plan_kind, + source_domain,tagged_root_tree_oid,scope,schema_version,metadata_codec,materialization_policy,fs_semantics,access_projection, + verification_revision,projection_revision,metadata_root,node_count,edge_count,total_bytes,state,canonical_bindings,bindings_digest, + primary_scope,storage_seal,graph_domain) VALUES($1,$2,$3,$4,'ROOTED',$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,'PREPARING',$19,$20,$21,$22,'qualified-v1')", + [prepare_id.clone().into(),operation.into(),manifest_digest.to_vec().into(),plan.encode()?.into(),identity.source_domain.clone().into(), + identity.tagged_root_tree_oid.clone().into(),identity.scope.clone().into(),(identity.schema_version as i16).into(), + (identity.metadata_codec as i16).into(),(identity.materialization_policy as i16).into(),(identity.fs_semantics as i16).into(), + (identity.access_projection as i16).into(),identity.verification_revision.into(),(identity.projection_revision as i16).into(), + plan.root.to_vec().into(),(plan.delta.len() as i32).into(),(plan.edges.len() as i32).into(),(plan.delta_bytes()? as i64).into(), + canonical_bindings.into(),bindings_digest.to_vec().into(),self.primary_scope.clone().into(),storage_seal.to_vec().into()])).await.map_err(internal)?; + let delta:Vec<_>=bindings.iter().map(|(page,(generation,size))|json!({"page":hex::encode(page),"generation":generation,"size":size})).collect(); + txn.execute_raw(sql("INSERT INTO mst2_metadata_prepare_page(prepare_id,page_id,generation,expected_size) + SELECT $1,decode(p.page,'hex'),p.generation,p.size FROM jsonb_to_recordset($2::jsonb) p(page text,generation bigint,size integer)", + [prepare_id.clone().into(),serde_json::to_string(&delta).map_err(internal)?.into()])).await.map_err(internal)?; + let reused:Vec<_>=plan.reused.iter().map(|(page,proof)|json!({"page":hex::encode(page),"generation":proof.generation, + "attestation_id":proof.attestation_id.to_string(),"attestation_digest":hex::encode(proof.attestation_digest)})).collect(); + txn.execute_raw(sql("INSERT INTO mst2_metadata_prepare_reuse_root(prepare_id,root_page,root_generation,attestation_id,attestation_digest) + SELECT $1,decode(r.page,'hex'),r.generation,r.attestation_id::uuid,decode(r.attestation_digest,'hex') + FROM jsonb_to_recordset($2::jsonb) r(page text,generation bigint,attestation_id text,attestation_digest text)", + [prepare_id.clone().into(),serde_json::to_string(&reused).map_err(internal)?.into()])).await.map_err(internal)?; + txn.execute_raw(sql("INSERT INTO mst2_metadata_root_anchor(anchor_id,anchor_kind,owner_key,prepare_id,root_page,root_generation,root_certificate_digest) + SELECT gen_random_uuid(),'REUSE',$1,$1,r.root_page,r.root_generation,a.root_certificate_digest + FROM mst2_metadata_prepare_reuse_root r JOIN mst2_metadata_source_root_attestation a USING(attestation_id) WHERE r.prepare_id=$1", + [prepare_id.clone().into()])).await.map_err(internal)?; + Ok(RootedPrepareIntent {prepare_id,operation_id:operation.into(),plan:Arc::new(plan.clone()),bindings,manifest_digest, + bindings_digest,primary_scope:self.primary_scope.clone(),storage_seal,root_generation}) + }.await; + committed( + txn, + result, + operation, + manifest_digest, + super::super::native_metadata_install::MetadataCommitPhase::Intent, + ) + .await + } + + async fn require_intent( + &self, + db: &C, + expected: &RootedPrepareIntent, + ) -> Result { + let (stored, state) = self + .load_intent(db, &expected.operation_id, &expected.plan) + .await? + .ok_or_else(|| unavailable("rooted preparation is missing"))?; + if stored != *expected { + return Err(integrity( + "rooted preparation crossed its immutable physical binding", + )); + } + if state == "ABORTED" { + return Err(unavailable("rooted preparation was definitively aborted")); + } + Ok(state) + } + + pub(crate) async fn install_pages( + &self, + intent: &RootedPrepareIntent, + payloads: &[MetadataPagePayload], + ) -> Result<(), MetadataInstallError> { + if payloads.is_empty() || payloads.len() > 64 { + return Err(integrity("rooted payload batch requires 1..=64 pages").into()); + } + let mut pages = BTreeMap::new(); + for payload in payloads { + let &(generation, size) = intent + .bindings + .get(&payload.id) + .ok_or_else(|| integrity("rooted payload is outside its fixed delta"))?; + if size != payload.size + || payload.bytes.len() as u64 != size + || page_id(&payload.bytes) != payload.id + { + return Err(integrity( + "rooted delta payload differs from its exact page digest or size", + ) + .into()); + } + Page::decode(&payload.bytes).map_err(internal)?; + if pages + .insert( + payload.id, + json!({"page":hex::encode(payload.id),"generation":generation,"size":size, + "payload":hex::encode(&payload.bytes)}), + ) + .is_some() + { + return Err(integrity("rooted payload batch has duplicate pages").into()); + } + } + let encoded = + serde_json::to_string(&pages.into_values().collect::>()).map_err(internal)?; + let txn = self.transaction().await?; + let result=async { + let state=self.require_intent(&txn,intent).await?; + if state=="PREPARING" { + txn.execute_raw(sql("INSERT INTO mst2_metadata_payload(page_id,generation,metadata_codec,byte_size,payload) + SELECT decode(p.page,'hex'),p.generation,1,p.size,decode(p.payload,'hex') + FROM jsonb_to_recordset($1::jsonb) p(page text,generation bigint,size integer,payload text) + WHERE NOT EXISTS(SELECT 1 FROM mst2_metadata_payload body WHERE body.page_id=decode(p.page,'hex')) + ON CONFLICT(page_id) DO NOTHING",[encoded.clone().into()])).await.map_err(internal)?; + } + if txn.query_one_raw(sql("SELECT p.page FROM jsonb_to_recordset($1::jsonb) p(page text,generation bigint,size integer,payload text) + LEFT JOIN mst2_metadata_payload body ON body.page_id=decode(p.page,'hex') + LEFT JOIN mst2_metadata_current cur ON cur.page_id=body.page_id AND cur.generation=body.generation + WHERE body.page_id IS NULL OR cur.page_id IS NULL OR body.generation<>p.generation OR body.metadata_codec<>1 + OR body.byte_size<>p.size OR body.payload<>decode(p.payload,'hex') LIMIT 1",[encoded.into()])) + .await.map_err(internal)?.is_some() {return Err(integrity("rooted durable delta payload conflicts with its exact incarnation"));} + Ok(()) + }.await; + committed( + txn, + result, + &intent.operation_id, + intent.manifest_digest, + super::super::native_metadata_install::MetadataCommitPhase::Payload, + ) + .await + } + + pub(crate) async fn finalize( + &self, + intent: &RootedPrepareIntent, + ) -> Result { + // Rehash/decode the actual delta outside the core route lock. A cold + // preparation additionally keeps the full Rust DAG validator as oracle. + let payloads = self.read_delta(intent).await?; + if intent.plan.reused.is_empty() { + use crate::ceres::snapshot::retention_dag::{ + MetadataDagCandidate, MetadataDagLimits, ValidatedMetadataDag, + }; + ValidatedMetadataDag::validate( + MetadataDagCandidate { + metadata_codec: intent.plan.identity.metadata_codec, + root: intent.plan.root, + pages: payloads, + edges: intent.plan.edges.iter().copied().collect(), + }, + MetadataDagLimits::default(), + )?; + } + let txn = self.transaction().await?; + let result=async { + txn.query_one_raw(sql("SELECT prepare_id FROM mst2_metadata_prepare WHERE prepare_id=$1 FOR UPDATE", + [intent.prepare_id.clone().into()])).await.map_err(internal)?; + let state=self.require_intent(&txn,intent).await?; + if state=="COMMITTED" {return self.receipt(&txn,intent).await;} + let ordered:Vec<_>=intent.plan.child_first_delta()?.iter().map(|page|json!({"page":hex::encode(page), + "generation":intent.bindings[page].0})).collect(); + if !ordered.is_empty() { + let count:i32=txn.query_one_raw(sql("SELECT mst2_metadata_certify_batch($1,$2::jsonb) AS certified", + [intent.prepare_id.clone().into(),serde_json::to_string(&ordered).map_err(internal)?.into()])) + .await.map_err(internal)?.ok_or_else(||integrity("rooted certification result is missing"))?.try_get("","certified").map_err(internal)?; + if count as usize!=ordered.len() {return Err(integrity("rooted certification did not cover its exact delta"));} + } + txn.execute_raw(sql("INSERT INTO mst2_metadata_root_anchor(anchor_id,anchor_kind,owner_key,prepare_id,root_page,root_generation,root_certificate_digest) + SELECT gen_random_uuid(),'PREPARE',$1,$1,c.page_id,c.generation,c.certificate_digest + FROM mst2_metadata_page_certificate c WHERE c.page_id=$2 AND c.generation=$3 + ON CONFLICT(anchor_kind,owner_key,root_page,root_generation) DO NOTHING", + [intent.prepare_id.clone().into(),intent.plan.root.to_vec().into(),intent.root_generation.into()])).await.map_err(internal)?; + let sources:Vec<_>=intent.plan.source_roots.iter().map(|(tree,page)|json!({"tree":tree,"page":hex::encode(page)})).collect(); + let sources=txn.query_all_raw(sql("SELECT source.tree,cur.page_id FROM jsonb_to_recordset($1::jsonb) source(tree text,page text) + JOIN mst2_metadata_current cur ON cur.page_id=decode(source.page,'hex') + JOIN mst2_metadata_page_certificate proof USING(page_id,generation) ORDER BY proof.rank,source.tree", + [serde_json::to_string(&sources).map_err(internal)?.into()])).await.map_err(internal)?; + if sources.len()!=intent.plan.source_roots.len() {return Err(unavailable("rooted source certificate order is incomplete"));} + for source in sources { + let tree:String=source.try_get("","tree").map_err(internal)?; + let page=digest_column(&source,"page_id")?; + let generation=intent.bindings.get(&page).map(|b|b.0).or_else(||intent.plan.reused.get(&page).map(|b|b.generation)) + .ok_or_else(||integrity("rooted directory source has no exact lifetime"))?; + let reused=txn.query_one_raw(sql("SELECT a.attestation_id FROM mst2_metadata_prepare_reuse_root r + JOIN mst2_metadata_source_root_attestation boundary ON boundary.attestation_id=r.attestation_id + JOIN mst2_metadata_source_root_attestation a ON a.root_page=r.root_page AND a.root_generation=r.root_generation + AND a.root_certificate_digest=boundary.root_certificate_digest + JOIN mst2_metadata_prepare origin ON origin.prepare_id=a.origin_prepare_id + JOIN mst2_metadata_current cur ON cur.page_id=a.root_page AND cur.generation=a.root_generation + JOIN mst2_metadata_lifetime life USING(page_id,generation) + JOIN $CORE$.mega_tree source ON source.tree_id=split_part(a.tagged_tree_oid,':',2) + WHERE r.prepare_id=$1 AND r.root_page=$2 AND r.root_generation=$3 AND a.tagged_tree_oid=$4 + AND origin.state='COMMITTED' AND life.state='LIVE' AND a.source_profile=mst2_metadata_native_profile($1) + AND $CORE$.mst2_route_source_tree_matches(split_part(a.tagged_tree_oid,':',2),a.source_revision,a.source_body_digest) + ORDER BY a.attestation_id LIMIT 1".replace("$CORE$",&identifier(&self.namespace.core_schema)), + [intent.prepare_id.clone().into(),page.to_vec().into(),generation.into(),tree.clone().into()])).await.map_err(internal)?; + if reused.is_some() {continue;} + txn.execute_raw(sql("INSERT INTO mst2_metadata_source_root_attestation(attestation_id,namespace_uuid,origin_prepare_id,tagged_tree_oid, + source_profile,profile_digest,source_body_digest,root_page,root_generation,root_certificate_digest,source_proof,attestation_digest) + SELECT gen_random_uuid(),(proof->>'namespace')::uuid,$1,$2,proof->'source_profile',decode(proof->>'profile_digest','hex'), + decode(proof->>'source_body_digest','hex'),$3,$4,decode(proof->>'root_certificate','hex'),proof,decode(proof->>'attestation','hex') + FROM (SELECT mst2_metadata_compute_source_proof($1,$2,$3,$4) AS proof) input", + [intent.prepare_id.clone().into(),tree.clone().into(),page.to_vec().into(),generation.into()])).await.map_err(internal)?; + } + let changed=txn.execute_raw(sql("UPDATE mst2_metadata_prepare SET state='COMMITTED',committed_at=clock_timestamp() + WHERE prepare_id=$1 AND state='PREPARING' AND storage_seal=$2", + [intent.prepare_id.clone().into(),intent.storage_seal.to_vec().into()])).await.map_err(internal)?; + if changed.rows_affected()!=1 {return Err(integrity("rooted finalize lost its exact prepare transition"));} + txn.execute_raw(sql("UPDATE mst2_metadata_lifetime life SET state='LIVE' FROM mst2_metadata_prepare_page member + WHERE member.prepare_id=$1 AND life.page_id=member.page_id AND life.generation=member.generation AND life.state='RESERVED'", + [intent.prepare_id.clone().into()])).await.map_err(internal)?; + txn.execute_raw(sql("INSERT INTO mst2_metadata_reuse_index(profile_digest,tagged_tree_oid,attestation_id,root_page,root_generation,attestation_digest) + SELECT a.profile_digest,a.tagged_tree_oid,a.attestation_id,a.root_page,a.root_generation,a.attestation_digest + FROM mst2_metadata_source_root_attestation a WHERE a.origin_prepare_id=$1 + ON CONFLICT(profile_digest,tagged_tree_oid) DO NOTHING",[intent.prepare_id.clone().into()])).await.map_err(internal)?; + self.receipt(&txn,intent).await + }.await; + committed( + txn, + result, + &intent.operation_id, + intent.manifest_digest, + super::super::native_metadata_install::MetadataCommitPhase::Finalize, + ) + .await + } + + async fn read_delta( + &self, + intent: &RootedPrepareIntent, + ) -> Result, SnapshotError> { + let rows=self.connection.query_all_raw(sql("SELECT member.page_id,member.generation,member.expected_size,body.metadata_codec, + body.byte_size,body.payload FROM mst2_metadata_prepare_page member JOIN mst2_metadata_payload body USING(page_id,generation) + JOIN mst2_metadata_current cur USING(page_id,generation) WHERE member.prepare_id=$1 ORDER BY member.page_id LIMIT 4097", + [intent.prepare_id.clone().into()])).await.map_err(internal)?; + if rows.len() != intent.bindings.len() { + return Err(unavailable("rooted durable delta is incomplete")); + } + let mut payloads = Vec::with_capacity(rows.len()); + for row in rows { + let page = digest_column(&row, "page_id")?; + let &(generation, size) = intent + .bindings + .get(&page) + .ok_or_else(|| integrity("rooted durable delta has an extra member"))?; + let bytes: Vec = row.try_get("", "payload").map_err(internal)?; + if row.try_get::("", "generation").map_err(internal)? != generation + || row.try_get::("", "expected_size").map_err(internal)? as u64 != size + || row.try_get::("", "byte_size").map_err(internal)? as u64 != size + || row.try_get::("", "metadata_codec").map_err(internal)? != 1 + || bytes.len() as u64 != size + || page_id(&bytes) != page + { + return Err(integrity( + "rooted durable delta crossed its fixed byte and lifetime binding", + )); + } + Page::decode(&bytes).map_err(internal)?; + payloads.push(MetadataPagePayload { + id: page, + size, + bytes, + }); + } + Ok(payloads) + } + + async fn receipt( + &self, + db: &C, + intent: &RootedPrepareIntent, + ) -> Result { + let row=db.query_one_raw(sql("SELECT c.certificate_digest,a.attestation_id::text,a.attestation_digest + FROM mst2_metadata_prepare q JOIN mst2_metadata_current cur ON cur.page_id=q.metadata_root AND cur.generation=$2 + JOIN mst2_metadata_lifetime life USING(page_id,generation) JOIN mst2_metadata_graph_node n USING(page_id,generation) + JOIN mst2_metadata_page_certificate c USING(page_id,generation) + JOIN mst2_metadata_source_root_attestation a ON a.root_page=c.page_id AND a.root_generation=c.generation + JOIN mst2_metadata_prepare origin ON origin.prepare_id=a.origin_prepare_id + JOIN $CORE$.mega_tree source ON source.tree_id=split_part(a.tagged_tree_oid,':',2) + WHERE q.prepare_id=$1 AND q.plan_kind='ROOTED' AND q.state='COMMITTED' AND life.state='LIVE' AND n.state='LIVE' + AND n.certificate_digest=c.certificate_digest AND a.root_certificate_digest=c.certificate_digest + AND origin.state='COMMITTED' AND a.source_profile=mst2_metadata_native_profile($1) + AND a.tagged_tree_oid=mst2_metadata_rooted_scope_tree($1) + AND $CORE$.mst2_route_source_tree_matches(split_part(a.tagged_tree_oid,':',2),a.source_revision,a.source_body_digest) + AND NOT EXISTS(SELECT 1 FROM mst2_metadata_gc_op gc WHERE gc.page_id=c.page_id AND gc.generation=c.generation) + AND (q.coverage_retired_at IS NULL AND EXISTS(SELECT 1 FROM mst2_metadata_root_anchor anchor + WHERE anchor.anchor_kind='PREPARE' AND anchor.prepare_id=q.prepare_id AND anchor.owner_key=q.prepare_id + AND anchor.root_page=c.page_id AND anchor.root_generation=c.generation + AND anchor.root_certificate_digest=c.certificate_digest) OR mst2_metadata_session_covers_prepare(q.prepare_id)) + ORDER BY (a.origin_prepare_id=q.prepare_id) DESC,a.attestation_id LIMIT 1".replace("$CORE$",&identifier(&self.namespace.core_schema)), + [intent.prepare_id.clone().into(),intent.root_generation.into()])).await.map_err(internal)? + .ok_or_else(||unavailable("rooted definitive receipt has no exact active canonical source root"))?; + Ok(RootedMetadataReceipt { + intent: intent.clone(), + certificate_digest: digest_column(&row, "certificate_digest")?, + attestation_id: uuid::Uuid::parse_str( + &row.try_get::("", "attestation_id") + .map_err(internal)?, + ) + .map_err(internal)?, + attestation_digest: digest_column(&row, "attestation_digest")?, + }) + } + + pub(crate) async fn recover( + &self, + operation: &str, + plan: &RootedMetadataInstallPlan, + ) -> Result, SnapshotError> { + let txn = self.transaction().await?; + let Some((intent, state)) = self.load_intent(&txn, operation, plan).await? else { + txn.commit().await.map_err(internal)?; + return Ok(None); + }; + let receipt = if state == "COMMITTED" { + Some(self.receipt(&txn, &intent).await?) + } else { + None + }; + txn.commit().await.map_err(internal)?; + Ok(receipt) + } +} diff --git a/src/jupiter/storage/qualified_metadata_rooted.sql b/src/jupiter/storage/qualified_metadata_rooted.sql new file mode 100644 index 00000000..8116bc61 --- /dev/null +++ b/src/jupiter/storage/qualified_metadata_rooted.sql @@ -0,0 +1,358 @@ +CREATE FUNCTION mst2_metadata_read_be(b bytea,p integer,w integer) RETURNS numeric +LANGUAGE plpgsql IMMUTABLE STRICT SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE v numeric:=0; i integer; +BEGIN + IF w NOT IN (2,4,8) OR p<0 OR p>octet_length(b)-w THEN RAISE EXCEPTION 'rooted integer is out of bounds'; END IF; + FOR i IN 0..w-1 LOOP v:=v*256+get_byte(b,p+i); END LOOP; + RETURN v; +END $$; + +CREATE FUNCTION mst2_metadata_rooted_string(b bytea,p integer,maximum integer) RETURNS jsonb +LANGUAGE plpgsql IMMUTABLE STRICT SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE n numeric; value text; +BEGIN + n:=mst2_metadata_read_be(b,p,4); p:=p+4; + IF n>maximum OR n>octet_length(b)-p THEN RAISE EXCEPTION 'rooted string exceeds its exact byte boundary'; END IF; + value:=convert_from(substring(b FROM p+1 FOR n::integer),'UTF8'); + RETURN jsonb_build_object('end',p+n::integer,'text',value); +END $$; + +CREATE FUNCTION mst2_metadata_decode_rooted_plan(b bytea) RETURNS jsonb +LANGUAGE plpgsql IMMUTABLE STRICT SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE domain bytea:=convert_to('mega.mst2.rooted-install.v1','UTF8')||decode('00','hex'); + cursor_pos integer; part jsonb; source_domain text; tree_oid text; scope text; hash_kind text; identity jsonb; + root bytea; count numeric; i integer; page bytea; parent bytea; child bytea; previous bytea; previous_tree text; + size numeric; generation numeric; attestation bytea; attestation_digest bytea; certificate_digest bytea; + delta_rows jsonb[]:=ARRAY[]::jsonb[]; edge_rows jsonb[]:=ARRAY[]::jsonb[]; + reuse_rows jsonb[]:=ARRAY[]::jsonb[]; source_rows jsonb[]:=ARRAY[]::jsonb[]; total_bytes bigint:=0; + delta_index jsonb; node_index jsonb; adjacency jsonb; +BEGIN + IF octet_length(b)>2097152 OR substring(b FROM 1 FOR octet_length(domain))<>domain THEN + RAISE EXCEPTION 'rooted preparation domain or byte budget is invalid'; + END IF; + cursor_pos:=octet_length(domain); + IF mst2_metadata_read_be(b,cursor_pos,2)<>1 THEN RAISE EXCEPTION 'rooted preparation version is unsupported'; END IF; + cursor_pos:=cursor_pos+2; + part:=mst2_metadata_rooted_string(b,cursor_pos,64); cursor_pos:=(part->>'end')::integer; source_domain:=part->>'text'; + part:=mst2_metadata_rooted_string(b,cursor_pos,128); cursor_pos:=(part->>'end')::integer; tree_oid:=part->>'text'; + part:=mst2_metadata_rooted_string(b,cursor_pos,4096); cursor_pos:=(part->>'end')::integer; scope:=part->>'text'; + IF source_domain<>'native-git' OR tree_oid !~ '^(sha1:[0-9a-f]{40}|sha256:[0-9a-f]{64}|blake3:[0-9a-f]{64})$' + OR scope NOT LIKE '/%' OR scope<>'/' AND (scope LIKE '%/' OR scope LIKE '%//%' OR EXISTS( + SELECT 1 FROM unnest(string_to_array(substring(scope FROM 2),'/')) name WHERE name IN ('','.','..') + OR octet_length(name)>255) OR cardinality(string_to_array(substring(scope FROM 2),'/'))>256) THEN + RAISE EXCEPTION 'rooted source identity or scope is invalid'; END IF; + hash_kind:=split_part(tree_oid,':',1); + identity:=jsonb_build_object('source_domain',source_domain,'tagged_root_tree_oid',tree_oid,'scope',scope, + 'schema_version',mst2_metadata_read_be(b,cursor_pos,2), + 'metadata_codec',mst2_metadata_read_be(b,cursor_pos+2,2), + 'materialization_policy',mst2_metadata_read_be(b,cursor_pos+4,2), + 'fs_semantics',mst2_metadata_read_be(b,cursor_pos+6,2), + 'access_projection',mst2_metadata_read_be(b,cursor_pos+8,2), + 'verification_revision',mst2_metadata_read_be(b,cursor_pos+10,4), + 'projection_revision',mst2_metadata_read_be(b,cursor_pos+14,2)); + cursor_pos:=cursor_pos+16; + IF (identity->>'schema_version')::integer<>2 OR (identity->>'metadata_codec')::integer<>1 + OR (identity->>'materialization_policy')::integer<>1 OR (identity->>'fs_semantics')::integer<>1 + OR (identity->>'access_projection')::integer<>0 OR (identity->>'verification_revision')::integer<>2 + OR (identity->>'projection_revision')::integer<>1 THEN RAISE EXCEPTION 'rooted source profile is not current native'; END IF; + IF cursor_pos>octet_length(b)-32 THEN RAISE EXCEPTION 'rooted metadata root is truncated'; END IF; + root:=substring(b FROM cursor_pos+1 FOR 32); cursor_pos:=cursor_pos+32; + count:=mst2_metadata_read_be(b,cursor_pos,4); cursor_pos:=cursor_pos+4; + IF count>4096 THEN RAISE EXCEPTION 'rooted delta exceeds its node budget'; END IF; + IF count>0 THEN FOR i IN 1..count::integer LOOP + IF cursor_pos>octet_length(b)-40 THEN RAISE EXCEPTION 'rooted delta member is truncated'; END IF; + page:=substring(b FROM cursor_pos+1 FOR 32); size:=mst2_metadata_read_be(b,cursor_pos+32,8); cursor_pos:=cursor_pos+40; + IF previous IS NOT NULL AND previous>=page OR size NOT BETWEEN 20 AND 16384 THEN + RAISE EXCEPTION 'rooted delta is not exactly ordered or has invalid size'; END IF; + previous:=page; total_bytes:=total_bytes+size::bigint; + IF total_bytes>67108864 THEN RAISE EXCEPTION 'rooted delta exceeds its metadata byte budget'; END IF; + delta_rows:=array_append(delta_rows,jsonb_build_object('page',encode(page,'hex'),'size',size)); + END LOOP; END IF; + previous:=NULL; count:=mst2_metadata_read_be(b,cursor_pos,4); cursor_pos:=cursor_pos+4; + IF count>16384 THEN RAISE EXCEPTION 'rooted delta exceeds its edge budget'; END IF; + IF count>0 THEN FOR i IN 1..count::integer LOOP + IF cursor_pos>octet_length(b)-64 THEN RAISE EXCEPTION 'rooted edge is truncated'; END IF; + parent:=substring(b FROM cursor_pos+1 FOR 32); child:=substring(b FROM cursor_pos+33 FOR 32); cursor_pos:=cursor_pos+64; + IF previous IS NOT NULL AND previous>=parent||child OR parent=child THEN RAISE EXCEPTION 'rooted edges are not unique and ordered'; END IF; + previous:=parent||child; + edge_rows:=array_append(edge_rows,jsonb_build_object('parent',encode(parent,'hex'),'child',encode(child,'hex'))); + END LOOP; END IF; + previous:=NULL; count:=mst2_metadata_read_be(b,cursor_pos,4); cursor_pos:=cursor_pos+4; + IF count>4096 OR coalesce(array_length(delta_rows,1),0)+count>4096 THEN RAISE EXCEPTION 'rooted delta and boundaries exceed node budget'; END IF; + IF count>0 THEN FOR i IN 1..count::integer LOOP + IF cursor_pos>octet_length(b)-120 THEN RAISE EXCEPTION 'rooted reuse boundary is truncated'; END IF; + page:=substring(b FROM cursor_pos+1 FOR 32); generation:=mst2_metadata_read_be(b,cursor_pos+32,8); + attestation:=substring(b FROM cursor_pos+41 FOR 16); attestation_digest:=substring(b FROM cursor_pos+57 FOR 32); + certificate_digest:=substring(b FROM cursor_pos+89 FOR 32); cursor_pos:=cursor_pos+120; + IF previous IS NOT NULL AND previous>=page OR generation NOT BETWEEN 1 AND 9223372036854775807 THEN + RAISE EXCEPTION 'rooted reuse boundaries are not exact positive ordered lifetimes'; END IF; + previous:=page; + reuse_rows:=array_append(reuse_rows,jsonb_build_object('page',encode(page,'hex'),'generation',generation, + 'attestation_id',encode(attestation,'hex')::uuid,'attestation_digest',encode(attestation_digest,'hex'), + 'certificate_digest',encode(certificate_digest,'hex'))); + END LOOP; END IF; + count:=mst2_metadata_read_be(b,cursor_pos,4); cursor_pos:=cursor_pos+4; + IF count NOT BETWEEN 1 AND 4096 THEN RAISE EXCEPTION 'rooted source-root budget is invalid'; END IF; + FOR i IN 1..count::integer LOOP + part:=mst2_metadata_rooted_string(b,cursor_pos,128); cursor_pos:=(part->>'end')::integer; tree_oid:=part->>'text'; + IF cursor_pos>octet_length(b)-32 THEN RAISE EXCEPTION 'rooted source root is truncated'; END IF; + page:=substring(b FROM cursor_pos+1 FOR 32); cursor_pos:=cursor_pos+32; + IF tree_oid !~ '^(sha1:[0-9a-f]{40}|sha256:[0-9a-f]{64}|blake3:[0-9a-f]{64})$' + OR split_part(tree_oid,':',1)<>hash_kind OR previous_tree IS NOT NULL AND convert_to(previous_tree,'UTF8')>=convert_to(tree_oid,'UTF8') THEN + RAISE EXCEPTION 'rooted source roots are not unique ordered same-profile identities'; END IF; + previous_tree:=tree_oid; source_rows:=array_append(source_rows,jsonb_build_object('tree_oid',tree_oid,'page',encode(page,'hex'))); + END LOOP; + IF cursor_pos<>octet_length(b) THEN RAISE EXCEPTION 'rooted preparation has trailing bytes'; END IF; + SELECT coalesce(jsonb_object_agg(value->>'page',true),'{}'::jsonb) INTO delta_index FROM unnest(delta_rows) d(value); + SELECT coalesce(jsonb_object_agg(value->>'page',true),'{}'::jsonb) INTO node_index FROM ( + SELECT value FROM unnest(delta_rows) UNION ALL SELECT value FROM unnest(reuse_rows)) nodes; + IF coalesce(array_length(delta_rows,1),0)+coalesce(array_length(reuse_rows,1),0)=0 + OR EXISTS(SELECT 1 FROM unnest(reuse_rows) r(value) WHERE delta_index ? (r.value->>'page')) + OR NOT node_index ? encode(root,'hex') + OR EXISTS(SELECT 1 FROM unnest(edge_rows) e(value) WHERE NOT delta_index ? (e.value->>'parent') + OR NOT node_index ? (e.value->>'child')) + OR EXISTS(SELECT 1 FROM unnest(source_rows) s(value) WHERE NOT node_index ? (s.value->>'page')) + OR NOT EXISTS(SELECT 1 FROM unnest(source_rows) s(value) WHERE value->>'page'=encode(root,'hex')) THEN + RAISE EXCEPTION 'rooted plan has overlap or an unbound graph/source endpoint'; + END IF; + SELECT coalesce(jsonb_object_agg(parent,children),'{}'::jsonb) INTO adjacency FROM ( + SELECT value->>'parent' AS parent,jsonb_agg(value->>'child') AS children FROM unnest(edge_rows) e(value) + GROUP BY value->>'parent') grouped; + IF EXISTS(WITH RECURSIVE reached(page) AS (SELECT encode(root,'hex') UNION + SELECT child FROM reached r CROSS JOIN LATERAL jsonb_array_elements_text(adjacency->r.page) children(child)) + SELECT 1 FROM (SELECT value FROM unnest(delta_rows) UNION ALL SELECT value FROM unnest(reuse_rows)) nodes + WHERE NOT EXISTS(SELECT 1 FROM reached r WHERE r.page=nodes.value->>'page')) THEN + RAISE EXCEPTION 'rooted plan includes members outside its bounded delta and boundary closure'; + END IF; + RETURN identity||jsonb_build_object('root',encode(root,'hex'),'delta',to_jsonb(delta_rows),'edges',to_jsonb(edge_rows), + 'reused',to_jsonb(reuse_rows),'source_roots',to_jsonb(source_rows),'total_delta_bytes',total_bytes); +END $$; + +CREATE FUNCTION mst2_metadata_decode_delta_bindings(b bytea,plan jsonb) RETURNS jsonb +LANGUAGE plpgsql IMMUTABLE STRICT SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE count numeric; cursor_pos integer:=12; i integer; page bytea; generation numeric; size numeric; + previous bytea; binding_rows jsonb[]:=ARRAY[]::jsonb[]; +BEGIN + IF octet_length(b) NOT BETWEEN 12 AND 196620 OR substring(b FROM 1 FOR 8)<>convert_to('MST2GEN1','UTF8') THEN + RAISE EXCEPTION 'rooted generation binding encoding is invalid'; END IF; + count:=mst2_metadata_read_be(b,8,4); + IF count<>jsonb_array_length(plan->'delta') OR octet_length(b)<>12+48*count THEN + RAISE EXCEPTION 'rooted delta binding count differs from its immutable plan'; END IF; + IF count>0 THEN FOR i IN 1..count::integer LOOP + page:=substring(b FROM cursor_pos+1 FOR 32); generation:=mst2_metadata_read_be(b,cursor_pos+32,8); + size:=mst2_metadata_read_be(b,cursor_pos+40,8); cursor_pos:=cursor_pos+48; + IF previous IS NOT NULL AND previous>=page OR generation NOT BETWEEN 1 AND 9223372036854775807 + OR encode(page,'hex') IS DISTINCT FROM plan->'delta'->(i-1)->>'page' + OR size IS DISTINCT FROM (plan->'delta'->(i-1)->>'size')::numeric THEN + RAISE EXCEPTION 'rooted generation binding is not its exact positive ordered delta member'; END IF; + previous:=page; binding_rows:=array_append(binding_rows,jsonb_build_object('page',encode(page,'hex'),'generation',generation,'size',size)); + END LOOP; END IF; + RETURN to_jsonb(binding_rows); +END $$; + +CREATE FUNCTION mst2_metadata_rooted_manifest(q mst2_metadata_prepare) RETURNS jsonb +LANGUAGE plpgsql IMMUTABLE STRICT SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE plan jsonb:=mst2_metadata_decode_rooted_plan(q.canonical_plan); bindings jsonb; +BEGIN + bindings:=mst2_metadata_decode_delta_bindings(q.canonical_bindings,plan); + IF q.plan_kind<>'ROOTED' OR q.source_domain IS DISTINCT FROM plan->>'source_domain' + OR q.tagged_root_tree_oid IS DISTINCT FROM plan->>'tagged_root_tree_oid' OR q.scope IS DISTINCT FROM plan->>'scope' + OR q.schema_version IS DISTINCT FROM (plan->>'schema_version')::smallint + OR q.metadata_codec IS DISTINCT FROM (plan->>'metadata_codec')::smallint + OR q.materialization_policy IS DISTINCT FROM (plan->>'materialization_policy')::smallint + OR q.fs_semantics IS DISTINCT FROM (plan->>'fs_semantics')::smallint + OR q.access_projection IS DISTINCT FROM (plan->>'access_projection')::smallint + OR q.verification_revision IS DISTINCT FROM (plan->>'verification_revision')::integer + OR q.projection_revision IS DISTINCT FROM (plan->>'projection_revision')::smallint + OR q.metadata_root IS DISTINCT FROM decode(plan->>'root','hex') + OR q.node_count<>jsonb_array_length(plan->'delta') OR q.edge_count<>jsonb_array_length(plan->'edges') + OR q.total_bytes<>(plan->>'total_delta_bytes')::bigint THEN + RAISE EXCEPTION 'rooted preparation fields differ from its independently decoded manifest'; END IF; + RETURN plan||jsonb_build_object('bindings',bindings); +END $$; + +CREATE FUNCTION mst2_metadata_check_rooted_members(pid text) RETURNS jsonb +LANGUAGE plpgsql VOLATILE STRICT SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE q mst2_metadata_prepare%ROWTYPE; plan jsonb; +BEGIN + SELECT * INTO STRICT q FROM mst2_metadata_prepare WHERE prepare_id=pid AND plan_kind='ROOTED'; + plan:=mst2_metadata_rooted_manifest(q); + IF EXISTS((SELECT decode(value->>'page','hex'),(value->>'generation')::bigint,(value->>'size')::integer + FROM jsonb_array_elements(plan->'bindings')) EXCEPT + (SELECT page_id,generation,expected_size FROM mst2_metadata_prepare_page WHERE prepare_id=pid)) + OR EXISTS((SELECT page_id,generation,expected_size FROM mst2_metadata_prepare_page WHERE prepare_id=pid) EXCEPT + (SELECT decode(value->>'page','hex'),(value->>'generation')::bigint,(value->>'size')::integer + FROM jsonb_array_elements(plan->'bindings'))) + OR EXISTS((SELECT decode(value->>'page','hex'),(value->>'generation')::bigint,(value->>'attestation_id')::uuid, + decode(value->>'attestation_digest','hex'),decode(value->>'certificate_digest','hex') + FROM jsonb_array_elements(plan->'reused')) EXCEPT + (SELECT r.root_page,r.root_generation,r.attestation_id,r.attestation_digest,a.root_certificate_digest + FROM mst2_metadata_prepare_reuse_root r JOIN mst2_metadata_source_root_attestation a USING(attestation_id) + WHERE r.prepare_id=pid)) + OR EXISTS((SELECT r.root_page,r.root_generation,r.attestation_id,r.attestation_digest,a.root_certificate_digest + FROM mst2_metadata_prepare_reuse_root r JOIN mst2_metadata_source_root_attestation a USING(attestation_id) + WHERE r.prepare_id=pid) EXCEPT + (SELECT decode(value->>'page','hex'),(value->>'generation')::bigint,(value->>'attestation_id')::uuid, + decode(value->>'attestation_digest','hex'),decode(value->>'certificate_digest','hex') + FROM jsonb_array_elements(plan->'reused'))) THEN + RAISE EXCEPTION 'rooted preparation has missing or extra exact delta/reuse lifetime bindings'; END IF; + RETURN plan; +END $$; + +CREATE FUNCTION mst2_metadata_rooted_members_complete() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +BEGIN + IF EXISTS(SELECT 1 FROM mst2_metadata_prepare WHERE prepare_id=NEW.prepare_id AND plan_kind='ROOTED') THEN + PERFORM mst2_metadata_check_rooted_members(NEW.prepare_id); + END IF; + RETURN NULL; +END $$; +CREATE CONSTRAINT TRIGGER mst2_metadata_rooted_prepare_complete AFTER INSERT OR UPDATE OF bindings_revision ON mst2_metadata_prepare + DEFERRABLE INITIALLY DEFERRED FOR EACH ROW EXECUTE FUNCTION mst2_metadata_rooted_members_complete(); + +CREATE FUNCTION mst2_metadata_rooted_members_added() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +BEGIN + -- Every actual membership statement queues one authoritative commit check + -- per affected prepare, including a late raw-SQL addition. No caller state + -- can disable the decoded exact-coverage proof. + UPDATE mst2_metadata_prepare q SET bindings_revision=q.bindings_revision+1 + WHERE q.plan_kind='ROOTED' AND q.prepare_id IN (SELECT DISTINCT prepare_id FROM added_members); + RETURN NULL; +END $$; +CREATE TRIGGER mst2_metadata_rooted_delta_added AFTER INSERT ON mst2_metadata_prepare_page + REFERENCING NEW TABLE AS added_members FOR EACH STATEMENT EXECUTE FUNCTION mst2_metadata_rooted_members_added(); +CREATE TRIGGER mst2_metadata_rooted_reuse_added AFTER INSERT ON mst2_metadata_prepare_reuse_root + REFERENCING NEW TABLE AS added_members FOR EACH STATEMENT EXECUTE FUNCTION mst2_metadata_rooted_members_added(); + +CREATE TABLE mst2_metadata_scope_source_reference ( + prepare_id text PRIMARY KEY REFERENCES mst2_metadata_prepare(prepare_id), + scope_tree_oid text NOT NULL CHECK(scope_tree_oid ~ '^(sha1:[0-9a-f]{40}|sha256:[0-9a-f]{64}|blake3:[0-9a-f]{64})$'), + ancestor_revisions jsonb NOT NULL CHECK((jsonb_typeof(ancestor_revisions)='array' AND jsonb_array_length(ancestor_revisions)<=256) IS TRUE) +); +CREATE FUNCTION mst2_metadata_derive_scope_source(pid text) RETURNS jsonb LANGUAGE plpgsql VOLATILE STRICT +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE q mst2_metadata_prepare%ROWTYPE; tree_oid text; component text; body bytea; item jsonb; scanned_bytes bigint:=0; + digest bytea; revision uuid; ancestors jsonb[]:=ARRAY[]::jsonb[]; +BEGIN + SELECT * INTO STRICT q FROM mst2_metadata_prepare WHERE prepare_id=pid AND plan_kind='ROOTED' AND state='PREPARING'; + tree_oid:=q.tagged_root_tree_oid; + IF q.scope='/' THEN RETURN jsonb_build_object('scope_tree_oid',tree_oid,'ancestor_revisions','[]'::jsonb); END IF; + FOREACH component IN ARRAY string_to_array(substring(q.scope FROM 2),'/') LOOP + SELECT sub_trees INTO body FROM $CORE_SCHEMA$.mega_tree WHERE tree_id=split_part(tree_oid,':',2); + IF NOT FOUND THEN RAISE EXCEPTION 'rooted source scope has a missing fixed ancestor'; END IF; + scanned_bytes:=scanned_bytes+octet_length(body); + IF scanned_bytes>67108864 THEN RAISE EXCEPTION 'rooted source-scope walk exceeds its fixed byte-work budget'; END IF; + digest:=sha256(body); + revision:=$CORE_SCHEMA$.mst2_route_capture_source_tree(split_part(tree_oid,':',2),digest); + ancestors:=array_append(ancestors,jsonb_build_object('tree_oid',tree_oid,'revision',revision::text,'body_digest',encode(digest,'hex'))); + SELECT value INTO item FROM jsonb_array_elements(mst2_metadata_decode_git_tree(body,split_part(tree_oid,':',1))) + WHERE decode(value->>'name','hex')=convert_to(component,'UTF8'); + IF NOT FOUND OR (item->>'kind')::integer<>4 THEN RAISE EXCEPTION 'rooted source scope is not its exact fixed directory'; END IF; + tree_oid:=item->>'oid'; + END LOOP; + RETURN jsonb_build_object('scope_tree_oid',tree_oid,'ancestor_revisions',to_jsonb(ancestors)); +END $$; +CREATE FUNCTION mst2_metadata_scope_source_guard() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE proof jsonb; +BEGIN + IF TG_OP<>'INSERT' THEN RAISE EXCEPTION 'rooted scope source proof is immutable'; END IF; + proof:=mst2_metadata_derive_scope_source(NEW.prepare_id); + IF NEW.scope_tree_oid IS DISTINCT FROM proof->>'scope_tree_oid' + OR NEW.ancestor_revisions IS DISTINCT FROM proof->'ancestor_revisions' THEN + RAISE EXCEPTION 'rooted scope source proof was not independently derived from actual core ancestors'; END IF; + RETURN NEW; +END $$; +CREATE TRIGGER mst2_metadata_scope_source_guard BEFORE INSERT OR UPDATE OR DELETE ON mst2_metadata_scope_source_reference + FOR EACH ROW EXECUTE FUNCTION mst2_metadata_scope_source_guard(); + +CREATE FUNCTION mst2_metadata_rooted_scope_tree(pid text) RETURNS text LANGUAGE plpgsql VOLATILE STRICT +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE bound mst2_metadata_scope_source_reference%ROWTYPE; proof jsonb; ancestor jsonb; +BEGIN + SELECT * INTO bound FROM mst2_metadata_scope_source_reference WHERE prepare_id=pid; + IF NOT FOUND THEN + proof:=mst2_metadata_derive_scope_source(pid); + INSERT INTO mst2_metadata_scope_source_reference(prepare_id,scope_tree_oid,ancestor_revisions) + VALUES(pid,proof->>'scope_tree_oid',proof->'ancestor_revisions') RETURNING * INTO bound; + END IF; + FOR ancestor IN SELECT value FROM jsonb_array_elements(bound.ancestor_revisions) LOOP + IF NOT $CORE_SCHEMA$.mst2_route_source_tree_matches(split_part(ancestor->>'tree_oid',':',2), + (ancestor->>'revision')::uuid,decode(ancestor->>'body_digest','hex')) THEN + RAISE EXCEPTION 'rooted fixed scope ancestor lost its exact current source revision'; END IF; + END LOOP; + RETURN bound.scope_tree_oid; +END $$; + +CREATE FUNCTION mst2_metadata_rooted_finalize_proof(pid text) RETURNS jsonb LANGUAGE plpgsql VOLATILE STRICT +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE q mst2_metadata_prepare%ROWTYPE; plan jsonb; selected_root_generation bigint; root_certificate bytea; + profile jsonb; source jsonb; source_generation bigint; scoped_tree text; + generation_index jsonb; +BEGIN + SELECT * INTO STRICT q FROM mst2_metadata_prepare WHERE prepare_id=pid AND plan_kind='ROOTED' AND state='PREPARING'; + plan:=mst2_metadata_check_rooted_members(pid); profile:=mst2_metadata_native_profile(pid); + SELECT jsonb_object_agg(value->>'page',value->'generation') INTO generation_index FROM ( + SELECT value FROM jsonb_array_elements(plan->'bindings') UNION ALL + SELECT value FROM jsonb_array_elements(plan->'reused')) members; + selected_root_generation:=(generation_index->>(plan->>'root'))::bigint; + SELECT c.certificate_digest INTO root_certificate FROM mst2_metadata_page_certificate c + JOIN mst2_metadata_graph_node n USING(page_id,generation) JOIN mst2_metadata_current cur USING(page_id,generation) + JOIN mst2_metadata_lifetime life USING(page_id,generation) + WHERE c.page_id=q.metadata_root AND c.generation=selected_root_generation AND n.state='LIVE' + AND n.certificate_digest=c.certificate_digest AND life.state IN ('RESERVED','LIVE') AND life.graph_domain='qualified-v1' + AND c.relative_path_bytes+CASE WHEN q.scope='/' THEN 0 ELSE octet_length(q.scope) END<=4096 + AND c.relative_components+CASE WHEN q.scope='/' THEN 0 ELSE cardinality(string_to_array(substring(q.scope FROM 2),'/')) END<=256 + AND NOT EXISTS(SELECT 1 FROM mst2_metadata_gc_op gc WHERE gc.page_id=c.page_id AND gc.generation=c.generation); + IF NOT FOUND OR NOT EXISTS(SELECT 1 FROM mst2_metadata_root_anchor a WHERE a.anchor_kind='PREPARE' + AND a.owner_key=pid AND a.prepare_id=pid AND a.root_page=q.metadata_root AND a.root_generation=selected_root_generation + AND a.root_certificate_digest=root_certificate) THEN RAISE EXCEPTION 'rooted finalize lacks its independently certified owned root'; END IF; + IF EXISTS(SELECT 1 FROM mst2_metadata_prepare_page m + LEFT JOIN mst2_metadata_current cur USING(page_id,generation) LEFT JOIN mst2_metadata_lifetime life USING(page_id,generation) + LEFT JOIN mst2_metadata_payload body USING(page_id,generation) LEFT JOIN mst2_metadata_graph_node n USING(page_id,generation) + LEFT JOIN mst2_metadata_page_certificate c USING(page_id,generation) + WHERE m.prepare_id=pid AND (cur.page_id IS NULL OR life.page_id IS NULL OR body.page_id IS NULL OR n.page_id IS NULL OR c.page_id IS NULL + OR life.state NOT IN ('RESERVED','LIVE') OR life.graph_domain<>'qualified-v1' OR life.metadata_codec<>q.metadata_codec + OR body.metadata_codec<>q.metadata_codec OR n.metadata_codec<>q.metadata_codec OR c.metadata_codec<>q.metadata_codec + OR life.expected_size<>m.expected_size OR body.byte_size<>m.expected_size OR n.bytes<>m.expected_size OR c.byte_size<>m.expected_size + OR n.state<>'LIVE' OR n.certificate_digest<>c.certificate_digest + OR life.state='RESERVED' AND c.origin_prepare_id<>pid + OR n.incoming_refs<>(SELECT count(*) FROM mst2_metadata_graph_edge e WHERE e.child_page=m.page_id AND e.child_generation=m.generation) + OR EXISTS(SELECT 1 FROM mst2_metadata_gc_op gc WHERE gc.page_id=m.page_id AND gc.generation=m.generation))) THEN + RAISE EXCEPTION 'rooted finalize lost its exact durable canonical delta graph'; END IF; + IF EXISTS((SELECT decode(value->>'parent','hex'),decode(value->>'child','hex') FROM jsonb_array_elements(plan->'edges')) EXCEPT + (SELECT e.parent_page,e.child_page FROM mst2_metadata_prepare_page m JOIN mst2_metadata_graph_edge e + ON e.parent_page=m.page_id AND e.parent_generation=m.generation WHERE m.prepare_id=pid)) + OR EXISTS((SELECT e.parent_page,e.child_page FROM mst2_metadata_prepare_page m JOIN mst2_metadata_graph_edge e + ON e.parent_page=m.page_id AND e.parent_generation=m.generation WHERE m.prepare_id=pid) EXCEPT + (SELECT decode(value->>'parent','hex'),decode(value->>'child','hex') FROM jsonb_array_elements(plan->'edges'))) + OR EXISTS(SELECT 1 FROM mst2_metadata_prepare_reuse_root r WHERE r.prepare_id=pid AND NOT EXISTS( + SELECT 1 FROM mst2_metadata_root_anchor a WHERE a.prepare_id=pid AND a.owner_key=pid AND a.anchor_kind='REUSE' + AND a.root_page=r.root_page AND a.root_generation=r.root_generation)) THEN + RAISE EXCEPTION 'rooted finalize differs from its exact delta edges or owned reuse boundaries'; END IF; + scoped_tree:=mst2_metadata_rooted_scope_tree(pid); + IF NOT EXISTS(SELECT 1 FROM jsonb_array_elements(plan->'source_roots') binding(value) + WHERE value->>'tree_oid'=scoped_tree AND value->>'page'=plan->>'root') THEN + RAISE EXCEPTION 'rooted metadata root is not bound to its independently selected fixed source scope'; END IF; + FOR source IN SELECT value FROM jsonb_array_elements(plan->'source_roots') LOOP + source_generation:=(generation_index->>(source->>'page'))::bigint; + IF NOT EXISTS(SELECT 1 FROM mst2_metadata_source_root_attestation a + JOIN mst2_metadata_current cur ON cur.page_id=a.root_page AND cur.generation=a.root_generation + JOIN mst2_metadata_lifetime life USING(page_id,generation) JOIN mst2_metadata_graph_node n USING(page_id,generation) + JOIN mst2_metadata_prepare origin ON origin.prepare_id=a.origin_prepare_id + JOIN $CORE_SCHEMA$.mega_tree t ON t.tree_id=split_part(a.tagged_tree_oid,':',2) + WHERE a.tagged_tree_oid=source->>'tree_oid' AND a.root_page=decode(source->>'page','hex') AND a.root_generation=source_generation + AND a.source_profile=profile + AND $CORE_SCHEMA$.mst2_route_source_tree_matches(split_part(a.tagged_tree_oid,':',2),a.source_revision,a.source_body_digest) AND n.state='LIVE' + AND n.certificate_digest=a.root_certificate_digest + AND (a.origin_prepare_id=pid AND origin.state='PREPARING' AND life.state IN ('RESERVED','LIVE') + OR origin.state='COMMITTED' AND life.state='LIVE' AND EXISTS(SELECT 1 FROM mst2_metadata_prepare_reuse_root r + WHERE r.prepare_id=pid AND r.root_page=a.root_page AND r.root_generation=a.root_generation + AND EXISTS(SELECT 1 FROM mst2_metadata_source_root_attestation boundary + WHERE boundary.attestation_id=r.attestation_id AND boundary.root_certificate_digest=a.root_certificate_digest)))) THEN + RAISE EXCEPTION 'rooted source binding lacks its independently attested exact current directory'; END IF; + END LOOP; + RETURN jsonb_build_object('root',plan->>'root','generation',selected_root_generation,'certificate',encode(root_certificate,'hex'), + 'scoped_tree_oid',scoped_tree,'delta_nodes',q.node_count,'delta_edges',q.edge_count,'delta_bytes',q.total_bytes); +END $$; diff --git a/src/jupiter/storage/qualified_metadata_serving.sql b/src/jupiter/storage/qualified_metadata_serving.sql new file mode 100644 index 00000000..fa3d729e --- /dev/null +++ b/src/jupiter/storage/qualified_metadata_serving.sql @@ -0,0 +1,581 @@ +-- Root ownership is derived from durable identities, never an application flag. +CREATE FUNCTION mst2_metadata_publication_valid(instance text,commit_id text,tree_id text, + sequence_id bigint,epoch_id bigint,receipt_id bigint,current_head boolean DEFAULT false) +RETURNS boolean LANGUAGE sql VOLATILE SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ + SELECT EXISTS(SELECT 1 FROM $CORE_SCHEMA$.mst2_native_publication c + JOIN $CORE_SCHEMA$.mst2_publication p ON p.id=c.receipt_id + JOIN $CORE_SCHEMA$.mst2_publication_outbox o ON o.operation_id=p.operation_id + JOIN $CORE_SCHEMA$.mega_commit source ON source.commit_id=c.root_commit AND source.tree=c.root_tree + JOIN $CORE_SCHEMA$.mega_commit previous ON previous.commit_id=c.old_root_commit AND previous.tree=c.old_root_tree + JOIN $CORE_SCHEMA$.mega_commit path_source ON path_source.commit_id=c.path_commit AND path_source.tree=c.path_tree + WHERE c.receipt_id=$6 AND c.namespace='/' AND c.instance_id=$1 + AND c.root_commit=$2 AND c.root_tree=$3 AND c.sequence=$4 AND c.writer_epoch=$5 + AND $4>0 AND $5>0 AND p.writer_epoch=$5 AND p.writer_kind='trunk_push' + AND p.native_certificate_version=1 AND p.request_digest_version=1 + AND p.request_digest ~ '^sha256:[0-9a-f]{64}$' AND c.origin_ref='refs/heads/main' + AND c.origin_path=p.namespace AND c.path_commit=p.new_oid AND c.old_root_commit=p.old_oid + AND c.old_path_commit IS DISTINCT FROM c.path_commit AND o.namespace=p.namespace AND o.sequence=p.sequence + AND (NOT $7 OR (EXISTS(SELECT 1 FROM $CORE_SCHEMA$.mst2_native_head h + WHERE h.namespace='/' AND h.state='READY' AND h.instance_id=$1 AND h.root_commit=$2 + AND h.root_tree=$3 AND h.sequence=$4 AND h.writer_epoch=$5 + AND h.certificate_receipt_id=$6) + AND (SELECT count(*) FROM $CORE_SCHEMA$.mega_refs r + WHERE r.path='/' AND r.ref_name='refs/heads/main' AND NOT r.is_cl)=1 + AND EXISTS(SELECT 1 FROM $CORE_SCHEMA$.mega_refs r WHERE r.path='/' + AND r.ref_name='refs/heads/main' AND NOT r.is_cl AND r.ref_commit_hash=$2 AND r.ref_tree_hash=$3)))) +$$; + +CREATE FUNCTION mst2_metadata_descriptor(pid text,instance text,commit_id text,tree_id text) +RETURNS bytea LANGUAGE plpgsql VOLATILE STRICT SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE p record; scope_bytes bytea; view_digest bytea; instance_bytes bytea; +BEGIN + SELECT tagged_root_tree_oid,scope,metadata_root INTO p FROM mst2_metadata_prepare WHERE prepare_id=pid AND plan_kind='ROOTED' + AND state='COMMITTED' AND graph_domain='qualified-v1' AND mst2_metadata_scope_matches(primary_scope); + IF NOT FOUND OR instance IS DISTINCT FROM (instance::uuid)::text + OR split_part(p.tagged_root_tree_oid,':',2) IS DISTINCT FROM tree_id + OR NOT EXISTS(SELECT 1 FROM $CORE_SCHEMA$.mega_commit c WHERE c.commit_id=$3 AND c.tree=$4) + OR octet_length(commit_id)<>octet_length(tree_id) + OR commit_id !~ '^([0-9a-f]{40}|[0-9a-f]{64})$' THEN + RAISE EXCEPTION 'qualified descriptor has no exact canonical instance and fixed source'; + END IF; + PERFORM mst2_metadata_native_profile(pid); + scope_bytes:=convert_to(p.scope,'UTF8'); + IF p.scope !~ '^/' OR octet_length(scope_bytes)>4096 OR p.scope<>'/' AND + (p.scope ~ '/$|//|/(\.|\.\.)(/|$)' OR cardinality(string_to_array(substring(p.scope FROM 2),'/'))>256) + OR EXISTS(SELECT 1 FROM unnest(string_to_array(substring(p.scope FROM 2),'/')) component + WHERE octet_length(component)>255) THEN + RAISE EXCEPTION 'qualified descriptor scope is not canonical'; + END IF; + instance_bytes:=decode(replace(instance,'-',''),'hex'); + view_digest:=sha256(convert_to('mega.mst2.namespaceview','UTF8')||decode('00','hex')||convert_to(commit_id,'UTF8')); + RETURN convert_to('MSD2','UTF8')||decode('00020001','hex')||instance_bytes||view_digest + ||decode(lpad(to_hex(octet_length(scope_bytes)),4,'0'),'hex')||scope_bytes + ||decode('0001000100000000','hex')||p.metadata_root; +END $$; + +CREATE FUNCTION mst2_metadata_root_live(p bytea,g bigint,c bytea) RETURNS boolean +LANGUAGE sql VOLATILE STRICT SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ + SELECT EXISTS(SELECT 1 FROM mst2_metadata_current cur JOIN mst2_metadata_lifetime life USING(page_id,generation) + JOIN mst2_metadata_graph_node node USING(page_id,generation) + JOIN mst2_metadata_page_certificate proof USING(page_id,generation) + JOIN mst2_metadata_payload body USING(page_id,generation) + WHERE cur.page_id=p AND cur.generation=g AND life.state='LIVE' AND life.graph_domain='qualified-v1' + AND life.metadata_codec=1 AND node.state='LIVE' AND node.metadata_codec=1 AND body.metadata_codec=1 + AND node.certificate_digest=c AND proof.certificate_digest=c AND proof.proof_revision=1 + AND proof.namespace_uuid='$NAMESPACE_UUID$'::uuid AND life.expected_size=proof.byte_size + AND node.bytes=proof.byte_size AND body.byte_size=proof.byte_size AND octet_length(body.payload)=proof.byte_size + AND p=sha256(convert_to('mega.mst2.metapage','UTF8')||decode('00','hex')||body.payload) + AND NOT EXISTS(SELECT 1 FROM mst2_metadata_gc_op gc WHERE gc.page_id=p AND gc.generation=g)) +$$; + +CREATE FUNCTION mst2_metadata_serving_source(pid text,a_id uuid,p bytea,g bigint,c bytea) +RETURNS boolean LANGUAGE plpgsql VOLATILE STRICT SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE a record; scope text; certificate record; +BEGIN + SELECT proof.tagged_tree_oid,proof.source_revision,proof.source_body_digest,proof.profile_digest,proof.source_profile + INTO a FROM mst2_metadata_source_root_attestation proof + JOIN mst2_metadata_prepare origin ON origin.prepare_id=proof.origin_prepare_id + WHERE proof.attestation_id=a_id AND proof.namespace_uuid='$NAMESPACE_UUID$'::uuid + AND proof.root_page=p AND proof.root_generation=g AND proof.root_certificate_digest=c + AND origin.state='COMMITTED' AND proof.source_profile=mst2_metadata_native_profile(pid) + AND proof.tagged_tree_oid=mst2_metadata_rooted_scope_tree(pid); + IF NOT FOUND THEN RETURN false; END IF; + SELECT relative_path_bytes,relative_components INTO certificate FROM mst2_metadata_page_certificate WHERE page_id=p AND generation=g AND certificate_digest=c; + IF NOT FOUND THEN RETURN false; END IF; + SELECT q.scope INTO STRICT scope FROM mst2_metadata_prepare q WHERE q.prepare_id=pid; + IF (CASE WHEN scope='/' THEN 0 ELSE octet_length(scope) END)::bigint+certificate.relative_path_bytes>4096 + OR (CASE WHEN scope='/' THEN 0 ELSE cardinality(string_to_array(substring(scope FROM 2),'/')) END)::bigint + +certificate.relative_components>256 THEN RETURN false; END IF; + RETURN $CORE_SCHEMA$.mst2_route_source_tree_matches(split_part(a.tagged_tree_oid,':',2),a.source_revision,a.source_body_digest) + AND a.profile_digest=sha256(convert_to('mega.mst2.native-profile.v1','UTF8')||decode('00','hex') + ||convert_to(a.source_profile::text,'UTF8')); +END $$; + +CREATE FUNCTION mst2_metadata_snapshot_candidate(sid text,descriptor bytea,instance text,commit_id text, + tree_id text,root bytea,profile jsonb) RETURNS boolean LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE p record; a record; +BEGIN + IF sid IS DISTINCT FROM 'sha256:'||encode(sha256(convert_to('mega.mst2.descriptor','UTF8') + ||decode('00','hex')||descriptor),'hex') THEN RETURN false; END IF; + FOR p IN SELECT q.prepare_id FROM mst2_metadata_prepare q WHERE q.metadata_root=root AND q.state='COMMITTED' + AND q.plan_kind='ROOTED' AND q.graph_domain='qualified-v1' AND q.coverage_retired_at IS NULL + AND q.tagged_root_tree_oid=profile->>'tagged_root_tree_oid' AND q.scope=profile->>'scope' + AND $CORE_SCHEMA$.mst2_route_profile(q.source_domain,q.tagged_root_tree_oid,q.scope,q.schema_version, + q.metadata_codec,q.materialization_policy,q.fs_semantics,q.access_projection, + q.verification_revision,q.projection_revision)=profile ORDER BY q.prepare_id LIMIT 1 LOOP + IF descriptor IS DISTINCT FROM mst2_metadata_descriptor(p.prepare_id,instance,commit_id,tree_id) THEN CONTINUE; END IF; + FOR a IN SELECT proof.attestation_id,proof.root_generation,proof.root_certificate_digest FROM mst2_metadata_source_root_attestation proof + JOIN mst2_metadata_current current_root ON current_root.page_id=proof.root_page AND current_root.generation=proof.root_generation + WHERE proof.root_page=root AND proof.tagged_tree_oid=mst2_metadata_rooted_scope_tree(p.prepare_id) + AND proof.source_profile=mst2_metadata_native_profile(p.prepare_id) + AND $CORE_SCHEMA$.mst2_route_source_tree_matches(split_part(proof.tagged_tree_oid,':',2),proof.source_revision,proof.source_body_digest) + ORDER BY proof.attestation_id LIMIT 1 LOOP + IF mst2_metadata_serving_source(p.prepare_id,a.attestation_id,root,a.root_generation,a.root_certificate_digest) + AND mst2_metadata_root_live(root,a.root_generation,a.root_certificate_digest) THEN RETURN true; END IF; + END LOOP; + END LOOP; + RETURN false; +END $$; + +CREATE FUNCTION mst2_metadata_incarnation_proof(sid text,inc uuid,require_live boolean DEFAULT false) +RETURNS boolean LANGUAGE plpgsql VOLATILE SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE s mst2_qualified_session_incarnation%ROWTYPE; p record; + a record; profile jsonb; +BEGIN + SELECT * INTO s FROM mst2_qualified_session_incarnation WHERE snapshot_id=sid AND session_incarnation=inc; + IF NOT FOUND OR s.namespace_uuid<>'$NAMESPACE_UUID$'::uuid OR s.authorization_epoch<>1 THEN RETURN false; END IF; + SELECT prepare_id,source_domain,tagged_root_tree_oid,scope,schema_version,metadata_codec, + materialization_policy,fs_semantics,access_projection,verification_revision,projection_revision + INTO p FROM mst2_metadata_prepare WHERE prepare_id=s.prepare_id AND state='COMMITTED' + AND plan_kind='ROOTED' AND storage_seal=s.storage_seal AND metadata_root=s.metadata_root; + IF NOT FOUND THEN RETURN false; END IF; + profile:=$CORE_SCHEMA$.mst2_route_profile(p.source_domain,p.tagged_root_tree_oid,p.scope,p.schema_version, + p.metadata_codec,p.materialization_policy,p.fs_semantics,p.access_projection,p.verification_revision,p.projection_revision); + IF s.source_profile IS DISTINCT FROM convert_to(profile::text,'UTF8') + OR s.canonical_descriptor IS DISTINCT FROM mst2_metadata_descriptor(p.prepare_id,s.instance_id,s.commit_oid,s.root_tree_oid) + OR s.snapshot_id IS DISTINCT FROM 'sha256:'||encode(sha256(convert_to('mega.mst2.descriptor','UTF8') + ||decode('00','hex')||s.canonical_descriptor),'hex') + OR NOT EXISTS(SELECT 1 FROM $CORE_SCHEMA$.mst2_snapshot_storage_route r + WHERE r.snapshot_id=s.snapshot_id AND r.namespace_uuid=s.namespace_uuid AND r.canonical_descriptor=s.canonical_descriptor + AND r.instance_id=s.instance_id AND r.commit_oid=s.commit_oid AND r.root_tree_oid=s.root_tree_oid + AND r.metadata_root=s.metadata_root AND r.source_profile=profile) + OR NOT mst2_metadata_publication_valid(s.instance_id,s.commit_oid,s.root_tree_oid, + s.publication_sequence,s.writer_epoch,s.certificate_receipt_id,false) THEN RETURN false; END IF; + SELECT attestation_id,root_certificate_digest INTO a FROM mst2_metadata_source_root_attestation WHERE attestation_id=s.attestation_id + AND attestation_digest=s.attestation_digest AND root_page=s.metadata_root AND root_generation=s.root_generation; + IF NOT FOUND OR NOT mst2_metadata_serving_source(s.prepare_id,a.attestation_id,s.metadata_root, + s.root_generation,a.root_certificate_digest) THEN RETURN false; END IF; + IF require_live AND (s.state<>'READY' OR NOT mst2_metadata_root_live(s.metadata_root,s.root_generation,a.root_certificate_digest) + OR NOT EXISTS(SELECT 1 FROM mst2_metadata_root_anchor anchor WHERE anchor.anchor_kind='SESSION' + AND anchor.owner_key=s.snapshot_id||':'||s.session_incarnation::text AND anchor.snapshot_id=s.snapshot_id + AND anchor.session_incarnation=s.session_incarnation AND anchor.root_page=s.metadata_root + AND anchor.root_generation=s.root_generation AND anchor.root_certificate_digest=a.root_certificate_digest)) THEN RETURN false; END IF; + RETURN true; +END $$; + +CREATE FUNCTION mst2_metadata_snapshot_route_proof(sid text,descriptor bytea,instance text,commit_id text, + tree_id text,root bytea,profile jsonb) RETURNS boolean LANGUAGE sql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ + SELECT EXISTS(SELECT 1 FROM mst2_qualified_session_incarnation s WHERE s.snapshot_id=sid + AND s.canonical_descriptor=descriptor AND s.instance_id=instance AND s.commit_oid=commit_id AND s.root_tree_oid=tree_id + AND s.metadata_root=root AND s.source_profile=convert_to(profile::text,'UTF8') + AND mst2_metadata_incarnation_proof(s.snapshot_id,s.session_incarnation,false)) +$$; + +CREATE FUNCTION mst2_metadata_lease_route_proof(lid text,sid text,namespace uuid,inc uuid,pid text,root bytea, + auth bigint,publication bigint,epoch bigint,receipt bigint) RETURNS boolean LANGUAGE sql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ + SELECT EXISTS(SELECT 1 FROM mst2_qualified_lease_binding l JOIN mst2_qualified_session_incarnation s + ON s.snapshot_id=l.snapshot_id AND s.session_incarnation=l.session_incarnation + WHERE l.lease_id=lid AND l.snapshot_id=sid AND l.namespace_uuid=namespace AND namespace='$NAMESPACE_UUID$'::uuid + AND l.session_incarnation=inc AND l.prepare_id=pid AND l.metadata_root=root + AND l.authorization_epoch=auth AND l.publication_sequence=publication AND l.writer_epoch=epoch + AND l.certificate_receipt_id=receipt AND l.storage_seal=s.storage_seal AND l.root_generation=s.root_generation + AND ROW(l.authorization_epoch,l.publication_sequence,l.writer_epoch,l.certificate_receipt_id) + IS NOT DISTINCT FROM ROW(s.authorization_epoch,s.publication_sequence,s.writer_epoch,s.certificate_receipt_id) + AND mst2_metadata_incarnation_proof(s.snapshot_id,s.session_incarnation,false)) +$$; + +CREATE UNIQUE INDEX mst2_qualified_snapshot_ready ON mst2_qualified_session_incarnation(snapshot_id) WHERE state='READY'; +CREATE INDEX mst2_metadata_committed_source ON mst2_metadata_prepare(metadata_root,tagged_root_tree_oid,scope,prepare_id) + WHERE state='COMMITTED' AND plan_kind='ROOTED'; +CREATE INDEX mst2_qualified_lease_expiry ON mst2_qualified_lease_binding(expires_at_unix,lease_id) WHERE state='ACTIVE'; +CREATE FUNCTION mst2_metadata_session_guard() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE p record; a record; profile jsonb; +BEGIN + IF TG_OP='DELETE' THEN RAISE EXCEPTION 'qualified session incarnation history is immutable'; END IF; + IF TG_OP='UPDATE' THEN + IF (to_jsonb(NEW)-'state') IS DISTINCT FROM (to_jsonb(OLD)-'state') OR OLD.state='RETIRED' AND NEW.state<>'RETIRED' + OR NEW.state NOT IN ('READY','RETIRED') THEN RAISE EXCEPTION 'qualified fixed incarnation cannot change or revive'; END IF; + IF NEW.state='RETIRED' AND EXISTS(SELECT 1 FROM mst2_qualified_lease_binding l + WHERE l.snapshot_id=OLD.snapshot_id AND l.session_incarnation=OLD.session_incarnation AND l.state='ACTIVE') THEN + RAISE EXCEPTION 'qualified ready session still has active lease identities'; END IF; + RETURN NEW; + END IF; + SELECT prepare_id,source_domain,tagged_root_tree_oid,scope,schema_version,metadata_codec, + materialization_policy,fs_semantics,access_projection,verification_revision,projection_revision + INTO p FROM mst2_metadata_prepare WHERE prepare_id=NEW.prepare_id AND state='COMMITTED' + AND plan_kind='ROOTED' AND storage_seal=NEW.storage_seal AND metadata_root=NEW.metadata_root AND coverage_retired_at IS NULL; + IF NOT FOUND OR NEW.namespace_uuid<>'$NAMESPACE_UUID$'::uuid OR NEW.state<>'READY' OR NEW.authorization_epoch<>1 + OR substr(NEW.session_incarnation::text,15,1)<>'4' OR substr(NEW.session_incarnation::text,20,1) NOT IN ('8','9','a','b') + OR EXISTS(SELECT 1 FROM mst2_qualified_session_incarnation WHERE snapshot_id=NEW.snapshot_id AND state='READY') THEN + RAISE EXCEPTION 'qualified incarnation requires a fresh server identity and definitive rooted preparation'; END IF; + profile:=$CORE_SCHEMA$.mst2_route_profile(p.source_domain,p.tagged_root_tree_oid,p.scope,p.schema_version, + p.metadata_codec,p.materialization_policy,p.fs_semantics,p.access_projection,p.verification_revision,p.projection_revision); + SELECT attestation_id,root_certificate_digest INTO a FROM mst2_metadata_source_root_attestation WHERE attestation_id=NEW.attestation_id + AND attestation_digest=NEW.attestation_digest AND root_page=NEW.metadata_root AND root_generation=NEW.root_generation; + IF NOT FOUND OR NEW.source_profile IS DISTINCT FROM convert_to(profile::text,'UTF8') + OR NEW.canonical_descriptor IS DISTINCT FROM mst2_metadata_descriptor(p.prepare_id,NEW.instance_id,NEW.commit_oid,NEW.root_tree_oid) + OR NEW.snapshot_id IS DISTINCT FROM 'sha256:'||encode(sha256(convert_to('mega.mst2.descriptor','UTF8') + ||decode('00','hex')||NEW.canonical_descriptor),'hex') + OR NOT mst2_metadata_serving_source(p.prepare_id,a.attestation_id,NEW.metadata_root,NEW.root_generation,a.root_certificate_digest) + OR NOT mst2_metadata_root_live(NEW.metadata_root,NEW.root_generation,a.root_certificate_digest) + OR NOT EXISTS(SELECT 1 FROM mst2_metadata_root_anchor anchor WHERE anchor.anchor_kind='PREPARE' + AND anchor.prepare_id=NEW.prepare_id AND anchor.owner_key=NEW.prepare_id AND anchor.root_page=NEW.metadata_root + AND anchor.root_generation=NEW.root_generation AND anchor.root_certificate_digest=a.root_certificate_digest) + OR NOT mst2_metadata_publication_valid(NEW.instance_id,NEW.commit_oid,NEW.root_tree_oid, + NEW.publication_sequence,NEW.writer_epoch,NEW.certificate_receipt_id,true) + OR NOT EXISTS(SELECT 1 FROM $CORE_SCHEMA$.mst2_snapshot_storage_route r WHERE r.snapshot_id=NEW.snapshot_id + AND r.namespace_uuid=NEW.namespace_uuid AND r.canonical_descriptor=NEW.canonical_descriptor + AND r.instance_id=NEW.instance_id AND r.commit_oid=NEW.commit_oid AND r.root_tree_oid=NEW.root_tree_oid + AND r.metadata_root=NEW.metadata_root AND r.source_profile=profile) THEN + RAISE EXCEPTION 'qualified incarnation differs from its independently derived source and permanent route'; END IF; + RETURN NEW; +END $$; +CREATE TRIGGER mst2_metadata_session_guard BEFORE INSERT OR UPDATE OR DELETE ON mst2_qualified_session_incarnation + FOR EACH ROW EXECUTE FUNCTION mst2_metadata_session_guard(); + +CREATE FUNCTION mst2_metadata_lease_guard() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE s mst2_qualified_session_incarnation%ROWTYPE; now_unix bigint:=floor(extract(epoch FROM clock_timestamp()))::bigint; +BEGIN + IF TG_OP='DELETE' THEN RAISE EXCEPTION 'qualified lease binding history is immutable'; END IF; + IF TG_OP='UPDATE' THEN + IF (to_jsonb(NEW)-ARRAY['state','expires_at_unix','lease_epoch']) IS DISTINCT FROM + (to_jsonb(OLD)-ARRAY['state','expires_at_unix','lease_epoch']) OR OLD.state<>'ACTIVE' AND NEW IS DISTINCT FROM OLD THEN + RAISE EXCEPTION 'qualified fixed lease cannot change or revive'; END IF; + IF NEW.state='ACTIVE' THEN + IF OLD.expires_at_unix<=now_unix OR NEW.expires_at_unix<=now_unix OR NEW.expires_at_unix>now_unix+3600 + OR NEW.lease_epoch<>OLD.lease_epoch THEN RAISE EXCEPTION 'qualified renewal requires its still-active bounded lease'; END IF; + ELSIF NEW.state IN ('RELEASED','EXPIRED') THEN + IF NEW.expires_at_unix<>OLD.expires_at_unix OR NEW.lease_epoch<>OLD.lease_epoch+1 + OR NEW.state='EXPIRED' AND OLD.expires_at_unix>now_unix THEN + RAISE EXCEPTION 'qualified terminal lease differs from its exact deadline and epoch'; END IF; + ELSE RAISE EXCEPTION 'qualified lease state is unsupported'; END IF; + RETURN NEW; + END IF; + SELECT * INTO s FROM mst2_qualified_session_incarnation + WHERE snapshot_id=NEW.snapshot_id AND session_incarnation=NEW.session_incarnation AND state='READY'; + IF NOT FOUND OR NEW.namespace_uuid<>'$NAMESPACE_UUID$'::uuid OR NEW.namespace_uuid<>s.namespace_uuid + OR NEW.state<>'ACTIVE' OR NEW.lease_epoch<>1 OR NEW.expires_at_unix<=now_unix OR NEW.expires_at_unix>now_unix+3600 + OR NEW.lease_id IS DISTINCT FROM (NEW.lease_id::uuid)::text OR substr(NEW.lease_id,15,1)<>'4' + OR substr(NEW.lease_id,20,1) NOT IN ('8','9','a','b') + OR ROW(NEW.prepare_id,NEW.storage_seal,NEW.metadata_root,NEW.root_generation,NEW.authorization_epoch, + NEW.publication_sequence,NEW.writer_epoch,NEW.certificate_receipt_id) IS DISTINCT FROM + ROW(s.prepare_id,s.storage_seal,s.metadata_root,s.root_generation,s.authorization_epoch, + s.publication_sequence,s.writer_epoch,s.certificate_receipt_id) + OR NOT mst2_metadata_incarnation_proof(s.snapshot_id,s.session_incarnation,true) THEN + RAISE EXCEPTION 'qualified lease differs from its exact active incarnation and source'; END IF; + RETURN NEW; +END $$; +CREATE TRIGGER mst2_metadata_lease_guard BEFORE INSERT OR UPDATE OR DELETE ON mst2_qualified_lease_binding + FOR EACH ROW EXECUTE FUNCTION mst2_metadata_lease_guard(); + +CREATE FUNCTION mst2_metadata_reader_guard() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE l mst2_qualified_lease_binding%ROWTYPE; now_unix bigint:=floor(extract(epoch FROM clock_timestamp()))::bigint; +BEGIN + IF TG_OP='DELETE' THEN RAISE EXCEPTION 'qualified reader operation history is immutable'; END IF; + IF TG_OP='UPDATE' THEN + IF (to_jsonb(NEW)-'state') IS DISTINCT FROM (to_jsonb(OLD)-'state') + OR OLD.state<>'ACTIVE' AND NEW.state<>OLD.state OR NEW.state NOT IN ('ACTIVE','FINISHED','EXPIRED') + OR NEW.state='EXPIRED' AND OLD.hard_deadline_unix>now_unix THEN + RAISE EXCEPTION 'qualified reader identity cannot change or be prematurely expired'; END IF; + RETURN NEW; + END IF; + SELECT * INTO l FROM mst2_qualified_lease_binding WHERE lease_id=NEW.lease_id AND state='ACTIVE' AND expires_at_unix>now_unix; + IF NOT FOUND OR NEW.state<>'ACTIVE' OR substr(NEW.operation_id::text,15,1)<>'4' + OR substr(NEW.operation_id::text,20,1) NOT IN ('8','9','a','b') + OR ROW(NEW.snapshot_id,NEW.session_incarnation,NEW.root_page,NEW.root_generation,NEW.lease_epoch) IS DISTINCT FROM + ROW(l.snapshot_id,l.session_incarnation,l.metadata_root,l.root_generation,l.lease_epoch) + OR NEW.hard_deadline_unix<=now_unix OR NEW.hard_deadline_unix>least(l.expires_at_unix,now_unix+60) + OR NOT mst2_metadata_incarnation_proof(l.snapshot_id,l.session_incarnation,true) THEN + RAISE EXCEPTION 'qualified reader lacks its exact active lease and bounded deadline'; END IF; + RETURN NEW; +END $$; +CREATE TRIGGER mst2_metadata_reader_guard BEFORE INSERT OR UPDATE OR DELETE ON mst2_metadata_reader_operation + FOR EACH ROW EXECUTE FUNCTION mst2_metadata_reader_guard(); + +CREATE FUNCTION mst2_metadata_serving_complete() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE s mst2_qualified_session_incarnation%ROWTYPE; l mst2_qualified_lease_binding%ROWTYPE; + r mst2_metadata_reader_operation%ROWTYPE; +BEGIN + IF TG_TABLE_NAME='mst2_qualified_session_incarnation' THEN + SELECT * INTO STRICT s FROM mst2_qualified_session_incarnation + WHERE snapshot_id=NEW.snapshot_id AND session_incarnation=NEW.session_incarnation; + IF NOT mst2_metadata_incarnation_proof(s.snapshot_id,s.session_incarnation,s.state='READY') + OR s.state='READY' AND NOT EXISTS(SELECT 1 FROM mst2_qualified_lease_binding lease + WHERE lease.snapshot_id=s.snapshot_id AND lease.session_incarnation=s.session_incarnation AND lease.state='ACTIVE') + OR s.state='RETIRED' AND EXISTS(SELECT 1 FROM mst2_metadata_root_anchor a + WHERE a.anchor_kind='SESSION' AND a.snapshot_id=s.snapshot_id AND a.session_incarnation=s.session_incarnation) THEN + RAISE EXCEPTION 'qualified session cannot commit without its exact final serving roots'; END IF; + ELSIF TG_TABLE_NAME='mst2_qualified_lease_binding' THEN + SELECT * INTO STRICT l FROM mst2_qualified_lease_binding WHERE lease_id=NEW.lease_id; + IF NOT EXISTS(SELECT 1 FROM $CORE_SCHEMA$.mst2_lease_storage_route route + WHERE route.lease_id=l.lease_id AND route.namespace_uuid=l.namespace_uuid AND route.snapshot_id=l.snapshot_id + AND route.session_incarnation=l.session_incarnation AND route.prepare_id=l.prepare_id AND route.metadata_root=l.metadata_root + AND route.authorization_epoch=l.authorization_epoch AND route.publication_sequence=l.publication_sequence + AND route.writer_epoch=l.writer_epoch AND route.certificate_receipt_id=l.certificate_receipt_id) + OR l.state='ACTIVE' AND (l.expires_at_unix<=floor(extract(epoch FROM clock_timestamp()))::bigint + OR NOT mst2_metadata_incarnation_proof(l.snapshot_id,l.session_incarnation,true) + OR NOT EXISTS(SELECT 1 FROM mst2_metadata_root_anchor a WHERE a.anchor_kind='LEASE' AND a.owner_key=l.lease_id + AND a.lease_id=l.lease_id AND a.root_page=l.metadata_root AND a.root_generation=l.root_generation)) + OR l.state<>'ACTIVE' AND NOT EXISTS(SELECT 1 FROM mst2_metadata_reader_operation operation + WHERE operation.lease_id=l.lease_id AND operation.state='ACTIVE') + AND EXISTS(SELECT 1 FROM mst2_metadata_root_anchor a WHERE a.anchor_kind='LEASE' AND a.lease_id=l.lease_id) THEN + RAISE EXCEPTION 'qualified lease cannot commit without its exact final route and owned protection'; END IF; + ELSE + SELECT * INTO STRICT r FROM mst2_metadata_reader_operation WHERE operation_id=NEW.operation_id; + IF r.state='ACTIVE' AND (r.hard_deadline_unix<=floor(extract(epoch FROM clock_timestamp()))::bigint + OR (SELECT count(*) FROM mst2_metadata_root_anchor a WHERE a.reader_operation_id=r.operation_id + AND a.anchor_kind IN ('REQUEST','READER') AND a.owner_key=r.operation_id::text + AND a.lease_id=r.lease_id AND a.snapshot_id=r.snapshot_id AND a.session_incarnation=r.session_incarnation + AND a.root_page=r.root_page AND a.root_generation=r.root_generation)<>2) + OR r.state<>'ACTIVE' AND EXISTS(SELECT 1 FROM mst2_metadata_root_anchor a WHERE a.reader_operation_id=r.operation_id) THEN + RAISE EXCEPTION 'qualified reader cannot commit without both exact owned roots or definitive cleanup'; END IF; + END IF; + RETURN NULL; +END $$; +CREATE CONSTRAINT TRIGGER mst2_metadata_session_complete AFTER INSERT OR UPDATE ON mst2_qualified_session_incarnation + DEFERRABLE INITIALLY DEFERRED FOR EACH ROW EXECUTE FUNCTION mst2_metadata_serving_complete(); +CREATE CONSTRAINT TRIGGER mst2_metadata_lease_complete AFTER INSERT OR UPDATE ON mst2_qualified_lease_binding + DEFERRABLE INITIALLY DEFERRED FOR EACH ROW EXECUTE FUNCTION mst2_metadata_serving_complete(); +CREATE CONSTRAINT TRIGGER mst2_metadata_reader_complete AFTER INSERT OR UPDATE ON mst2_metadata_reader_operation + DEFERRABLE INITIALLY DEFERRED FOR EACH ROW EXECUTE FUNCTION mst2_metadata_serving_complete(); + +CREATE FUNCTION mst2_metadata_session_row(sid text,lid text,instance text) +RETURNS TABLE(canonical_descriptor bytea,commit_oid text,root_tree_oid text,expires_at_unix bigint, + authorization_epoch bigint,session_incarnation uuid,root_generation bigint,certificate_digest bytea,attestation_id uuid) +LANGUAGE plpgsql VOLATILE SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE l mst2_qualified_lease_binding%ROWTYPE; s mst2_qualified_session_incarnation%ROWTYPE; + c bytea; +BEGIN + SELECT * INTO l FROM mst2_qualified_lease_binding lease WHERE lease.lease_id=lid AND lease.snapshot_id=sid + AND lease.state='ACTIVE' AND lease.expires_at_unix>floor(extract(epoch FROM clock_timestamp()))::bigint; + IF NOT FOUND THEN RETURN; END IF; + SELECT * INTO s FROM mst2_qualified_session_incarnation session WHERE session.snapshot_id=sid + AND session.session_incarnation=l.session_incarnation AND session.state='READY' AND session.instance_id=instance; + IF NOT FOUND THEN RETURN; END IF; + SELECT proof.root_certificate_digest INTO c FROM mst2_metadata_source_root_attestation proof + WHERE proof.attestation_id=s.attestation_id AND proof.attestation_digest=s.attestation_digest; + IF NOT FOUND OR NOT mst2_metadata_incarnation_proof(sid,s.session_incarnation,true) + OR NOT $CORE_SCHEMA$.mst2_route_qualified_namespace_valid('$NAMESPACE_UUID$'::uuid) + OR NOT EXISTS(SELECT 1 FROM $CORE_SCHEMA$.mst2_lease_storage_route route + WHERE route.lease_id=lid AND route.snapshot_id=sid AND route.namespace_uuid=l.namespace_uuid + AND route.session_incarnation=l.session_incarnation AND route.prepare_id=l.prepare_id AND route.metadata_root=l.metadata_root + AND route.authorization_epoch=l.authorization_epoch AND route.publication_sequence=l.publication_sequence + AND route.writer_epoch=l.writer_epoch AND route.certificate_receipt_id=l.certificate_receipt_id) + OR l.namespace_uuid IS DISTINCT FROM s.namespace_uuid + OR ROW(l.prepare_id,l.storage_seal,l.metadata_root,l.root_generation,l.authorization_epoch, + l.publication_sequence,l.writer_epoch,l.certificate_receipt_id) IS DISTINCT FROM + ROW(s.prepare_id,s.storage_seal,s.metadata_root,s.root_generation,s.authorization_epoch, + s.publication_sequence,s.writer_epoch,s.certificate_receipt_id) + OR NOT EXISTS(SELECT 1 FROM mst2_metadata_root_anchor anchor WHERE anchor.anchor_kind='LEASE' AND anchor.owner_key=lid + AND anchor.lease_id=lid AND anchor.snapshot_id=sid AND anchor.session_incarnation=l.session_incarnation + AND anchor.root_page=l.metadata_root AND anchor.root_generation=l.root_generation AND anchor.root_certificate_digest=c) THEN + RAISE EXCEPTION 'qualified durable session route, source, current lifetime or root ownership changed'; END IF; + RETURN QUERY SELECT s.canonical_descriptor,s.commit_oid,s.root_tree_oid,l.expires_at_unix, + l.authorization_epoch,s.session_incarnation,s.root_generation,c,s.attestation_id; +END $$; + +CREATE FUNCTION mst2_metadata_handoff(request jsonb) RETURNS TABLE(lease_id text,expires_at_unix bigint) +LANGUAGE plpgsql VOLATILE SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE sid text:=request->>'snapshot_id'; descriptor bytea:=decode(request->>'canonical_descriptor','hex'); + instance text:=request->>'instance_id'; commit_id text:=request->>'commit_oid'; tree_id text:=request->>'root_tree_oid'; + lid text:=request->>'lease_id'; inc uuid:=(request->>'session_incarnation')::uuid; pid text:=request->>'prepare_id'; + p record; s mst2_qualified_session_incarnation%ROWTYPE; + a record; profile jsonb; now_unix bigint; deadline bigint; +BEGIN + PERFORM $CORE_SCHEMA$.mst2_route_enter($CORE_LITERAL$); + PERFORM mst2_metadata_cleanup_expired(64); + now_unix:=floor(extract(epoch FROM clock_timestamp()))::bigint; + deadline:=now_unix+least(3600,greatest(1,(request->>'lease_seconds')::bigint)); + IF request->>'authorization_epoch' IS DISTINCT FROM '1' OR lid IS NULL OR inc IS NULL OR descriptor IS NULL + OR NOT mst2_metadata_publication_valid(instance,commit_id,tree_id,(request->>'publication_sequence')::bigint, + (request->>'writer_epoch')::bigint,(request->>'certificate_receipt_id')::bigint,true) THEN + RAISE EXCEPTION 'qualified handoff differs from the independently observed current publication'; END IF; + SELECT * INTO s FROM mst2_qualified_session_incarnation WHERE snapshot_id=sid AND state='READY'; + IF FOUND THEN + IF s.state<>'READY' OR s.canonical_descriptor IS DISTINCT FROM descriptor OR s.instance_id IS DISTINCT FROM instance + OR s.commit_oid IS DISTINCT FROM commit_id OR s.root_tree_oid IS DISTINCT FROM tree_id + OR ROW(s.publication_sequence,s.writer_epoch,s.certificate_receipt_id) IS DISTINCT FROM + ROW((request->>'publication_sequence')::bigint,(request->>'writer_epoch')::bigint,(request->>'certificate_receipt_id')::bigint) + OR NOT mst2_metadata_incarnation_proof(sid,s.session_incarnation,true) THEN + RAISE EXCEPTION 'qualified handoff cannot revive or replace a historical incarnation'; END IF; + ELSE + SELECT metadata_root,storage_seal,source_domain,tagged_root_tree_oid,scope,schema_version,metadata_codec, + materialization_policy,fs_semantics,access_projection,verification_revision,projection_revision + INTO p FROM mst2_metadata_prepare WHERE prepare_id=pid AND state='COMMITTED' AND plan_kind='ROOTED' + AND storage_seal=decode(request->>'storage_seal','hex') AND coverage_retired_at IS NULL; + IF NOT FOUND THEN RAISE EXCEPTION 'qualified handoff has no exact definitive rooted receipt'; END IF; + SELECT attestation_id,attestation_digest,root_generation,root_certificate_digest + INTO a FROM mst2_metadata_source_root_attestation WHERE attestation_id=(request->>'attestation_id')::uuid + AND attestation_digest=decode(request->>'attestation_digest','hex') AND root_page=p.metadata_root + AND root_generation=(request->>'root_generation')::bigint AND root_certificate_digest=decode(request->>'certificate_digest','hex'); + IF NOT FOUND OR descriptor IS DISTINCT FROM mst2_metadata_descriptor(pid,instance,commit_id,tree_id) + OR NOT mst2_metadata_serving_source(pid,a.attestation_id,p.metadata_root,a.root_generation,a.root_certificate_digest) + OR NOT mst2_metadata_root_live(p.metadata_root,a.root_generation,a.root_certificate_digest) + OR NOT EXISTS(SELECT 1 FROM mst2_metadata_root_anchor anchor WHERE anchor.anchor_kind='PREPARE' + AND anchor.prepare_id=pid AND anchor.owner_key=pid AND anchor.root_page=p.metadata_root + AND anchor.root_generation=a.root_generation AND anchor.root_certificate_digest=a.root_certificate_digest) THEN + RAISE EXCEPTION 'qualified handoff lacks its continuously owned exact canonical source root'; END IF; + profile:=$CORE_SCHEMA$.mst2_route_profile(p.source_domain,p.tagged_root_tree_oid,p.scope,p.schema_version, + p.metadata_codec,p.materialization_policy,p.fs_semantics,p.access_projection,p.verification_revision,p.projection_revision); + INSERT INTO $CORE_SCHEMA$.mst2_snapshot_storage_route(snapshot_id,namespace_uuid,canonical_descriptor,instance_id, + commit_oid,root_tree_oid,metadata_root,source_profile) + VALUES(sid,'$NAMESPACE_UUID$'::uuid,descriptor,instance,commit_id,tree_id,p.metadata_root,profile) + ON CONFLICT(snapshot_id) DO NOTHING; + INSERT INTO mst2_qualified_session_incarnation(snapshot_id,session_incarnation,namespace_uuid,prepare_id,storage_seal, + metadata_root,root_generation,source_profile,instance_id,commit_oid,root_tree_oid,authorization_epoch, + publication_sequence,writer_epoch,certificate_receipt_id,canonical_descriptor,attestation_id,attestation_digest,state) + VALUES(sid,inc,'$NAMESPACE_UUID$'::uuid,pid,p.storage_seal,p.metadata_root,a.root_generation,convert_to(profile::text,'UTF8'), + instance,commit_id,tree_id,1,(request->>'publication_sequence')::bigint,(request->>'writer_epoch')::bigint, + (request->>'certificate_receipt_id')::bigint,descriptor,a.attestation_id,a.attestation_digest,'READY') RETURNING * INTO s; + INSERT INTO mst2_metadata_root_anchor(anchor_id,anchor_kind,owner_key,root_page,root_generation,root_certificate_digest, + snapshot_id,session_incarnation) VALUES(gen_random_uuid(),'SESSION',sid||':'||inc::text, + s.metadata_root,s.root_generation,a.root_certificate_digest,sid,inc); + END IF; + SELECT root_certificate_digest INTO STRICT a FROM mst2_metadata_source_root_attestation WHERE attestation_id=s.attestation_id; + INSERT INTO mst2_qualified_lease_binding(lease_id,snapshot_id,session_incarnation,prepare_id,storage_seal,metadata_root, + root_generation,namespace_uuid,authorization_epoch,publication_sequence,writer_epoch,certificate_receipt_id, + expires_at_unix,state,lease_epoch) VALUES(lid,sid,s.session_incarnation,s.prepare_id,s.storage_seal,s.metadata_root, + s.root_generation,s.namespace_uuid,s.authorization_epoch,s.publication_sequence,s.writer_epoch,s.certificate_receipt_id, + deadline,'ACTIVE',1); + INSERT INTO $CORE_SCHEMA$.mst2_lease_storage_route(lease_id,snapshot_id,namespace_uuid,session_incarnation,prepare_id, + metadata_root,authorization_epoch,publication_sequence,writer_epoch,certificate_receipt_id) + VALUES(lid,sid,s.namespace_uuid,s.session_incarnation,s.prepare_id,s.metadata_root,s.authorization_epoch, + s.publication_sequence,s.writer_epoch,s.certificate_receipt_id); + INSERT INTO mst2_metadata_root_anchor(anchor_id,anchor_kind,owner_key,root_page,root_generation,root_certificate_digest, + snapshot_id,session_incarnation,lease_id) VALUES(gen_random_uuid(),'LEASE',lid,s.metadata_root,s.root_generation, + a.root_certificate_digest,sid,s.session_incarnation,lid); + IF pid IS NOT NULL THEN + SELECT coverage_retired_at INTO p FROM mst2_metadata_prepare WHERE prepare_id=pid AND state='COMMITTED' AND plan_kind='ROOTED' + AND storage_seal=decode(request->>'storage_seal','hex') AND metadata_root=s.metadata_root; + IF NOT FOUND OR s.root_generation IS DISTINCT FROM (request->>'root_generation')::bigint + OR a.root_certificate_digest IS DISTINCT FROM decode(request->>'certificate_digest','hex') + OR descriptor IS DISTINCT FROM mst2_metadata_descriptor(pid,instance,commit_id,tree_id) + OR NOT mst2_metadata_session_covers_prepare(pid) + OR NOT EXISTS(SELECT 1 FROM mst2_metadata_source_root_attestation receipt + WHERE receipt.attestation_id=(request->>'attestation_id')::uuid + AND receipt.attestation_digest=decode(request->>'attestation_digest','hex') + AND receipt.root_page=s.metadata_root AND receipt.root_generation=s.root_generation + AND receipt.root_certificate_digest=a.root_certificate_digest + AND mst2_metadata_serving_source(pid,receipt.attestation_id,s.metadata_root,s.root_generation,a.root_certificate_digest)) THEN + RAISE EXCEPTION 'qualified losing preparation cannot retire a different serving identity'; END IF; + IF p.coverage_retired_at IS NULL THEN + UPDATE mst2_metadata_prepare SET coverage_retired_at=clock_timestamp() WHERE prepare_id=pid; + DELETE FROM mst2_metadata_root_anchor WHERE prepare_id=pid AND anchor_kind IN ('PREPARE','REUSE'); + END IF; + END IF; + RETURN QUERY SELECT lid,deadline; +END $$; + +CREATE FUNCTION mst2_metadata_cleanup_lease(lid text) RETURNS void LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE l mst2_qualified_lease_binding%ROWTYPE; +BEGIN + SELECT * INTO l FROM mst2_qualified_lease_binding WHERE lease_id=lid; + IF NOT FOUND THEN RETURN; END IF; + IF l.state<>'ACTIVE' AND NOT EXISTS(SELECT 1 FROM mst2_metadata_reader_operation WHERE lease_id=lid AND state='ACTIVE') THEN + DELETE FROM mst2_metadata_root_anchor WHERE lease_id=lid AND anchor_kind='LEASE'; + END IF; + IF NOT EXISTS(SELECT 1 FROM mst2_qualified_lease_binding lease WHERE lease.snapshot_id=l.snapshot_id + AND lease.session_incarnation=l.session_incarnation AND lease.state='ACTIVE') THEN + UPDATE mst2_qualified_session_incarnation SET state='RETIRED' + WHERE snapshot_id=l.snapshot_id AND session_incarnation=l.session_incarnation AND state='READY'; + DELETE FROM mst2_metadata_root_anchor WHERE snapshot_id=l.snapshot_id + AND session_incarnation=l.session_incarnation AND anchor_kind='SESSION'; + END IF; +END $$; + +CREATE FUNCTION mst2_metadata_cleanup_expired(maximum integer DEFAULT 64) RETURNS void LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE item record; now_unix bigint; bound integer:=least(64,greatest(0,maximum)); +BEGIN + PERFORM $CORE_SCHEMA$.mst2_route_enter($CORE_LITERAL$); + now_unix:=floor(extract(epoch FROM clock_timestamp()))::bigint; + FOR item IN SELECT * FROM ( + (SELECT 'READER'::text AS kind,operation_id::text AS owner,lease_id,hard_deadline_unix AS deadline + FROM mst2_metadata_reader_operation WHERE state='ACTIVE' AND hard_deadline_unix<=now_unix + ORDER BY hard_deadline_unix,operation_id LIMIT bound) + UNION ALL + (SELECT 'LEASE'::text,lease_id,lease_id,expires_at_unix + FROM mst2_qualified_lease_binding WHERE state='ACTIVE' AND expires_at_unix<=now_unix + ORDER BY expires_at_unix,lease_id LIMIT bound) + ) expired ORDER BY deadline,kind,owner LIMIT bound LOOP + IF item.kind='READER' THEN + UPDATE mst2_metadata_reader_operation SET state='EXPIRED' WHERE operation_id=item.owner::uuid AND state='ACTIVE'; + DELETE FROM mst2_metadata_root_anchor WHERE reader_operation_id=item.owner::uuid AND anchor_kind IN ('REQUEST','READER'); + ELSE + UPDATE mst2_qualified_lease_binding SET state='EXPIRED',lease_epoch=lease_epoch+1 WHERE lease_id=item.owner AND state='ACTIVE'; + END IF; + PERFORM mst2_metadata_cleanup_lease(item.lease_id); + END LOOP; +END $$; + +CREATE FUNCTION mst2_metadata_renew_lease(lid text,seconds bigint,instance text) +RETURNS TABLE(snapshot_id text,expires_at_unix bigint) LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE l mst2_qualified_lease_binding%ROWTYPE; now_unix bigint; deadline bigint; +BEGIN + PERFORM $CORE_SCHEMA$.mst2_route_enter($CORE_LITERAL$); + PERFORM mst2_metadata_cleanup_expired(64); + now_unix:=floor(extract(epoch FROM clock_timestamp()))::bigint; + SELECT * INTO l FROM mst2_qualified_lease_binding WHERE lease_id=lid AND state='ACTIVE'; + IF NOT FOUND THEN RETURN; END IF; + IF l.expires_at_unix<=now_unix THEN + UPDATE mst2_qualified_lease_binding SET state='EXPIRED',lease_epoch=lease_epoch+1 WHERE lease_id=lid; + PERFORM mst2_metadata_cleanup_lease(lid); RETURN; + END IF; + IF NOT EXISTS(SELECT 1 FROM mst2_metadata_session_row(l.snapshot_id,lid,instance)) THEN RETURN; END IF; + deadline:=greatest(l.expires_at_unix,now_unix+least(3600,greatest(1,seconds))); + UPDATE mst2_qualified_lease_binding lease SET expires_at_unix=deadline WHERE lease.lease_id=lid; + RETURN QUERY SELECT l.snapshot_id,deadline; +END $$; + +CREATE FUNCTION mst2_metadata_release_lease(lid text) RETURNS boolean LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE l mst2_qualified_lease_binding%ROWTYPE; changed boolean; +BEGIN + PERFORM $CORE_SCHEMA$.mst2_route_enter($CORE_LITERAL$); + PERFORM mst2_metadata_cleanup_expired(64); + SELECT * INTO l FROM mst2_qualified_lease_binding WHERE lease_id=lid; + IF NOT FOUND THEN RETURN false; END IF; + IF NOT mst2_metadata_lease_route_proof(lid,l.snapshot_id,l.namespace_uuid,l.session_incarnation,l.prepare_id,l.metadata_root, + l.authorization_epoch,l.publication_sequence,l.writer_epoch,l.certificate_receipt_id) THEN + RAISE EXCEPTION 'qualified release lost its fixed historical source identity'; END IF; + changed:=l.state='ACTIVE'; + IF changed THEN UPDATE mst2_qualified_lease_binding SET state='RELEASED',lease_epoch=lease_epoch+1 WHERE lease_id=lid; END IF; + PERFORM mst2_metadata_cleanup_lease(lid); + RETURN changed; +END $$; + +CREATE FUNCTION mst2_metadata_begin_reader(sid text,lid text,instance text) +RETURNS TABLE(operation_id uuid,root_generation bigint,certificate_digest bytea) LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE l mst2_qualified_lease_binding%ROWTYPE; s record; op uuid:=gen_random_uuid(); deadline bigint; kind text; +BEGIN + PERFORM $CORE_SCHEMA$.mst2_route_enter($CORE_LITERAL$); + PERFORM mst2_metadata_cleanup_expired(64); + SELECT * INTO s FROM mst2_metadata_session_row(sid,lid,instance); + IF NOT FOUND THEN RETURN; END IF; + SELECT * INTO STRICT l FROM mst2_qualified_lease_binding WHERE lease_id=lid; + deadline:=least(l.expires_at_unix,floor(extract(epoch FROM clock_timestamp()))::bigint+60); + INSERT INTO mst2_metadata_reader_operation(operation_id,lease_id,snapshot_id,session_incarnation,root_page,root_generation, + lease_epoch,hard_deadline_unix,state) VALUES(op,lid,sid,l.session_incarnation,l.metadata_root,l.root_generation,l.lease_epoch,deadline,'ACTIVE'); + FOREACH kind IN ARRAY ARRAY['REQUEST','READER'] LOOP + INSERT INTO mst2_metadata_root_anchor(anchor_id,anchor_kind,owner_key,root_page,root_generation,root_certificate_digest, + snapshot_id,session_incarnation,lease_id,reader_operation_id) VALUES(gen_random_uuid(),kind,op::text, + l.metadata_root,l.root_generation,s.certificate_digest,sid,l.session_incarnation,lid,op); + END LOOP; + RETURN QUERY SELECT op,l.root_generation,s.certificate_digest::bytea; +END $$; + +CREATE FUNCTION mst2_metadata_finish_reader(op uuid) RETURNS void LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE r mst2_metadata_reader_operation%ROWTYPE; +BEGIN + PERFORM $CORE_SCHEMA$.mst2_route_enter($CORE_LITERAL$); + SELECT * INTO r FROM mst2_metadata_reader_operation WHERE operation_id=op; + IF NOT FOUND THEN RETURN; END IF; + IF r.state='ACTIVE' THEN UPDATE mst2_metadata_reader_operation SET state='FINISHED' WHERE operation_id=op; END IF; + DELETE FROM mst2_metadata_root_anchor WHERE reader_operation_id=op AND anchor_kind IN ('REQUEST','READER'); + PERFORM mst2_metadata_cleanup_lease(r.lease_id); +END $$; + +DROP TRIGGER mst2_01_family_closed ON mst2_qualified_session_incarnation; +DROP TRIGGER mst2_01_family_closed ON mst2_qualified_lease_binding; +DROP TRIGGER mst2_01_family_closed ON mst2_metadata_reader_operation; diff --git a/src/jupiter/storage/qualified_metadata_session.rs b/src/jupiter/storage/qualified_metadata_session.rs new file mode 100644 index 00000000..8e29b301 --- /dev/null +++ b/src/jupiter/storage/qualified_metadata_session.rs @@ -0,0 +1,291 @@ +//! Rooted session handoff and lease operations over the captured physical Q. + +use mst2_codec::descriptor::ServingDescriptor; + +use super::*; +use crate::{ + ceres::snapshot::{ + descriptor::BuiltDescriptor, + rooted_metadata_projection::PreparedRootedNativeMetadata, + runtime::{LeaseRenewed, SnapshotContext}, + view::{SnapshotView, hex}, + }, + jupiter::storage::native_publication_storage::NativePublicationHead, +}; + +fn gone() -> SnapshotError { + SnapshotError::new( + crate::ceres::snapshot::error::SnapshotErrorCode::LeaseExpired, + "lease is not active for this snapshot; re-resolve", + ) +} + +pub(super) async fn finish( + txn: DatabaseTransaction, + result: Result, +) -> Result { + match result { + Ok(value) => { + txn.commit().await.map_err(|_| { + SnapshotError::new( + crate::ceres::snapshot::error::SnapshotErrorCode::TemporaryUnavailable, + "qualified transaction outcome unknown; retry the original operation", + ) + })?; + Ok(value) + } + Err(error) => { + let _ = txn.rollback().await; + Err(error) + } + } +} + +pub(super) fn context( + row: &QueryResult, + sid: &str, + lease: &str, + instance: &str, +) -> Result { + let bytes: Vec = row.try_get("", "canonical_descriptor").map_err(internal)?; + let descriptor = ServingDescriptor::decode(&bytes).map_err(internal)?; + let commit: String = row.try_get("", "commit_oid").map_err(internal)?; + let tree: String = row.try_get("", "root_tree_oid").map_err(internal)?; + let view = SnapshotView::from_commit(&commit, &tree); + if format!( + "sha256:{}", + hex(&descriptor.snapshot_id().map_err(internal)?) + ) != sid + || uuid::Uuid::from_bytes(descriptor.instance_uuid).to_string() != instance + || format!("sha256:{}", hex(&descriptor.namespace_view_id)) != view.view_id + { + return Err(integrity( + "qualified durable descriptor differs from its fixed source", + )); + } + let deadline = u64::try_from( + row.try_get::("", "expires_at_unix") + .map_err(internal)?, + ) + .map_err(internal)?; + let epoch = u64::try_from( + row.try_get::("", "authorization_epoch") + .map_err(internal)?, + ) + .map_err(internal)?; + let metadata_root = format!("sha256:{}", hex(&descriptor.metadata_root)); + Ok(SnapshotContext { + built: BuiltDescriptor { + descriptor, + instance_id: instance.into(), + snapshot_id: sid.into(), + metadata_root, + }, + commit_oid: commit, + root_tree_oid: tree, + lease_id: lease.into(), + lease_expires_at_unix: deadline, + authorization_epoch: epoch, + }) +} + +impl RootedQualifiedMetadataRepository { + pub(crate) async fn install( + &self, + built: &BuiltDescriptor, + prepared: &PreparedRootedNativeMetadata, + ) -> Result { + if prepared.plan.root != built.descriptor.metadata_root + || prepared.plan.identity.scope != built.descriptor.scope + { + return Err(integrity("rooted projection differs from the serving descriptor").into()); + } + // Each attempt has its own durable identity. A previous retired + // incarnation may have the same cold plan but a different generation; + // its historical receipt cannot install or resurrect this attempt. + let operation = format!("rooted-http:{}:{}", built.snapshot_id, uuid::Uuid::new_v4()); + let intent = self.begin_intent(&operation, &prepared.plan).await?; + for pages in prepared.payloads.chunks(64) { + self.install_pages(&intent, pages).await?; + } + self.finalize(&intent).await + } + + pub(crate) async fn open_session( + &self, + expected: &NativePublicationHead, + built: &BuiltDescriptor, + receipt: Option<&RootedMetadataReceipt>, + seconds: u64, + ) -> Result, SnapshotError> { + let txn = self.transaction().await?; + let result = async { + if receipt.is_none() { + txn.execute_raw(sql("SELECT mst2_metadata_cleanup_expired(64)",[])).await.map_err(database_error)?; + } + let present = txn.query_one_raw(sql("SELECT session_incarnation FROM mst2_qualified_session_incarnation + WHERE snapshot_id=$1 AND state='READY'", [built.snapshot_id.clone().into()])).await.map_err(database_error)?; + if present.is_none() && receipt.is_none() { return Ok(None); } + if let Some(receipt) = receipt { + self.require_intent(&txn, &receipt.intent).await?; + if self.receipt(&txn, &receipt.intent).await? != *receipt + || receipt.metadata_root() != built.descriptor.metadata_root + { return Err(integrity("qualified handoff receipt differs from its definitive canonical root")); } + } + let request = json!({ + "snapshot_id":built.snapshot_id,"canonical_descriptor":hex::encode(built.descriptor.encode().map_err(internal)?), + "instance_id":expected.instance_id,"commit_oid":expected.root.commit,"root_tree_oid":expected.root.tree, + "publication_sequence":expected.token.sequence,"writer_epoch":expected.token.epoch, + "certificate_receipt_id":expected.token.certificate,"authorization_epoch":1, + "lease_id":uuid::Uuid::new_v4().to_string(),"lease_seconds":seconds.clamp(1,3600), + "session_incarnation":uuid::Uuid::new_v4().to_string(), + "prepare_id":receipt.map(|r|r.intent.prepare_id.as_str()), + "storage_seal":receipt.map(|r|hex::encode(r.intent.storage_seal)), + "root_generation":receipt.map(|r|r.intent.root_generation), + "attestation_id":receipt.map(|r|r.attestation_id.to_string()), + "attestation_digest":receipt.map(|r|hex::encode(r.attestation_digest)), + "certificate_digest":receipt.map(|r|hex::encode(r.certificate_digest)), + }); + let row = txn.query_one_raw(sql("SELECT * FROM mst2_metadata_handoff($1::jsonb)", [request.into()])) + .await.map_err(database_error)?.ok_or_else(|| integrity("qualified handoff returned no lease"))?; + let lease: String = row.try_get("", "lease_id").map_err(internal)?; + let row = self.session_row(&txn, &built.snapshot_id, &lease, &expected.instance_id).await?; + let context = context(&row, &built.snapshot_id, &lease, &expected.instance_id)?; + if context.built.descriptor != built.descriptor || context.commit_oid != expected.root.commit + || context.root_tree_oid != expected.root.tree { return Err(integrity("qualified handoff fixed source changed")); } + Ok(Some(context)) + }.await; + finish(txn, result).await + } + + pub(super) async fn session_row( + &self, + db: &C, + sid: &str, + lease: &str, + instance: &str, + ) -> Result { + db.query_one_raw(sql( + "SELECT * FROM mst2_metadata_session_row($1,$2,$3)", + [sid.into(), lease.into(), instance.into()], + )) + .await + .map_err(database_error)? + .ok_or_else(gone) + } + + pub(crate) async fn context( + &self, + sid: &str, + lease: &str, + instance: &str, + ) -> Result { + // Read-only revalidation grants no mutable authority. The helper checks + // the exact route, incarnation, publication, LIVE root and owned roots. + let txn = self.read_transaction().await?; + let result = async { + let row = self.session_row(&txn, sid, lease, instance).await?; + context(&row, sid, lease, instance) + } + .await; + finish(txn, result).await + } + + pub(crate) async fn renew( + &self, + lease: &str, + seconds: u64, + instance: &str, + ) -> Result { + let txn = self.transaction().await?; + let result = async { + let row = txn + .query_one_raw(sql( + "SELECT * FROM mst2_metadata_renew_lease($1,$2,$3)", + [ + lease.into(), + (seconds.clamp(1, 3600) as i64).into(), + instance.into(), + ], + )) + .await + .map_err(database_error)? + .ok_or_else(gone)?; + Ok(LeaseRenewed { + lease_id: lease.into(), + snapshot_id: row.try_get("", "snapshot_id").map_err(internal)?, + expires_at_unix: u64::try_from( + row.try_get::("", "expires_at_unix") + .map_err(internal)?, + ) + .map_err(internal)?, + }) + } + .await; + finish(txn, result).await + } + + pub(crate) async fn release(&self, lease: &str) -> Result { + let txn = self.transaction().await?; + let result = async { + txn.query_one_raw(sql( + "SELECT mst2_metadata_release_lease($1) AS released", + [lease.into()], + )) + .await + .map_err(database_error)? + .ok_or_else(|| integrity("qualified release result missing"))? + .try_get("", "released") + .map_err(internal) + } + .await; + finish(txn, result).await + } + + pub(super) async fn read_transaction(&self) -> Result { + let txn = self + .connection + .begin_with_config(Some(IsolationLevel::ReadCommitted), None) + .await + .map_err(database_error)?; + if registered( + &txn, + &(self.namespace.core_schema.clone(), self.namespace.core_oid), + ) + .await + .map_err(|error| match error { + crate::common::errors::MegaError::Db(error) => database_error(error), + error => internal(error), + })? + .as_ref() + != Some(&self.namespace) + { + return Err(integrity( + "qualified read physical namespace or authority catalog changed", + )); + } + let valid: bool = txn + .query_one_raw(sql( + "SELECT mst2_metadata_scope_matches($1) AS valid", + [self.primary_scope.clone().into()], + )) + .await + .map_err(database_error)? + .ok_or_else(|| integrity("qualified read scope missing"))? + .try_get("", "valid") + .map_err(internal)?; + if !valid { + return Err(integrity("qualified read left its captured primary scope")); + } + #[cfg(test)] + if super::reader::READER_TEMP_SOURCE_SHADOW + .try_with(|shadow| *shadow) + .unwrap_or(false) + { + txn.execute_unprepared("CREATE TEMP TABLE mega_tree(id bigint,tree_id text,sub_trees bytea) ON COMMIT DROP; + CREATE TEMP TABLE mst2_rooted_source_tree_revision(tree_id text,tree_row_id bigint,revision uuid,body_digest bytea,valid boolean) ON COMMIT DROP") + .await.map_err(database_error)?; + } + Ok(txn) + } +} diff --git a/src/jupiter/storage/qualified_metadata_source_read.sql b/src/jupiter/storage/qualified_metadata_source_read.sql new file mode 100644 index 00000000..232a0e6d --- /dev/null +++ b/src/jupiter/storage/qualified_metadata_source_read.sql @@ -0,0 +1,132 @@ +-- Indexed fixed-source names are independently derived by the attestation +-- proof. Selected windows revalidate current file facts without source scans. +CREATE TABLE mst2_metadata_source_entry_reference ( + attestation_id uuid NOT NULL REFERENCES mst2_metadata_source_root_attestation(attestation_id), + name bytea NOT NULL CHECK(octet_length(name) BETWEEN 1 AND 255), + git_oid text NOT NULL CHECK(git_oid ~ '^(sha1:[0-9a-f]{40}|sha256:[0-9a-f]{64}|blake3:[0-9a-f]{64})$'), + kind smallint NOT NULL CHECK(kind BETWEEN 1 AND 4), + byte_size bigint,content_digest bytea,child_root bytea,child_generation bigint,child_certificate_digest bytea, + PRIMARY KEY(attestation_id,name), + CHECK(((kind=4 AND byte_size IS NULL AND content_digest IS NULL AND octet_length(child_root)=32 + AND child_generation>0 AND octet_length(child_certificate_digest)=32) + OR (kind<>4 AND byte_size BETWEEN 0 AND 8796093022208 AND octet_length(content_digest)=32 + AND child_root IS NULL AND child_generation IS NULL AND child_certificate_digest IS NULL)) IS TRUE), + CHECK((kind<>3 OR byte_size BETWEEN 1 AND 4095) IS TRUE) +); + +CREATE FUNCTION mst2_metadata_source_entry_guard() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +BEGIN + RAISE EXCEPTION 'qualified fixed source entry history is immutable'; +END $$; +CREATE TRIGGER mst2_metadata_source_entry_guard BEFORE UPDATE OR DELETE ON mst2_metadata_source_entry_reference + FOR EACH ROW EXECUTE FUNCTION mst2_metadata_source_entry_guard(); + +CREATE FUNCTION mst2_metadata_source_entries_proof() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE source_id uuid; expected jsonb; supplied jsonb; +BEGIN + FOR source_id IN SELECT DISTINCT attestation_id FROM added_source_entries LOOP + SELECT source_proof->'source_entries' INTO expected FROM mst2_metadata_source_root_attestation + WHERE attestation_id=source_id; + SELECT jsonb_object_agg(encode(reference.name,'hex'), + jsonb_build_object('kind',reference.kind,'git_oid',reference.git_oid)||CASE WHEN reference.kind=4 + THEN jsonb_build_object('child_root',encode(reference.child_root,'hex'), + 'child_generation',reference.child_generation,'child_certificate',encode(reference.child_certificate_digest,'hex')) + ELSE jsonb_build_object('size',reference.byte_size,'content_digest',encode(reference.content_digest,'hex')) END) + INTO supplied FROM added_source_entries reference WHERE reference.attestation_id=source_id; + IF expected IS NULL OR expected IS DISTINCT FROM supplied THEN + RAISE EXCEPTION 'fixed source entry batch differs from independently derived complete body and fact proof'; END IF; + END LOOP; + RETURN NULL; +END $$; +CREATE TRIGGER mst2_metadata_source_entries_proof AFTER INSERT ON mst2_metadata_source_entry_reference + REFERENCING NEW TABLE AS added_source_entries FOR EACH STATEMENT EXECUTE FUNCTION mst2_metadata_source_entries_proof(); + +CREATE FUNCTION mst2_metadata_source_entries_install() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +BEGIN + INSERT INTO mst2_metadata_source_entry_reference(attestation_id,name,git_oid,kind,byte_size,content_digest, + child_root,child_generation,child_certificate_digest) + SELECT NEW.attestation_id,decode(key,'hex'),value->>'git_oid',(value->>'kind')::smallint, + (value->>'size')::bigint,decode(value->>'content_digest','hex'),decode(value->>'child_root','hex'), + (value->>'child_generation')::bigint,decode(value->>'child_certificate','hex') + FROM jsonb_each(NEW.source_proof->'source_entries'); + RETURN NULL; +END $$; +CREATE TRIGGER mst2_metadata_source_entries_install AFTER INSERT ON mst2_metadata_source_root_attestation + FOR EACH ROW EXECUTE FUNCTION mst2_metadata_source_entries_install(); + +CREATE FUNCTION mst2_metadata_source_entries_complete() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +BEGIN + IF (SELECT count(*) FROM mst2_metadata_source_entry_reference WHERE attestation_id=NEW.attestation_id) + IS DISTINCT FROM (NEW.source_proof->>'source_entry_count')::bigint THEN + RAISE EXCEPTION 'source attestation cannot commit with incomplete exact name references'; END IF; + RETURN NULL; +END $$; +CREATE CONSTRAINT TRIGGER mst2_metadata_source_entries_complete AFTER INSERT ON mst2_metadata_source_root_attestation + DEFERRABLE INITIALLY DEFERRED FOR EACH ROW EXECUTE FUNCTION mst2_metadata_source_entries_complete(); + +CREATE FUNCTION mst2_metadata_read_source_entries(op uuid,source_id uuid,p bytea,g bigint,c bytea,names jsonb) +RETURNS TABLE(name bytea,git_oid text,kind smallint,byte_size bigint,content_digest bytea,child_root bytea, + child_generation bigint,child_certificate_digest bytea,child_attestation_id uuid,fact_state text) +LANGUAGE plpgsql VOLATILE SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE a record; profile jsonb; +BEGIN + IF jsonb_typeof(names)<>'array' OR jsonb_array_length(names)>256 + OR EXISTS(SELECT 1 FROM jsonb_array_elements_text(names) wanted + WHERE wanted !~ '^([0-9a-f]{2}){1,255}$') THEN + RAISE EXCEPTION 'qualified source-name read exceeds its bounded exact request'; END IF; + SELECT source.namespace_uuid,source.source_profile,source.tagged_tree_oid,source.source_body_digest,source.source_revision, + source.root_page,source.root_generation,source.root_certificate_digest + INTO a FROM mst2_metadata_source_root_attestation source + JOIN mst2_metadata_prepare origin ON origin.prepare_id=source.origin_prepare_id + WHERE source.attestation_id=source_id AND origin.state='COMMITTED' + AND source.root_page=p AND source.root_generation=g AND source.root_certificate_digest=c + AND source.namespace_uuid='$NAMESPACE_UUID$'::uuid; + IF NOT FOUND THEN RAISE EXCEPTION 'qualified selected directory has no definitive source attestation'; END IF; + SELECT mst2_metadata_native_profile(session.prepare_id) INTO profile + FROM mst2_metadata_reader_operation reader JOIN mst2_qualified_session_incarnation session + ON session.snapshot_id=reader.snapshot_id AND session.session_incarnation=reader.session_incarnation + WHERE reader.operation_id=op AND reader.state='ACTIVE' + AND reader.hard_deadline_unix>floor(extract(epoch FROM clock_timestamp()))::bigint + AND EXISTS(SELECT 1 FROM mst2_metadata_root_anchor anchor WHERE anchor.reader_operation_id=reader.operation_id + AND anchor.anchor_kind='READER' AND anchor.root_page=reader.root_page AND anchor.root_generation=reader.root_generation); + IF NOT FOUND OR profile IS DISTINCT FROM a.source_profile + OR NOT mst2_metadata_root_live(a.root_page,a.root_generation,a.root_certificate_digest) THEN + RAISE EXCEPTION 'qualified source read lost its exact reader profile and current directory'; END IF; + IF NOT $CORE_SCHEMA$.mst2_route_source_tree_matches(split_part(a.tagged_tree_oid,':',2),a.source_revision,a.source_body_digest) THEN + RAISE EXCEPTION 'qualified selected directory source body changed'; END IF; + RETURN QUERY SELECT reference.name,reference.git_oid,reference.kind,reference.byte_size,reference.content_digest, + reference.child_root,reference.child_generation,reference.child_certificate_digest,child.attestation_id, + CASE WHEN reference.kind=4 THEN CASE WHEN child.attestation_id IS NULL THEN 'SOURCE_UNAVAILABLE' ELSE 'READY' END + WHEN fact.git_oid IS NULL THEN 'MISSING' + WHEN fact.state<>'VERIFIED' OR fact.verification_version NOT IN (1,2) + OR fact.size NOT BETWEEN 0 AND 8796093022208 OR octet_length(fact.raw_sha256)<>32 THEN 'INVALID' + WHEN fact.verification_version=1 THEN 'MISSING' + WHEN fact.size IS DISTINCT FROM reference.byte_size OR reference.kind=3 AND fact.size NOT BETWEEN 1 AND 4095 + OR CASE WHEN octet_length(fact.raw_sha256)=32 THEN fact.raw_sha256 ELSE NULL END + IS DISTINCT FROM reference.content_digest THEN 'INVALID' + ELSE 'READY' END + FROM (SELECT DISTINCT decode(value,'hex') AS name FROM jsonb_array_elements_text(names)) wanted + JOIN mst2_metadata_source_entry_reference reference ON reference.attestation_id=source_id AND reference.name=wanted.name + LEFT JOIN LATERAL ( + SELECT verified.git_oid,verified.state,verified.verification_version,verified.size,verified.raw_sha256 + FROM $CORE_SCHEMA$.mst2_verified_object verified WHERE reference.kind<>4 AND verified.storage_domain='git' + AND verified.object_kind='blob' AND verified.git_oid=split_part(reference.git_oid,':',2) + FOR SHARE OF verified NOWAIT + ) fact ON true + LEFT JOIN LATERAL ( + SELECT candidate.attestation_id FROM mst2_metadata_source_root_attestation candidate + JOIN mst2_metadata_prepare origin ON origin.prepare_id=candidate.origin_prepare_id + JOIN $CORE_SCHEMA$.mega_tree source_tree ON source_tree.tree_id=split_part(candidate.tagged_tree_oid,':',2) + WHERE reference.kind=4 AND candidate.namespace_uuid=a.namespace_uuid AND origin.state='COMMITTED' + AND candidate.tagged_tree_oid=reference.git_oid AND candidate.source_profile=a.source_profile + AND candidate.root_page=reference.child_root AND candidate.root_generation=reference.child_generation + AND candidate.root_certificate_digest=reference.child_certificate_digest + AND $CORE_SCHEMA$.mst2_route_source_tree_matches(split_part(candidate.tagged_tree_oid,':',2),candidate.source_revision,candidate.source_body_digest) + AND mst2_metadata_root_live(candidate.root_page,candidate.root_generation,candidate.root_certificate_digest) + ORDER BY candidate.attestation_id LIMIT 1 + ) child ON true ORDER BY reference.name; +END $$; diff --git a/src/jupiter/storage/qualified_metadata_source_revision_tests.rs b/src/jupiter/storage/qualified_metadata_source_revision_tests.rs new file mode 100644 index 00000000..9bbb6c54 --- /dev/null +++ b/src/jupiter/storage/qualified_metadata_source_revision_tests.rs @@ -0,0 +1,329 @@ +use super::{canonical_tests::seeded_rooted_plan, *}; +use crate::ceres::snapshot::rooted_metadata_projection::RootedReuseLookup; + +async fn revision(core: &DatabaseConnection, oid: &str) -> String { + core.query_one_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + "SELECT revision::text FROM mst2_rooted_source_tree_revision WHERE tree_id=$1", + [oid.into()], + )) + .await + .unwrap() + .unwrap() + .try_get_by_index(0) + .unwrap() +} + +#[tokio::test] +async fn source_revision_rejects_forgery_preserves_unchanged_writes_and_requires_fresh_attestation() +{ + let (config, core, namespace, q, _guard) = fixture().await; + let (plan, payload) = seeded_rooted_plan(&core, 'a', "file", 1).await; + let writer = RootedQualifiedMetadataRepository::open(&core, &config) + .await + .unwrap(); + let intent = writer + .begin_intent("source-revision-first", &plan) + .await + .unwrap(); + writer + .install_pages(&intent, std::slice::from_ref(&payload)) + .await + .unwrap(); + writer.finalize(&intent).await.unwrap(); + let oid = "a".repeat(40); + let original = revision(&core, &oid).await; + for statement in [ + "UPDATE mst2_rooted_source_tree_revision SET revision=gen_random_uuid()", + "UPDATE mst2_rooted_source_tree_revision SET body_digest=decode(repeat('11',32),'hex')", + "UPDATE mst2_rooted_source_tree_revision SET valid=false", + "DELETE FROM mst2_rooted_source_tree_revision", + "TRUNCATE mst2_rooted_source_tree_revision", + ] { + assert!( + core.execute_unprepared(statement).await.is_err(), + "{statement}" + ); + } + assert_eq!(revision(&core, &oid).await, original); + core.execute_unprepared( + "CREATE TABLE source_revision_spoof(id integer); + CREATE FUNCTION source_revision_spoof_trigger() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN UPDATE mst2_rooted_source_tree_revision SET valid=false; RETURN NEW; END $$; + CREATE TRIGGER source_revision_spoof AFTER INSERT ON source_revision_spoof + FOR EACH ROW EXECUTE FUNCTION source_revision_spoof_trigger()", + ) + .await + .unwrap(); + assert!( + core.execute_unprepared("INSERT INTO source_revision_spoof VALUES(1)") + .await + .is_err() + ); + assert_eq!(revision(&core, &oid).await, original); + core.execute_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + "UPDATE mega_tree SET sub_trees=sub_trees,pack_offset=pack_offset WHERE tree_id=$1", + [oid.clone().into()], + )) + .await + .unwrap(); + assert_eq!(revision(&core, &oid).await, original); + assert!( + writer + .lookup_reuse(&plan.identity.tagged_root_tree_oid, &plan.identity) + .await + .unwrap() + .is_some() + ); + core.execute_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + "UPDATE mega_tree SET sub_trees=set_byte(sub_trees,7,120) WHERE tree_id=$1", + [oid.clone().into()], + )) + .await + .unwrap(); + assert_ne!(revision(&core, &oid).await, original); + assert_eq!( + count( + &core, + "SELECT count(*) FROM mst2_rooted_source_tree_revision WHERE valid" + ) + .await, + 0 + ); + assert!( + writer + .lookup_reuse(&plan.identity.tagged_root_tree_oid, &plan.identity) + .await + .unwrap() + .is_none() + ); + let rejected=core.execute_raw(Statement::from_sql_and_values(DbBackend::Postgres, + "SELECT mst2_route_capture_source_tree($1,(SELECT source_body_digest FROM {q}.mst2_metadata_source_root_attestation LIMIT 1))" + .replace("{q}",&identifier(&namespace.schema)),[oid.clone().into()])).await; + assert!(rejected.is_err()); + // Restoring the same old bytes does not silently resurrect the old revision. + core.execute_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + "UPDATE mega_tree SET sub_trees=set_byte(sub_trees,7,102) WHERE tree_id=$1", + [oid.clone().into()], + )) + .await + .unwrap(); + assert_eq!( + count( + &core, + "SELECT count(*) FROM mst2_rooted_source_tree_revision WHERE valid" + ) + .await, + 0 + ); + let fresh = writer + .begin_intent("source-revision-reattest", &plan) + .await + .unwrap(); + writer.install_pages(&fresh, &[payload]).await.unwrap(); + writer.finalize(&fresh).await.unwrap(); + assert_eq!( + count( + &core, + "SELECT count(*) FROM mst2_rooted_source_tree_revision WHERE valid" + ) + .await, + 1 + ); + assert_ne!(revision(&core, &oid).await, original); + assert_eq!( + count( + &q, + "SELECT count(*) FROM mst2_metadata_source_root_attestation" + ) + .await, + 2 + ); + assert_eq!(count(&q,&format!("SELECT count(*) FROM mst2_metadata_source_root_attestation a WHERE NOT {}.mst2_route_source_tree_matches( + split_part(a.tagged_tree_oid,':',2),a.source_revision,a.source_body_digest)",identifier(&namespace.core_schema))).await,1); +} + +#[tokio::test] +async fn deleted_and_reinserted_source_oid_requires_new_independent_revision() { + let (config, core, namespace, q, _guard) = fixture().await; + let (plan, payload) = seeded_rooted_plan(&core, 'a', "file", 1).await; + let writer = RootedQualifiedMetadataRepository::open(&core, &config) + .await + .unwrap(); + let intent = writer + .begin_intent("source-delete-original", &plan) + .await + .unwrap(); + writer + .install_pages(&intent, std::slice::from_ref(&payload)) + .await + .unwrap(); + writer.finalize(&intent).await.unwrap(); + let oid = "a".repeat(40); + let original = revision(&core, &oid).await; + let saved: String = core + .query_one_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + "SELECT row_to_json(t)::text FROM mega_tree t WHERE tree_id=$1", + [oid.clone().into()], + )) + .await + .unwrap() + .unwrap() + .try_get_by_index(0) + .unwrap(); + core.execute_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + "DELETE FROM mega_tree WHERE tree_id=$1", + [oid.clone().into()], + )) + .await + .unwrap(); + let deleted = revision(&core, &oid).await; + assert_ne!(deleted, original); + assert_eq!( + count( + &core, + "SELECT count(*) FROM mst2_rooted_source_tree_revision WHERE valid" + ) + .await, + 0 + ); + assert!( + writer + .lookup_reuse(&plan.identity.tagged_root_tree_oid, &plan.identity) + .await + .unwrap() + .is_none() + ); + // Even an identical OID, row ID, and byte body cannot revive the old stamp. + core.execute_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + "INSERT INTO mega_tree SELECT (json_populate_record(NULL::mega_tree,$1::json)).*", + [saved.into()], + )) + .await + .unwrap(); + assert_eq!(revision(&core, &oid).await, deleted); + assert_eq!( + count( + &core, + "SELECT count(*) FROM mst2_rooted_source_tree_revision WHERE valid" + ) + .await, + 0 + ); + assert!( + writer + .lookup_reuse(&plan.identity.tagged_root_tree_oid, &plan.identity) + .await + .unwrap() + .is_none() + ); + let fresh = writer + .begin_intent("source-delete-reattest", &plan) + .await + .unwrap(); + writer.install_pages(&fresh, &[payload]).await.unwrap(); + writer.finalize(&fresh).await.unwrap(); + assert_ne!(revision(&core, &oid).await, deleted); + assert_eq!( + count( + &core, + "SELECT count(*) FROM mst2_rooted_source_tree_revision WHERE valid" + ) + .await, + 1 + ); + assert_eq!(count(&q,&format!("SELECT count(*) FROM mst2_metadata_source_root_attestation a WHERE NOT {}.mst2_route_source_tree_matches( + split_part(a.tagged_tree_oid,':',2),a.source_revision,a.source_body_digest)",identifier(&namespace.core_schema))).await,1); +} + +#[tokio::test] +async fn current_source_revision_uses_captured_core_relations_under_temp_shadow() { + let (config, core, namespace, q, _guard) = fixture().await; + let (plan, payload) = seeded_rooted_plan(&core, 'a', "file", 1).await; + let writer = RootedQualifiedMetadataRepository::open(&core, &config) + .await + .unwrap(); + let intent = writer + .begin_intent("source-temp-shadow", &plan) + .await + .unwrap(); + writer.install_pages(&intent, &[payload]).await.unwrap(); + writer.finalize(&intent).await.unwrap(); + q.execute_unprepared("CREATE TEMP TABLE mega_tree(id bigint,tree_id text,sub_trees bytea); + CREATE TEMP TABLE mst2_rooted_source_tree_revision(tree_id text,tree_row_id bigint,revision uuid,body_digest bytea,valid boolean)") + .await.unwrap(); + assert_eq!(count(&q,&format!("SELECT {}.mst2_route_source_tree_matches(split_part(tagged_tree_oid,':',2),source_revision,source_body_digest)::bigint + FROM mst2_metadata_source_root_attestation",identifier(&namespace.core_schema))).await,1); +} + +#[tokio::test] +async fn current_source_revision_share_fence_orders_real_source_update_after_read() { + let (config, core, namespace, q, _guard) = fixture().await; + let (plan, payload) = seeded_rooted_plan(&core, 'a', "file", 1).await; + let writer = RootedQualifiedMetadataRepository::open(&core, &config) + .await + .unwrap(); + let intent = writer + .begin_intent("source-share-fence", &plan) + .await + .unwrap(); + writer.install_pages(&intent, &[payload]).await.unwrap(); + writer.finalize(&intent).await.unwrap(); + let read = q.begin().await.unwrap(); + assert_eq!(count(&read,&format!("SELECT {}.mst2_route_source_tree_matches(split_part(tagged_tree_oid,':',2),source_revision,source_body_digest)::bigint + FROM mst2_metadata_source_root_attestation",identifier(&namespace.core_schema))).await,1); + let core_writer = core.clone(); + let (ready, received) = tokio::sync::oneshot::channel(); + let pending = tokio::spawn(async move { + let txn = core_writer.begin().await.unwrap(); + let pid: i32 = txn + .query_one_raw(Statement::from_string( + DbBackend::Postgres, + "SELECT pg_backend_pid()", + )) + .await + .unwrap() + .unwrap() + .try_get_by_index(0) + .unwrap(); + ready.send(pid).unwrap(); + txn.execute_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + "UPDATE mega_tree SET sub_trees=set_byte(sub_trees,7,120) WHERE tree_id=$1", + ["a".repeat(40).into()], + )) + .await + .unwrap(); + txn.commit().await + }); + let pid = received.await.unwrap(); + tokio::time::timeout(Duration::from_secs(4),async { + loop { + let waiting:bool=core.query_one_raw(Statement::from_sql_and_values(DbBackend::Postgres, + "SELECT coalesce(wait_event_type='Lock',false) FROM pg_stat_activity WHERE pid=$1",[pid.into()])) + .await.unwrap().unwrap().try_get_by_index(0).unwrap(); + if waiting {break;} tokio::task::yield_now().await; + } + }).await.unwrap(); + assert!(!pending.is_finished()); + read.commit().await.unwrap(); + tokio::time::timeout(Duration::from_secs(4), pending) + .await + .unwrap() + .unwrap() + .unwrap(); + assert_eq!( + count( + &core, + "SELECT count(*) FROM mst2_rooted_source_tree_revision WHERE valid" + ) + .await, + 0 + ); +} diff --git a/src/jupiter/storage/qualified_source_revision.sql b/src/jupiter/storage/qualified_source_revision.sql new file mode 100644 index 00000000..855112ce --- /dev/null +++ b/src/jupiter/storage/qualified_source_revision.sql @@ -0,0 +1,124 @@ +-- This records only trees actually attested by Q, not the global Git inventory. +CREATE TABLE mst2_rooted_source_tree_revision ( + tree_id text PRIMARY KEY CHECK(tree_id ~ '^([0-9a-f]{40}|[0-9a-f]{64})$'), + tree_row_id bigint NOT NULL,revision uuid NOT NULL,body_digest bytea NOT NULL CHECK(octet_length(body_digest)=32), + valid boolean NOT NULL, + CHECK(substr(revision::text,15,1)='4' AND substr(revision::text,20,1) IN ('8','9','a','b')) +); + +CREATE FUNCTION mst2_route_source_tree_revision_guard() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$CORE_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE source_id bigint; body bytea; +BEGIN + IF TG_OP='DELETE' OR TG_OP='TRUNCATE' THEN RAISE EXCEPTION 'rooted source revision watermarks are immutable'; END IF; + IF pg_is_in_recovery() OR current_setting('transaction_isolation')<>'read committed' + OR TG_RELID<>'mst2_rooted_source_tree_revision'::regclass THEN + RAISE EXCEPTION 'rooted source revision requires its captured primary relation'; END IF; + IF TG_OP='UPDATE' THEN + IF NEW.tree_id IS DISTINCT FROM OLD.tree_id THEN RAISE EXCEPTION 'rooted source revision cannot retarget its OID'; END IF; + IF NEW.valid IS FALSE THEN + -- A nested caller cannot invent invalidation: independently observe the + -- actual core mutation after it happened in this still-uncommitted writer. + IF pg_trigger_depth()<2 OR NOT OLD.valid OR NEW.tree_row_id IS DISTINCT FROM OLD.tree_row_id + OR NEW.body_digest IS DISTINCT FROM OLD.body_digest OR NEW.revision IS DISTINCT FROM OLD.revision THEN + RAISE EXCEPTION 'rooted source invalidation is outside its actual source writer'; END IF; + PERFORM mst2_route_enter($CORE_LITERAL$); + SELECT id,sub_trees INTO source_id,body FROM mega_tree WHERE tree_id=OLD.tree_id; + IF FOUND AND source_id=OLD.tree_row_id AND octet_length(body)<=67108864 + AND sha256(body)=OLD.body_digest THEN + RAISE EXCEPTION 'rooted source invalidation has no actual changed or deleted core bytes'; END IF; + NEW.revision:=gen_random_uuid(); RETURN NEW; + END IF; + IF OLD.valid OR NEW.revision IS DISTINCT FROM OLD.revision OR NEW.valid IS DISTINCT FROM true THEN + RAISE EXCEPTION 'ordinary DML cannot rewrite a valid rooted source revision'; END IF; + END IF; + PERFORM mst2_route_enter($CORE_LITERAL$); + SELECT id,sub_trees INTO source_id,body FROM mega_tree WHERE tree_id=NEW.tree_id FOR SHARE; + IF NOT FOUND OR octet_length(body)>67108864 THEN RAISE EXCEPTION 'rooted source capture has no bounded actual core tree'; END IF; + IF NEW.body_digest IS DISTINCT FROM sha256(body) THEN + RAISE EXCEPTION 'rooted source revision was not independently derived from its actual core bytes'; END IF; + NEW.tree_row_id:=source_id; NEW.revision:=gen_random_uuid(); NEW.valid:=true; + RETURN NEW; +END $$; +CREATE TRIGGER mst2_route_source_tree_revision_guard BEFORE INSERT OR UPDATE OR DELETE ON mst2_rooted_source_tree_revision + FOR EACH ROW EXECUTE FUNCTION mst2_route_source_tree_revision_guard(); +CREATE TRIGGER mst2_route_source_tree_revision_truncate BEFORE TRUNCATE ON mst2_rooted_source_tree_revision + FOR EACH STATEMENT EXECUTE FUNCTION mst2_route_source_tree_revision_guard(); + +CREATE FUNCTION mst2_route_source_tree_inventory_guard() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$CORE_SCHEMA$,pg_catalog,pg_temp AS $$ +BEGIN + IF (SELECT count(*) FROM (SELECT 1 FROM mst2_rooted_source_tree_revision LIMIT 65537) bounded)>65536 THEN + RAISE EXCEPTION 'rooted attested source inventory capacity is exceeded'; END IF; + RETURN NULL; +END $$; +CREATE TRIGGER mst2_route_source_tree_inventory_guard AFTER INSERT ON mst2_rooted_source_tree_revision + FOR EACH STATEMENT EXECUTE FUNCTION mst2_route_source_tree_inventory_guard(); + +CREATE FUNCTION mst2_route_source_tree_writer_enter() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$CORE_SCHEMA$,pg_catalog,pg_temp AS $$ +BEGIN + IF TG_RELID<>'mega_tree'::regclass THEN RAISE EXCEPTION 'rooted source writer left its captured core relation'; END IF; + IF TG_OP='TRUNCATE' THEN RAISE EXCEPTION 'core tree truncation cannot bypass rooted source revision invalidation'; END IF; + PERFORM mst2_route_enter($CORE_LITERAL$); + RETURN NULL; +END $$; +CREATE TRIGGER mst2_route_source_tree_writer_enter BEFORE INSERT OR UPDATE OR DELETE OR TRUNCATE ON mega_tree + FOR EACH STATEMENT EXECUTE FUNCTION mst2_route_source_tree_writer_enter(); + +CREATE FUNCTION mst2_route_source_tree_invalidate() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$CORE_SCHEMA$,pg_catalog,pg_temp AS $$ +BEGIN + IF TG_RELID<>'mega_tree'::regclass THEN RAISE EXCEPTION 'rooted invalidation left its captured core relation'; END IF; + IF TG_OP='UPDATE' AND NEW.tree_id IS NOT DISTINCT FROM OLD.tree_id AND NEW.id IS NOT DISTINCT FROM OLD.id + AND NEW.sub_trees IS NOT DISTINCT FROM OLD.sub_trees THEN RETURN NEW; END IF; + UPDATE mst2_rooted_source_tree_revision SET valid=false WHERE tree_id=OLD.tree_id AND valid; + IF TG_OP='DELETE' THEN RETURN OLD; END IF; + RETURN NEW; +END $$; +CREATE TRIGGER mst2_route_source_tree_invalidate AFTER UPDATE OR DELETE ON mega_tree + FOR EACH ROW EXECUTE FUNCTION mst2_route_source_tree_invalidate(); +CREATE FUNCTION mst2_route_source_tree_inserted() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$CORE_SCHEMA$,pg_catalog,pg_temp AS $$ +BEGIN + IF TG_RELID<>'mega_tree'::regclass THEN RAISE EXCEPTION 'rooted insertion left its captured core relation'; END IF; + UPDATE mst2_rooted_source_tree_revision SET valid=false WHERE tree_id=NEW.tree_id AND valid; + RETURN NULL; +END $$; +CREATE TRIGGER mst2_route_source_tree_inserted AFTER INSERT ON mega_tree + FOR EACH ROW EXECUTE FUNCTION mst2_route_source_tree_inserted(); + +CREATE FUNCTION mst2_route_capture_source_tree(oid_text text,expected_digest bytea) RETURNS uuid LANGUAGE plpgsql VOLATILE +SET search_path=$CORE_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE captured mst2_rooted_source_tree_revision%ROWTYPE; actual_id bigint; +BEGIN + PERFORM mst2_route_enter($CORE_LITERAL$); + SELECT id INTO actual_id FROM mega_tree WHERE tree_id=oid_text FOR SHARE; + IF NOT FOUND OR expected_digest IS NULL OR octet_length(expected_digest)<>32 THEN + RAISE EXCEPTION 'rooted source capture has no exact actual row and body digest'; END IF; + SELECT * INTO captured FROM mst2_rooted_source_tree_revision WHERE tree_id=oid_text FOR UPDATE; + IF FOUND THEN + IF captured.valid THEN + IF captured.tree_row_id<>actual_id OR captured.body_digest IS DISTINCT FROM expected_digest THEN + RAISE EXCEPTION 'rooted source bytes changed behind their exact immutable revision'; END IF; + RETURN captured.revision; + END IF; + UPDATE mst2_rooted_source_tree_revision SET valid=true,body_digest=expected_digest WHERE tree_id=oid_text + RETURNING revision INTO captured.revision; + ELSE + INSERT INTO mst2_rooted_source_tree_revision(tree_id,tree_row_id,revision,body_digest,valid) + VALUES(oid_text,actual_id,gen_random_uuid(),expected_digest,true) RETURNING revision INTO captured.revision; + END IF; + RETURN captured.revision; +END $$; + +CREATE FUNCTION mst2_route_source_tree_matches(oid_text text,exact_revision uuid,exact_digest bytea) +RETURNS boolean LANGUAGE plpgsql VOLATILE STRICT SET search_path=$CORE_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE captured mst2_rooted_source_tree_revision%ROWTYPE; +BEGIN + IF pg_is_in_recovery() OR current_setting('transaction_isolation')<>'read committed' THEN RETURN false; END IF; + SELECT revision.* INTO captured FROM mst2_rooted_source_tree_revision revision + JOIN mega_tree source ON source.id=revision.tree_row_id AND source.tree_id=revision.tree_id + WHERE revision.tree_id=oid_text FOR SHARE OF revision NOWAIT; + RETURN FOUND AND captured.valid AND captured.revision=exact_revision AND captured.body_digest=exact_digest; +END $$; diff --git a/src/jupiter/tests.rs b/src/jupiter/tests.rs index 4f39f1ad..aa3dd08d 100644 --- a/src/jupiter/tests.rs +++ b/src/jupiter/tests.rs @@ -223,6 +223,45 @@ fn drop_test_schema(admin_url: &str, schema: &str) { [Value::from(schema)], )) .await?; + // A bootstrap-created physical Q family belongs to this exact test + // core schema. Drop the pair even on panic, before its core FK targets. + let registry = admin + .query_one_raw(Statement::from_sql_and_values( + DatabaseBackend::Postgres, + "SELECT pg_catalog.to_regclass($1) IS NOT NULL AS present", + [format!("{schema}.mst2_metadata_namespace").into()], + )) + .await? + .unwrap() + .try_get::("", "present")?; + if registry { + let families = admin + .query_all_raw(Statement::from_string( + DatabaseBackend::Postgres, + format!( + "SELECT n.nspname FROM {schema}.mst2_metadata_namespace q + JOIN pg_catalog.pg_namespace n ON n.oid=q.metadata_schema_oid + JOIN pg_catalog.pg_namespace c ON c.oid=q.core_schema_oid + WHERE q.graph_domain='qualified-v1' AND c.nspname='{schema}' + AND q.core_schema='{schema}' AND q.family_identity='v3-rooted-qualified-1'" + ), + )) + .await?; + for family in families { + let q_schema: String = family.try_get("", "nspname")?; + if !q_schema.starts_with("mst2q_") { + return Err(sea_orm::DbErr::Custom( + "test Q schema escaped its generated prefix".into(), + )); + } + let quoted = format!("\"{}\"", q_schema.replace('"', "\"\"")); + admin + .execute_unprepared(&format!( + "SET lock_timeout='120s'; DROP SCHEMA {quoted} CASCADE" + )) + .await?; + } + } // The timeout turns a lock held by anything else into a leaked schema // and a message, not a hung test run. admin @@ -380,6 +419,8 @@ pub async fn test_storage_with_config(temp_dir: impl AsRef, config: Config native_projection_cache: Arc::default(), native_snapshot_sessions: Arc::default(), native_chunk_maps: Arc::default(), + shadow_qualified_metadata: Arc::default(), + rooted_qualified_metadata: Arc::default(), projection_observation_sink: None, cl_service: CLService::mock(), push_queue_service: PushQueueService::new(