From 6cf856422a3e78379061e5c4fcfcd84a574bdf0a Mon Sep 17 00:00:00 2001 From: Xiaoyang Han Date: Thu, 8 Oct 2026 06:38:14 +0800 Subject: [PATCH] perf(v3): serve fixed snapshots from rooted delta metadata Commit publication builds canonical local directory proofs and reuses unchanged fixed-root metadata. Default qualified-family serving reads bounded DIR windows and selected LOOKUP routes with current source revision facts; warm body callers avoid Git-tree projection. Permanent snapshot routes, exact publication and lease identities, source mutation invalidation, root ownership and bounded background recovery/GC remain enforced. The authority catalog validates exact registered physical families, full qualified shape and source revision triggers, including precise treatment of legitimate qualified foreign-key RI triggers. Direct lease lock contention maps to retryable HTTP 503. Publication-disabled formal MST/2 sessions retain fixed-root HEAD/body/OBJECT/CHUNK service. Migration 000200 and actual PostgreSQL/HTTP regression sources are integrated without raising existing projection limits. Independent review covers all 46 paths, all four corrected findings and every changed assertion site. Format, diff and locked offline metadata pass; native PostgreSQL execution and performance measurement remain separate. --- src/api/router/snapshot_content.rs | 170 ++- src/api/router/snapshot_content_tests.rs | 220 +++- .../snapshot_generation_upgrade_tests.rs | 18 +- .../snapshot_persisted_metadata_tests.rs | 23 +- src/api/router/snapshot_raw_blob.rs | 8 +- src/api/router/snapshot_rooted_metadata.rs | 185 +++ .../router/snapshot_rooted_metadata_tests.rs | 784 ++++++++++++ src/api/router/snapshot_router.rs | 229 +++- src/api/router/snapshot_session_tests.rs | 42 +- .../router/snapshot_storage_route_fixture.rs | 12 +- .../router/snapshot_storage_route_tests.rs | 44 +- src/ceres/snapshot/mod.rs | 2 + src/ceres/snapshot/projection_observation.rs | 94 +- src/ceres/snapshot/rooted_metadata_install.rs | 848 +++++++++++++ .../snapshot/rooted_metadata_projection.rs | 1108 +++++++++++++++++ ...000200_add_mst2_rooted_qualified_family.rs | 125 ++ ...0261008_000200_rooted_qualified_family.sql | 161 +++ .../m20261008_000200_rooted_routes.sql | 171 +++ src/jupiter/migration/mod.rs | 22 +- src/jupiter/storage/init.rs | 20 + src/jupiter/storage/mod.rs | 63 +- .../storage/native_metadata_generations.rs | 6 +- .../storage/native_metadata_install.rs | 5 + .../storage/native_metadata_qualified.rs | 113 +- .../storage/native_snapshot_session.rs | 74 +- .../storage/qualified_family_catalog.sql | 57 + .../storage/qualified_family_shape.sql | 65 + .../storage/qualified_metadata_anchors.sql | 426 +++++++ .../storage/qualified_metadata_canonical.sql | 227 ++++ .../qualified_metadata_canonical_tests.rs | 688 ++++++++++ .../qualified_metadata_certificates.sql | 293 +++++ .../storage/qualified_metadata_family.rs | 624 ++++++++++ .../storage/qualified_metadata_family.sql | 491 ++++++++ .../qualified_metadata_family_tests.rs | 855 +++++++++++++ src/jupiter/storage/qualified_metadata_gc.rs | 540 ++++++++ src/jupiter/storage/qualified_metadata_gc.sql | 574 +++++++++ .../storage/qualified_metadata_gc_tests.rs | 456 +++++++ .../storage/qualified_metadata_reader.rs | 961 ++++++++++++++ .../storage/qualified_metadata_rooted.rs | 766 ++++++++++++ .../storage/qualified_metadata_rooted.sql | 358 ++++++ .../storage/qualified_metadata_serving.sql | 581 +++++++++ .../storage/qualified_metadata_session.rs | 291 +++++ .../qualified_metadata_source_read.sql | 132 ++ ...ualified_metadata_source_revision_tests.rs | 329 +++++ .../storage/qualified_source_revision.sql | 124 ++ src/jupiter/tests.rs | 41 + 46 files changed, 13256 insertions(+), 170 deletions(-) create mode 100644 src/api/router/snapshot_rooted_metadata.rs create mode 100644 src/api/router/snapshot_rooted_metadata_tests.rs create mode 100644 src/ceres/snapshot/rooted_metadata_install.rs create mode 100644 src/ceres/snapshot/rooted_metadata_projection.rs create mode 100644 src/jupiter/migration/m20261008_000200_add_mst2_rooted_qualified_family.rs create mode 100644 src/jupiter/migration/m20261008_000200_rooted_qualified_family.sql create mode 100644 src/jupiter/migration/m20261008_000200_rooted_routes.sql create mode 100644 src/jupiter/storage/qualified_family_catalog.sql create mode 100644 src/jupiter/storage/qualified_family_shape.sql create mode 100644 src/jupiter/storage/qualified_metadata_anchors.sql create mode 100644 src/jupiter/storage/qualified_metadata_canonical.sql create mode 100644 src/jupiter/storage/qualified_metadata_canonical_tests.rs create mode 100644 src/jupiter/storage/qualified_metadata_certificates.sql create mode 100644 src/jupiter/storage/qualified_metadata_family.rs create mode 100644 src/jupiter/storage/qualified_metadata_family.sql create mode 100644 src/jupiter/storage/qualified_metadata_family_tests.rs create mode 100644 src/jupiter/storage/qualified_metadata_gc.rs create mode 100644 src/jupiter/storage/qualified_metadata_gc.sql create mode 100644 src/jupiter/storage/qualified_metadata_gc_tests.rs create mode 100644 src/jupiter/storage/qualified_metadata_reader.rs create mode 100644 src/jupiter/storage/qualified_metadata_rooted.rs create mode 100644 src/jupiter/storage/qualified_metadata_rooted.sql create mode 100644 src/jupiter/storage/qualified_metadata_serving.sql create mode 100644 src/jupiter/storage/qualified_metadata_session.rs create mode 100644 src/jupiter/storage/qualified_metadata_source_read.sql create mode 100644 src/jupiter/storage/qualified_metadata_source_revision_tests.rs create mode 100644 src/jupiter/storage/qualified_source_revision.sql diff --git a/src/api/router/snapshot_content.rs b/src/api/router/snapshot_content.rs index 6a1f191f..275f7dd7 100644 --- a/src/api/router/snapshot_content.rs +++ b/src/api/router/snapshot_content.rs @@ -50,7 +50,7 @@ pub(super) async fn fixed_root_tree( +async fn resolve_legacy_file_metadata( handler: &T, root_tree: &git_internal::internal::object::tree::Tree, scope: &str, @@ -89,6 +89,132 @@ pub(super) async fn resolve_file_metadata( + handler: &T, + ctx: &crate::ceres::snapshot::runtime::SnapshotContext, +) -> Result, Response> { + use crate::jupiter::storage::qualified_metadata_family::SnapshotMetadataFamily; + if !handler.get_context().config().mst2.publication_enabled { + return fixed_root_tree(handler, &ctx.root_tree_oid).await.map(Some); + } + match handler + .get_context() + .snapshot_metadata_family(&ctx.lease_id, true) + .await + .map_err(mst2_error_response)? + { + Some(SnapshotMetadataFamily::Rooted) => Ok(None), + Some(SnapshotMetadataFamily::Generic) => { + fixed_root_tree(handler, &ctx.root_tree_oid).await.map(Some) + } + None => Err(mst2_error_response(SnapshotError::new( + SnapshotErrorCode::SnapshotGone, + "fixed content lease has no permanent storage route", + ))), + } +} + +#[allow(clippy::result_large_err)] +async fn resolve_file_metadata( + handler: &T, + root_tree: Option<&git_internal::internal::object::tree::Tree>, + ctx: &crate::ceres::snapshot::runtime::SnapshotContext, + path: &str, + expected_digest: Option<&str>, +) -> Result { + if let Some(root_tree) = root_tree { + return resolve_legacy_file_metadata( + handler, + root_tree, + &ctx.built.descriptor.scope, + path, + expected_digest, + ) + .await; + } + use crate::jupiter::storage::qualified_metadata_family::RootedLookupStatus; + let storage = handler.get_context(); + let repository = storage + .rooted_qualified_metadata_writer() + .await + .map_err(internal)?; + let (entry, git_oid) = match repository + .fixed_path_metadata(ctx, path) + .await + .map_err(mst2_error_response)? + { + RootedLookupStatus::File { entry, git_oid } => (entry, git_oid), + RootedLookupStatus::Directory(_) => { + return Err(mst2_error_response(SnapshotError::new( + SnapshotErrorCode::NotDirectory, + format!("{path} is a directory"), + ))); + } + RootedLookupStatus::Absent => { + return Err(mst2_error_response(SnapshotError::new( + SnapshotErrorCode::PathNotFound, + format!("{path} absent in the fixed view"), + ))); + } + RootedLookupStatus::NotDirectory { symlink } => { + return Err(mst2_error_response(SnapshotError::new( + if symlink { + SnapshotErrorCode::SymlinkTraversal + } else { + SnapshotErrorCode::NotDirectory + }, + format!("{path}: intermediate component is not a directory"), + ))); + } + }; + let fs_kind = match entry.kind { + mst2_codec::metapage::EntryKind::Regular => FsKind::Regular, + mst2_codec::metapage::EntryKind::Executable => FsKind::Executable, + mst2_codec::metapage::EntryKind::Symlink => FsKind::Symlink, + mst2_codec::metapage::EntryKind::Directory => { + return Err(mst2_error_response(internal( + "fixed source file has a directory kind", + ))); + } + }; + let oid = git_oid + .split_once(':') + .ok_or_else(|| mst2_error_response(internal("fixed source OID has no hash kind")))? + .1 + .to_owned(); + let file = verified_file_metadata(handler, fs_kind, oid, path, expected_digest).await?; + if file.size != entry.size || file.digest != entry.content_id { + return Err(mst2_error_response(SnapshotError::new( + SnapshotErrorCode::IntegrityError, + "fixed content metadata changed from its certified current source occurrence", + ))); + } + Ok(file) +} + +#[allow(clippy::result_large_err)] +pub(super) async fn resolve_snapshot_file_metadata( + state: &crate::api::MonoApiServiceState, + ctx: &crate::ceres::snapshot::runtime::SnapshotContext, + path: &str, + expected_digest: Option<&str>, +) -> Result { + let handler = state + .api_handler(std::path::Path::new("/")) + .await + .map_err(internal)?; + let root_tree = fixed_content_root(handler.as_ref(), ctx).await?; + resolve_file_metadata( + handler.as_ref(), + root_tree.as_ref(), + ctx, + path, + expected_digest, + ) + .await +} + #[allow(clippy::result_large_err)] pub(super) async fn verified_file_metadata( handler: &T, @@ -264,11 +390,11 @@ pub(super) async fn blob_head( .api_handler(std::path::Path::new("/")) .await .map_err(internal)?; - let root_tree = fixed_root_tree(handler.as_ref(), &ctx.root_tree_oid).await?; + let root_tree = fixed_content_root(handler.as_ref(), &ctx).await?; let f = resolve_file_metadata( handler.as_ref(), - &root_tree, - &ctx.built.descriptor.scope, + root_tree.as_ref(), + &ctx, &q.path, q.expected_digest.as_deref(), ) @@ -331,20 +457,19 @@ pub(super) async fn objects( .api_handler(std::path::Path::new("/")) .await .map_err(internal)?; - let root_tree = fixed_root_tree(handler.as_ref(), &ctx.root_tree_oid).await?; - let scope = ctx.built.descriptor.scope.clone(); + let root_tree = fixed_content_root(handler.as_ref(), &ctx).await?; // Copied references: each per-item future borrows the shared walk state // without moving it (the stream is an FnMut over owned items). let handler_ref = handler.as_ref(); - let root_ref = &root_tree; - let scope_ref = &scope; + let root_ref = root_tree.as_ref(); + let ctx_ref = &ctx; let resolved: Vec> = futures::stream::iter(req.items.clone()) .map(move |item| async move { validate_scope_relative_path(&item.path).map_err(mst2_error_response)?; let f = resolve_file_metadata( handler_ref, root_ref, - scope_ref, + ctx_ref, &item.path, Some(&item.expected_digest), ) @@ -468,13 +593,13 @@ pub(super) struct ChunkMapQuery { #[allow(clippy::result_large_err)] async fn project_for( handler: &T, - root_tree: &git_internal::internal::object::tree::Tree, - scope: &str, + root_tree: Option<&git_internal::internal::object::tree::Tree>, + ctx: &crate::ceres::snapshot::runtime::SnapshotContext, path: &str, expected_digest: Option<&str>, ) -> Result, Response> { - let f = resolve_file_metadata(handler, root_tree, scope, path, expected_digest).await?; + let f = resolve_file_metadata(handler, root_tree, ctx, path, expected_digest).await?; project_resolved(handler, &f).await } @@ -552,12 +677,11 @@ pub(super) async fn chunk_map( .api_handler(std::path::Path::new("/")) .await .map_err(internal)?; - let root_tree = fixed_root_tree(handler.as_ref(), &ctx.root_tree_oid).await?; - let scope = ctx.built.descriptor.scope.clone(); + let root_tree = fixed_content_root(handler.as_ref(), &ctx).await?; let proj = project_for( handler.as_ref(), - &root_tree, - &scope, + root_tree.as_ref(), + &ctx, &q.path, q.expected_digest.as_deref(), ) @@ -619,12 +743,11 @@ pub(super) async fn chunk_map_pages( .api_handler(std::path::Path::new("/")) .await .map_err(internal)?; - let root_tree = fixed_root_tree(handler.as_ref(), &ctx.root_tree_oid).await?; - let scope = ctx.built.descriptor.scope.clone(); + let root_tree = fixed_content_root(handler.as_ref(), &ctx).await?; let proj = project_for( handler.as_ref(), - &root_tree, - &scope, + root_tree.as_ref(), + &ctx, &q.path, q.expected_digest.as_deref(), ) @@ -894,8 +1017,7 @@ pub(super) async fn chunks( .api_handler(std::path::Path::new("/")) .await .map_err(internal)?; - let root_tree = fixed_root_tree(handler.as_ref(), &ctx.root_tree_oid).await?; - let scope = ctx.built.descriptor.scope.clone(); + let root_tree = fixed_content_root(handler.as_ref(), &ctx).await?; let mut planned: Vec = Vec::new(); let mut resolved: Vec = Vec::new(); let mut units: Vec<(String, u64)> = Vec::new(); @@ -907,8 +1029,8 @@ pub(super) async fn chunks( parse_decimal_count(&item.chunk_index, "chunk_index").map_err(mst2_error_response)?; let file = resolve_file_metadata( handler.as_ref(), - &root_tree, - &scope, + root_tree.as_ref(), + &ctx, &item.path, Some(&item.expected_digest), ) diff --git a/src/api/router/snapshot_content_tests.rs b/src/api/router/snapshot_content_tests.rs index 617cf2ea..65234d8b 100644 --- a/src/api/router/snapshot_content_tests.rs +++ b/src/api/router/snapshot_content_tests.rs @@ -107,6 +107,8 @@ mod generation_qualified_fixture; mod install_capability_fixture; type ReceiptWriteHold = (Arc, Arc); +#[path = "snapshot_rooted_metadata_tests.rs"] +mod rooted_metadata; #[derive(Default)] struct ReadCounts { @@ -393,6 +395,7 @@ impl LogStorage for CountingStorage { struct Fixture { app: Router, state: MonoApiServiceState, + generic_history: bool, snapshot: String, lease: String, oid: String, @@ -503,12 +506,47 @@ impl Fixture { rebuildable: bool, directory_count: usize, objects: &[(String, Vec)], + ) -> Self { + Self::new_in_metadata_family(rebuildable, directory_count, objects, false).await + } + + async fn new_generic_history_with_pg_config(rebuildable: bool) -> Self { + Self::new_generic_history_with_pg_config_and_directories(rebuildable, 0).await + } + + async fn new_generic_history_with_pg_config_and_directories( + rebuildable: bool, + directory_count: usize, + ) -> Self { + Self::new_in_metadata_family(rebuildable, directory_count, &[], true).await + } + + async fn new_in_metadata_family( + rebuildable: bool, + directory_count: usize, + objects: &[(String, Vec)], + generic_history: bool, + ) -> Self { + Self::new_in_publication_mode(rebuildable, directory_count, objects, generic_history, true) + .await + } + + async fn new_without_publication() -> Self { + Self::new_in_publication_mode(true, 0, &[], false, false).await + } + + async fn new_in_publication_mode( + rebuildable: bool, + directory_count: usize, + objects: &[(String, Vec)], + generic_history: bool, + publication_enabled: bool, ) -> Self { let temp = tempfile::tempdir().unwrap(); let mut config = isolated_config(temp.path().join("config")); config.monorepo.push_policy = PushPolicy::Trunk; config.mst2.enabled = true; - config.mst2.publication_enabled = true; + config.mst2.publication_enabled = publication_enabled; config.mst2.instance_uuid = Some(uuid::Uuid::new_v4().to_string()); config.mst2.auth_token = Some(TOKEN.to_string()); let backend = build_object_storage(&config.object_storage).await.unwrap(); @@ -516,22 +554,37 @@ impl Fixture { let (mut storage, schema) = if rebuildable { let (database, schema) = test_db_config(temp.path()).await; config.database = database; - let connection = crate::jupiter::storage::init::database_connection(&config.database) - .await - .unwrap(); - ( + let assembly = async { + let connection = + crate::jupiter::storage::init::database_connection(&config.database) + .await + .unwrap(); crate::jupiter::storage::Storage::new_with_connection( Arc::new(config), Arc::new(connection), backend.clone(), ) .await - .unwrap(), - Some(schema), - ) + .unwrap() + }; + let storage = if generic_history { + crate::jupiter::storage::init::with_generic_history_bootstrap(assembly).await + } else { + assembly.await + }; + (storage, Some(schema)) } else { (test_storage_with_config(temp.path(), config).await, None) }; + if generic_history { + let q_rows:i64=storage.mono_storage().get_connection().query_one_raw(sea_orm::Statement::from_string( + sea_orm::DbBackend::Postgres,"SELECT count(*) FROM mst2_metadata_namespace WHERE graph_domain='qualified-v1'")) + .await.unwrap().unwrap().try_get_by_index(0).unwrap(); + assert_eq!( + q_rows, 0, + "G history fixtures must not create or erase actual Q ownership" + ); + } storage.git_service = GitService { obj_storage: MegaObjectStorageWrapper::new(Arc::new(CountingStorage { inner: backend, @@ -664,18 +717,24 @@ impl Fixture { ) .await .unwrap(); - mono.initialize_native_publication(storage.config().mst2.instance_uuid.as_deref().unwrap()) - .await - .unwrap(); - publish_native_push(&storage, "/project", old_tip.id, &new_tip).await; - let head = mono - .read_native_publication_head(storage.config().mst2.instance_uuid.as_deref().unwrap()) + if publication_enabled { + mono.initialize_native_publication( + storage.config().mst2.instance_uuid.as_deref().unwrap(), + ) .await .unwrap(); - assert_eq!(head.token.sequence, 1); - assert!(head.token.certificate.is_some()); - assert_eq!(head.root.commit, commit.id.to_string()); - assert_eq!(head.root.tree, root.id.to_string()); + publish_native_push(&storage, "/project", old_tip.id, &new_tip).await; + let head = mono + .read_native_publication_head( + storage.config().mst2.instance_uuid.as_deref().unwrap(), + ) + .await + .unwrap(); + assert_eq!(head.token.sequence, 1); + assert!(head.token.certificate.is_some()); + assert_eq!(head.root.commit, commit.id.to_string()); + assert_eq!(head.root.tree, root.id.to_string()); + } let state = MonoApiServiceState { entity_store: storage.entity_store.clone(), storage, @@ -686,7 +745,12 @@ impl Fixture { }), listen_addr: "127.0.0.1:0".to_string(), }; - let app = Router::new().nest("/api/v2", routers(state.clone()).with_state(state.clone())); + let routes = if generic_history { + crate::api::router::snapshot_router::generic_history_routers(state.clone()) + } else { + routers(state.clone()) + }; + let app = Router::new().nest("/api/v2", routes.with_state(state.clone())); let response = app .clone() .oneshot( @@ -724,6 +788,7 @@ impl Fixture { digest: Sha256::digest(&raw).into(), raw, counts, + generic_history, _temp: temp, _schema: schema, } @@ -915,6 +980,123 @@ async fn mst2_fixed_head_uses_verified_facts_without_body_reads_and_preserves_ra assert_eq!(fixture.counts.receipt_writes.load(Ordering::SeqCst), 1); } +#[tokio::test] +async fn mst2_publication_disabled_resolve_preserves_head_body_object_map_page_and_chunk_content() { + let fixture = Fixture::new_without_publication().await; + assert!(!fixture.state.storage.config().mst2.publication_enabled); + assert_eq!( + fixture + .state + .storage + .snapshot_metadata_family(&fixture.lease, true) + .await + .unwrap(), + None + ); + let routes: i64 = fixture + .state + .storage + .mono_storage() + .get_connection() + .query_one_raw(sea_orm::Statement::from_string( + sea_orm::DbBackend::Postgres, + "SELECT count(*) FROM mst2_snapshot_storage_route", + )) + .await + .unwrap() + .unwrap() + .try_get_by_index(0) + .unwrap(); + assert_eq!( + routes, 0, + "runtime resolve does not create a permanent SID route" + ); + let head = fixture.send("HEAD", "blob?path=/file", Body::empty()).await; + assert_eq!(head.status(), 200); + assert_eq!( + head.headers()["content-length"], + fixture.raw.len().to_string() + ); + assert_eq!( + head.headers()["etag"], + format!("\"{}\"", fixture.digest_string()) + ); + assert!(to_bytes(head.into_body(), 1024).await.unwrap().is_empty()); + fixture.counts.assert(0, 0); + let body = fixture.send("GET", "blob?path=/file", Body::empty()).await; + assert_eq!(body.status(), 200); + assert_eq!( + to_bytes(body.into_body(), 2 * 1024 * 1024) + .await + .unwrap() + .as_ref(), + fixture.raw + ); + let link_digest = digest(b"file"); + let request=json!({"items":[{"path":"/link","expected_digest":format!("sha256:{}",hex_of(&link_digest))}],"encoding":"identity"}).to_string(); + let objects = fixture + .send("POST", "objects", Body::from(request.clone())) + .await; + assert_eq!(objects.status(), 200); + let wire = to_bytes(objects.into_body(), 2 * 1024 * 1024) + .await + .unwrap(); + let frames = parse_stream(&wire).unwrap(); + let [Frame::Object(object), Frame::End(end)] = frames.as_slice() else { + panic!("expected OBJECT and terminal END"); + }; + assert_eq!(object.objects, vec![(link_digest, b"file".to_vec())]); + assert_eq!(end.request_item_count, 1); + assert_eq!(end.logical_bytes, 4); + assert_eq!( + end.request_body_sha256, + <[u8; 32]>::from(Sha256::digest(request.as_bytes())) + ); + let map = fixture.map("/file").await; + let map_id = map["map"]["map_id"].as_str().unwrap(); + assert_eq!(map["map"]["file_content_id"], fixture.digest_string()); + let page = success_json( + fixture + .send( + "GET", + &format!("chunk-map/pages?path=/file&map_id={map_id}&page_index=0"), + Body::empty(), + ) + .await, + ) + .await; + let leaf = ChunkLeaf::decode( + &STANDARD + .decode(page["leaf_base64"].as_str().unwrap()) + .unwrap(), + ) + .unwrap(); + let hashes: Vec<[u8; 32]> = fixture + .raw + .chunks(CHUNK_SIZE as usize) + .map(|bytes| Sha256::digest(bytes).into()) + .collect(); + assert_eq!(leaf.chunk_sha256, hashes); + let chunks = fixture + .send( + "POST", + "chunks", + Body::from(fixture.chunk_body("/file", map_id, "0").to_string()), + ) + .await; + assert_eq!(chunks.status(), 200); + let wire = to_bytes(chunks.into_body(), 2 * 1024 * 1024).await.unwrap(); + let frames = parse_stream(&wire).unwrap(); + let [Frame::Chunk(chunk), Frame::End(end)] = frames.as_slice() else { + panic!("expected CHUNK and terminal END"); + }; + assert_eq!(chunk.chunk_bytes, &fixture.raw[..CHUNK_SIZE as usize]); + assert_eq!(chunk.file_content_id, fixture.digest); + assert_eq!(chunk.chunk_index, 0); + assert_eq!(end.request_item_count, 1); + assert_eq!(end.logical_bytes, CHUNK_SIZE); +} + #[tokio::test] async fn mst2_fixed_warm_map_and_leaf_aliases_skip_body_reads_and_chunks_use_current_ranges() { let fixture = Fixture::new().await; diff --git a/src/api/router/snapshot_generation_upgrade_tests.rs b/src/api/router/snapshot_generation_upgrade_tests.rs index 08684866..28f4a203 100644 --- a/src/api/router/snapshot_generation_upgrade_tests.rs +++ b/src/api/router/snapshot_generation_upgrade_tests.rs @@ -17,7 +17,7 @@ async fn scalar(db: &sea_orm::DatabaseConnection, sql: &str) -> i64 { #[tokio::test] async fn mst2_generation_additive_upgrade_preserves_legacy_v3_sid_lease_and_new_resolve() { - let fixture = Fixture::new_with_pg_config(true).await; + let fixture = Fixture::new_generic_history_with_pg_config(true).await; let mono = fixture.state.storage.mono_storage(); let db = mono.get_connection(); assert_eq!( @@ -102,10 +102,12 @@ async fn mst2_generation_additive_upgrade_preserves_legacy_v3_sid_lease_and_new_ let connection = crate::jupiter::storage::init::postgres_connection(&config.database) .await .unwrap(); - let storage = crate::jupiter::storage::Storage::new_with_connection( - config, - Arc::new(connection), - fixture.state.storage.git_service.obj_storage.clone(), + let storage = crate::jupiter::storage::init::with_generic_history_bootstrap( + crate::jupiter::storage::Storage::new_with_connection( + config, + Arc::new(connection), + fixture.state.storage.git_service.obj_storage.clone(), + ), ) .await .unwrap(); @@ -113,7 +115,11 @@ async fn mst2_generation_additive_upgrade_preserves_legacy_v3_sid_lease_and_new_ storage, ..fixture.state.clone() }; - let app = Router::new().nest("/api/v2", routers(state.clone()).with_state(state)); + let app = Router::new().nest( + "/api/v2", + crate::api::router::snapshot_router::generic_history_routers(state.clone()) + .with_state(state), + ); let restored = success_json( app.clone() .oneshot(fixture.request("GET", "descriptor", Body::empty())) diff --git a/src/api/router/snapshot_persisted_metadata_tests.rs b/src/api/router/snapshot_persisted_metadata_tests.rs index 057d9d26..ba95602d 100644 --- a/src/api/router/snapshot_persisted_metadata_tests.rs +++ b/src/api/router/snapshot_persisted_metadata_tests.rs @@ -65,7 +65,7 @@ fn assert_metadata(bytes: &[u8], body: &[u8], count: u32, expected: &[([u8; 32], #[tokio::test] async fn mst2_persisted_meta_matches_canonical_routes_after_rebuild_and_advance_without_git_reads() { - let fixture = Fixture::new_with_pg_config_and_directories(true, 140).await; + let fixture = Fixture::new_generic_history_with_pg_config_and_directories(true, 140).await; let context = fixture .state .storage @@ -184,7 +184,7 @@ async fn mst2_persisted_meta_matches_canonical_routes_after_rebuild_and_advance_ #[tokio::test] async fn mst2_persisted_meta_absence_scope_digest_limits_and_release_oracles() { - let fixture = Fixture::new_with_pg_config(true).await; + let fixture = Fixture::new_generic_history_with_pg_config(true).await; for (items, status, code) in [ ( json!([{"directory_path":"/missing"}]), @@ -336,7 +336,7 @@ async fn damage_page(fixture: &Fixture, id: [u8; 32], remove: bool) { #[tokio::test] async fn mst2_persisted_meta_warm_missing_and_corrupt_pages_never_reproject_from_git() { for remove in [true, false] { - let fixture = Fixture::new_with_pg_config(true).await; + let fixture = Fixture::new_generic_history_with_pg_config(true).await; let (id, _) = page_row(&fixture, "/nested").await; damage_page(&fixture, id, remove).await; let body = metadata_body(json!([{"directory_path":"/nested"}]), "identity"); @@ -398,7 +398,7 @@ async fn wait_retention_waiter(txn: &sea_orm::DatabaseTransaction) { #[tokio::test] async fn mst2_persisted_meta_rechecks_deadline_and_release_after_waiting_for_retention_lock() { for expire in [true, false] { - let fixture = Fixture::new_with_pg_config(true).await; + let fixture = Fixture::new_generic_history_with_pg_config(true).await; let mono = fixture.state.storage.mono_storage(); let held = mono.get_connection().begin().await.unwrap(); held.execute_unprepared( @@ -441,7 +441,7 @@ async fn bound_fixture() -> ( Fixture, crate::ceres::snapshot::retention_dag::MetadataPagePayload, ) { - let mut fixture = Fixture::new_with_pg_config(true).await; + let mut fixture = Fixture::new_generic_history_with_pg_config(true).await; advance(&fixture).await; let mono = fixture.state.storage.mono_storage(); let head = mono @@ -546,7 +546,7 @@ async fn mst2_persisted_meta_serves_bound_generic_pages_with_null_members_and_ex #[tokio::test] async fn mst2_persisted_meta_rejects_touched_graph_damage_and_prepare_membership_loss() { for case in 0..5 { - let fixture = Fixture::new_with_pg_config(true).await; + let fixture = Fixture::new_generic_history_with_pg_config(true).await; let (id, _) = page_row(&fixture, "/nested").await; let mono = fixture.state.storage.mono_storage(); let db = mono.get_connection(); @@ -618,7 +618,7 @@ async fn mst2_persisted_meta_reader_holds_protection_until_all_route_bytes_are_o use crate::jupiter::storage::native_snapshot_session::with_metadata_read_barriers; for release_lease in [false, true] { - let fixture = Fixture::new_with_pg_config(true).await; + let fixture = Fixture::new_generic_history_with_pg_config(true).await; let expected = vec![ page_row(&fixture, "/nested").await, page_row(&fixture, "/directory").await, @@ -691,10 +691,11 @@ async fn mst2_persisted_meta_reader_holds_protection_until_all_route_bytes_are_o assert_eq!(observer_config.max_connection, 2); observer_config.max_connection = 1; observer_config.min_connection = 0; - let observer_connection = - crate::jupiter::storage::init::database_connection(&observer_config) - .await - .unwrap(); + let observer_connection = crate::jupiter::storage::init::with_generic_history_bootstrap( + crate::jupiter::storage::init::database_connection(&observer_config), + ) + .await + .unwrap(); let observer = observer_connection.begin().await.unwrap(); tokio::select! { () = wait_retention_waiter(&observer) => {}, diff --git a/src/api/router/snapshot_raw_blob.rs b/src/api/router/snapshot_raw_blob.rs index 6aee3b5c..ea438ec1 100644 --- a/src/api/router/snapshot_raw_blob.rs +++ b/src/api/router/snapshot_raw_blob.rs @@ -91,11 +91,9 @@ pub(super) async fn blob_with_budgets( .api_handler(std::path::Path::new("/")) .await .map_err(internal)?; - let root = content::fixed_root_tree(handler.as_ref(), &context.root_tree_oid).await?; - let file = content::resolve_file_metadata( - handler.as_ref(), - &root, - &context.built.descriptor.scope, + let file = content::resolve_snapshot_file_metadata( + &state, + &context, &query.path, query.expected_digest.as_deref(), ) diff --git a/src/api/router/snapshot_rooted_metadata.rs b/src/api/router/snapshot_rooted_metadata.rs new file mode 100644 index 00000000..a35c7bd1 --- /dev/null +++ b/src/api/router/snapshot_rooted_metadata.rs @@ -0,0 +1,185 @@ +//! JSON metadata endpoints use the same certified, protected fixed-root reader. + +use mst2_codec::metapage::{Entry, EntryKind}; + +use super::*; +use crate::{ + ceres::snapshot::runtime::SnapshotContext, + jupiter::storage::qualified_metadata_family::RootedLookupStatus, +}; + +fn entry_json(entry: &Entry) -> Result { + let name = std::str::from_utf8(&entry.name).map_err(internal)?; + let kind = match entry.kind { + EntryKind::Regular => "regular", + EntryKind::Executable => "executable", + EntryKind::Symlink => "symlink", + EntryKind::Directory => "directory", + }; + let mut value = json!({"name":name,"fs_kind":kind}); + if entry.is_dir() { + value["directory_root"] = json!(format!("sha256:{}", hex_of(&entry.child_root))); + value["node_class"] = json!("native_tree"); + value["lifecycle"] = json!("mutable"); + } else { + value["size"] = json!(entry.size.to_string()); + value["content_digest"] = json!(format!("sha256:{}", hex_of(&entry.content_id))); + } + Ok(value) +} + +fn cursor_name(sid: &str, path: &str, query: &DirectoryQuery) -> Result, Response> { + let Some(cursor) = query.cursor.as_deref() else { + return Ok(None); + }; + let invalid = || { + mst2_error_response(SnapshotError::new( + SnapshotErrorCode::CursorInvalid, + "cursor bound to different parameters or invalid", + )) + }; + let (payload, signature) = cursor.rsplit_once('.').ok_or_else(invalid)?; + if runtime().sign_cursor(payload) != signature { + return Err(invalid()); + } + let bytes = base64::engine::general_purpose::STANDARD + .decode(payload) + .map_err(|_| invalid())?; + let value: serde_json::Value = serde_json::from_slice(&bytes).map_err(|_| invalid())?; + if value["s"].as_str() != Some(sid) + || value["p"].as_str() != Some(path) + || value["l"].as_u64() != Some(query.limit as u64) + { + return Err(invalid()); + } + let name = value["a"].as_str().ok_or_else(invalid)?; + if name.is_empty() || name.len() > 255 || name.contains('/') || name.contains('\0') { + return Err(invalid()); + } + Ok(Some(name.into())) +} + +#[allow(clippy::result_large_err)] +pub(super) async fn directory_response( + state: &MonoApiServiceState, + ctx: &SnapshotContext, + sid: &str, + query: &DirectoryQuery, +) -> Result { + let absolute = abs_view_path(&ctx.built.descriptor.scope, &query.path); + let last = cursor_name(sid, &absolute, query)?; + let repository = state + .storage + .rooted_qualified_metadata_writer() + .await + .map_err(internal)?; + let window = repository + .directory_window(ctx, &query.path, last.as_deref(), query.limit as usize) + .await + .map_err(mst2_error_response)?; + let entries = window + .entries + .iter() + .map(entry_json) + .collect::, _>>()?; + let next = if window.has_more { + let name = window.entries.last().ok_or_else(|| { + mst2_error_response(internal( + "qualified directory returned an empty continuation", + )) + })?; + let payload = json!({"s":sid,"p":absolute,"l":query.limit,"a":std::str::from_utf8(&name.name).map_err(internal)?}); + let encoded = base64_of(&serde_json::to_vec(&payload).map_err(internal)?); + Some(format!("{encoded}.{}", runtime().sign_cursor(&encoded))) + } else { + None + }; + let ancestors = if query.ancestors.as_deref() == Some("chain") { + window + .ancestors + .iter() + .filter(|(path, _)| path != &query.path) + .map(|(path, root)| { + json!({"path":path, + "directory_root":format!("sha256:{}",hex_of(root)),"node_class":"native_tree"}) + }) + .collect::>() + } else { + Vec::new() + }; + revalidate_request(state, ctx) + .await + .map_err(mst2_error_response)?; + let body = json!({"snapshot_id":sid,"path":query.path,"metadata_root":ctx.built.metadata_root, + "directory_root":format!("sha256:{}",hex_of(&window.directory_root)),"node_class":"native_tree","lifecycle":"mutable", + "range_start_exclusive":last,"entries":entries,"entry_count":window.entry_count.to_string(),"next_cursor":next, + "proof_pages":window.proof_pages.iter().map(|(root,bytes)|json!({"digest":format!("sha256:{}",hex_of(root)), + "data_base64":base64_of(bytes)})).collect::>(),"ancestor_chain":ancestors}); + let mut response = Json(body).into_response(); + let tag = format!( + "\"{}:{}:{}:{}\"", + &sid[..16.min(sid.len())], + hex_of(&window.directory_root), + query.limit, + query.cursor.as_deref().unwrap_or("") + ); + if let Ok(value) = HeaderValue::from_str(&tag) { + response.headers_mut().insert("etag", value); + } + response.headers_mut().insert( + "cache-control", + HeaderValue::from_static("private, no-cache, no-transform"), + ); + Ok(response) +} + +#[allow(clippy::result_large_err)] +pub(super) async fn lookup_response( + state: &MonoApiServiceState, + ctx: &SnapshotContext, + sid: &str, + request: &LookupRequest, +) -> Result { + let repository = state + .storage + .rooted_qualified_metadata_writer() + .await + .map_err(internal)?; + let batch = repository + .lookup_metadata(ctx, &request.paths) + .await + .map_err(mst2_error_response)?; + let mut results = Vec::with_capacity(batch.results.len()); + for (path, status) in request.paths.iter().zip(batch.results) { + let mut item = json!({"path":path}); + match status { + RootedLookupStatus::Directory(root) => { + let mut node = json!({"fs_kind":"directory","directory_root":format!("sha256:{}",hex_of(&root)), + "node_class":"native_tree","lifecycle":"mutable"}); + if path != "/" { + node["name"] = json!(path.rsplit('/').next()); + } + item["status"] = json!("found"); + item["node"] = node; + } + RootedLookupStatus::File { entry, .. } => { + item["status"] = json!("found"); + item["node"] = entry_json(&entry)?; + } + RootedLookupStatus::Absent => item["status"] = json!("absent"), + RootedLookupStatus::NotDirectory { symlink } => { + item["status"] = json!(if symlink { + "symlink_traversal" + } else { + "not_directory" + }) + } + } + results.push(item); + } + revalidate_request(state, ctx) + .await + .map_err(mst2_error_response)?; + Ok(Json(json!({"snapshot_id":sid,"results":results,"proof_pages":batch.proof_pages.iter().map(|(root,bytes)|json!({ + "digest":format!("sha256:{}",hex_of(root)),"data_base64":base64_of(bytes)})).collect::>()})).into_response()) +} diff --git a/src/api/router/snapshot_rooted_metadata_tests.rs b/src/api/router/snapshot_rooted_metadata_tests.rs new file mode 100644 index 00000000..7c944913 --- /dev/null +++ b/src/api/router/snapshot_rooted_metadata_tests.rs @@ -0,0 +1,784 @@ +use sea_orm::{DbBackend, Statement, TransactionTrait}; +use tokio::sync::Barrier; + +use super::*; +use crate::jupiter::storage::qualified_metadata_family::{ + SnapshotMetadataFamily, with_rooted_reader_barriers, with_rooted_source_fact_barriers, + with_rooted_source_temporary_shadow, +}; + +async fn q_schema(fixture: &Fixture) -> String { + fixture + .state + .storage + .mono_storage() + .get_connection() + .query_one_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + "SELECT namespace.metadata_schema FROM mst2_snapshot_storage_route route + JOIN mst2_metadata_namespace namespace USING(namespace_uuid) WHERE route.snapshot_id=$1 + AND namespace.graph_domain='qualified-v1'", + [fixture.snapshot.clone().into()], + )) + .await + .unwrap() + .unwrap() + .try_get_by_index(0) + .unwrap() +} + +async fn q_count(fixture: &Fixture, query: &str) -> i64 { + let schema = q_schema(fixture).await; + let query = query.replace("{q}", &format!("\"{}\"", schema.replace('"', "\"\""))); + fixture + .state + .storage + .mono_storage() + .get_connection() + .query_one_raw(Statement::from_string(DbBackend::Postgres, query)) + .await + .unwrap() + .unwrap() + .try_get_by_index(0) + .unwrap() +} + +async fn resolve(fixture: &Fixture) -> Value { + success_json( + fixture + .app + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri("/api/v2/snapshots/resolve") + .header("authorization", format!("Bearer {TOKEN}")) + .header("content-type", "application/json") + .body(Body::from( + json!({"target":{"kind":"latest"},"scope":"/project"}).to_string(), + )) + .unwrap(), + ) + .await + .unwrap(), + ) + .await +} + +async fn release(fixture: &Fixture, lease: &str) -> Value { + success_json( + fixture + .app + .clone() + .oneshot( + Request::builder() + .method("DELETE") + .uri(format!("/api/v2/snapshots/leases/{lease}")) + .header("authorization", format!("Bearer {TOKEN}")) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(), + ) + .await +} + +async fn collect_unowned(fixture: &Fixture) { + let writer = fixture + .state + .storage + .rooted_qualified_metadata_writer() + .await + .unwrap(); + for _ in 0..8 { + let work = writer.maintenance_tick(64).await.unwrap(); + assert!(work.collector_enabled && work.examined <= 64); + if q_count(fixture, "SELECT count(*) FROM {q}.mst2_metadata_payload").await == 0 { + return; + } + } + panic!("bounded maintenance failed to collect the small unowned fixture"); +} + +#[tokio::test] +async fn default_rooted_http_handoff_renew_release_and_fresh_incarnation_keep_exact_ownership() { + let fixture = Fixture::new_with_pg_config(true).await; + assert_eq!( + fixture + .state + .storage + .snapshot_metadata_family(&fixture.lease, true) + .await + .unwrap(), + Some(SnapshotMetadataFamily::Rooted) + ); + assert_eq!( + q_count( + &fixture, + "SELECT count(*) FROM {q}.mst2_qualified_session_incarnation WHERE state='READY'" + ) + .await, + 1 + ); + let first_generation = q_count( + &fixture, + "SELECT root_generation FROM {q}.mst2_qualified_session_incarnation WHERE state='READY'", + ) + .await; + assert_eq!(q_count(&fixture,"SELECT count(*) FROM {q}.mst2_metadata_root_anchor WHERE anchor_kind IN ('PREPARE','REUSE')").await,0); + assert_eq!(q_count(&fixture,"SELECT count(*) FROM {q}.mst2_metadata_prepare WHERE plan_kind='ROOTED' AND state='COMMITTED' AND coverage_retired_at IS NOT NULL").await,1); + let original_payloads = + q_count(&fixture, "SELECT count(*) FROM {q}.mst2_metadata_payload").await; + let next = resolve(&fixture).await; + assert_eq!(next["descriptor"]["snapshot_id"], fixture.snapshot); + let second = next["lease_id"].as_str().unwrap(); + assert_ne!(second, fixture.lease); + assert_eq!( + q_count(&fixture, "SELECT count(*) FROM {q}.mst2_metadata_prepare").await, + 1 + ); + assert_eq!( + q_count(&fixture, "SELECT count(*) FROM {q}.mst2_metadata_payload").await, + original_payloads + ); + let previous_deadline = fixture + .state + .storage + .snapshot_context(&fixture.snapshot, second) + .await + .unwrap() + .lease_expires_at_unix; + let renewed = fixture + .state + .storage + .snapshot_renew(second, 1) + .await + .unwrap(); + assert_eq!(renewed.snapshot_id, fixture.snapshot); + assert!(renewed.expires_at_unix >= previous_deadline); + assert_eq!(release(&fixture, &fixture.lease).await["released"], true); + assert_eq!(release(&fixture, &fixture.lease).await["released"], false); + assert_eq!(q_count(&fixture,"SELECT count(*) FROM {q}.mst2_qualified_lease_binding WHERE state='RELEASED' AND lease_epoch=2").await,1); + let mut request = fixture.request("GET", "descriptor", Body::empty()); + request + .headers_mut() + .insert("x-mega-snapshot-lease", second.parse().unwrap()); + success_json(fixture.app.clone().oneshot(request).await.unwrap()).await; + assert_eq!(release(&fixture, second).await["released"], true); + assert_eq!( + q_count( + &fixture, + "SELECT count(*) FROM {q}.mst2_metadata_root_anchor" + ) + .await, + 0 + ); + assert_eq!( + q_count( + &fixture, + "SELECT count(*) FROM {q}.mst2_qualified_session_incarnation WHERE state='RETIRED'" + ) + .await, + 1 + ); + collect_unowned(&fixture).await; + assert_eq!( + q_count( + &fixture, + "SELECT count(*) FROM {q}.mst2_metadata_graph_node" + ) + .await, + 0 + ); + assert!( + q_count( + &fixture, + "SELECT count(*) FROM {q}.mst2_metadata_page_certificate" + ) + .await + > 0 + ); + let fresh = resolve(&fixture).await; + assert_eq!(fresh["descriptor"]["snapshot_id"], fixture.snapshot); + assert_ne!(fresh["lease_id"], next["lease_id"]); + assert_eq!( + q_count( + &fixture, + "SELECT count(*) FROM {q}.mst2_qualified_session_incarnation" + ) + .await, + 2 + ); + assert_eq!( + q_count( + &fixture, + "SELECT count(*) FROM {q}.mst2_qualified_session_incarnation WHERE state='READY'" + ) + .await, + 1 + ); + assert_eq!( + q_count(&fixture, "SELECT count(*) FROM {q}.mst2_metadata_payload").await, + original_payloads + ); + assert_eq!( + q_count( + &fixture, + "SELECT root_generation FROM {q}.mst2_qualified_session_incarnation WHERE state='READY'" + ) + .await, + first_generation + 1 + ); + fixture.counts.assert(0, 0); +} + +#[tokio::test] +async fn rooted_wide_directory_windows_and_lookup_survive_rebuild_with_valid_proofs() { + let fixture = Fixture::new_with_pg_config_and_directories(true, 140).await; + let config = fixture.state.storage.config(); + let connection = crate::jupiter::storage::init::postgres_connection(&config.database) + .await + .unwrap(); + let storage = crate::jupiter::storage::Storage::new_with_connection( + config, + Arc::new(connection), + fixture.state.storage.git_service.obj_storage.clone(), + ) + .await + .unwrap(); + let state = MonoApiServiceState { + storage, + ..fixture.state.clone() + }; + let app = Router::new().nest("/api/v2", routers(state.clone()).with_state(state)); + let mut cursor = None; + let mut names = Vec::new(); + for _ in 0..12 { + let mut suffix = "directory?path=/&limit=17".to_string(); + if let Some(current) = &cursor { + suffix.push_str(&format!("&cursor={current}")); + } + let response = app + .clone() + .oneshot(fixture.request("GET", &suffix, Body::empty())) + .await + .unwrap(); + let window = success_json(response).await; + assert_eq!(window["entry_count"], "147"); + let entries = window["entries"].as_array().unwrap(); + assert!(!entries.is_empty() && entries.len() <= 17); + names.extend( + entries + .iter() + .map(|entry| entry["name"].as_str().unwrap().to_owned()), + ); + for proof in window["proof_pages"].as_array().unwrap() { + let bytes = STANDARD + .decode(proof["data_base64"].as_str().unwrap()) + .unwrap(); + mst2_codec::metapage::Page::decode(&bytes).unwrap(); + assert_eq!( + proof["digest"], + format!("sha256:{}", hex_of(&mst2_codec::metapage::page_id(&bytes))) + ); + } + cursor = window["next_cursor"].as_str().map(str::to_owned); + if cursor.is_none() { + break; + } + } + assert!(cursor.is_none()); + assert_eq!(names.len(), 147); + assert!( + names + .windows(2) + .all(|pair| pair[0].as_bytes() < pair[1].as_bytes()) + ); + let lookup=success_json(app.oneshot(fixture.request("POST","lookup", + Body::from(json!({"paths":["/nested/file","/wide-139/file-139","/missing","/file/child","/link/child"]}).to_string()))) + .await.unwrap()).await; + let statuses: Vec<_> = lookup["results"] + .as_array() + .unwrap() + .iter() + .map(|row| row["status"].as_str().unwrap()) + .collect(); + assert_eq!( + statuses, + [ + "found", + "found", + "absent", + "not_directory", + "symlink_traversal" + ] + ); + assert_eq!( + q_count( + &fixture, + "SELECT count(*) FROM {q}.mst2_metadata_reader_operation WHERE state='ACTIVE'" + ) + .await, + 0 + ); + assert_eq!(q_count(&fixture,"SELECT count(*) FROM {q}.mst2_metadata_root_anchor WHERE anchor_kind IN ('REQUEST','READER')").await,0); + fixture.counts.assert(0, 0); +} + +#[tokio::test] +async fn rooted_reader_release_race_keeps_independent_roots_until_owned_buffers_finish() { + let fixture = Fixture::new_with_pg_config(true).await; + let admitted = Arc::new(Barrier::new(2)); + let resume = Arc::new(Barrier::new(2)); + let request = fixture.request( + "POST", + "metadata/pages", + Body::from( + json!({"encoding":"identity","items":[{"directory_path":"/nested"}]}).to_string(), + ), + ); + let app = fixture.app.clone(); + let pending = tokio::spawn(with_rooted_reader_barriers( + admitted.clone(), + resume.clone(), + async move { app.oneshot(request).await.unwrap() }, + )); + tokio::time::timeout(Duration::from_secs(4), admitted.wait()) + .await + .unwrap(); + assert_eq!(q_count(&fixture,"SELECT count(*) FROM {q}.mst2_metadata_root_anchor WHERE anchor_kind IN ('REQUEST','READER')").await,2); + let work = fixture + .state + .storage + .rooted_qualified_metadata_writer() + .await + .unwrap() + .maintenance_tick(64) + .await + .unwrap(); + assert_eq!(work.payload_pages_removed, 0); + assert_eq!(release(&fixture, &fixture.lease).await["released"], true); + assert_eq!(q_count(&fixture,"SELECT count(*) FROM {q}.mst2_metadata_root_anchor WHERE anchor_kind IN ('REQUEST','READER')").await,2); + let work = fixture + .state + .storage + .rooted_qualified_metadata_writer() + .await + .unwrap() + .maintenance_tick(64) + .await + .unwrap(); + assert_eq!(work.payload_pages_removed, 0); + assert_eq!( + q_count( + &fixture, + "SELECT count(*) FROM {q}.mst2_metadata_root_anchor WHERE anchor_kind='LEASE'" + ) + .await, + 1 + ); + resume.wait().await; + error( + tokio::time::timeout(Duration::from_secs(4), pending) + .await + .unwrap() + .unwrap(), + 410, + "LEASE_EXPIRED", + false, + ) + .await; + assert_eq!( + q_count( + &fixture, + "SELECT count(*) FROM {q}.mst2_metadata_reader_operation WHERE state='FINISHED'" + ) + .await, + 1 + ); + assert_eq!( + q_count( + &fixture, + "SELECT count(*) FROM {q}.mst2_metadata_root_anchor" + ) + .await, + 0 + ); + collect_unowned(&fixture).await; + fixture.counts.assert(0, 0); +} + +async fn source_oid(fixture: &Fixture) -> String { + let schema = q_schema(fixture).await; + let quoted = format!("\"{}\"", schema.replace('"', "\"\"")); + fixture.state.storage.mono_storage().get_connection().query_one_raw(Statement::from_string(DbBackend::Postgres, + format!("SELECT split_part(a.tagged_tree_oid,':',2) FROM {quoted}.mst2_qualified_session_incarnation s + JOIN {quoted}.mst2_metadata_source_root_attestation a ON a.attestation_id=s.attestation_id WHERE s.state='READY'"))) + .await.unwrap().unwrap().try_get_by_index(0).unwrap() +} + +async fn source_revision(fixture: &Fixture, oid: &str) -> String { + fixture + .state + .storage + .mono_storage() + .get_connection() + .query_one_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + "SELECT revision::text FROM mst2_rooted_source_tree_revision WHERE tree_id=$1", + [oid.into()], + )) + .await + .unwrap() + .unwrap() + .try_get_by_index(0) + .unwrap() +} + +#[tokio::test] +async fn actual_q_body_callers_preserve_unchanged_source_revision_and_ignore_temp_shadow() { + let fixture = Fixture::new_with_pg_config_and_directories(true, 140).await; + fixture.map("/file").await; + fixture.counts.reset(); + let oid = source_oid(&fixture).await; + let revision = source_revision(&fixture, &oid).await; + fixture + .state + .storage + .mono_storage() + .get_connection() + .execute_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + "UPDATE mega_tree SET sub_trees=sub_trees,pack_offset=pack_offset WHERE tree_id=$1", + [oid.clone().into()], + )) + .await + .unwrap(); + assert_eq!(source_revision(&fixture, &oid).await, revision); + let response = + with_rooted_source_temporary_shadow(fixture.send("HEAD", "blob?path=/file", Body::empty())) + .await; + assert_eq!(response.status(), 200); + assert_eq!( + response.headers()["x-mega-content-size"], + fixture.raw.len().to_string() + ); + fixture.counts.assert(0, 0); + let response = + with_rooted_source_temporary_shadow(fixture.send("GET", "blob?path=/file", Body::empty())) + .await; + assert_eq!(response.status(), 200); + assert_eq!( + to_bytes(response.into_body(), fixture.raw.len() + 1) + .await + .unwrap() + .as_ref(), + fixture.raw + ); + assert_eq!( + q_count( + &fixture, + "SELECT count(*) FROM {q}.mst2_metadata_reader_operation WHERE state='ACTIVE'" + ) + .await, + 0 + ); +} + +#[tokio::test] +async fn actual_q_body_callers_reject_changed_or_deleted_source_before_opening_a_body() { + for delete in [false, true] { + let fixture = Fixture::new_with_pg_config(true).await; + let oid = source_oid(&fixture).await; + let revision = source_revision(&fixture, &oid).await; + fixture + .state + .storage + .mono_storage() + .get_connection() + .execute_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + if delete { + "DELETE FROM mega_tree WHERE tree_id=$1" + } else { + "UPDATE mega_tree SET sub_trees=decode('00','hex') WHERE tree_id=$1" + }, + [oid.clone().into()], + )) + .await + .unwrap(); + assert_ne!(source_revision(&fixture, &oid).await, revision); + let response = fixture.send("GET", "blob?path=/file", Body::empty()).await; + assert!(response.status().is_client_error() || response.status().is_server_error()); + fixture.counts.assert(0, 0); + } +} + +#[tokio::test] +async fn actual_q_body_callers_recheck_only_returned_current_file_facts() { + let fixture = Fixture::new_with_pg_config(true).await; + fixture.state.storage.mono_storage().get_connection().execute_raw(Statement::from_sql_and_values(DbBackend::Postgres, + "UPDATE mst2_verified_object SET verification_version=1 WHERE storage_domain='git' AND object_kind='blob' AND git_oid=$1", + [fixture.oid.clone().into()])).await.unwrap(); + let untouched = fixture + .send("HEAD", "blob?path=/empty", Body::empty()) + .await; + assert_eq!(untouched.status(), 200); + assert_eq!(untouched.headers()["x-mega-content-size"], "0"); + error( + fixture.send("GET", "blob?path=/file", Body::empty()).await, + 503, + "METADATA_NOT_READY", + false, + ) + .await; + fixture.counts.assert(0, 0); +} + +#[tokio::test] +async fn actual_q_body_caller_source_mutation_after_reader_admission_cannot_serve_stale_content() { + let fixture = Fixture::new_with_pg_config(true).await; + let oid = source_oid(&fixture).await; + let admitted = Arc::new(Barrier::new(2)); + let resume = Arc::new(Barrier::new(2)); + let app = fixture.app.clone(); + let request = fixture.request("GET", "blob?path=/file", Body::empty()); + let pending = tokio::spawn(with_rooted_reader_barriers( + admitted.clone(), + resume.clone(), + async move { app.oneshot(request).await.unwrap() }, + )); + tokio::time::timeout(Duration::from_secs(4), admitted.wait()) + .await + .unwrap(); + fixture + .state + .storage + .mono_storage() + .get_connection() + .execute_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + "UPDATE mega_tree SET sub_trees=decode('00','hex') WHERE tree_id=$1", + [oid.into()], + )) + .await + .unwrap(); + resume.wait().await; + let response = tokio::time::timeout(Duration::from_secs(4), pending) + .await + .unwrap() + .unwrap(); + assert!(response.status().is_client_error() || response.status().is_server_error()); + assert_eq!(q_count(&fixture,"SELECT count(*) FROM {q}.mst2_metadata_root_anchor WHERE anchor_kind IN ('REQUEST','READER')").await,0); + assert_eq!( + q_count( + &fixture, + "SELECT count(*) FROM {q}.mst2_metadata_reader_operation WHERE state='FINISHED'" + ) + .await, + 1 + ); + fixture.counts.assert(0, 0); +} + +#[tokio::test] +async fn actual_q_body_caller_retries_a_busy_current_file_fact_without_opening_a_body() { + let fixture = Fixture::new_with_pg_config(true).await; + let writer = fixture + .state + .storage + .mono_storage() + .get_connection() + .begin() + .await + .unwrap(); + writer.execute_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + "UPDATE mst2_verified_object SET raw_sha256=raw_sha256 WHERE storage_domain='git' AND object_kind='blob' AND git_oid=$1", + [fixture.oid.clone().into()], + )).await.unwrap(); + error( + fixture.send("GET", "blob?path=/file", Body::empty()).await, + 503, + "TEMPORARY_UNAVAILABLE", + true, + ) + .await; + assert_eq!(q_count(&fixture,"SELECT count(*) FROM {q}.mst2_metadata_root_anchor WHERE anchor_kind IN ('REQUEST','READER')").await,0); + fixture.counts.assert(0, 0); + writer.rollback().await.unwrap(); + assert_eq!( + fixture + .send("HEAD", "blob?path=/file", Body::empty()) + .await + .status(), + 200 + ); + fixture.counts.assert(0, 0); +} + +#[tokio::test] +async fn actual_q_body_caller_holds_the_selected_current_fact_until_the_read_transaction_finishes() +{ + let fixture = Fixture::new_with_pg_config(true).await; + let admitted = Arc::new(Barrier::new(2)); + let resume = Arc::new(Barrier::new(2)); + let app = fixture.app.clone(); + let request = fixture.request("HEAD", "blob?path=/file", Body::empty()); + let pending = tokio::spawn(with_rooted_source_fact_barriers( + admitted.clone(), + resume.clone(), + async move { app.oneshot(request).await.unwrap() }, + )); + tokio::time::timeout(Duration::from_secs(4), admitted.wait()) + .await + .unwrap(); + let core_writer = fixture + .state + .storage + .mono_storage() + .get_connection() + .clone(); + let oid = fixture.oid.clone(); + let (ready, received) = tokio::sync::oneshot::channel(); + let update = tokio::spawn(async move { + let txn = core_writer.begin().await.unwrap(); + let pid: i32 = txn + .query_one_raw(Statement::from_string( + DbBackend::Postgres, + "SELECT pg_backend_pid()", + )) + .await + .unwrap() + .unwrap() + .try_get_by_index(0) + .unwrap(); + ready.send(pid).unwrap(); + txn.execute_raw(Statement::from_sql_and_values(DbBackend::Postgres, + "UPDATE mst2_verified_object SET raw_sha256=raw_sha256 WHERE storage_domain='git' AND object_kind='blob' AND git_oid=$1", + [oid.into()])).await.unwrap(); + txn.commit().await.unwrap(); + }); + let pid = received.await.unwrap(); + tokio::time::timeout(Duration::from_secs(4),async { + loop { + let waiting:bool=fixture.state.storage.mono_storage().get_connection().query_one_raw(Statement::from_sql_and_values( + DbBackend::Postgres,"SELECT coalesce(wait_event_type='Lock',false) FROM pg_stat_activity WHERE pid=$1",[pid.into()])) + .await.unwrap().unwrap().try_get_by_index(0).unwrap(); + if waiting {break;} tokio::task::yield_now().await; + } + }).await.unwrap(); + assert!(!update.is_finished()); + resume.wait().await; + let response = tokio::time::timeout(Duration::from_secs(4), pending) + .await + .unwrap() + .unwrap(); + assert_eq!(response.status(), 200); + tokio::time::timeout(Duration::from_secs(4), update) + .await + .unwrap() + .unwrap(); + assert_eq!(q_count(&fixture,"SELECT count(*) FROM {q}.mst2_metadata_root_anchor WHERE anchor_kind IN ('REQUEST','READER')").await,0); + fixture.counts.assert(0, 0); +} + +async fn durable_lease_state(fixture: &Fixture) -> Value { + let schema = q_schema(fixture).await; + let quoted = format!("\"{}\"", schema.replace('"', "\"\"")); + fixture.state.storage.mono_storage().get_connection().query_one_raw(Statement::from_string(DbBackend::Postgres, + format!("SELECT jsonb_build_object( + 'sessions',(SELECT jsonb_agg(to_jsonb(s) ORDER BY s.snapshot_id,s.session_incarnation) FROM {quoted}.mst2_qualified_session_incarnation s), + 'leases',(SELECT jsonb_agg(to_jsonb(l) ORDER BY l.lease_id) FROM {quoted}.mst2_qualified_lease_binding l), + 'anchors',(SELECT jsonb_agg(to_jsonb(a) ORDER BY a.anchor_id) FROM {quoted}.mst2_metadata_root_anchor a), + 'roots',(SELECT jsonb_agg(to_jsonb(r) ORDER BY r.prepare_id,r.page_id,r.generation) FROM {quoted}.mst2_metadata_graph_root r), + 'routes',(SELECT jsonb_agg(to_jsonb(r) ORDER BY r.lease_id) FROM mst2_lease_storage_route r))"))) + .await.unwrap().unwrap().try_get_by_index(0).unwrap() +} + +#[tokio::test] +async fn actual_q_lease_http_and_direct_handoff_retry_source_lock_without_partial_durable_mutation() +{ + let fixture = Fixture::new_with_pg_config(true).await; + let storage = &fixture.state.storage; + let context = storage + .snapshot_context(&fixture.snapshot, &fixture.lease) + .await + .unwrap(); + let config = storage.config(); + let instance = config.mst2.instance_uuid.as_deref().unwrap(); + let head = storage + .mono_storage() + .read_native_publication_head(instance) + .await + .unwrap(); + let repository = storage.rooted_qualified_metadata_writer().await.unwrap(); + let before = durable_lease_state(&fixture).await; + let revision = source_revision(&fixture, &source_oid(&fixture).await).await; + let writer = storage + .mono_storage() + .get_connection() + .begin() + .await + .unwrap(); + let locked=writer.query_one_raw(Statement::from_sql_and_values(DbBackend::Postgres, + "SELECT revision::text FROM mst2_rooted_source_tree_revision WHERE revision=$1::uuid FOR UPDATE", + [revision.clone().into()])).await.unwrap().unwrap(); + assert_eq!(locked.try_get_by_index::(0).unwrap(), revision); + let direct_errors = tokio::time::timeout(Duration::from_secs(4), async { + [ + repository + .open_session(&head, &context.built, None, 600) + .await + .unwrap_err(), + repository + .renew(&fixture.lease, 600, instance) + .await + .unwrap_err(), + repository.release(&fixture.lease).await.unwrap_err(), + ] + }) + .await + .unwrap(); + for error in direct_errors { + assert_eq!(error.code, SnapshotErrorCode::TemporaryUnavailable); + } + assert_eq!(durable_lease_state(&fixture).await, before); + for (method, uri) in [ + ("POST", "/api/v2/snapshots/resolve".to_owned()), + ( + "POST", + format!("/api/v2/snapshots/leases/{}/renew", fixture.lease), + ), + ( + "DELETE", + format!("/api/v2/snapshots/leases/{}", fixture.lease), + ), + ] { + let body = if uri.ends_with("/resolve") { + Body::from(json!({"target":{"kind":"latest"},"scope":"/project"}).to_string()) + } else { + Body::empty() + }; + let request = Request::builder() + .method(method) + .uri(uri) + .header("authorization", format!("Bearer {TOKEN}")) + .header("content-type", "application/json") + .body(body) + .unwrap(); + let response = + tokio::time::timeout(Duration::from_secs(4), fixture.app.clone().oneshot(request)) + .await + .unwrap() + .unwrap(); + error(response, 503, "TEMPORARY_UNAVAILABLE", true).await; + assert_eq!(durable_lease_state(&fixture).await, before); + } + writer.rollback().await.unwrap(); + assert_eq!(durable_lease_state(&fixture).await, before); + let next = resolve(&fixture).await; + assert_ne!(next["lease_id"], fixture.lease); + assert_eq!(release(&fixture, &fixture.lease).await["released"], true); +} diff --git a/src/api/router/snapshot_router.rs b/src/api/router/snapshot_router.rs index a6838aea..4b568aeb 100644 --- a/src/api/router/snapshot_router.rs +++ b/src/api/router/snapshot_router.rs @@ -76,6 +76,19 @@ pub fn routers(api_state: MonoApiServiceState) -> Router { )) } +/// Explicit fixture for the historical G authority and corruption contracts. +/// Production resolve always installs the rooted family for a new SID. +#[cfg(test)] +pub(crate) fn generic_history_routers( + api_state: MonoApiServiceState, +) -> Router { + routers(api_state).layer(axum::middleware::from_fn( + |request: axum::extract::Request, next: axum::middleware::Next| async move { + GENERIC_HISTORY_RESOLVE.scope(true, next.run(request)).await + }, + )) +} + #[path = "snapshot_content.rs"] mod content; @@ -212,6 +225,7 @@ tokio::task_local! { static NATIVE_RESOLVE_BARRIERS: (std::sync::Arc, std::sync::Arc); static NATIVE_HANDOFF_BARRIERS: (std::sync::Arc, std::sync::Arc); static REJECT_NATIVE_OBSERVATION_SOURCE: bool; + static GENERIC_HISTORY_RESOLVE: bool; } #[cfg(test)] @@ -770,48 +784,141 @@ async fn resolve( // The scope root doubles as metadata_root; building it also validates // that the scope exists and is a directory in this view. let projection_started = std::time::Instant::now(); - let (scope_page, projection_work) = - build_directory_page_with_work(handler.as_ref(), &root_tree, &req.scope) + #[cfg(test)] + let prepared_generic = if selected_native_head.is_some() + && GENERIC_HISTORY_RESOLVE + .try_with(|value| *value) + .unwrap_or(false) + { + let (page, work) = build_directory_page_with_work(handler.as_ref(), &root_tree, &req.scope) .await .map_err(mst2_error_response)?; - let projection_elapsed = projection_started.elapsed(); - - let built = build_descriptor(&config.mst2, &view, &req.scope, scope_page.page_id) + let prepared = crate::ceres::snapshot::pages::prepare_native_metadata_retention( + handler.as_ref(), + &root_tree, + &req.scope, + crate::ceres::snapshot::retention_dag::MetadataDagLimits::default(), + ) + .await .map_err(mst2_error_response)?; - let ctx = if let Some(head) = selected_native_head.as_ref() { - let sessions = state.storage.snapshot_sessions().await; - if let Some(context) = sessions - .open(head, &built, None, req.lease_seconds) + Some((page.page_id, work, prepared)) + } else { + None + }; + #[cfg(not(test))] + let prepared_generic: Option<( + [u8; 32], + crate::ceres::snapshot::pages::ProjectionWork, + crate::ceres::snapshot::pages::PreparedNativeMetadataRetention, + )> = None; + let (metadata_root, projection_work, prepared_rooted) = if let Some((root, work, _)) = + prepared_generic.as_ref() + { + (*root, Some(work.clone()), None) + } else if selected_native_head.is_some() { + let repository = state + .storage + .rooted_qualified_metadata_writer() .await - .map_err(mst2_error_response)? - { - context - } else { - let prepared = crate::ceres::snapshot::pages::prepare_native_metadata_retention( + .map_err(internal)?; + let prepared = + crate::ceres::snapshot::rooted_metadata_projection::prepare_rooted_native_metadata( handler.as_ref(), &root_tree, &req.scope, - crate::ceres::snapshot::retention_dag::MetadataDagLimits::default(), + repository, ) .await .map_err(mst2_error_response)?; - let receipt = sessions - .install(&built, &prepared) + (prepared.plan.root, None, Some(prepared)) + } else { + let (page, work) = build_directory_page_with_work(handler.as_ref(), &root_tree, &req.scope) + .await + .map_err(mst2_error_response)?; + (page.page_id, Some(work), None) + }; + let projection_elapsed = projection_started.elapsed(); + + let built = build_descriptor(&config.mst2, &view, &req.scope, metadata_root) + .map_err(mst2_error_response)?; + let ctx = if let Some(head) = selected_native_head.as_ref() { + use crate::jupiter::storage::qualified_metadata_family::SnapshotMetadataFamily; + let family = state + .storage + .snapshot_metadata_family(&built.snapshot_id, false) + .await + .map_err(mst2_error_response)?; + if family == Some(SnapshotMetadataFamily::Generic) || prepared_generic.is_some() { + // A permanent SID route retains its original physical family. + let existing = state + .storage + .snapshot_sessions() + .await + .open(head, &built, None, req.lease_seconds) .await .map_err(mst2_error_response)?; - #[cfg(test)] - if let Ok((prepared, release)) = NATIVE_HANDOFF_BARRIERS.try_with(|value| value.clone()) - { - prepared.wait().await; - release.wait().await; + if let Some(existing) = existing { + existing + } else { + let (_, _, prepared) = prepared_generic.as_ref().ok_or_else(|| { + mst2_error_response(internal("existing generic route has no durable session")) + })?; + let sessions = state.storage.snapshot_sessions().await; + let receipt = sessions + .install(&built, prepared) + .await + .map_err(mst2_error_response)?; + #[cfg(test)] + if let Ok((prepared, release)) = NATIVE_HANDOFF_BARRIERS.try_with(Clone::clone) { + prepared.wait().await; + release.wait().await; + } + sessions + .open(head, &built, Some(&receipt), req.lease_seconds) + .await + .map_err(mst2_error_response)? + .ok_or_else(|| { + mst2_error_response(internal( + "generic history fixture handoff returned no context", + )) + })? } - sessions - .open(head, &built, Some(&receipt), req.lease_seconds) + } else { + let repository = state + .storage + .rooted_qualified_metadata_writer() + .await + .map_err(internal)?; + if let Some(context) = repository + .open_session(head, &built, None, req.lease_seconds) .await .map_err(mst2_error_response)? - .ok_or_else(|| { - mst2_error_response(internal("durable session handoff returned no context")) - })? + { + context + } else { + let prepared = prepared_rooted.as_ref().ok_or_else(|| { + mst2_error_response(internal("rooted resolve has no source projection")) + })?; + let receipt = repository + .install(&built, prepared) + .await + .map_err(crate::jupiter::storage::native_snapshot_session::install_error) + .map_err(mst2_error_response)?; + #[cfg(test)] + if let Ok((prepared, release)) = + NATIVE_HANDOFF_BARRIERS.try_with(|value| value.clone()) + { + prepared.wait().await; + release.wait().await; + } + repository + .open_session(head, &built, Some(&receipt), req.lease_seconds) + .await + .map_err(mst2_error_response)? + .ok_or_else(|| { + mst2_error_response(internal("durable session handoff returned no context")) + })? + } } } else { runtime() @@ -821,20 +928,26 @@ async fn resolve( if let Some(source) = native_source { let request_id = current_request_id(); - match source.observe( - ResolvedProjection { - descriptor: &ctx.built.descriptor, - snapshot_id: &ctx.built.snapshot_id, - metadata_root: &ctx.built.metadata_root, - context_commit: &ctx.commit_oid, - context_root_tree: &ctx.root_tree_oid, - fixed_root_tree: root_tree.id, - requested_scope: &req.scope, - request_id: &request_id, - }, - projection_work, - projection_elapsed, - ) { + let resolved = ResolvedProjection { + descriptor: &ctx.built.descriptor, + snapshot_id: &ctx.built.snapshot_id, + metadata_root: &ctx.built.metadata_root, + context_commit: &ctx.commit_oid, + context_root_tree: &ctx.root_tree_oid, + fixed_root_tree: root_tree.id, + requested_scope: &req.scope, + request_id: &request_id, + }; + let observation = if let Some(prepared) = prepared_rooted { + source.observe_rooted(resolved, prepared.work, projection_elapsed) + } else { + source.observe( + resolved, + projection_work.unwrap_or_default(), + projection_elapsed, + ) + }; + match observation { Ok(observation) => { if let Some(sink) = &state.storage.projection_observation_sink { let _ = sink.enqueue(&observation); @@ -963,6 +1076,9 @@ struct DirectoryQuery { ancestors: Option, } +#[path = "snapshot_rooted_metadata.rs"] +mod rooted_metadata; + fn default_limit() -> u32 { 128 } @@ -986,6 +1102,18 @@ async fn directory( "limit must be 1..256", ))); } + if state.storage.config().mst2.publication_enabled + && state + .storage + .snapshot_metadata_family(&ctx.lease_id, true) + .await + .map_err(mst2_error_response)? + == Some( + crate::jupiter::storage::qualified_metadata_family::SnapshotMetadataFamily::Rooted, + ) + { + return rooted_metadata::directory_response(&state, &ctx, &snapshot_id, &q).await; + } let handler = state .api_handler(std::path::Path::new("/")) @@ -1201,11 +1329,24 @@ async fn lookup( ))); } + let ctx = request_context(&state, &snapshot_id).map_err(mst2_error_response)?; + if state.storage.config().mst2.publication_enabled + && state + .storage + .snapshot_metadata_family(&ctx.lease_id, true) + .await + .map_err(mst2_error_response)? + == Some( + crate::jupiter::storage::qualified_metadata_family::SnapshotMetadataFamily::Rooted, + ) + { + return rooted_metadata::lookup_response(&state, &ctx, &snapshot_id, &req).await; + } + let handler = state .api_handler(std::path::Path::new("/")) .await .map_err(internal)?; - let ctx = request_context(&state, &snapshot_id).map_err(mst2_error_response)?; let root_tree = handler .get_tree_by_hash(&ctx.root_tree_oid) .await @@ -1371,9 +1512,7 @@ async fn metadata_pages( .collect(); let batch = state .storage - .snapshot_sessions() - .await - .metadata_routes(&ctx, &items) + .snapshot_metadata_routes(&ctx, &items) .await .map_err(mst2_error_response)?; tracing::debug!( diff --git a/src/api/router/snapshot_session_tests.rs b/src/api/router/snapshot_session_tests.rs index ae84d3c6..36101a05 100644 --- a/src/api/router/snapshot_session_tests.rs +++ b/src/api/router/snapshot_session_tests.rs @@ -84,12 +84,16 @@ async fn rebuilt(fixture: &Fixture) -> MonoApiServiceState { let connection = crate::jupiter::storage::init::postgres_connection(&config.database) .await .unwrap(); - let storage = crate::jupiter::storage::Storage::new_with_connection( + let assembly = crate::jupiter::storage::Storage::new_with_connection( config, Arc::new(connection), fixture.state.storage.git_service.obj_storage.clone(), - ) - .await + ); + let storage = if fixture.generic_history { + crate::jupiter::storage::init::with_generic_history_bootstrap(assembly).await + } else { + assembly.await + } .unwrap(); MonoApiServiceState { storage, @@ -102,7 +106,11 @@ async fn rebuilt(fixture: &Fixture) -> MonoApiServiceState { } fn app(state: &MonoApiServiceState) -> Router { - Router::new().nest("/api/v2", routers(state.clone()).with_state(state.clone())) + Router::new().nest( + "/api/v2", + crate::api::router::snapshot_router::generic_history_routers(state.clone()) + .with_state(state.clone()), + ) } async fn lease_control(fixture: &Fixture, lease: &str, method: &str, renew: bool) -> Value { @@ -150,7 +158,7 @@ async fn advance(fixture: &Fixture) { #[tokio::test] async fn mst2_durable_http_resolve_installs_complete_dag_and_warm_leases_share_it() { - let fixture = Fixture::new_with_pg_config(true).await; + let fixture = Fixture::new_generic_history_with_pg_config(true).await; let mono = fixture.state.storage.mono_storage(); let db = mono.get_connection(); let pages = scalar(db, "SELECT count(*) FROM mst2_metadata_payload").await; @@ -306,7 +314,7 @@ async fn mst2_durable_http_resolve_installs_complete_dag_and_warm_leases_share_i #[tokio::test] async fn mst2_durable_http_old_sid_and_original_lease_survive_real_publication_and_fresh_service() { - let fixture = Fixture::new_with_pg_config(true).await; + let fixture = Fixture::new_generic_history_with_pg_config(true).await; let old = success_json(fixture.send("GET", "descriptor", Body::empty()).await).await; advance(&fixture).await; let next = success_json( @@ -376,7 +384,7 @@ async fn mst2_durable_http_old_sid_and_original_lease_survive_real_publication_a #[tokio::test] async fn mst2_durable_http_renew_release_expiry_and_last_lease_retire_protection() { - let fixture = Fixture::new_with_pg_config(true).await; + let fixture = Fixture::new_generic_history_with_pg_config(true).await; let another = success_json( fixture .app @@ -491,7 +499,7 @@ async fn mst2_durable_http_renew_release_expiry_and_last_lease_retire_protection #[tokio::test] async fn mst2_durable_http_live_to_deleting_winner_rejects_paused_resolve_without_leak() { - let fixture = Fixture::new_with_pg_config(true).await; + let fixture = Fixture::new_generic_history_with_pg_config(true).await; let captured = Arc::new(Barrier::new(2)); let release = Arc::new(Barrier::new(2)); let resolving = { @@ -542,7 +550,7 @@ async fn mst2_durable_http_live_to_deleting_winner_rejects_paused_resolve_withou #[tokio::test] async fn mst2_durable_http_lease_winner_blocks_gc_until_the_last_release() { - let fixture = Fixture::new_with_pg_config(true).await; + let fixture = Fixture::new_generic_history_with_pg_config(true).await; let another = success_json( fixture .app @@ -595,7 +603,7 @@ async fn mst2_durable_http_lease_winner_blocks_gc_until_the_last_release() { #[tokio::test] async fn mst2_durable_http_renew_waits_for_lock_before_checking_database_deadline() { - let fixture = Fixture::new_with_pg_config(true).await; + let fixture = Fixture::new_generic_history_with_pg_config(true).await; let mono = fixture.state.storage.mono_storage(); let db = mono.get_connection(); let held = db.begin().await.unwrap(); @@ -671,7 +679,7 @@ async fn mst2_durable_http_renew_waits_for_lock_before_checking_database_deadlin #[tokio::test] async fn mst2_durable_http_publication_advance_rejects_mixed_resolve_then_retries_current() { - let fixture = Fixture::new_with_pg_config(true).await; + let fixture = Fixture::new_generic_history_with_pg_config(true).await; let captured = Arc::new(Barrier::new(2)); let release = Arc::new(Barrier::new(2)); let resolving = { @@ -725,7 +733,7 @@ async fn mst2_durable_http_publication_advance_rejects_mixed_resolve_then_retrie #[tokio::test] async fn mst2_durable_http_install_fault_never_hands_off_or_returns_success() { - let fixture = Fixture::new_with_pg_config(true).await; + let fixture = Fixture::new_generic_history_with_pg_config(true).await; let mono = fixture.state.storage.mono_storage(); let db = mono.get_connection(); let prepared = root_metadata(&fixture).await; @@ -828,7 +836,7 @@ async fn mst2_durable_http_install_fault_never_hands_off_or_returns_success() { #[tokio::test] async fn mst2_durable_http_frame_and_raw_delivery_recheck_after_release() { for raw in [false, true] { - let fixture = Fixture::new_with_pg_config(true).await; + let fixture = Fixture::new_generic_history_with_pg_config(true).await; let response = if raw { fixture.send("GET", "blob?path=/file", Body::empty()).await } else { @@ -862,7 +870,7 @@ async fn mst2_durable_http_frame_and_raw_delivery_recheck_after_release() { #[tokio::test] async fn mst2_durable_http_current_state_wrong_lease_and_corrupt_source_reject_warm_reads() { - let fixture = Fixture::new_with_pg_config(true).await; + let fixture = Fixture::new_generic_history_with_pg_config(true).await; let warm = success_json(fixture.send("GET", "descriptor", Body::empty()).await).await; let mono = fixture.state.storage.mono_storage(); let db = mono.get_connection(); @@ -935,7 +943,7 @@ async fn mst2_durable_http_current_state_wrong_lease_and_corrupt_source_reject_w #[tokio::test] async fn mst2_durable_http_large_dag_batches_handoff_without_scanning_pages_or_edges() { - let fixture = Fixture::new_with_pg_config_and_directories(true, 80).await; + let fixture = Fixture::new_generic_history_with_pg_config_and_directories(true, 80).await; let mono = fixture.state.storage.mono_storage(); let db = mono.get_connection(); let projected = root_metadata(&fixture).await; @@ -1083,7 +1091,7 @@ async fn mst2_durable_http_handoff_rejects_changed_plan_or_receipt_summary() { "projection_revision=projection_revision+1", "total_bytes=total_bytes+1", ] { - let fixture = Fixture::new_with_pg_config_and_directories(true, 80).await; + let fixture = Fixture::new_generic_history_with_pg_config_and_directories(true, 80).await; let prepared = Arc::new(Barrier::new(2)); let release = Arc::new(Barrier::new(2)); let resolving = { @@ -1254,7 +1262,7 @@ async fn overwrite_primary_scope_for_test(db: &DatabaseConnection, storage_uuid: #[tokio::test] async fn mst2_durable_http_warm_reads_renew_and_frame_delivery_reject_primary_scope_drift() { - let fixture = Fixture::new_with_pg_config(true).await; + let fixture = Fixture::new_generic_history_with_pg_config(true).await; success_json(fixture.send("GET", "descriptor", Body::empty()).await).await; let response = fixture .send( diff --git a/src/api/router/snapshot_storage_route_fixture.rs b/src/api/router/snapshot_storage_route_fixture.rs index abc3aa48..71fb3204 100644 --- a/src/api/router/snapshot_storage_route_fixture.rs +++ b/src/api/router/snapshot_storage_route_fixture.rs @@ -30,12 +30,19 @@ pub(super) async fn restore_pre_route_schema(db: &DatabaseConnection) { // The original contexts, leases, plans, payloads and protection stay intact. txn.execute_unprepared( "DO $$ DECLARE r record; functions text; BEGIN + IF EXISTS(SELECT 1 FROM mst2_metadata_namespace WHERE graph_domain='qualified-v1') THEN + RAISE EXCEPTION 'route upgrade fixture cannot tear down a provisioned qualified family'; + END IF; + IF EXISTS(SELECT 1 FROM mst2_rooted_source_tree_revision) THEN + RAISE EXCEPTION 'route upgrade fixture cannot erase actual qualified source history'; + END IF; + DROP FUNCTION mst2_metadata_has_generic_overlap(bytea); FOR r IN SELECT c.relname,t.tgname FROM pg_trigger t JOIN pg_class c ON c.oid=t.tgrelid JOIN pg_namespace n ON n.oid=c.relnamespace JOIN pg_proc p ON p.oid=t.tgfoid WHERE n.nspname=current_schema() AND NOT t.tgisinternal AND left(p.proname,11)='mst2_route_' LOOP EXECUTE format('DROP TRIGGER %I ON %I.%I',r.tgname,current_schema(),r.relname); END LOOP; - DROP TABLE mst2_lease_storage_route,mst2_generic_session_storage_binding,mst2_snapshot_storage_route,mst2_metadata_namespace; + DROP TABLE mst2_lease_storage_route,mst2_generic_session_storage_binding,mst2_snapshot_storage_route,mst2_metadata_namespace,mst2_qualified_family_policy,mst2_rooted_source_tree_revision; FOR r IN SELECT conname FROM pg_constraint WHERE conrelid='mst2_snapshot_context'::regclass AND contype='u' AND pg_get_constraintdef(oid)='UNIQUE (snapshot_id, prepare_id, metadata_root)' @@ -46,7 +53,8 @@ pub(super) async fn restore_pre_route_schema(db: &DatabaseConnection) { IF functions IS NULL THEN RAISE EXCEPTION 'route upgrade fixture has no route functions'; END IF; EXECUTE 'DROP FUNCTION '||functions; END $$; - DELETE FROM seaql_migrations WHERE version='m20261007_000600_add_mst2_storage_routes'", + DELETE FROM seaql_migrations WHERE version IN ( + 'm20261007_000600_add_mst2_storage_routes','m20261008_000200_add_mst2_rooted_qualified_family')", ) .await .unwrap(); diff --git a/src/api/router/snapshot_storage_route_tests.rs b/src/api/router/snapshot_storage_route_tests.rs index 25d40ec3..63b59fc6 100644 --- a/src/api/router/snapshot_storage_route_tests.rs +++ b/src/api/router/snapshot_storage_route_tests.rs @@ -176,7 +176,11 @@ async fn lease_control(fixture: &Fixture, lease: &str, renew: bool) -> Response async fn rebuilt(fixture: &Fixture) -> Router { let state = rebuilt_state(fixture).await; - Router::new().nest("/api/v2", routers(state.clone()).with_state(state)) + Router::new().nest( + "/api/v2", + crate::api::router::snapshot_router::generic_history_routers(state.clone()) + .with_state(state), + ) } async fn rebuilt_state(fixture: &Fixture) -> MonoApiServiceState { @@ -187,10 +191,12 @@ async fn rebuilt_state(fixture: &Fixture) -> MonoApiServiceState { let connection = crate::jupiter::storage::init::postgres_connection(&database) .await .unwrap(); - let storage = crate::jupiter::storage::Storage::new_with_connection( - config, - Arc::new(connection), - fixture.state.storage.git_service.obj_storage.clone(), + let storage = crate::jupiter::storage::init::with_generic_history_bootstrap( + crate::jupiter::storage::Storage::new_with_connection( + config, + Arc::new(connection), + fixture.state.storage.git_service.obj_storage.clone(), + ), ) .await .unwrap(); @@ -326,7 +332,7 @@ async fn rejected(db: &DatabaseConnection, sql: &str) { #[tokio::test] async fn mst2_generic_storage_routes_actual_resolve_warm_and_fresh_service_keep_exact_tuple() { - let fixture = Fixture::new_with_pg_config(true).await; + let fixture = Fixture::new_generic_history_with_pg_config(true).await; let mono = fixture.state.storage.mono_storage(); let db = mono.get_connection(); assert_exact_bindings(db, 1).await; @@ -374,7 +380,7 @@ async fn mst2_generic_storage_routes_actual_resolve_warm_and_fresh_service_keep_ #[tokio::test] async fn mst2_generic_storage_routes_every_member_delete_and_truncate_are_immutable() { - let fixture = Fixture::new_with_pg_config(true).await; + let fixture = Fixture::new_generic_history_with_pg_config(true).await; let mono = fixture.state.storage.mono_storage(); let db = mono.get_connection(); let original = routes(db).await; @@ -431,7 +437,7 @@ fn lease_insert(lease: &str) -> String { #[tokio::test] async fn mst2_generic_storage_routes_raw_lease_derives_atomically_and_half_rows_roll_back() { - let fixture = Fixture::new_with_pg_config(true).await; + let fixture = Fixture::new_generic_history_with_pg_config(true).await; let mono = fixture.state.storage.mono_storage(); let db = mono.get_connection(); let original = routes(db).await; @@ -488,7 +494,7 @@ async fn mst2_generic_storage_routes_raw_lease_derives_atomically_and_half_rows_ #[tokio::test] async fn mst2_generic_storage_routes_renew_terminal_and_unknown_release_preserve_route_and_roots() { - let fixture = Fixture::new_with_pg_config(true).await; + let fixture = Fixture::new_generic_history_with_pg_config(true).await; let mono = fixture.state.storage.mono_storage(); let db = mono.get_connection(); let original = routes(db).await; @@ -592,7 +598,7 @@ async fn lock_count(held: &DatabaseTransaction, pid: i64, key: i32) -> i64 { #[tokio::test] async fn mst2_generic_storage_routes_raw_statement_waits_mono_then_route_then_retention() { - let fixture = Fixture::new_with_pg_config(true).await; + let fixture = Fixture::new_generic_history_with_pg_config(true).await; let mono = fixture.state.storage.mono_storage(); let db = mono.get_connection(); for held_key in [MONO_WRITE_LOCK_KEY1, ROUTE_LOCK_KEY, RETENTION_LOCK_KEY] { @@ -692,8 +698,8 @@ async fn mst2_generic_storage_routes_raw_statement_waits_mono_then_route_then_re #[tokio::test] async fn mst2_generic_storage_routes_schema_isolation_wrong_caller_rr_and_temp_shadow_fail_closed() { - let fixture = Fixture::new_with_pg_config(true).await; - let alien = Fixture::new_with_pg_config(true).await; + let fixture = Fixture::new_generic_history_with_pg_config(true).await; + let alien = Fixture::new_generic_history_with_pg_config(true).await; let mono = fixture.state.storage.mono_storage(); let db = mono.get_connection(); let original = routes(db).await; @@ -801,7 +807,7 @@ async fn mst2_generic_storage_routes_schema_isolation_wrong_caller_rr_and_temp_s #[tokio::test] async fn mst2_generic_storage_routes_additive_backfill_keeps_active_terminal_sources_bytes_and_roots() { - let fixture = Fixture::new_with_pg_config(true).await; + let fixture = Fixture::new_generic_history_with_pg_config(true).await; let mono = fixture.state.storage.mono_storage(); let db = mono.get_connection(); let warm = success_json( @@ -862,9 +868,13 @@ async fn mst2_generic_storage_routes_additive_backfill_keeps_active_terminal_sou #[tokio::test] async fn mst2_generic_storage_routes_actual_http_ignores_temp_source_and_ledger_shadows() { - let fixture = Fixture::new_with_pg_config(true).await; + let fixture = Fixture::new_generic_history_with_pg_config(true).await; let state = rebuilt_state(&fixture).await; - let app = Router::new().nest("/api/v2", routers(state.clone()).with_state(state.clone())); + let app = Router::new().nest( + "/api/v2", + crate::api::router::snapshot_router::generic_history_routers(state.clone()) + .with_state(state.clone()), + ); let mono = state.storage.mono_storage(); let db = mono.get_connection(); let original = routes(db).await; @@ -936,7 +946,7 @@ async fn mst2_generic_storage_routes_actual_http_ignores_temp_source_and_ledger_ #[tokio::test] async fn mst2_generic_storage_routes_temp_prepare_shadow_rejects_qualified_context_and_registered_generic_rebind() { - let fixture = Fixture::new_with_pg_config(true).await; + let fixture = Fixture::new_generic_history_with_pg_config(true).await; let mono = fixture.state.storage.mono_storage(); let db = mono.get_connection(); let original_routes = routes(db).await; @@ -1118,7 +1128,7 @@ async fn mst2_generic_storage_routes_temp_prepare_shadow_rejects_qualified_conte #[tokio::test] async fn mst2_generic_storage_routes_corrupt_original_incarnation_rejects_reads_renew_release_without_repair() { - let fixture = Fixture::new_with_pg_config(true).await; + let fixture = Fixture::new_generic_history_with_pg_config(true).await; let mono = fixture.state.storage.mono_storage(); let db = mono.get_connection(); let source = sources(db).await; diff --git a/src/ceres/snapshot/mod.rs b/src/ceres/snapshot/mod.rs index 3fae7f32..267a78bb 100644 --- a/src/ceres/snapshot/mod.rs +++ b/src/ceres/snapshot/mod.rs @@ -19,5 +19,7 @@ pub mod publication; pub mod resolver; pub mod retention; pub mod retention_dag; +pub(crate) mod rooted_metadata_install; +pub(crate) mod rooted_metadata_projection; pub mod runtime; pub mod view; diff --git a/src/ceres/snapshot/projection_observation.rs b/src/ceres/snapshot/projection_observation.rs index 819e0e86..532b3f18 100644 --- a/src/ceres/snapshot/projection_observation.rs +++ b/src/ceres/snapshot/projection_observation.rs @@ -74,6 +74,7 @@ pub(crate) struct NativeProjectionObservation { request_id: String, projection_elapsed_micros: u64, work: ProjectionWork, + rooted_work: Option, } /// Closed wire fields, borrowed only from the already validated observation. @@ -106,7 +107,9 @@ pub(crate) struct ProjectionWireRecord<'a> { page_counter_scope: &'static str, codec_radix_work: &'static str, #[serde(flatten)] - work: &'a ProjectionWork, + work: Option<&'a ProjectionWork>, + #[serde(skip_serializing_if = "Option::is_none")] + rooted_work: Option<&'a super::rooted_metadata_projection::RootedProjectionWork>, message: &'static str, } @@ -190,15 +193,32 @@ impl NativeResolveSource { projection_elapsed_micros: u64::try_from(projection_elapsed.as_micros()) .map_err(|_| invalid())?, work, + rooted_work: None, }) } + + pub(crate) fn observe_rooted( + self, + resolved: ResolvedProjection<'_>, + work: super::rooted_metadata_projection::RootedProjectionWork, + projection_elapsed: Duration, + ) -> Result { + let mut observation = + self.observe(resolved, ProjectionWork::default(), projection_elapsed)?; + observation.rooted_work = Some(work); + Ok(observation) + } } impl NativeProjectionObservation { pub(crate) fn wire_record(&self) -> ProjectionWireRecord<'_> { ProjectionWireRecord { - observation_revision: 1, - phase: "resolve_directory_projection", + observation_revision: if self.rooted_work.is_some() { 2 } else { 1 }, + phase: if self.rooted_work.is_some() { + "resolve_rooted_projection" + } else { + "resolve_directory_projection" + }, source_domain: "native-git", request_id: &self.request_id, instance_id: self.source.instance.to_string(), @@ -219,14 +239,37 @@ impl NativeProjectionObservation { snapshot_id: &self.snapshot_id, metadata_root: &self.metadata_root, projection_elapsed_micros: self.projection_elapsed_micros, - page_counter_scope: "returned-directory-root-pages", - codec_radix_work: "NOT_EXPOSED", - work: &self.work, + page_counter_scope: if self.rooted_work.is_some() { + "physical-delta-and-reuse-boundaries" + } else { + "returned-directory-root-pages" + }, + codec_radix_work: if self.rooted_work.is_some() { + "OPERATION_CALLS_ONLY_NOT_TOTAL_SQL" + } else { + "NOT_EXPOSED" + }, + work: self.rooted_work.is_none().then_some(&self.work), + rooted_work: self.rooted_work.as_ref(), message: "native resolve directory projection succeeded", } } pub(crate) fn emit(self) { + if let Some(work) = &self.rooted_work { + tracing::debug!(target:"mst2::native_projection_observation", + observation_revision=2u16,phase="resolve_rooted_projection",source_domain="native-git", + request_id=%self.request_id,instance_id=%self.source.instance, + root_commit_oid=%self.source.commit.to_tagged_string(),root_tree_oid=%self.source.tree.to_tagged_string(), + native_certificate_receipt_id=self.source.certificate_receipt_id, + native_writer_epoch=self.source.writer_epoch,native_publication_sequence=self.source.publication_sequence, + scope=?self.scope,snapshot_id=%self.snapshot_id,metadata_root=%self.metadata_root, + projection_elapsed_micros=self.projection_elapsed_micros,rooted_work=?work, + "native rooted projection succeeded; counters exclude total SQL body and catalog work"); + #[cfg(test)] + let _ = OBSERVATIONS.try_with(|observations| observations.lock().unwrap().push(self)); + return; + } tracing::debug!( target: "mst2::native_projection_observation", observation_revision = 1u16, @@ -406,6 +449,45 @@ mod tests { assert_eq!(observation.snapshot_id, fixture.snapshot_id); } + #[test] + fn rooted_observation_exposes_delta_boundaries_without_legacy_or_total_sql_counters() { + let fixture = Fixture::new(); + let commit = fixture.commit.to_string(); + let tree = fixture.tree.to_string(); + let work = super::super::rooted_metadata_projection::RootedProjectionWork { + tree_fetches: 1, + delta_pages: 2, + reused_roots: 7, + ..Default::default() + }; + let observation = fixture + .source() + .observe_rooted( + fixture.resolved(&commit, &tree), + work, + Duration::from_micros(456), + ) + .unwrap(); + let wire = serde_json::to_value(observation.wire_record()).unwrap(); + assert_eq!(wire["observation_revision"], 2); + assert_eq!(wire["phase"], "resolve_rooted_projection"); + assert_eq!( + wire["page_counter_scope"], + "physical-delta-and-reuse-boundaries" + ); + assert_eq!( + wire["codec_radix_work"], + "OPERATION_CALLS_ONLY_NOT_TOTAL_SQL" + ); + assert_eq!(wire["rooted_work"]["tree_fetches"], 1); + assert_eq!(wire["rooted_work"]["delta_pages"], 2); + assert_eq!(wire["rooted_work"]["reused_roots"], 7); + assert!(wire.get("directories_rebuilt").is_none()); + assert!(wire.get("directory_root_pages_returned").is_none()); + assert_eq!(wire["snapshot_id"], fixture.snapshot_id); + assert_eq!(wire["projection_elapsed_micros"], 456); + } + #[test] fn observation_rejects_each_changed_binding_or_unbounded_trace_token() { let fixture = Fixture::new(); diff --git a/src/ceres/snapshot/rooted_metadata_install.rs b/src/ceres/snapshot/rooted_metadata_install.rs new file mode 100644 index 00000000..5086bfa5 --- /dev/null +++ b/src/ceres/snapshot/rooted_metadata_install.rs @@ -0,0 +1,848 @@ +//! Bounded native metadata delta and certified reused-root installation identity. +//! Reused roots are opaque boundaries; their database proofs grant no authority here. + +use std::collections::{BTreeMap, BTreeSet}; + +use mst2_codec::metapage::{HEADER_LEN, PAGE_MAX_BYTES}; +use sha2::{Digest, Sha256}; +use uuid::Uuid; + +use super::{ + error::{SnapshotError, SnapshotErrorCode}, + metadata_install::{MAX_PLAN_BYTES, MetadataInstallIdentity}, + projection_observation::NATIVE_PROJECTION_REVISION, + retention_dag::{MetadataDagLimits, MetadataPageId}, +}; + +const DOMAIN: &[u8] = b"mega.mst2.rooted-install.v1\0"; +const ENCODING_VERSION: u16 = 1; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct RootedReuseRoot { + pub(crate) generation: i64, + pub(crate) attestation_id: Uuid, + pub(crate) attestation_digest: [u8; 32], + pub(crate) certificate_digest: [u8; 32], +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct RootedMetadataInstallPlan { + pub(crate) identity: MetadataInstallIdentity, + pub(crate) root: MetadataPageId, + pub(crate) delta: BTreeMap, + pub(crate) edges: BTreeSet<(MetadataPageId, MetadataPageId)>, + pub(crate) reused: BTreeMap, + pub(crate) source_roots: BTreeMap, +} + +impl RootedMetadataInstallPlan { + pub(crate) fn new( + identity: MetadataInstallIdentity, + root: MetadataPageId, + delta: BTreeMap, + edges: BTreeSet<(MetadataPageId, MetadataPageId)>, + reused: BTreeMap, + source_roots: BTreeMap, + ) -> Result { + let plan = Self { + identity, + root, + delta, + edges, + reused, + source_roots, + }; + plan.validate()?; + Ok(plan) + } + + pub(crate) fn digest(&self) -> Result<[u8; 32], SnapshotError> { + Ok(Sha256::digest(self.encode()?).into()) + } + + pub(crate) fn delta_bytes(&self) -> Result { + self.delta + .values() + .try_fold(0u64, |total, size| total.checked_add(*size)) + .ok_or_else(|| limit("rooted metadata delta byte count overflow")) + } + + pub(crate) fn encode(&self) -> Result, SnapshotError> { + self.validate()?; + let mut bytes = Vec::with_capacity(self.encoded_len()); + bytes.extend_from_slice(DOMAIN); + bytes.extend_from_slice(&ENCODING_VERSION.to_be_bytes()); + for value in [ + &self.identity.source_domain, + &self.identity.tagged_root_tree_oid, + &self.identity.scope, + ] { + write_string(&mut bytes, value); + } + for value in [ + self.identity.schema_version, + self.identity.metadata_codec, + self.identity.materialization_policy, + self.identity.fs_semantics, + self.identity.access_projection, + ] { + bytes.extend_from_slice(&value.to_be_bytes()); + } + bytes.extend_from_slice(&self.identity.verification_revision.to_be_bytes()); + bytes.extend_from_slice(&self.identity.projection_revision.to_be_bytes()); + bytes.extend_from_slice(&self.root); + bytes.extend_from_slice(&(self.delta.len() as u32).to_be_bytes()); + for (page, size) in &self.delta { + bytes.extend_from_slice(page); + bytes.extend_from_slice(&size.to_be_bytes()); + } + bytes.extend_from_slice(&(self.edges.len() as u32).to_be_bytes()); + for (parent, child) in &self.edges { + bytes.extend_from_slice(parent); + bytes.extend_from_slice(child); + } + bytes.extend_from_slice(&(self.reused.len() as u32).to_be_bytes()); + for (page, reused) in &self.reused { + bytes.extend_from_slice(page); + bytes.extend_from_slice(&reused.generation.to_be_bytes()); + bytes.extend_from_slice(reused.attestation_id.as_bytes()); + bytes.extend_from_slice(&reused.attestation_digest); + bytes.extend_from_slice(&reused.certificate_digest); + } + bytes.extend_from_slice(&(self.source_roots.len() as u32).to_be_bytes()); + for (tree_oid, page) in &self.source_roots { + write_string(&mut bytes, tree_oid); + bytes.extend_from_slice(page); + } + Ok(bytes) + } + + pub(crate) fn decode(bytes: &[u8], expected_digest: &[u8; 32]) -> Result { + if bytes.len() > MAX_PLAN_BYTES { + return Err(limit("stored rooted metadata plan exceeds its byte budget")); + } + let digest: [u8; 32] = Sha256::digest(bytes).into(); + if &digest != expected_digest { + return Err(integrity("stored rooted metadata plan digest mismatch")); + } + let mut reader = PlanReader(bytes); + if reader.take(DOMAIN.len())? != DOMAIN || reader.u16()? != ENCODING_VERSION { + return Err(integrity("unsupported rooted metadata plan encoding")); + } + let identity = MetadataInstallIdentity { + source_domain: reader.string(64)?, + tagged_root_tree_oid: reader.string(128)?, + scope: reader.string(4096)?, + schema_version: reader.u16()?, + metadata_codec: reader.u16()?, + materialization_policy: reader.u16()?, + fs_semantics: reader.u16()?, + access_projection: reader.u16()?, + verification_revision: i32::from_be_bytes(reader.array()?), + projection_revision: reader.u16()?, + }; + let root = reader.array()?; + let limits = MetadataDagLimits::default(); + let count = reader.count(limits.nodes)?; + let mut delta = BTreeMap::new(); + let mut last = None; + for _ in 0..count { + let page = reader.array()?; + let size = reader.u64()?; + if last.is_some_and(|previous| previous >= page) { + return Err(integrity("rooted metadata delta is not uniquely ordered")); + } + last = Some(page); + delta.insert(page, size); + } + let count = reader.count(limits.edges)?; + let mut edges = BTreeSet::new(); + let mut last = None; + for _ in 0..count { + let edge = (reader.array()?, reader.array()?); + if last.is_some_and(|previous| previous >= edge) { + return Err(integrity("rooted metadata edges are not uniquely ordered")); + } + last = Some(edge); + edges.insert(edge); + } + let count = reader.count(limits.nodes - delta.len())?; + let mut reused = BTreeMap::new(); + let mut last = None; + for _ in 0..count { + let page = reader.array()?; + let proof = RootedReuseRoot { + generation: i64::from_be_bytes(reader.array()?), + attestation_id: Uuid::from_bytes(reader.array()?), + attestation_digest: reader.array()?, + certificate_digest: reader.array()?, + }; + if last.is_some_and(|previous| previous >= page) { + return Err(integrity( + "rooted metadata reused roots are not uniquely ordered", + )); + } + last = Some(page); + reused.insert(page, proof); + } + let count = reader.count(limits.nodes)?; + let mut source_roots: BTreeMap = BTreeMap::new(); + for _ in 0..count { + let tree_oid = reader.string(128)?; + let page = reader.array()?; + if source_roots + .last_key_value() + .is_some_and(|(previous, _)| previous >= &tree_oid) + { + return Err(integrity( + "rooted metadata source roots are not uniquely ordered", + )); + } + source_roots.insert(tree_oid, page); + } + if !reader.0.is_empty() { + return Err(integrity("rooted metadata plan has trailing bytes")); + } + Self::new(identity, root, delta, edges, reused, source_roots) + } + + pub(crate) fn validate(&self) -> Result<(), SnapshotError> { + self.validate_and_order().map(|_| ()) + } + + /// Child-first delta order. Reused roots have no descendants in this plan. + pub(crate) fn child_first_delta(&self) -> Result, SnapshotError> { + self.validate_and_order() + } + + fn validate_and_order(&self) -> Result, SnapshotError> { + let identity = &self.identity; + if identity.source_domain != "native-git" + || identity.schema_version != mst2_codec::descriptor::SCHEMA_VERSION + || identity.metadata_codec != mst2_codec::descriptor::METADATA_CODEC + || identity.materialization_policy + != mst2_codec::descriptor::MATERIALIZATION_POLICY_GIT_RAW_V1 + || identity.fs_semantics != mst2_codec::descriptor::FS_SEMANTICS_LINUX_CODE_V1 + || identity.access_projection != mst2_codec::descriptor::ACCESS_PROJECTION_EXACT_FULL + || identity.verification_revision + != crate::jupiter::storage::mono_storage::MST2_VERIFICATION_VERSION + || identity.projection_revision != NATIVE_PROJECTION_REVISION + { + return Err(integrity("unsupported rooted native metadata profile")); + } + let hash_kind = tagged_hash_kind(&identity.tagged_root_tree_oid)?; + super::view::validate_scope_relative_path(&identity.scope) + .map_err(|_| integrity("noncanonical rooted metadata scope"))?; + let limits = MetadataDagLimits::default(); + if self.delta.len() > limits.nodes + || self.reused.len() > limits.nodes - self.delta.len() + || self.delta.is_empty() && self.reused.is_empty() + || self.edges.len() > limits.edges + || self.source_roots.is_empty() + || self.source_roots.len() > limits.nodes + { + return Err(limit("rooted metadata plan exceeds its group budget")); + } + if self.delta_bytes()? > limits.payload_bytes { + return Err(limit("rooted metadata delta exceeds its payload budget")); + } + if !self.contains(&self.root) + || self.delta.iter().any(|(page, size)| { + self.reused.contains_key(page) + || !(HEADER_LEN as u64..=PAGE_MAX_BYTES as u64).contains(size) + }) + || self.reused.values().any(|proof| proof.generation <= 0) + || self.delta.is_empty() + && (!self.reused.contains_key(&self.root) || !self.edges.is_empty()) + { + return Err(integrity( + "rooted metadata root, delta or reuse boundary is invalid", + )); + } + for (tree_oid, page) in &self.source_roots { + if tagged_hash_kind(tree_oid)? != hash_kind || !self.contains(page) { + return Err(integrity( + "rooted metadata source binding crossed its profile or boundary", + )); + } + } + if !self.source_roots.values().any(|page| page == &self.root) { + return Err(integrity( + "rooted metadata root lacks a source tree binding", + )); + } + if self.encoded_len() > MAX_PLAN_BYTES { + return Err(limit("rooted metadata plan exceeds its byte budget")); + } + + let mut remaining: BTreeMap<_, usize> = self.delta.keys().map(|page| (*page, 0)).collect(); + let mut parents: BTreeMap<_, Vec<_>> = BTreeMap::new(); + let mut children: BTreeMap<_, Vec<_>> = BTreeMap::new(); + for &(parent, child) in &self.edges { + if parent == child || !self.delta.contains_key(&parent) || !self.contains(&child) { + return Err(integrity( + "rooted metadata edge crossed its delta or reused boundary", + )); + } + children.entry(parent).or_default().push(child); + if self.delta.contains_key(&child) { + *remaining + .get_mut(&parent) + .ok_or_else(|| integrity("rooted metadata delta parent is missing"))? += 1; + parents.entry(child).or_default().push(parent); + } + } + let mut ready: BTreeSet<_> = remaining + .iter() + .filter_map(|(page, count)| (*count == 0).then_some(*page)) + .collect(); + let mut ordered = Vec::with_capacity(self.delta.len()); + while let Some(page) = ready.pop_first() { + ordered.push(page); + for parent in parents.get(&page).into_iter().flatten() { + let count = remaining + .get_mut(parent) + .ok_or_else(|| integrity("rooted metadata delta ancestor is missing"))?; + *count = count + .checked_sub(1) + .ok_or_else(|| integrity("rooted metadata delta edge accounting underflow"))?; + if *count == 0 { + ready.insert(*parent); + } + } + } + if ordered.len() != self.delta.len() { + return Err(integrity("rooted metadata delta contains a cycle")); + } + let mut reachable = BTreeSet::new(); + let mut pending = vec![self.root]; + while let Some(page) = pending.pop() { + if reachable.insert(page) { + pending.extend(children.get(&page).into_iter().flatten()); + } + } + if reachable.len() != self.delta.len() + self.reused.len() { + return Err(integrity( + "rooted metadata plan contains unreachable boundaries", + )); + } + Ok(ordered) + } + + fn contains(&self, page: &MetadataPageId) -> bool { + self.delta.contains_key(page) || self.reused.contains_key(page) + } + + fn encoded_len(&self) -> usize { + DOMAIN.len() + + 2 + + 3 * 4 + + self.identity.source_domain.len() + + self.identity.tagged_root_tree_oid.len() + + self.identity.scope.len() + + 5 * 2 + + 4 + + 2 + + 32 + + 4 * 4 + + self.delta.len() * 40 + + self.edges.len() * 64 + + self.reused.len() * 120 + + self + .source_roots + .keys() + .map(|tree_oid| 4 + tree_oid.len() + 32) + .sum::() + } +} + +fn tagged_hash_kind(oid: &str) -> Result<&str, SnapshotError> { + let (kind, hex) = oid + .split_once(':') + .ok_or_else(|| integrity("untagged rooted metadata source tree"))?; + let length = match kind { + "sha1" => 40, + "sha256" | "blake3" => 64, + _ => return Err(integrity("unsupported rooted metadata source hash kind")), + }; + if hex.len() != length + || !hex + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + { + return Err(integrity( + "noncanonical rooted metadata source tree identity", + )); + } + Ok(kind) +} + +fn write_string(bytes: &mut Vec, value: &str) { + bytes.extend_from_slice(&(value.len() as u32).to_be_bytes()); + bytes.extend_from_slice(value.as_bytes()); +} + +struct PlanReader<'a>(&'a [u8]); + +impl<'a> PlanReader<'a> { + fn take(&mut self, count: usize) -> Result<&'a [u8], SnapshotError> { + if count > self.0.len() { + return Err(integrity("truncated rooted metadata plan")); + } + let (bytes, rest) = self.0.split_at(count); + self.0 = rest; + Ok(bytes) + } + + fn array(&mut self) -> Result<[u8; N], SnapshotError> { + self.take(N)? + .try_into() + .map_err(|_| integrity("invalid rooted metadata plan field")) + } + + fn u16(&mut self) -> Result { + Ok(u16::from_be_bytes(self.array()?)) + } + + fn u32(&mut self) -> Result { + Ok(u32::from_be_bytes(self.array()?)) + } + + fn u64(&mut self) -> Result { + Ok(u64::from_be_bytes(self.array()?)) + } + + fn count(&mut self, maximum: usize) -> Result { + let count = self.u32()? as usize; + if count > maximum { + return Err(limit("rooted metadata plan count exceeds its budget")); + } + Ok(count) + } + + fn string(&mut self, maximum: usize) -> Result { + let count = self.count(maximum)?; + String::from_utf8(self.take(count)?.to_vec()) + .map_err(|_| integrity("non-UTF8 rooted metadata plan identity")) + } +} + +fn integrity(message: &str) -> SnapshotError { + SnapshotError::new(SnapshotErrorCode::IntegrityError, message) +} + +fn limit(message: &str) -> SnapshotError { + SnapshotError::new(SnapshotErrorCode::LimitExceeded, message) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn page(value: u32) -> MetadataPageId { + let mut id = [0; 32]; + id[28..].copy_from_slice(&value.to_be_bytes()); + id + } + + fn identity() -> MetadataInstallIdentity { + MetadataInstallIdentity { + source_domain: "native-git".into(), + tagged_root_tree_oid: format!("sha1:{}", "a".repeat(40)), + scope: "/".into(), + schema_version: mst2_codec::descriptor::SCHEMA_VERSION, + metadata_codec: mst2_codec::descriptor::METADATA_CODEC, + materialization_policy: mst2_codec::descriptor::MATERIALIZATION_POLICY_GIT_RAW_V1, + fs_semantics: mst2_codec::descriptor::FS_SEMANTICS_LINUX_CODE_V1, + access_projection: mst2_codec::descriptor::ACCESS_PROJECTION_EXACT_FULL, + verification_revision: crate::jupiter::storage::mono_storage::MST2_VERIFICATION_VERSION, + projection_revision: NATIVE_PROJECTION_REVISION, + } + } + + fn reuse(value: u128) -> RootedReuseRoot { + RootedReuseRoot { + generation: 7, + attestation_id: Uuid::from_u128(value), + attestation_digest: [11; 32], + certificate_digest: [12; 32], + } + } + + fn plan() -> RootedMetadataInstallPlan { + let identity = identity(); + RootedMetadataInstallPlan::new( + identity.clone(), + page(1), + BTreeMap::from([(page(1), 20), (page(2), 57), (page(3), 16384)]), + BTreeSet::from([ + (page(1), page(2)), + (page(1), page(3)), + (page(1), page(5)), + (page(2), page(4)), + (page(3), page(4)), + ]), + BTreeMap::from([(page(4), reuse(1)), (page(5), reuse(2))]), + BTreeMap::from([ + (identity.tagged_root_tree_oid, page(1)), + (format!("sha1:{}", "b".repeat(40)), page(4)), + ]), + ) + .unwrap() + } + + fn decode(bytes: &[u8]) -> Result { + RootedMetadataInstallPlan::decode(bytes, &Sha256::digest(bytes).into()) + } + + fn records(bytes: &[u8]) -> [Vec>; 4] { + let mut reader = PlanReader(bytes); + reader.take(DOMAIN.len() + 2).unwrap(); + for _ in 0..3 { + reader.string(4096).unwrap(); + } + reader.take(16 + 32).unwrap(); + let mut sections: [Vec>; 4] = std::array::from_fn(|_| Vec::new()); + for (section, width) in sections[..3].iter_mut().zip([40, 64, 120]) { + let count = reader.u32().unwrap(); + for _ in 0..count { + let start = bytes.len() - reader.0.len(); + reader.take(width).unwrap(); + section.push(start..start + width); + } + } + let count = reader.u32().unwrap(); + for _ in 0..count { + let start = bytes.len() - reader.0.len(); + reader.string(128).unwrap(); + reader.take(32).unwrap(); + sections[3].push(start..bytes.len() - reader.0.len()); + } + assert!(reader.0.is_empty()); + sections + } + + #[test] + fn rooted_plan_roundtrip_and_shared_reuse_need_only_delta_order() { + let plan = plan(); + let bytes = plan.encode().unwrap(); + assert_eq!(bytes.len(), plan.encoded_len()); + assert_eq!(bytes.len(), 1004); + assert_eq!( + hex::encode(plan.digest().unwrap()), + "e74e98e8737cfcbdfb264d9f12e15ba2126e96b1e5f35cb0cfd89fb24dde808b" + ); + assert_eq!(decode(&bytes).unwrap(), plan); + assert_eq!( + plan.child_first_delta().unwrap(), + [page(2), page(3), page(1)] + ); + assert_eq!(plan.delta_bytes().unwrap(), 20 + 57 + 16384); + for range in &records(&bytes)[2] { + assert_eq!(range.len(), 120); + } + let original = plan.digest().unwrap(); + for field in 0..4 { + let mut changed = plan.clone(); + let proof = changed.reused.get_mut(&page(4)).unwrap(); + match field { + 0 => proof.generation += 1, + 1 => proof.attestation_id = Uuid::from_u128(9), + 2 => proof.attestation_digest[0] ^= 1, + _ => proof.certificate_digest[0] ^= 1, + } + assert_ne!(changed.digest().unwrap(), original); + } + let mut changed = plan.clone(); + changed.identity.scope = "/目录/a\\b".into(); + assert_ne!(changed.digest().unwrap(), original); + assert_eq!(decode(&changed.encode().unwrap()).unwrap(), changed); + changed = plan; + changed + .source_roots + .insert(format!("sha1:{}", "c".repeat(40)), page(3)); + assert_ne!(changed.digest().unwrap(), original); + } + + #[test] + fn rooted_plan_stored_order_duplicates_digest_domain_version_truncation_and_eof_reject() { + let plan = plan(); + let bytes = plan.encode().unwrap(); + let mut wrong_digest = plan.digest().unwrap(); + wrong_digest[0] ^= 1; + assert!(RootedMetadataInstallPlan::decode(&bytes, &wrong_digest).is_err()); + for section in records(&bytes) { + let mut changed = bytes.clone(); + let a = §ion[0]; + let b = §ion[1]; + assert_eq!(a.len(), b.len()); + changed[a.clone()].copy_from_slice(&bytes[b.clone()]); + changed[b.clone()].copy_from_slice(&bytes[a.clone()]); + assert!(decode(&changed).is_err()); + changed[b.clone()].copy_from_slice(&bytes[b.clone()]); + assert!(decode(&changed).is_err()); + } + for length in [0, DOMAIN.len(), bytes.len() - 1] { + assert!(decode(&bytes[..length]).is_err()); + } + let mut changed = bytes.clone(); + changed.push(0); + assert!(decode(&changed).is_err()); + changed = bytes.clone(); + changed[0] ^= 1; + assert!(decode(&changed).is_err()); + changed = bytes; + changed[DOMAIN.len() + 1] = 2; + assert!(decode(&changed).is_err()); + let oversized = vec![0; MAX_PLAN_BYTES + 1]; + assert_eq!( + decode(&oversized).unwrap_err().code, + SnapshotErrorCode::LimitExceeded + ); + } + + #[test] + fn rooted_plan_decode_bounds_declared_counts_strings_and_reused_generations() { + let bytes = plan().encode().unwrap(); + let sections = records(&bytes); + for section in §ions { + let mut changed = bytes.clone(); + let count_at = section[0].start - 4; + changed[count_at..count_at + 4].copy_from_slice(&u32::MAX.to_be_bytes()); + assert_eq!( + decode(&changed).unwrap_err().code, + SnapshotErrorCode::LimitExceeded + ); + } + let mut changed = bytes.clone(); + let source_at = sections[3][0].start; + changed[source_at..source_at + 4].copy_from_slice(&u32::MAX.to_be_bytes()); + assert_eq!( + decode(&changed).unwrap_err().code, + SnapshotErrorCode::LimitExceeded + ); + changed = bytes.clone(); + changed[DOMAIN.len() + 2 + 4] = 0xff; + assert_eq!( + decode(&changed).unwrap_err().code, + SnapshotErrorCode::IntegrityError + ); + changed = bytes; + let generation_at = sections[2][0].start + 32; + changed[generation_at..generation_at + 8].copy_from_slice(&i64::MIN.to_be_bytes()); + assert_eq!( + decode(&changed).unwrap_err().code, + SnapshotErrorCode::IntegrityError + ); + } + + #[test] + fn rooted_plan_rejects_cycles_unknown_endpoints_reused_parents_and_unreachable_members() { + let original = plan(); + for invalid in 0..7 { + let mut changed = original.clone(); + match invalid { + 0 => { + changed.edges.insert((page(2), page(1))); + } + 1 => { + changed.edges.insert((page(4), page(2))); + } + 2 => { + changed.edges.insert((page(2), page(99))); + } + 3 => { + changed.edges.insert((page(99), page(2))); + } + 4 => { + changed.delta.insert(page(6), 20); + } + 5 => { + changed.reused.insert(page(6), reuse(6)); + } + _ => { + changed.reused.insert(page(2), reuse(2)); + } + } + assert_eq!( + changed.encode().unwrap_err().code, + SnapshotErrorCode::IntegrityError + ); + } + } + + #[test] + fn rooted_plan_zero_delta_requires_its_single_reused_root_and_source_binding() { + let identity = identity(); + let plan = RootedMetadataInstallPlan::new( + identity.clone(), + page(4), + BTreeMap::new(), + BTreeSet::new(), + BTreeMap::from([(page(4), reuse(4))]), + BTreeMap::from([(identity.tagged_root_tree_oid, page(4))]), + ) + .unwrap(); + assert!(plan.child_first_delta().unwrap().is_empty()); + assert_eq!(plan.delta_bytes().unwrap(), 0); + assert_eq!(plan.encode().unwrap().len(), 363); + assert_eq!( + hex::encode(plan.digest().unwrap()), + "81b4cbdaaacd7efe8687477d05bd4db190b974f5aa99ce7e8101326857d0e7b1" + ); + assert_eq!(decode(&plan.encode().unwrap()).unwrap(), plan); + for invalid in 0..5 { + let mut changed = plan.clone(); + match invalid { + 0 => { + changed.root = page(9); + } + 1 => { + changed.edges.insert((page(4), page(4))); + } + 2 => { + changed.reused.insert(page(5), reuse(5)); + } + 3 => { + changed.delta.insert(page(5), 20); + } + _ => { + changed.source_roots.clear(); + } + } + assert!(changed.encode().is_err()); + } + } + + #[test] + fn rooted_plan_rejects_source_profile_scope_size_and_generation_mismatches() { + let original = plan(); + for invalid in 0..18 { + let mut changed = original.clone(); + match invalid { + 0 => changed.identity.metadata_codec += 1, + 1 => changed.identity.projection_revision += 1, + 2 => changed.identity.verification_revision += 1, + 3 => changed.identity.scope = "/a/..".into(), + 4 => changed.identity.scope = format!("/{}", "a".repeat(256)), + 5 => changed.identity.tagged_root_tree_oid = format!("sha1:{}", "A".repeat(40)), + 6 => { + changed.delta.insert(page(2), HEADER_LEN as u64 - 1); + } + 7 => { + changed.delta.insert(page(2), PAGE_MAX_BYTES as u64 + 1); + } + 8 => { + changed.reused.get_mut(&page(4)).unwrap().generation = 0; + } + 9 => { + changed + .source_roots + .insert(format!("sha256:{}", "a".repeat(64)), page(3)); + } + 10 => { + changed + .source_roots + .insert(format!("sha1:{}", "d".repeat(40)), page(99)); + } + 11 => { + changed + .source_roots + .retain(|_, page_id| *page_id != page(1)); + } + 12 => { + changed.source_roots.insert("sha1:bad".into(), page(3)); + } + 13 => changed.identity.source_domain = "other".into(), + 14 => changed.identity.schema_version += 1, + 15 => changed.identity.materialization_policy += 1, + 16 => changed.identity.fs_semantics += 1, + _ => changed.identity.access_projection += 1, + } + assert!(changed.encode().is_err()); + } + for kind in ["sha256", "blake3"] { + let mut changed = original.clone(); + changed.identity.tagged_root_tree_oid = format!("{kind}:{}", "a".repeat(64)); + changed.source_roots = + BTreeMap::from([(changed.identity.tagged_root_tree_oid.clone(), page(1))]); + assert_eq!(decode(&changed.encode().unwrap()).unwrap(), changed); + } + } + + #[test] + fn rooted_plan_admits_exact_group_payload_and_source_count_boundaries() { + let identity = identity(); + let limits = MetadataDagLimits::default(); + let delta: BTreeMap<_, _> = (1..=limits.nodes as u32) + .map(|value| (page(value), PAGE_MAX_BYTES as u64)) + .collect(); + let edges = (2..=limits.nodes as u32) + .map(|value| (page(1), page(value))) + .collect(); + let full_plan = RootedMetadataInstallPlan::new( + identity.clone(), + page(1), + delta, + edges, + BTreeMap::new(), + BTreeMap::from([(identity.tagged_root_tree_oid, page(1))]), + ) + .unwrap(); + assert_eq!(full_plan.delta_bytes().unwrap(), limits.payload_bytes); + assert_eq!(decode(&full_plan.encode().unwrap()).unwrap(), full_plan); + let mut changed = full_plan.clone(); + changed + .delta + .insert(page(limits.nodes as u32 + 1), HEADER_LEN as u64); + assert_eq!( + changed.encode().unwrap_err().code, + SnapshotErrorCode::LimitExceeded + ); + changed = full_plan.clone(); + changed + .reused + .insert(page(limits.nodes as u32 + 1), reuse(9)); + assert_eq!( + changed.encode().unwrap_err().code, + SnapshotErrorCode::LimitExceeded + ); + + let mut source_plan = plan(); + source_plan.source_roots = (0..limits.nodes) + .map(|value| (format!("sha1:{value:040x}"), page(1))) + .collect(); + assert_eq!(decode(&source_plan.encode().unwrap()).unwrap(), source_plan); + source_plan + .source_roots + .insert(format!("sha1:{:040x}", limits.nodes), page(1)); + assert_eq!( + source_plan.encode().unwrap_err().code, + SnapshotErrorCode::LimitExceeded + ); + } + + #[test] + fn rooted_plan_admits_exact_edge_boundary_then_rejects_one_more_edge() { + let mut value = plan(); + value.delta = (1..=4096).map(|id| (page(id), HEADER_LEN as u64)).collect(); + value.reused.clear(); + value.source_roots.retain(|_, root| *root == page(1)); + value.edges = (2..=4096).map(|id| (page(1), page(id))).collect(); + 'fill: for parent in 2..=4096 { + for child in parent + 1..=4096 { + if value.edges.len() == MetadataDagLimits::default().edges { + break 'fill; + } + value.edges.insert((page(parent), page(child))); + } + } + assert_eq!(value.edges.len(), MetadataDagLimits::default().edges); + assert_eq!(decode(&value.encode().unwrap()).unwrap(), value); + assert!(value.edges.insert((page(4095), page(4096)))); + assert_eq!( + value.encode().unwrap_err().code, + SnapshotErrorCode::LimitExceeded + ); + } +} diff --git a/src/ceres/snapshot/rooted_metadata_projection.rs b/src/ceres/snapshot/rooted_metadata_projection.rs new file mode 100644 index 00000000..086db637 --- /dev/null +++ b/src/ceres/snapshot/rooted_metadata_projection.rs @@ -0,0 +1,1108 @@ +//! Native source projection that stops at certified reused-directory boundaries. +//! Hints and operation-local memoization are not database installation authority. + +use std::{ + borrow::Cow, + collections::{BTreeMap, BTreeSet, VecDeque}, +}; + +use async_trait::async_trait; +use git_internal::{hash::ObjectHash, internal::object::tree::Tree}; +use mst2_codec::{ + descriptor::{ + ACCESS_PROJECTION_EXACT_FULL, FS_SEMANTICS_LINUX_CODE_V1, + MATERIALIZATION_POLICY_GIT_RAW_V1, METADATA_CODEC, SCHEMA_VERSION, + }, + metapage::{Entry, EntryKind, HEADER_LEN, PAGE_MAX_BYTES, Page, page_id}, +}; +use sea_orm::ActiveValue::Set; +use sha2::{Digest, Sha256}; + +use super::{ + error::{SnapshotError, SnapshotErrorCode}, + metadata_install::MetadataInstallIdentity, + projection_observation::NATIVE_PROJECTION_REVISION, + resolver::{FsKind, direct_entries}, + retention_dag::{MetadataDagLimits, MetadataPageId, MetadataPagePayload}, + rooted_metadata_install::{RootedMetadataInstallPlan, RootedReuseRoot}, + view::validate_scope_relative_path, +}; +use crate::{ + ceres::api_service::ApiHandler, + common::errors::MegaError, + jupiter::storage::{Storage, mono_storage::MST2_VERIFICATION_VERSION}, +}; + +#[async_trait] +pub(crate) trait RootedReuseLookup: Send + Sync { + async fn lookup_reuse( + &self, + tree_oid: &str, + identity: &MetadataInstallIdentity, + ) -> Result, SnapshotError>; +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct CertifiedReusableDirectory { + pub(crate) page_id: MetadataPageId, + pub(crate) proof: RootedReuseRoot, + pub(crate) relative_path_bytes: usize, + pub(crate) relative_components: usize, + pub(crate) closure_nodes_upper: usize, + pub(crate) closure_edges_upper: usize, + pub(crate) closure_bytes_upper: u64, + pub(crate) closure_entries_upper: usize, +} + +/// API calls and materialized outputs only, not total database or codec work. +#[derive(Debug, Clone, Default, PartialEq, Eq, serde::Serialize)] +pub(crate) struct RootedProjectionWork { + pub(crate) input_tree_entries: u64, + pub(crate) scope_path_entries_examined: u64, + pub(crate) tree_fetches: u64, + pub(crate) reuse_lookups: u64, + pub(crate) directory_memo_hits: u64, + pub(crate) directories_scanned: u64, + pub(crate) direct_entries_scanned: u64, + pub(crate) verified_blob_queries: u64, + pub(crate) verified_blob_facts_loaded: u64, + pub(crate) verified_blob_persistence_batches: u64, + pub(crate) blob_fetches: u64, + pub(crate) raw_bytes_fetched: u64, + pub(crate) raw_bytes_hashed: u64, + pub(crate) directory_root_builds: u64, + pub(crate) radix_route_builds: u64, + pub(crate) codec_input_entries: u64, + pub(crate) delta_pages: u64, + pub(crate) delta_bytes: u64, + pub(crate) reused_roots: u64, + pub(crate) reused_closure_nodes_upper: u64, + pub(crate) reused_closure_edges_upper: u64, + pub(crate) reused_closure_bytes_upper: u64, + pub(crate) reused_closure_entries_upper: u64, + pub(crate) plan_bytes: u64, +} + +#[derive(Debug)] +pub(crate) struct PreparedRootedNativeMetadata { + pub(crate) plan: RootedMetadataInstallPlan, + pub(crate) payloads: Vec, + pub(crate) work: RootedProjectionWork, +} + +pub(crate) async fn prepare_rooted_native_metadata< + T: ApiHandler + ?Sized, + R: RootedReuseLookup + ?Sized, +>( + handler: &T, + root_tree: &Tree, + scope: &str, + reuse: &R, +) -> Result { + validate_scope_relative_path(scope)?; + if !handler.native_snapshot_projection() { + return Err(SnapshotError::new( + SnapshotErrorCode::ScopeInvalid, + "rooted metadata projection requires native Git source semantics", + )); + } + let identity = MetadataInstallIdentity { + source_domain: "native-git".into(), + tagged_root_tree_oid: root_tree.id.to_tagged_string(), + scope: scope.to_owned(), + schema_version: SCHEMA_VERSION, + metadata_codec: METADATA_CODEC, + materialization_policy: MATERIALIZATION_POLICY_GIT_RAW_V1, + fs_semantics: FS_SEMANTICS_LINUX_CODE_V1, + access_projection: ACCESS_PROJECTION_EXACT_FULL, + verification_revision: MST2_VERIFICATION_VERSION, + projection_revision: NATIVE_PROJECTION_REVISION, + }; + let storage = handler.get_context(); + let mut state = ProjectionState::new(identity); + state.work.input_tree_entries = root_tree.tree_items.len() as u64; + let scoped_oid = resolve_scope_oid(handler, root_tree, scope, &mut state.work).await?; + let loaded = (scope == "/").then_some(root_tree); + let root = project_directory( + handler, reuse, &storage, scoped_oid, loaded, scope, &mut state, + ) + .await?; + state.finish(root.page_id) +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +struct PathBounds { + bytes: usize, + components: usize, +} + +impl PathBounds { + fn include(&mut self, name: &str, child: Self) -> Result<(), SnapshotError> { + let bytes = name + .len() + .checked_add(1) + .and_then(|value| value.checked_add(child.bytes)) + .filter(|value| *value <= 4096) + .ok_or_else(path_limit)?; + let components = child + .components + .checked_add(1) + .filter(|value| *value <= 256) + .ok_or_else(path_limit)?; + self.bytes = self.bytes.max(bytes); + self.components = self.components.max(components); + Ok(()) + } + + fn validate_at(self, prefix: &str) -> Result<(), SnapshotError> { + validate_scope_relative_path(prefix)?; + let (bytes, components) = if prefix == "/" { + (0, 0) + } else { + (prefix.len(), prefix[1..].split('/').count()) + }; + bytes + .checked_add(self.bytes) + .filter(|value| *value <= 4096) + .ok_or_else(path_limit)?; + components + .checked_add(self.components) + .filter(|value| *value <= 256) + .ok_or_else(path_limit)?; + Ok(()) + } +} + +#[derive(Debug, Clone, Copy)] +struct DirectorySummary { + page_id: MetadataPageId, + bounds: PathBounds, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct BlobFact { + size: u64, + digest: [u8; 32], +} + +#[derive(Default)] +struct ClosureUpper { + nodes: usize, + edges: usize, + bytes: u64, + entries: usize, +} + +impl ClosureUpper { + fn add(&mut self, directory: &CertifiedReusableDirectory) -> Result<(), SnapshotError> { + self.nodes = self + .nodes + .checked_add(directory.closure_nodes_upper) + .ok_or_else(budget_limit)?; + self.edges = self + .edges + .checked_add(directory.closure_edges_upper) + .ok_or_else(budget_limit)?; + self.bytes = self + .bytes + .checked_add(directory.closure_bytes_upper) + .ok_or_else(budget_limit)?; + self.entries = self + .entries + .checked_add(directory.closure_entries_upper) + .ok_or_else(budget_limit)?; + Ok(()) + } +} + +struct ProjectionState { + identity: MetadataInstallIdentity, + required_trees: BTreeSet, + active_trees: BTreeSet, + directories: BTreeMap, + bounds: BTreeMap, + payloads: BTreeMap, + page_entries: BTreeMap, + edges: BTreeSet<(MetadataPageId, MetadataPageId)>, + reused: BTreeMap, + source_roots: BTreeMap, + blob_facts: BTreeMap, + source_entries: usize, + source_encoded_bytes: u64, + codec_entry_visits: usize, + payload_bytes: u64, + delta_entries: usize, + reuse_upper: ClosureUpper, + work: RootedProjectionWork, +} + +impl ProjectionState { + fn new(identity: MetadataInstallIdentity) -> Self { + Self { + identity, + required_trees: BTreeSet::new(), + active_trees: BTreeSet::new(), + directories: BTreeMap::new(), + bounds: BTreeMap::new(), + payloads: BTreeMap::new(), + page_entries: BTreeMap::new(), + edges: BTreeSet::new(), + reused: BTreeMap::new(), + source_roots: BTreeMap::new(), + blob_facts: BTreeMap::new(), + source_entries: 0, + source_encoded_bytes: 0, + codec_entry_visits: 0, + payload_bytes: 0, + delta_entries: 0, + reuse_upper: ClosureUpper::default(), + work: RootedProjectionWork::default(), + } + } + + fn require_tree(&mut self, tree_oid: &str) -> Result<(), SnapshotError> { + if !self.required_trees.contains(tree_oid) { + if self.required_trees.len() >= MetadataDagLimits::default().nodes { + return Err(budget_limit()); + } + self.required_trees.insert(tree_oid.to_owned()); + } + Ok(()) + } + + fn admit_entries(&mut self, entries: &[(String, FsKind, String)]) -> Result<(), SnapshotError> { + let limits = MetadataDagLimits::default(); + self.source_encoded_bytes = self + .source_encoded_bytes + .checked_add(HEADER_LEN as u64) + .filter(|value| *value <= limits.payload_bytes) + .ok_or_else(budget_limit)?; + let mut previous: Option<&str> = None; + for (name, kind, _) in entries { + if name.is_empty() + || name.len() > 255 + || name == "." + || name == ".." + || name.contains('/') + || name.contains('\0') + || previous.is_some_and(|value| value >= name.as_str()) + { + return Err(integrity( + "source tree has an invalid or duplicate UTF-8 name", + )); + } + previous = Some(name); + let value_bytes = if *kind == FsKind::Directory { 32 } else { 40 }; + self.source_encoded_bytes = self + .source_encoded_bytes + .checked_add((3 + name.len() + value_bytes) as u64) + .filter(|value| *value <= limits.payload_bytes) + .ok_or_else(budget_limit)?; + } + Ok(()) + } + + fn record_bounds( + &mut self, + page: MetadataPageId, + bounds: PathBounds, + ) -> Result<(), SnapshotError> { + if self + .bounds + .get(&page) + .is_some_and(|previous| previous != &bounds) + { + return Err(integrity( + "one canonical directory has inconsistent descendant path bounds", + )); + } + self.bounds.insert(page, bounds); + Ok(()) + } + + fn record_reuse( + &mut self, + tree_oid: &str, + directory: CertifiedReusableDirectory, + path: &str, + ) -> Result { + let limits = MetadataDagLimits::default(); + if directory.page_id == [0; 32] + || directory.proof.generation <= 0 + || directory.closure_nodes_upper == 0 + || directory.closure_nodes_upper > limits.nodes + || directory.closure_edges_upper > limits.edges + || !(HEADER_LEN as u64..=limits.payload_bytes).contains(&directory.closure_bytes_upper) + || directory.closure_entries_upper > limits.entries + || directory.relative_path_bytes > 4096 + || directory.relative_components > 256 + { + return Err(integrity( + "reused directory hint has an invalid certificate summary", + )); + } + let summary = DirectorySummary { + page_id: directory.page_id, + bounds: PathBounds { + bytes: directory.relative_path_bytes, + components: directory.relative_components, + }, + }; + summary.bounds.validate_at(path)?; + self.record_bounds(directory.page_id, summary.bounds)?; + if !self.payloads.contains_key(&directory.page_id) { + if let Some(previous) = self.reused.get(&directory.page_id) { + if previous.proof.generation != directory.proof.generation + || previous.proof.certificate_digest != directory.proof.certificate_digest + || previous.closure_nodes_upper != directory.closure_nodes_upper + || previous.closure_edges_upper != directory.closure_edges_upper + || previous.closure_bytes_upper != directory.closure_bytes_upper + || previous.closure_entries_upper != directory.closure_entries_upper + { + return Err(integrity( + "one reused boundary has inconsistent lifetime or certificate summaries", + )); + } + } else { + self.reuse_upper.add(&directory)?; + self.reused.insert(directory.page_id, directory); + self.check_budget()?; + } + } + self.source_roots + .insert(tree_oid.to_owned(), summary.page_id); + Ok(summary) + } + + fn charge_codec(&mut self, entries: usize, route_length: usize) -> Result<(), SnapshotError> { + let count = entries + .checked_mul(route_length.checked_add(1).ok_or_else(budget_limit)?) + .ok_or_else(budget_limit)?; + self.codec_entry_visits = self + .codec_entry_visits + .checked_add(count) + .filter(|value| *value <= MetadataDagLimits::default().prepare_entry_visits) + .ok_or_else(budget_limit)?; + self.work.codec_input_entries = self + .work + .codec_input_entries + .checked_add(entries as u64) + .ok_or_else(budget_limit)?; + Ok(()) + } + + fn add_edge( + &mut self, + parent: MetadataPageId, + child: MetadataPageId, + ) -> Result<(), SnapshotError> { + if self.edges.insert((parent, child)) { + self.check_budget()?; + } + Ok(()) + } + + fn collect_directory( + &mut self, + entries: &[Entry], + root_bytes: Vec, + ) -> Result { + let root = page_id(&root_bytes); + let mut routes = VecDeque::from([(Vec::::new(), root, Some(root_bytes))]); + while let Some((route, expected, provided)) = routes.pop_front() { + if self.payloads.contains_key(&expected) || self.reused.contains_key(&expected) { + continue; + } + let bytes = if let Some(bytes) = provided { + bytes + } else { + self.charge_codec(entries.len(), route.len())?; + self.work.radix_route_builds += 1; + Page::pages_along_route(entries, &route) + .map_err(codec_error)? + .pop() + .ok_or_else(|| integrity("canonical radix route returned no page"))? + }; + if page_id(&bytes) != expected || !(HEADER_LEN..=PAGE_MAX_BYTES).contains(&bytes.len()) + { + return Err(integrity( + "canonical radix page differs from its parent binding", + )); + } + let (page, _) = Page::decode(&bytes).map_err(codec_error)?; + let page_entries = match page { + Page::Leaf { entries } => { + for entry in entries.iter().filter(|entry| entry.is_dir()) { + self.add_edge(expected, entry.child_root)?; + } + entries.len() + } + Page::Branch { + terminal, children, .. + } => { + if let Some(entry) = terminal.as_ref().filter(|entry| entry.is_dir()) { + self.add_edge(expected, entry.child_root)?; + } + for child in children { + self.add_edge(expected, child.child_page_id)?; + let mut child_route = route.clone(); + child_route.push(child.label); + routes.push_back((child_route, child.child_page_id, None)); + } + usize::from(terminal.is_some()) + } + }; + self.payload_bytes = self + .payload_bytes + .checked_add(bytes.len() as u64) + .ok_or_else(budget_limit)?; + self.delta_entries = self + .delta_entries + .checked_add(page_entries) + .ok_or_else(budget_limit)?; + self.page_entries.insert(expected, page_entries); + self.payloads.insert( + expected, + MetadataPagePayload { + id: expected, + size: bytes.len() as u64, + bytes, + }, + ); + self.check_budget()?; + } + Ok(root) + } + + fn check_budget(&self) -> Result<(), SnapshotError> { + let limits = MetadataDagLimits::default(); + if self + .payloads + .len() + .checked_add(self.reuse_upper.nodes) + .is_none_or(|value| value > limits.nodes) + || self + .edges + .len() + .checked_add(self.reuse_upper.edges) + .is_none_or(|value| value > limits.edges) + || self + .payload_bytes + .checked_add(self.reuse_upper.bytes) + .is_none_or(|value| value > limits.payload_bytes) + || self + .delta_entries + .checked_add(self.reuse_upper.entries) + .is_none_or(|value| value > limits.entries) + { + return Err(budget_limit()); + } + Ok(()) + } + + fn finish( + mut self, + root: MetadataPageId, + ) -> Result { + let mut children: BTreeMap<_, Vec<_>> = BTreeMap::new(); + for &(parent, child) in &self.edges { + children.entry(parent).or_default().push(child); + } + let mut reachable = BTreeSet::new(); + let mut pending = vec![root]; + while let Some(page) = pending.pop() { + if reachable.insert(page) && !self.reused.contains_key(&page) { + pending.extend(children.get(&page).into_iter().flatten()); + } + } + self.payloads.retain(|page, _| reachable.contains(page)); + self.page_entries.retain(|page, _| reachable.contains(page)); + self.reused.retain(|page, _| reachable.contains(page)); + self.source_roots.retain(|_, page| reachable.contains(page)); + self.edges + .retain(|(parent, child)| reachable.contains(parent) && reachable.contains(child)); + self.payload_bytes = self + .payloads + .values() + .try_fold(0u64, |total, page| total.checked_add(page.size)) + .ok_or_else(budget_limit)?; + self.delta_entries = self + .page_entries + .values() + .try_fold(0usize, |total, count| total.checked_add(*count)) + .ok_or_else(budget_limit)?; + self.reuse_upper = ClosureUpper::default(); + for directory in self.reused.values() { + self.reuse_upper.add(directory)?; + } + self.check_budget()?; + let plan = RootedMetadataInstallPlan::new( + self.identity, + root, + self.payloads + .iter() + .map(|(page, payload)| (*page, payload.size)) + .collect(), + self.edges, + self.reused + .into_iter() + .map(|(page, directory)| (page, directory.proof)) + .collect(), + self.source_roots, + )?; + self.work.delta_pages = plan.delta.len() as u64; + self.work.delta_bytes = self.payload_bytes; + self.work.reused_roots = plan.reused.len() as u64; + self.work.reused_closure_nodes_upper = self.reuse_upper.nodes as u64; + self.work.reused_closure_edges_upper = self.reuse_upper.edges as u64; + self.work.reused_closure_bytes_upper = self.reuse_upper.bytes; + self.work.reused_closure_entries_upper = self.reuse_upper.entries as u64; + self.work.plan_bytes = plan.encode()?.len() as u64; + Ok(PreparedRootedNativeMetadata { + plan, + payloads: self.payloads.into_values().collect(), + work: self.work, + }) + } +} + +async fn resolve_scope_oid( + handler: &T, + root: &Tree, + scope: &str, + work: &mut RootedProjectionWork, +) -> Result { + if scope == "/" { + return Ok(root.id); + } + let mut current = Cow::Borrowed(root); + let mut components = scope[1..].split('/').peekable(); + while let Some(component) = components.next() { + let position = current + .tree_items + .iter() + .position(|item| item.name == component); + work.scope_path_entries_examined += + position.map_or(current.tree_items.len(), |index| index + 1) as u64; + let item = position + .map(|index| ¤t.tree_items[index]) + .ok_or_else(|| { + SnapshotError::new( + SnapshotErrorCode::PathNotFound, + "name absent in enumerated parent directory", + ) + })?; + let kind = FsKind::from_git_mode(item.mode).ok_or_else(|| { + SnapshotError::new( + SnapshotErrorCode::UnsupportedEntry, + "gitlink entries are not supported in this profile", + ) + })?; + if kind != FsKind::Directory { + return Err(SnapshotError::new( + SnapshotErrorCode::NotDirectory, + "rooted metadata scope is not a directory", + )); + } + let expected = item.id; + if expected.kind() != root.id.kind() { + return Err(integrity("scope tree crossed its source hash kind")); + } + if components.peek().is_none() { + return Ok(expected); + } + work.tree_fetches += 1; + let fetched = handler + .get_tree_by_hash(&expected.to_string()) + .await + .map_err(source_error)?; + if fetched.id != expected { + return Err(integrity("fetched scope ancestor identity mismatch")); + } + current = Cow::Owned(fetched); + } + Err(integrity("rooted metadata scope walk has no target")) +} + +async fn project_directory( + handler: &T, + reuse: &R, + storage: &Storage, + oid: ObjectHash, + loaded: Option<&Tree>, + path: &str, + state: &mut ProjectionState, +) -> Result { + validate_scope_relative_path(path)?; + let tagged = oid.to_tagged_string(); + if let Some(summary) = state.directories.get(&tagged).copied() { + summary.bounds.validate_at(path)?; + state.work.directory_memo_hits += 1; + return Ok(summary); + } + state.require_tree(&tagged)?; + if !state.active_trees.insert(tagged.clone()) { + return Err(integrity("native source trees contain a cycle")); + } + state.work.reuse_lookups += 1; + if let Some(hint) = reuse.lookup_reuse(&tagged, &state.identity).await? { + let summary = state.record_reuse(&tagged, hint, path)?; + state.active_trees.remove(&tagged); + state.directories.insert(tagged, summary); + return Ok(summary); + } + let fetched; + let tree = if let Some(tree) = loaded { + tree + } else { + state.work.tree_fetches += 1; + fetched = handler + .get_tree_by_hash(&oid.to_string()) + .await + .map_err(source_error)?; + &fetched + }; + if tree.id != oid + || tree + .tree_items + .iter() + .any(|item| item.id.kind() != oid.kind()) + { + return Err(integrity( + "fetched source directory identity or child hash kind mismatch", + )); + } + state.source_entries = state + .source_entries + .checked_add(tree.tree_items.len()) + .filter(|value| *value <= MetadataDagLimits::default().entries) + .ok_or_else(budget_limit)?; + let direct = direct_entries(tree)?; + state.admit_entries(&direct)?; + state.work.directories_scanned += 1; + state.work.direct_entries_scanned += direct.len() as u64; + let requested: Vec<_> = direct + .iter() + .filter(|(_, kind, oid)| *kind != FsKind::Directory && !state.blob_facts.contains_key(oid)) + .map(|(_, _, oid)| oid.clone()) + .collect::>() + .into_iter() + .collect(); + for batch in requested.chunks(64) { + state.work.verified_blob_queries += 1; + let facts = storage + .mono_storage() + .get_verified_blobs(batch.to_vec()) + .await + .map_err(source_error)?; + state.work.verified_blob_facts_loaded += facts.len() as u64; + for (oid, fact) in facts { + state.blob_facts.insert(oid, verified_fact(&fact)?); + } + } + let mut pending_facts = Vec::new(); + let mut bounds = PathBounds::default(); + let mut entries = Vec::with_capacity(direct.len()); + for (name, kind, raw_oid) in direct { + let child_path = if path == "/" { + format!("/{name}") + } else { + format!("{path}/{name}") + }; + validate_scope_relative_path(&child_path)?; + match kind { + FsKind::Directory => { + let child_oid = ObjectHash::from_hex_for_kind(oid.kind(), &raw_oid) + .map_err(|error| integrity(&error.to_string()))?; + let child = Box::pin(project_directory( + handler, + reuse, + storage, + child_oid, + None, + &child_path, + state, + )) + .await?; + bounds.include(&name, child.bounds)?; + entries.push(Entry::dir(name.as_bytes(), child.page_id)); + } + FsKind::Regular | FsKind::Executable | FsKind::Symlink => { + bounds.include(&name, PathBounds::default())?; + let fact = if let Some(fact) = state.blob_facts.get(&raw_oid).copied() { + fact + } else { + state.work.blob_fetches += 1; + let raw = super::pages::fetch_raw_blob(handler, &raw_oid).await?; + state.work.raw_bytes_fetched = state + .work + .raw_bytes_fetched + .checked_add(raw.len() as u64) + .ok_or_else(budget_limit)?; + let size = u64::try_from(raw.len()) + .map_err(|_| integrity("raw file size is not representable"))?; + if size > 8_796_093_022_208 { + return Err(integrity( + "raw file exceeds the native verified-object size profile", + )); + } + let digest: [u8; 32] = Sha256::digest(&raw).into(); + state.work.raw_bytes_hashed = state + .work + .raw_bytes_hashed + .checked_add(size) + .ok_or_else(budget_limit)?; + drop(raw); + let fact = BlobFact { size, digest }; + state.blob_facts.insert(raw_oid.clone(), fact); + pending_facts.push((raw_oid.clone(), fact)); + if pending_facts.len() == 64 { + persist_facts(storage, &pending_facts, &mut state.work).await?; + pending_facts.clear(); + } + fact + }; + let entry_kind = match kind { + FsKind::Regular => EntryKind::Regular, + FsKind::Executable => EntryKind::Executable, + FsKind::Symlink => EntryKind::Symlink, + FsKind::Directory => { + return Err(integrity("file projection received a directory kind")); + } + }; + entries.push(Entry::file( + entry_kind, + name.as_bytes(), + fact.size, + fact.digest, + )); + } + } + } + if !pending_facts.is_empty() { + persist_facts(storage, &pending_facts, &mut state.work).await?; + } + bounds.validate_at(path)?; + state.charge_codec(entries.len(), 0)?; + state.work.directory_root_builds += 1; + let root_bytes = Page::build(&entries).map_err(codec_error)?; + let page_id = state.collect_directory(&entries, root_bytes)?; + state.record_bounds(page_id, bounds)?; + state.source_roots.insert(tagged.clone(), page_id); + let summary = DirectorySummary { page_id, bounds }; + state.active_trees.remove(&tagged); + state.directories.insert(tagged, summary); + Ok(summary) +} + +fn verified_fact( + fact: &crate::callisto::mst2_verified_object::Model, +) -> Result { + if fact.state != "VERIFIED" + || fact.verification_version != MST2_VERIFICATION_VERSION + || !(0..=8_796_093_022_208).contains(&fact.size) + { + return Err(integrity( + "native projection received an invalid current verified blob fact", + )); + } + Ok(BlobFact { + size: u64::try_from(fact.size).map_err(|_| integrity("verified blob size is invalid"))?, + digest: fact + .raw_sha256 + .as_slice() + .try_into() + .map_err(|_| integrity("verified blob digest is invalid"))?, + }) +} + +async fn persist_facts( + storage: &Storage, + facts: &[(String, BlobFact)], + work: &mut RootedProjectionWork, +) -> Result<(), SnapshotError> { + let rows = facts + .iter() + .map( + |(oid, fact)| crate::callisto::mst2_verified_object::ActiveModel { + id: sea_orm::ActiveValue::NotSet, + storage_domain: Set("git".into()), + git_oid: Set(oid.clone()), + object_kind: Set("blob".into()), + raw_sha256: Set(fact.digest.to_vec()), + size: Set(fact.size as i64), + verification_version: Set(MST2_VERIFICATION_VERSION), + state: Set("VERIFIED".into()), + created_at: Set(chrono::Utc::now().fixed_offset()), + }, + ) + .collect(); + work.verified_blob_persistence_batches += 1; + storage + .mono_storage() + .insert_verified_blobs(rows) + .await + .map_err(source_error)?; + work.verified_blob_queries += 1; + let stored = storage + .mono_storage() + .get_verified_blobs(facts.iter().map(|(oid, _)| oid.clone()).collect()) + .await + .map_err(source_error)?; + work.verified_blob_facts_loaded += stored.len() as u64; + for (oid, expected) in facts { + let actual = stored + .get(oid) + .ok_or_else(|| integrity("verified blob persistence lost its current fact"))?; + if verified_fact(actual)? != *expected { + return Err(integrity( + "verified blob persistence conflicts with the hashed raw source", + )); + } + } + Ok(()) +} + +fn source_error(error: MegaError) -> SnapshotError { + let code = match error { + MegaError::ObjStorageNotFound(_) => SnapshotErrorCode::ObjectUnavailable, + MegaError::ObjStorageInconsistent(_) => SnapshotErrorCode::IntegrityError, + _ => SnapshotErrorCode::Internal, + }; + tracing::warn!(error = %error, "rooted native metadata source operation failed"); + SnapshotError::new(code, "rooted native metadata source operation failed") +} + +fn codec_error(error: mst2_codec::CodecError) -> SnapshotError { + integrity(&error.to_string()) +} +fn integrity(message: &str) -> SnapshotError { + SnapshotError::new(SnapshotErrorCode::IntegrityError, message) +} +fn budget_limit() -> SnapshotError { + SnapshotError::new( + SnapshotErrorCode::LimitExceeded, + "rooted metadata projection exceeds its fixed budget", + ) +} +fn path_limit() -> SnapshotError { + SnapshotError::new( + SnapshotErrorCode::ScopeInvalid, + "rooted metadata subtree exceeds its full-path budget", + ) +} + +#[cfg(test)] +mod tests { + use git_internal::hash::HashKind; + use uuid::Uuid; + + use super::*; + + fn state() -> ProjectionState { + ProjectionState::new(MetadataInstallIdentity { + source_domain: "native-git".into(), + tagged_root_tree_oid: ObjectHash::from_hex_for_kind(HashKind::Sha1, &"a".repeat(40)) + .unwrap() + .to_tagged_string(), + scope: "/".into(), + schema_version: SCHEMA_VERSION, + metadata_codec: METADATA_CODEC, + materialization_policy: MATERIALIZATION_POLICY_GIT_RAW_V1, + fs_semantics: FS_SEMANTICS_LINUX_CODE_V1, + access_projection: ACCESS_PROJECTION_EXACT_FULL, + verification_revision: MST2_VERIFICATION_VERSION, + projection_revision: NATIVE_PROJECTION_REVISION, + }) + } + + fn reusable(page: MetadataPageId) -> CertifiedReusableDirectory { + CertifiedReusableDirectory { + page_id: page, + proof: RootedReuseRoot { + generation: 1, + attestation_id: Uuid::from_u128(1), + attestation_digest: [1; 32], + certificate_digest: [2; 32], + }, + relative_path_bytes: 0, + relative_components: 0, + closure_nodes_upper: 1, + closure_edges_upper: 0, + closure_bytes_upper: HEADER_LEN as u64, + closure_entries_upper: 0, + } + } + + #[test] + fn reused_path_bounds_recheck_moved_aliases_and_checked_overflow() { + let bounds = PathBounds { + bytes: 4093, + components: 255, + }; + assert!(bounds.validate_at("/").is_ok()); + assert!(bounds.validate_at("/a").is_ok()); + assert_eq!( + bounds.validate_at("/a/b").unwrap_err().code, + SnapshotErrorCode::ScopeInvalid + ); + assert!( + PathBounds { + bytes: usize::MAX, + components: 0 + } + .validate_at("/a") + .is_err() + ); + let mut parent = PathBounds::default(); + parent + .include( + "a", + PathBounds { + bytes: 3, + components: 1, + }, + ) + .unwrap(); + parent.include("longer", PathBounds::default()).unwrap(); + assert_eq!( + parent, + PathBounds { + bytes: 7, + components: 2 + } + ); + assert!( + parent + .include( + "a", + PathBounds { + bytes: usize::MAX, + components: 0 + } + ) + .is_err() + ); + } + + #[test] + fn canonical_radix_payloads_and_shared_reuse_edges_form_a_delta_only_plan() { + let empty = Page::build(&[]).unwrap(); + let reused_page = page_id(&empty); + let mut state = state(); + state + .record_reuse( + &format!("sha1:{}", "b".repeat(40)), + reusable(reused_page), + "/left", + ) + .unwrap(); + let mut entries: Vec<_> = (0..260) + .map(|index| { + Entry::file( + EntryKind::Regular, + format!("file-{index:03}").as_bytes(), + index, + [3; 32], + ) + }) + .collect(); + entries.push(Entry::dir(b"left", reused_page)); + entries.push(Entry::dir(b"right", reused_page)); + entries.sort_by(|left, right| left.name.cmp(&right.name)); + let bytes = Page::build(&entries).unwrap(); + let root = state.collect_directory(&entries, bytes).unwrap(); + state + .source_roots + .insert(state.identity.tagged_root_tree_oid.clone(), root); + let prepared = state.finish(root).unwrap(); + assert!(prepared.payloads.len() > 1); + assert_eq!(prepared.plan.reused.len(), 1); + assert!(!prepared.plan.delta.contains_key(&reused_page)); + assert_eq!(prepared.work.tree_fetches, 0); + assert_eq!(prepared.work.blob_fetches, 0); + assert_eq!( + prepared.plan.child_first_delta().unwrap().last(), + Some(&root) + ); + let mut decoded_edges = BTreeSet::new(); + for payload in &prepared.payloads { + assert_eq!(page_id(&payload.bytes), payload.id); + match Page::decode(&payload.bytes).unwrap().0 { + Page::Leaf { entries } => { + for entry in entries.iter().filter(|entry| entry.is_dir()) { + decoded_edges.insert((payload.id, entry.child_root)); + } + } + Page::Branch { + terminal, children, .. + } => { + if let Some(entry) = terminal.filter(Entry::is_dir) { + decoded_edges.insert((payload.id, entry.child_root)); + } + for child in children { + decoded_edges.insert((payload.id, child.child_page_id)); + } + } + } + } + assert_eq!(decoded_edges, prepared.plan.edges); + assert_eq!( + RootedMetadataInstallPlan::decode( + &prepared.plan.encode().unwrap(), + &prepared.plan.digest().unwrap() + ) + .unwrap(), + prepared.plan + ); + } + + #[test] + fn reuse_budget_is_deduplicated_conservative_and_never_expands_descendants() { + let empty = Page::build(&[]).unwrap(); + let page = page_id(&empty); + let hint = reusable(page); + let mut state = state(); + state + .record_reuse(&format!("sha1:{}", "b".repeat(40)), hint.clone(), "/one") + .unwrap(); + let mut alias = hint.clone(); + alias.proof.attestation_id = Uuid::from_u128(2); + alias.proof.attestation_digest = [9; 32]; + state + .record_reuse(&format!("sha1:{}", "c".repeat(40)), alias, "/two") + .unwrap(); + assert_eq!(state.reuse_upper.nodes, 1); + assert_eq!(state.reuse_upper.bytes, HEADER_LEN as u64); + assert_eq!(state.reused.get(&page).unwrap().proof, hint.proof); + assert!(state.payloads.is_empty()); + assert!(state.edges.is_empty()); + let mut different_lifetime = hint.clone(); + different_lifetime.proof.generation = 2; + assert!( + state + .record_reuse( + &format!("sha1:{}", "f".repeat(40)), + different_lifetime, + "/lifetime" + ) + .is_err() + ); + let mut bad = hint; + bad.relative_path_bytes = 1; + assert!( + state + .record_reuse(&format!("sha1:{}", "d".repeat(40)), bad, "/three") + .is_err() + ); + let mut huge = reusable([4; 32]); + huge.closure_nodes_upper = MetadataDagLimits::default().nodes; + assert_eq!( + state + .record_reuse(&format!("sha1:{}", "e".repeat(40)), huge, "/four") + .unwrap_err() + .code, + SnapshotErrorCode::LimitExceeded + ); + } +} diff --git a/src/jupiter/migration/m20261008_000200_add_mst2_rooted_qualified_family.rs b/src/jupiter/migration/m20261008_000200_add_mst2_rooted_qualified_family.rs new file mode 100644 index 00000000..fd613dce --- /dev/null +++ b/src/jupiter/migration/m20261008_000200_add_mst2_rooted_qualified_family.rs @@ -0,0 +1,125 @@ +//! A bounded namespace registry; physical Q provisioning occurs at bootstrap. + +use sea_orm::{ConnectionTrait, DbBackend, Statement}; +use sea_orm_migration::prelude::*; + +#[derive(DeriveMigrationName)] +pub struct Migration; + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + let connection = manager.get_connection(); + let row = connection.query_one_raw(Statement::from_string(DbBackend::Postgres, + "SELECT current_schema() AS schema,n.oid::bigint AS oid FROM pg_catalog.pg_namespace n WHERE n.nspname=current_schema()")) + .await?.ok_or_else(|| DbErr::Custom("qualified family core schema is missing".into()))?; + let schema: String = row.try_get("", "schema")?; + let oid: i64 = row.try_get("", "oid")?; + let quoted = format!("\"{}\"", schema.replace('"', "\"\"")); + let literal = format!("'{}'", schema.replace('\'', "''")); + let old = include_str!("m20261007_000600_storage_routes.sql"); + let start = old + .find("CREATE FUNCTION mst2_route_insert_guard()") + .ok_or_else(|| { + DbErr::Custom("generic route insertion guard source is missing".into()) + })?; + let end = old[start..] + .find("CREATE FUNCTION mst2_route_context_insert()") + .ok_or_else(|| { + DbErr::Custom("generic route insertion guard boundary is missing".into()) + })? + + start; + let guard = old[start..end].replace("CREATE FUNCTION", "CREATE OR REPLACE FUNCTION") + .replace("WHERE s.snapshot_id=NEW.snapshot_id AND NEW.canonical_descriptor=s.canonical_descriptor", + "WHERE n.singleton=1 AND n.graph_domain='generic-v1' AND s.snapshot_id=NEW.snapshot_id AND NEW.canonical_descriptor=s.canonical_descriptor"); + let catalog = include_str!("../storage/qualified_family_catalog.sql") + .replace("$CORE_OID$", "c_oid") + .replace("$Q_OID$", "q_oid") + .replace("$EXEMPT_Q_OID$", "exempt_q_oid"); + let shape = include_str!("../storage/qualified_family_shape.sql"); + let implementation = hex::encode( + super::super::storage::qualified_metadata_family::implementation_fingerprint(), + ); + let sql = include_str!("m20261008_000200_rooted_qualified_family.sql") + .replace("$CATALOG_SQL$", &catalog) + .replace("$SHAPE_SQL$", shape) + .replace("$GENERIC_INSERT_GUARD$", &guard) + .replace( + "$SOURCE_REVISION_SQL$", + include_str!("../storage/qualified_source_revision.sql"), + ) + .replace( + "$ROOTED_ROUTES_SQL$", + include_str!("m20261008_000200_rooted_routes.sql"), + ) + .replace("$CORE_SCHEMA$", "ed) + .replace("$CORE_LITERAL$", &literal) + .replace("$IMPLEMENTATION_SHA$", &implementation) + .replace("$CORE_OID$", &oid.to_string()); + connection.execute_unprepared(&sql).await?; + // Build the expected physical shape from trusted source exactly once, + // under this forward migration, without registering a namespace or + // preserving any template relations/history after the transaction. + let template_uuid = uuid::Uuid::new_v4().to_string(); + let template_schema = format!("mst2q_{}", template_uuid.replace('-', "")); + let template_quoted = format!("\"{template_schema}\""); + connection + .execute_unprepared(&format!("CREATE SCHEMA {template_quoted}")) + .await?; + let template_oid: i64 = connection + .query_one_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + "SELECT oid::bigint AS oid FROM pg_catalog.pg_namespace WHERE nspname=$1", + [template_schema.clone().into()], + )) + .await? + .ok_or_else(|| DbErr::Custom("qualified family template schema is missing".into()))? + .try_get("", "oid")?; + let template_storage = uuid::Uuid::new_v4().to_string(); + connection + .execute_unprepared( + &super::super::storage::qualified_metadata_family::render_family( + &schema, + oid, + &template_schema, + template_oid, + &template_uuid, + &template_storage, + ), + ) + .await?; + let expected_shape: Vec = connection.query_one_raw(Statement::from_sql_and_values(DbBackend::Postgres, + format!("SELECT {quoted}.mst2_route_family_shape($1::bigint::oid,$2::uuid,$3) AS fingerprint"), + [template_oid.into(),template_uuid.into(),template_storage.into()])).await? + .ok_or_else(||DbErr::Custom("qualified family template shape is missing".into()))?.try_get("","fingerprint")?; + connection.execute_unprepared(&format!("SET LOCAL search_path={quoted},pg_catalog,pg_temp; DROP SCHEMA {template_quoted} CASCADE")).await?; + let authority_catalog: Vec = connection + .query_one_raw(Statement::from_string( + DbBackend::Postgres, + format!("SELECT {quoted}.mst2_route_family_catalog({oid},0::oid) AS fingerprint"), + )) + .await? + .ok_or_else(|| { + DbErr::Custom("qualified family core authority catalog is missing".into()) + })? + .try_get("", "fingerprint")?; + connection + .execute_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + format!("INSERT INTO {quoted}.mst2_qualified_family_policy VALUES(1,$1,$2,$3)"), + [ + hex::decode(implementation) + .map_err(|e| DbErr::Custom(e.to_string()))? + .into(), + expected_shape.into(), + authority_catalog.into(), + ], + )) + .await?; + Ok(()) + } + + async fn down(&self, _manager: &SchemaManager) -> Result<(), DbErr> { + Ok(()) + } +} diff --git a/src/jupiter/migration/m20261008_000200_rooted_qualified_family.sql b/src/jupiter/migration/m20261008_000200_rooted_qualified_family.sql new file mode 100644 index 00000000..22038ef2 --- /dev/null +++ b/src/jupiter/migration/m20261008_000200_rooted_qualified_family.sql @@ -0,0 +1,161 @@ +SELECT mst2_route_enter(current_schema()); +SET LOCAL search_path=$CORE_SCHEMA$,pg_catalog,pg_temp; +LOCK TABLE mst2_metadata_namespace IN ACCESS EXCLUSIVE MODE; + +ALTER TABLE mst2_metadata_namespace + DROP CONSTRAINT mst2_metadata_namespace_pkey, + ALTER COLUMN singleton DROP NOT NULL, + DROP CONSTRAINT mst2_metadata_namespace_family_identity_check, + DROP CONSTRAINT mst2_metadata_namespace_graph_domain_check, + DROP CONSTRAINT mst2_metadata_namespace_admission_state_check, + DROP CONSTRAINT mst2_metadata_namespace_collector_state_check, + DROP CONSTRAINT mst2_metadata_namespace_check, + ADD COLUMN metadata_storage_uuid text, + ADD COLUMN implementation_fingerprint bytea, + ADD COLUMN catalog_fingerprint bytea, + ADD CONSTRAINT mst2_namespace_family_pair CHECK (( + (singleton=1 AND family_identity='v3-generic-session-1' AND graph_domain='generic-v1' + AND admission_state='G_ADMITTED_Q_CLOSED' AND collector_state='CLOSED' AND core_schema=metadata_schema + AND core_schema_oid=metadata_schema_oid AND metadata_storage_uuid IS NULL + AND implementation_fingerprint IS NULL AND catalog_fingerprint IS NULL) + OR (singleton IS NULL AND family_identity='v3-rooted-qualified-1' AND graph_domain='qualified-v1' + AND admission_state='ROOTED_Q_ADMITTED' AND collector_state='ENABLED' AND core_schema<>metadata_schema + AND core_schema_oid<>metadata_schema_oid AND metadata_storage_uuid IS NOT NULL + AND octet_length(implementation_fingerprint)=32 AND octet_length(catalog_fingerprint)=32) + ) IS TRUE); +CREATE UNIQUE INDEX idx_mst2_namespace_generic_slot ON mst2_metadata_namespace(singleton) + WHERE singleton IS NOT NULL; +CREATE UNIQUE INDEX idx_mst2_namespace_domain ON mst2_metadata_namespace(graph_domain); +CREATE UNIQUE INDEX idx_mst2_namespace_metadata_schema ON mst2_metadata_namespace(metadata_schema_oid); + +CREATE OR REPLACE FUNCTION mst2_route_enter(caller_schema text) RETURNS uuid LANGUAGE plpgsql VOLATILE +SET search_path=$CORE_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE n mst2_metadata_namespace%ROWTYPE; g mst2_metadata_namespace%ROWTYPE; mono_held boolean; route_held boolean; +BEGIN + IF NOT mst2_route_scope_valid(caller_schema) OR (SELECT count(*) FROM mst2_metadata_namespace)>2 THEN + RAISE EXCEPTION 'storage route captured primary scope is unavailable'; + END IF; + SELECT * INTO STRICT g FROM mst2_metadata_namespace WHERE singleton=1; + mono_held:=mst2_route_lock_held(1297043024,g.mono_lock_key2); + route_held:=mst2_route_lock_held(1296718001,pg_catalog.hashtext(g.core_schema)); + IF mst2_route_lock_held(1296718001,pg_catalog.hashtext(g.core_schema),false) AND NOT mono_held THEN + RAISE EXCEPTION 'storage route lock was acquired before core mono'; + END IF; + IF EXISTS(SELECT 1 FROM mst2_metadata_namespace x + WHERE mst2_route_lock_held(1296717362,pg_catalog.hashtext(x.metadata_schema),false)) + AND NOT (mono_held AND route_held) THEN + RAISE EXCEPTION 'storage route cannot acquire core locks after retention'; + END IF; + PERFORM pg_catalog.set_config('lock_timeout','5000ms',true); + PERFORM pg_catalog.pg_advisory_xact_lock(1297043024,g.mono_lock_key2); + PERFORM pg_catalog.pg_advisory_xact_lock(1296718001,pg_catalog.hashtext(g.core_schema)); + FOR n IN SELECT * FROM mst2_metadata_namespace ORDER BY namespace_uuid LOOP + PERFORM pg_catalog.pg_advisory_xact_lock(1296717362,pg_catalog.hashtext(n.metadata_schema)); + END LOOP; + RETURN g.namespace_uuid; +END $$; + +-- The new namespace is included before acquiring any retention lock. +CREATE FUNCTION mst2_route_family_candidate_enter(caller_schema text,candidate uuid,candidate_schema text) +RETURNS void LANGUAGE plpgsql VOLATILE SET search_path=$CORE_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE g mst2_metadata_namespace%ROWTYPE; r record; +BEGIN + IF NOT mst2_route_scope_valid(caller_schema) OR candidate IS NULL + OR substr(candidate::text,15,1)<>'4' OR substr(candidate::text,20,1) NOT IN ('8','9','a','b') + OR candidate_schema IS DISTINCT FROM 'mst2q_'||replace(candidate::text,'-','') THEN + RAISE EXCEPTION 'qualified provisioning candidate is not a fresh server namespace'; + END IF; + IF EXISTS(SELECT 1 FROM pg_catalog.pg_locks WHERE locktype='advisory' AND pid=pg_catalog.pg_backend_pid() + AND database=(SELECT oid FROM pg_catalog.pg_database WHERE datname=pg_catalog.current_database()) + AND classid=1296717362::oid AND objsubid=2 AND granted) THEN + RAISE EXCEPTION 'qualified provisioning cannot extend a previously locked retention set'; + END IF; + SELECT * INTO STRICT g FROM mst2_metadata_namespace WHERE singleton=1; + IF mst2_route_lock_held(1296718001,pg_catalog.hashtext(g.core_schema),false) + AND NOT mst2_route_lock_held(1297043024,g.mono_lock_key2) THEN + RAISE EXCEPTION 'storage route lock was acquired before core mono'; + END IF; + PERFORM pg_catalog.set_config('lock_timeout','5000ms',true); + PERFORM pg_catalog.pg_advisory_xact_lock(1297043024,g.mono_lock_key2); + PERFORM pg_catalog.pg_advisory_xact_lock(1296718001,pg_catalog.hashtext(g.core_schema)); + -- A racing successful bootstrap can register while this caller waits. + IF EXISTS(SELECT 1 FROM mst2_metadata_namespace WHERE graph_domain='qualified-v1') THEN + RAISE EXCEPTION 'qualified provisioning candidate lost bootstrap serialization'; + END IF; + FOR r IN SELECT namespace_uuid,metadata_schema FROM mst2_metadata_namespace + UNION ALL SELECT candidate,candidate_schema ORDER BY namespace_uuid LOOP + PERFORM pg_catalog.pg_advisory_xact_lock(1296717362,pg_catalog.hashtext(r.metadata_schema)); + END LOOP; +END $$; + +CREATE FUNCTION mst2_route_family_catalog(c_oid oid,q_oid oid,exempt_q_oid oid DEFAULT 0::oid) RETURNS bytea LANGUAGE sql VOLATILE +SET search_path=pg_catalog,pg_temp AS $catalog$ $CATALOG_SQL$ $catalog$; +CREATE FUNCTION mst2_route_family_shape(q_oid oid,n_uuid uuid,s_uuid text) RETURNS bytea LANGUAGE sql VOLATILE +SET search_path=pg_catalog,pg_temp AS $shape$ $SHAPE_SQL$ $shape$; +CREATE TABLE mst2_qualified_family_policy ( + singleton smallint PRIMARY KEY CHECK(singleton=1),implementation_fingerprint bytea NOT NULL CHECK(octet_length(implementation_fingerprint)=32), + expected_shape bytea NOT NULL CHECK(octet_length(expected_shape)=32), + authority_catalog bytea NOT NULL CHECK(octet_length(authority_catalog)=32) +); +CREATE TRIGGER mst2_route_family_policy_immutable BEFORE UPDATE OR DELETE ON mst2_qualified_family_policy + FOR EACH ROW EXECUTE FUNCTION mst2_route_immutable(); +CREATE TRIGGER mst2_route_family_policy_truncate_guard BEFORE TRUNCATE ON mst2_qualified_family_policy + FOR EACH STATEMENT EXECUTE FUNCTION mst2_route_immutable(); + +CREATE FUNCTION mst2_route_family_registration_guard() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$CORE_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE g mst2_metadata_namespace%ROWTYPE; stamp record; +BEGIN + SELECT * INTO STRICT g FROM mst2_metadata_namespace WHERE singleton=1; + IF (SELECT count(*) FROM mst2_metadata_namespace)<>1 OR NEW.singleton IS NOT NULL + OR substr(NEW.namespace_uuid::text,15,1)<>'4' OR substr(NEW.namespace_uuid::text,20,1) NOT IN ('8','9','a','b') + OR NEW.graph_domain<>'qualified-v1' OR NEW.family_identity<>'v3-rooted-qualified-1' + OR NEW.admission_state<>'ROOTED_Q_ADMITTED' OR NEW.collector_state<>'ENABLED' + OR ROW(NEW.core_schema,NEW.core_schema_oid,NEW.database_name,NEW.database_oid,NEW.storage_uuid, + NEW.server_address,NEW.server_port,NEW.mono_lock_key2) IS DISTINCT FROM + ROW(g.core_schema,g.core_schema_oid,g.database_name,g.database_oid,g.storage_uuid, + g.server_address,g.server_port,g.mono_lock_key2) + OR NEW.metadata_schema IS DISTINCT FROM 'mst2q_'||replace(NEW.namespace_uuid::text,'-','') + OR NEW.metadata_storage_uuid IS NOT DISTINCT FROM g.storage_uuid + OR NEW.metadata_storage_uuid IS DISTINCT FROM (NEW.metadata_storage_uuid::uuid)::text + OR substr(NEW.metadata_storage_uuid,15,1)<>'4' OR substr(NEW.metadata_storage_uuid,20,1) NOT IN ('8','9','a','b') + OR NOT EXISTS(SELECT 1 FROM pg_catalog.pg_namespace n + WHERE n.oid=NEW.metadata_schema_oid AND n.nspname=NEW.metadata_schema) + OR NEW.implementation_fingerprint IS DISTINCT FROM decode('$IMPLEMENTATION_SHA$','hex') + OR NOT mst2_route_lock_held(1297043024,g.mono_lock_key2) + OR NOT mst2_route_lock_held(1296718001,pg_catalog.hashtext(g.core_schema)) + OR NOT mst2_route_lock_held(1296717362,pg_catalog.hashtext(NEW.metadata_schema)) + OR NOT mst2_route_lock_held(1296717362,pg_catalog.hashtext(g.metadata_schema)) THEN + RAISE EXCEPTION 'qualified namespace registration has no exact admitted rooted family scope and lock set'; + END IF; + EXECUTE pg_catalog.format('SELECT * FROM %I.mst2_metadata_family_identity WHERE singleton=1',NEW.metadata_schema) + INTO STRICT stamp; + IF ROW(stamp.namespace_uuid,stamp.storage_uuid,stamp.core_schema_oid,stamp.metadata_schema_oid, + stamp.family_identity,stamp.implementation_fingerprint) IS DISTINCT FROM + ROW(NEW.namespace_uuid,NEW.metadata_storage_uuid,NEW.core_schema_oid,NEW.metadata_schema_oid, + NEW.family_identity,NEW.implementation_fingerprint) + OR NEW.catalog_fingerprint IS DISTINCT FROM mst2_route_family_catalog(NEW.core_schema_oid,NEW.metadata_schema_oid) THEN + RAISE EXCEPTION 'qualified namespace registration fingerprint disagrees with its physical family'; + END IF; + IF NOT EXISTS(SELECT 1 FROM mst2_qualified_family_policy p WHERE p.singleton=1 + AND p.implementation_fingerprint=NEW.implementation_fingerprint + AND p.authority_catalog=mst2_route_family_catalog(NEW.core_schema_oid,0::oid,NEW.metadata_schema_oid) + AND p.expected_shape=mst2_route_family_shape(NEW.metadata_schema_oid,NEW.namespace_uuid,NEW.metadata_storage_uuid)) THEN + RAISE EXCEPTION 'qualified namespace does not have the trusted complete physical family shape'; + END IF; + RETURN NEW; +END $$; +DROP TRIGGER mst2_route_namespace_registration_closed ON mst2_metadata_namespace; +CREATE TRIGGER mst2_route_namespace_registration_guard BEFORE INSERT ON mst2_metadata_namespace + FOR EACH ROW EXECUTE FUNCTION mst2_route_family_registration_guard(); + +CREATE FUNCTION mst2_metadata_has_generic_overlap(p bytea) RETURNS boolean LANGUAGE sql VOLATILE +SET search_path=$CORE_SCHEMA$,pg_catalog,pg_temp AS $$ + SELECT EXISTS(SELECT 1 FROM mst2_retention_node WHERE node_id='page:sha256:'||encode(p,'hex')) + OR EXISTS(SELECT 1 FROM mst2_retention_gc_op WHERE node_id='page:sha256:'||encode(p,'hex')) +$$; + +-- Existing generic route derivation must never select the newly registered Q row. +$GENERIC_INSERT_GUARD$ +$SOURCE_REVISION_SQL$ +$ROOTED_ROUTES_SQL$ diff --git a/src/jupiter/migration/m20261008_000200_rooted_routes.sql b/src/jupiter/migration/m20261008_000200_rooted_routes.sql new file mode 100644 index 00000000..e4cef146 --- /dev/null +++ b/src/jupiter/migration/m20261008_000200_rooted_routes.sql @@ -0,0 +1,171 @@ +CREATE FUNCTION mst2_route_qualified_namespace_valid(id uuid) RETURNS boolean LANGUAGE sql VOLATILE +SET search_path=$CORE_SCHEMA$,pg_catalog,pg_temp AS $$ + SELECT mst2_route_scope_valid($CORE_LITERAL$) AND EXISTS(SELECT 1 FROM mst2_metadata_namespace q + JOIN mst2_metadata_namespace g ON g.singleton=1 + JOIN mst2_qualified_family_policy policy ON policy.singleton=1 + JOIN pg_catalog.pg_namespace physical ON physical.oid=q.metadata_schema_oid AND physical.nspname=q.metadata_schema + WHERE q.namespace_uuid=id AND q.singleton IS NULL AND q.graph_domain='qualified-v1' + AND q.family_identity='v3-rooted-qualified-1' AND q.admission_state='ROOTED_Q_ADMITTED' AND q.collector_state='ENABLED' + AND q.metadata_schema='mst2q_'||replace(q.namespace_uuid::text,'-','') + AND q.metadata_schema_oid<>q.core_schema_oid + AND ROW(q.core_schema,q.core_schema_oid,q.database_name,q.database_oid,q.storage_uuid, + q.server_address,q.server_port,q.mono_lock_key2) IS NOT DISTINCT FROM + ROW(g.core_schema,g.core_schema_oid,g.database_name,g.database_oid,g.storage_uuid, + g.server_address,g.server_port,g.mono_lock_key2) + AND q.implementation_fingerprint=policy.implementation_fingerprint + AND policy.authority_catalog=mst2_route_family_catalog(q.core_schema_oid,0::oid,q.metadata_schema_oid) + AND q.catalog_fingerprint=mst2_route_family_catalog(q.core_schema_oid,q.metadata_schema_oid) + AND policy.expected_shape=mst2_route_family_shape(q.metadata_schema_oid,q.namespace_uuid,q.metadata_storage_uuid)) +$$; + +CREATE OR REPLACE FUNCTION mst2_route_statement_barrier() RETURNS trigger LANGUAGE plpgsql VOLATILE AS $$ +DECLARE q_id uuid; caller text:=pg_catalog.current_schema(); +BEGIN + IF TG_TABLE_SCHEMA<>$CORE_LITERAL$ OR NOT EXISTS(SELECT 1 FROM pg_catalog.pg_class + WHERE oid=TG_RELID AND relnamespace=$CORE_OID$) THEN + RAISE EXCEPTION 'storage route mutation is outside its captured core schema'; + END IF; + IF caller IS DISTINCT FROM $CORE_LITERAL$ THEN + SELECT n.namespace_uuid INTO q_id FROM $CORE_SCHEMA$.mst2_metadata_namespace n + WHERE n.metadata_schema=caller AND n.graph_domain='qualified-v1'; + IF NOT FOUND OR NOT $CORE_SCHEMA$.mst2_route_qualified_namespace_valid(q_id) THEN + RAISE EXCEPTION 'storage route mutation caller has no exact captured physical family'; END IF; + END IF; + PERFORM $CORE_SCHEMA$.mst2_route_enter($CORE_LITERAL$); + RETURN NULL; +END $$; + +CREATE FUNCTION mst2_route_qualified_snapshot_proof(sid text) RETURNS boolean LANGUAGE plpgsql VOLATILE +SET search_path=$CORE_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE r mst2_snapshot_storage_route%ROWTYPE; n mst2_metadata_namespace%ROWTYPE; valid boolean; +BEGIN + SELECT * INTO r FROM mst2_snapshot_storage_route WHERE snapshot_id=sid; + IF NOT FOUND THEN RETURN false; END IF; + SELECT * INTO n FROM mst2_metadata_namespace WHERE namespace_uuid=r.namespace_uuid; + IF NOT FOUND OR NOT mst2_route_qualified_namespace_valid(n.namespace_uuid) THEN RETURN false; END IF; + EXECUTE pg_catalog.format('SELECT %I.mst2_metadata_snapshot_route_proof($1,$2,$3,$4,$5,$6,$7)',n.metadata_schema) + INTO valid USING r.snapshot_id,r.canonical_descriptor,r.instance_id,r.commit_oid,r.root_tree_oid,r.metadata_root,r.source_profile; + RETURN coalesce(valid,false); +END $$; + +CREATE FUNCTION mst2_route_qualified_lease_proof(lid text) RETURNS boolean LANGUAGE plpgsql VOLATILE +SET search_path=$CORE_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE r mst2_lease_storage_route%ROWTYPE; n mst2_metadata_namespace%ROWTYPE; valid boolean; +BEGIN + SELECT * INTO r FROM mst2_lease_storage_route WHERE lease_id=lid; + IF NOT FOUND THEN RETURN false; END IF; + SELECT * INTO n FROM mst2_metadata_namespace WHERE namespace_uuid=r.namespace_uuid; + IF NOT FOUND OR NOT mst2_route_qualified_namespace_valid(n.namespace_uuid) THEN RETURN false; END IF; + EXECUTE pg_catalog.format('SELECT %I.mst2_metadata_lease_route_proof($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)',n.metadata_schema) + INTO valid USING r.lease_id,r.snapshot_id,r.namespace_uuid,r.session_incarnation,r.prepare_id,r.metadata_root, + r.authorization_epoch,r.publication_sequence,r.writer_epoch,r.certificate_receipt_id; + RETURN coalesce(valid,false); +END $$; + +-- Existing G proof functions remain authoritative for their original family. +CREATE FUNCTION mst2_route_family_for_snapshot(sid text,caller_schema text) +RETURNS TABLE(namespace_uuid uuid,graph_domain text) LANGUAGE plpgsql VOLATILE +SET search_path=$CORE_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE route mst2_snapshot_storage_route%ROWTYPE; namespace mst2_metadata_namespace%ROWTYPE; +BEGIN + IF NOT mst2_route_scope_valid(caller_schema) THEN RAISE EXCEPTION 'storage route captured primary scope is unavailable'; END IF; + SELECT * INTO route FROM mst2_snapshot_storage_route WHERE snapshot_id=sid; + IF NOT FOUND THEN RETURN; END IF; + SELECT * INTO namespace FROM mst2_metadata_namespace n WHERE n.namespace_uuid=route.namespace_uuid; + IF NOT FOUND OR namespace.graph_domain='generic-v1' AND NOT mst2_route_snapshot_proof(sid) + OR namespace.graph_domain='qualified-v1' AND NOT mst2_route_qualified_snapshot_proof(sid) + OR namespace.graph_domain NOT IN ('generic-v1','qualified-v1') THEN + RAISE EXCEPTION 'permanent snapshot route has a corrupt exact family or fixed source binding'; END IF; + RETURN QUERY SELECT namespace.namespace_uuid,namespace.graph_domain; +END $$; + +CREATE FUNCTION mst2_route_family_for_lease(lid text,caller_schema text) +RETURNS TABLE(namespace_uuid uuid,graph_domain text) LANGUAGE plpgsql VOLATILE +SET search_path=$CORE_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE route mst2_lease_storage_route%ROWTYPE; namespace mst2_metadata_namespace%ROWTYPE; +BEGIN + IF NOT mst2_route_scope_valid(caller_schema) THEN RAISE EXCEPTION 'storage route captured primary scope is unavailable'; END IF; + SELECT * INTO route FROM mst2_lease_storage_route WHERE lease_id=lid; + IF NOT FOUND THEN RETURN; END IF; + SELECT * INTO namespace FROM mst2_metadata_namespace n WHERE n.namespace_uuid=route.namespace_uuid; + IF NOT FOUND OR namespace.graph_domain='generic-v1' AND NOT mst2_route_lease_proof(lid) + OR namespace.graph_domain='qualified-v1' AND NOT mst2_route_qualified_lease_proof(lid) + OR namespace.graph_domain NOT IN ('generic-v1','qualified-v1') THEN + RAISE EXCEPTION 'permanent lease route has a corrupt exact family or fixed source binding'; END IF; + RETURN QUERY SELECT namespace.namespace_uuid,namespace.graph_domain; +END $$; + +CREATE OR REPLACE FUNCTION mst2_route_insert_guard() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$CORE_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE namespace mst2_metadata_namespace%ROWTYPE; valid boolean; +BEGIN + SELECT * INTO namespace FROM mst2_metadata_namespace WHERE namespace_uuid=NEW.namespace_uuid; + IF NOT FOUND THEN RAISE EXCEPTION 'storage route target namespace is not registered'; END IF; + IF namespace.graph_domain='qualified-v1' THEN + IF NOT mst2_route_qualified_namespace_valid(namespace.namespace_uuid) THEN + RAISE EXCEPTION 'qualified route has no exact complete physical family catalog'; END IF; + IF TG_TABLE_NAME='mst2_snapshot_storage_route' THEN + EXECUTE pg_catalog.format('SELECT %I.mst2_metadata_snapshot_candidate($1,$2,$3,$4,$5,$6,$7)',namespace.metadata_schema) + INTO valid USING NEW.snapshot_id,NEW.canonical_descriptor,NEW.instance_id,NEW.commit_oid, + NEW.root_tree_oid,NEW.metadata_root,NEW.source_profile; + ELSIF TG_TABLE_NAME='mst2_lease_storage_route' THEN + EXECUTE pg_catalog.format('SELECT %I.mst2_metadata_lease_route_proof($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)',namespace.metadata_schema) + INTO valid USING NEW.lease_id,NEW.snapshot_id,NEW.namespace_uuid,NEW.session_incarnation,NEW.prepare_id, + NEW.metadata_root,NEW.authorization_epoch,NEW.publication_sequence,NEW.writer_epoch,NEW.certificate_receipt_id; + ELSE RAISE EXCEPTION 'qualified route cannot use a generic binding relation'; END IF; + IF NOT coalesce(valid,false) THEN RAISE EXCEPTION 'qualified route is not independently derived from its exact source'; END IF; + RETURN NEW; + END IF; + IF namespace.singleton IS DISTINCT FROM 1 OR namespace.graph_domain<>'generic-v1' THEN + RAISE EXCEPTION 'storage route target family is unsupported'; END IF; + IF TG_TABLE_NAME='mst2_snapshot_storage_route' THEN + IF NOT EXISTS(SELECT 1 FROM mst2_snapshot_context s JOIN mst2_metadata_prepare p ON p.prepare_id=s.prepare_id + JOIN mst2_metadata_namespace n ON n.namespace_uuid=NEW.namespace_uuid + WHERE n.singleton=1 AND n.graph_domain='generic-v1' AND s.snapshot_id=NEW.snapshot_id AND NEW.canonical_descriptor=s.canonical_descriptor + AND NEW.instance_id=s.instance_id AND NEW.commit_oid=s.commit_oid AND NEW.root_tree_oid=s.root_tree_oid + AND NEW.metadata_root=s.metadata_root AND NEW.source_profile=mst2_route_profile(p.source_domain,p.tagged_root_tree_oid, + p.scope,p.schema_version,p.metadata_codec,p.materialization_policy,p.fs_semantics,p.access_projection, + p.verification_revision,p.projection_revision) + AND p.state='COMMITTED' AND p.metadata_root=s.metadata_root AND p.source_domain='native-git' + AND (p.graph_domain IS NULL OR p.graph_domain='generic-v1') + AND p.tagged_root_tree_oid IN ('sha1:'||s.root_tree_oid,'sha256:'||s.root_tree_oid)) THEN + RAISE EXCEPTION 'storage route is not derived from its actual generic context'; + END IF; + ELSIF TG_TABLE_NAME='mst2_generic_session_storage_binding' THEN + IF NOT EXISTS(SELECT 1 FROM mst2_snapshot_context s JOIN mst2_snapshot_storage_route r USING(snapshot_id) + WHERE s.snapshot_id=NEW.snapshot_id AND r.namespace_uuid=NEW.namespace_uuid + AND s.prepare_id=NEW.prepare_id AND s.metadata_root=NEW.metadata_root) THEN + RAISE EXCEPTION 'storage route generic incarnation is not its actual context'; + END IF; + ELSIF TG_TABLE_NAME='mst2_lease_storage_route' THEN + IF NOT EXISTS(SELECT 1 FROM mst2_snapshot_lease l JOIN mst2_generic_session_storage_binding b USING(snapshot_id) + WHERE l.lease_id=NEW.lease_id AND l.snapshot_id=NEW.snapshot_id AND b.namespace_uuid=NEW.namespace_uuid + AND b.session_incarnation=NEW.session_incarnation AND b.prepare_id=NEW.prepare_id AND b.metadata_root=NEW.metadata_root + AND l.authorization_epoch=NEW.authorization_epoch AND l.publication_sequence=NEW.publication_sequence + AND l.writer_epoch=NEW.writer_epoch AND l.certificate_receipt_id=NEW.certificate_receipt_id + AND mst2_route_snapshot_proof(l.snapshot_id)) THEN + RAISE EXCEPTION 'storage route lease is not its exact generic incarnation and source'; + END IF; + ELSE RAISE EXCEPTION 'storage route insertion target is not registered'; END IF; + RETURN NEW; +END $$; + +CREATE FUNCTION mst2_route_permanent_complete() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$CORE_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE domain text; valid boolean; +BEGIN + SELECT graph_domain INTO domain FROM mst2_metadata_namespace WHERE namespace_uuid=NEW.namespace_uuid; + IF TG_TABLE_NAME='mst2_snapshot_storage_route' THEN + valid:=CASE domain WHEN 'generic-v1' THEN mst2_route_snapshot_proof(NEW.snapshot_id) + WHEN 'qualified-v1' THEN mst2_route_qualified_snapshot_proof(NEW.snapshot_id) ELSE false END; + ELSE + valid:=CASE domain WHEN 'generic-v1' THEN mst2_route_lease_proof(NEW.lease_id) + WHEN 'qualified-v1' THEN mst2_route_qualified_lease_proof(NEW.lease_id) ELSE false END; + END IF; + IF NOT coalesce(valid,false) THEN RAISE EXCEPTION 'permanent storage route cannot commit without its exact actual incarnation'; END IF; + RETURN NULL; +END $$; +CREATE CONSTRAINT TRIGGER mst2_route_permanent_complete AFTER INSERT ON mst2_snapshot_storage_route + DEFERRABLE INITIALLY DEFERRED FOR EACH ROW EXECUTE FUNCTION mst2_route_permanent_complete(); +CREATE CONSTRAINT TRIGGER mst2_route_permanent_complete AFTER INSERT ON mst2_lease_storage_route + DEFERRABLE INITIALLY DEFERRED FOR EACH ROW EXECUTE FUNCTION mst2_route_permanent_complete(); diff --git a/src/jupiter/migration/mod.rs b/src/jupiter/migration/mod.rs index f5981f02..2fbc20ce 100644 --- a/src/jupiter/migration/mod.rs +++ b/src/jupiter/migration/mod.rs @@ -152,6 +152,7 @@ mod m20261007_000400_add_mst2_qualified_metadata_gc; mod m20261007_000500_add_mst2_install_capability; mod m20261007_000600_add_mst2_storage_routes; mod m20261008_000100_add_mst2_chunk_maps; +mod m20261008_000200_add_mst2_rooted_qualified_family; mod runner; pub use m20260905_000100_add_push_queue::ensure_queue_control_seed; pub use runner::apply_migrations; @@ -292,6 +293,7 @@ impl MigratorTrait for Migrator { Box::new(m20261007_000500_add_mst2_install_capability::Migration), Box::new(m20261007_000600_add_mst2_storage_routes::Migration), Box::new(m20261008_000100_add_mst2_chunk_maps::Migration), + Box::new(m20261008_000200_add_mst2_rooted_qualified_family::Migration), ] } } @@ -1202,7 +1204,7 @@ mod tests { async fn import_repo_alias_rows_canonicalized() { let names = migration_names(); assert_eq!( - &names[names.len() - 14..names.len() - 5], + &names[names.len() - 16..names.len() - 7], &[ "m20260923_000200_canonicalize_import_repo_paths".to_string(), "m20260925_000100_media_paging".to_string(), @@ -1217,25 +1219,33 @@ mod tests { "native retention, metadata installation and view tables follow media paging" ); assert_eq!( - &names[names.len() - 5], + &names[names.len() - 7], "m20261007_000200_add_mst2_metadata_generations" ); assert_eq!( - &names[names.len() - 4], + &names[names.len() - 6], "m20261007_000300_add_mst2_metadata_lifetime_history" ); assert_eq!( - &names[names.len() - 3], + &names[names.len() - 5], "m20261007_000400_add_mst2_qualified_metadata_gc" ); assert_eq!( - &names[names.len() - 2], + &names[names.len() - 4], "m20261007_000500_add_mst2_install_capability" ); assert_eq!( - names.last().unwrap(), + &names[names.len() - 3], "m20261007_000600_add_mst2_storage_routes" ); + assert_eq!( + &names[names.len() - 2], + "m20261008_000100_add_mst2_chunk_maps" + ); + assert_eq!( + names.last().unwrap(), + "m20261008_000200_add_mst2_rooted_qualified_family" + ); let db = alias_db().await; insert_repo(&db, 1, "/third-party//a").await; diff --git a/src/jupiter/storage/init.rs b/src/jupiter/storage/init.rs index d8480141..47044027 100644 --- a/src/jupiter/storage/init.rs +++ b/src/jupiter/storage/init.rs @@ -10,6 +10,21 @@ use crate::{ jupiter::migration::{apply_migrations, ensure_queue_control_seed}, }; +#[cfg(test)] +tokio::task_local! { + static GENERIC_HISTORY_BOOTSTRAP: (); +} + +#[cfg(test)] +pub(crate) async fn with_generic_history_bootstrap(future: F) -> F::Output { + GENERIC_HISTORY_BOOTSTRAP.scope((), future).await +} + +#[cfg(test)] +pub(super) fn generic_history_bootstrap_active() -> bool { + GENERIC_HISTORY_BOOTSTRAP.try_with(|_| ()).is_ok() +} + /// Create a PostgreSQL database connection. /// /// After a successful connection, applies any pending database migrations and @@ -25,6 +40,11 @@ pub async fn database_connection(db_config: &DbConfig) -> Result>, pub(crate) native_snapshot_sessions: Arc>, + #[cfg(test)] + pub(crate) shadow_qualified_metadata: + Arc>, + pub(crate) rooted_qualified_metadata: Arc< + tokio::sync::OnceCell>, + >, pub(crate) projection_observation_sink: Option>, pub cl_service: CLService, @@ -194,6 +201,46 @@ pub struct Storage { } impl Storage { + pub(crate) async fn rooted_qualified_metadata_writer( + &self, + ) -> Result<&qualified_metadata_family::RootedQualifiedMetadataRepository, MegaError> { + self.rooted_qualified_metadata + .get_or_try_init(|| async { + let repository = Arc::new( + qualified_metadata_family::RootedQualifiedMetadataRepository::open( + self.mono_storage().get_connection(), + &self.config().database, + ) + .await?, + ); + repository + .maintenance_tick(64) + .await + .map_err(|error| MegaError::Other(error.to_string()))?; + qualified_metadata_family::RootedQualifiedMetadataRepository::start_maintenance( + &repository, + ); + Ok::<_, MegaError>(repository) + }) + .await + .map(Arc::as_ref) + } + + #[cfg(test)] + pub(crate) async fn shadow_qualified_metadata_writer( + &self, + ) -> Result<&qualified_metadata_family::ShadowQualifiedMetadataWriter, MegaError> { + self.shadow_qualified_metadata + .get_or_try_init(|| async { + qualified_metadata_family::ShadowQualifiedMetadataWriter::open( + self.mono_storage().get_connection(), + &self.config().database, + ) + .await + }) + .await + } + pub async fn new( config: Arc, object_store: MegaObjectStorageWrapper, @@ -325,11 +372,14 @@ impl Storage { let storage_event_emitter = crate::jupiter::service::storage_event_emitter::StorageEventEmitter::from_config_disabled(&config); - Ok(Storage { + let storage = Storage { app_service: app_service.into(), native_projection_cache: Arc::default(), native_snapshot_sessions: Arc::default(), native_chunk_maps: Arc::default(), + #[cfg(test)] + shadow_qualified_metadata: Arc::default(), + rooted_qualified_metadata: Arc::default(), projection_observation_sink: None, config_handle, config, @@ -349,7 +399,13 @@ impl Storage { view_runtime, entity_store: Arc::new(SharedEntityStore::default()), vault: None, - }) + }; + #[cfg(test)] + if init::generic_history_bootstrap_active() { + return Ok(storage); + } + storage.rooted_qualified_metadata_writer().await?; + Ok(storage) } pub fn config_handle(&self) -> ConfigHandle { @@ -709,6 +765,9 @@ impl Storage { native_projection_cache: Arc::default(), native_snapshot_sessions: Arc::default(), native_chunk_maps: Arc::default(), + #[cfg(test)] + shadow_qualified_metadata: Arc::default(), + rooted_qualified_metadata: Arc::default(), projection_observation_sink: None, // app_service: AppService::mock(), cl_service: CLService::mock(), diff --git a/src/jupiter/storage/native_metadata_generations.rs b/src/jupiter/storage/native_metadata_generations.rs index a2cad697..dde51fc2 100644 --- a/src/jupiter/storage/native_metadata_generations.rs +++ b/src/jupiter/storage/native_metadata_generations.rs @@ -398,7 +398,11 @@ impl PostgresMetadataGenerationRepository { let fixed = self.require_fixed_plan(txn, intent).await?; check_generation_payloads(txn, &fixed).await?; let legacy = if intent.graph_domain == GraphDomain::Qualified { - qualified::finalize_graph(txn, &fixed, &observation.dag).await? + if self.inner.qualified_family.is_some() { + qualified::finalize_canonical_graph(txn, &fixed, &observation.dag).await? + } else { + qualified::finalize_graph(txn, &fixed, &observation.dag).await? + } } else { self.inner .finalize_stored_plan_in_txn(txn, &intent.legacy, &observation.dag, fixed.stored) diff --git a/src/jupiter/storage/native_metadata_install.rs b/src/jupiter/storage/native_metadata_install.rs index 5b97f663..6ecf72c3 100644 --- a/src/jupiter/storage/native_metadata_install.rs +++ b/src/jupiter/storage/native_metadata_install.rs @@ -139,6 +139,7 @@ pub struct PostgresMetadataInstallRepository { connection: DatabaseConnection, barrier_timeout: Duration, storage_scope: PrimaryStorageScope, + qualified_family: Option, } #[derive(Debug, Clone, PartialEq, Eq)] @@ -160,6 +161,7 @@ impl PostgresMetadataInstallRepository { connection, barrier_timeout: Duration::from_secs(5), storage_scope, + qualified_family: None, }) } @@ -654,6 +656,9 @@ impl PostgresMetadataInstallRepository { if txn.get_database_backend() != DbBackend::Postgres { return Err(internal("metadata installation requires PostgreSQL")); } + if let Some(family) = &self.qualified_family { + family.enter(txn).await?; + } if read_storage_scope(txn).await? != self.storage_scope { return Err(internal( "metadata recovery connection is outside the captured primary storage scope", diff --git a/src/jupiter/storage/native_metadata_qualified.rs b/src/jupiter/storage/native_metadata_qualified.rs index 5a29c9c6..221f9c98 100644 --- a/src/jupiter/storage/native_metadata_qualified.rs +++ b/src/jupiter/storage/native_metadata_qualified.rs @@ -115,6 +115,22 @@ impl GcRecord { } impl PostgresQualifiedMetadataRepository { + #[cfg(test)] + pub(crate) async fn registered_shadow( + connection: DatabaseConnection, + family: crate::jupiter::storage::qualified_metadata_family::VerifiedQualifiedNamespace, + ) -> Result { + let mut inner = PostgresMetadataInstallRepository::new(connection).await?; + inner.qualified_family = Some(family); + Ok(Self { + inner: PostgresMetadataGenerationRepository { + inner, + graph_domain: "qualified-v1", + }, + }) + } + + #[cfg(test)] pub async fn new(connection: DatabaseConnection) -> Result { Ok(Self { inner: PostgresMetadataGenerationRepository { @@ -635,8 +651,7 @@ pub(super) async fn allocate_lifetimes( n.state AS graph_state,n.metadata_codec AS graph_codec,n.bytes AS graph_bytes, b.page_id IS NOT NULL AS payload_present,b.generation AS payload_generation,b.metadata_codec AS payload_codec,b.byte_size AS payload_size, EXISTS(SELECT 1 FROM mst2_metadata_gc_op o WHERE o.page_id=l.page_id AND o.generation=l.generation) AS tombstone, - EXISTS(SELECT 1 FROM mst2_retention_node x WHERE x.node_id=l.node_id) - OR EXISTS(SELECT 1 FROM mst2_retention_gc_op x WHERE x.node_id=l.node_id) AS generic_graph + mst2_metadata_has_generic_overlap(l.page_id) AS generic_graph FROM jsonb_to_recordset($1::jsonb) p(id text,size integer) JOIN mst2_metadata_current c ON c.page_id=decode(p.id,'hex') JOIN mst2_metadata_lifetime l USING(page_id,generation) @@ -740,18 +755,16 @@ pub(super) async fn check_lifetimes( OR ($3='COMMITTED' AND l.state<>'LIVE') OR (l.state='LIVE' AND n.page_id IS NULL) OR n.state<>'LIVE' OR n.metadata_codec<>$2 OR n.bytes<>m.expected_size OR EXISTS(SELECT 1 FROM mst2_metadata_gc_op o WHERE o.page_id=m.page_id AND o.generation=m.generation) - OR EXISTS(SELECT 1 FROM mst2_retention_node x WHERE x.node_id=l.node_id) - OR EXISTS(SELECT 1 FROM mst2_retention_gc_op x WHERE x.node_id=l.node_id)) LIMIT 1", + OR mst2_metadata_has_generic_overlap(l.page_id)) LIMIT 1", [stored.record.prepare_id.clone().into(),stored.record.metadata_codec.into(),stored.record.state.clone().into()], )).await.map_err(internal)?.is_some() { return Err(unavailable("fixed qualified incarnation is no longer installable")); } Ok(()) } -pub(super) async fn finalize_graph( - txn: &DatabaseTransaction, +fn validate_fixed_observation( fixed: &FixedPlan, dag: &ValidatedMetadataDag, -) -> Result { +) -> Result<(), SnapshotError> { let actual_pages: BTreeSet<_> = dag.payloads().iter().map(|p| (p.id, p.size)).collect(); let actual_edges: BTreeSet<_> = dag .edges() @@ -780,6 +793,92 @@ pub(super) async fn finalize_graph( "qualified DAG observation differs from immutable plan", )); } + Ok(()) +} + +pub(super) async fn finalize_canonical_graph( + txn: &DatabaseTransaction, + fixed: &FixedPlan, + dag: &ValidatedMetadataDag, +) -> Result { + validate_fixed_observation(fixed, dag)?; + if fixed.stored.record.state == "COMMITTED" { + verify_graph(txn, fixed).await?; + return fixed.stored.receipt(); + } + let mut pending: BTreeMap<_, usize> = fixed.stored.plan.pages.keys().map(|p| (*p, 0)).collect(); + let mut parents: BTreeMap<_, Vec<_>> = BTreeMap::new(); + for &(parent, child) in &fixed.stored.plan.edges { + *pending + .get_mut(&parent) + .ok_or_else(|| integrity("canonical parent is outside its fixed plan"))? += 1; + parents.entry(child).or_default().push(parent); + } + let mut ready: BTreeSet<_> = pending + .iter() + .filter_map(|(p, count)| (*count == 0).then_some(*p)) + .collect(); + let mut ordered = Vec::with_capacity(pending.len()); + while let Some(page) = ready.pop_first() { + ordered.push(json!({"page":hex::encode(page),"generation":fixed.bindings.0[&page].0})); + for parent in parents.get(&page).into_iter().flatten() { + let count = pending + .get_mut(parent) + .ok_or_else(|| integrity("canonical ancestor is outside its fixed plan"))?; + *count = count + .checked_sub(1) + .ok_or_else(|| integrity("canonical edge accounting underflow"))?; + if *count == 0 { + ready.insert(*parent); + } + } + } + if ordered.len() != pending.len() { + return Err(integrity("canonical certification order contains a cycle")); + } + let certified: i32 = txn + .query_one_raw(statement( + "SELECT mst2_metadata_certify_batch($1,$2::jsonb) AS certified", + [ + fixed.intent.prepare_id().into(), + serde_json::to_string(&ordered).map_err(internal)?.into(), + ], + )) + .await + .map_err(internal)? + .ok_or_else(|| integrity("canonical certification batch is missing"))? + .try_get("", "certified") + .map_err(internal)?; + if certified as usize != ordered.len() { + return Err(integrity( + "canonical certification did not cover its exact cold DAG", + )); + } + retain_existing_roots(txn, &fixed.intent).await?; + verify_graph(txn, fixed).await?; + let changed = txn + .execute_raw(statement( + "UPDATE mst2_metadata_prepare SET state='COMMITTED',committed_at=clock_timestamp() + WHERE prepare_id=$1 AND state='PREPARING' AND storage_seal=$2", + [ + fixed.intent.prepare_id().into(), + fixed.intent.storage_seal.to_vec().into(), + ], + )) + .await + .map_err(internal)?; + if changed.rows_affected() != 1 { + return Err(integrity("canonical finalize lost its prepare CAS")); + } + fixed.stored.receipt() +} + +pub(super) async fn finalize_graph( + txn: &DatabaseTransaction, + fixed: &FixedPlan, + dag: &ValidatedMetadataDag, +) -> Result { + validate_fixed_observation(fixed, dag)?; if fixed.stored.record.state == "COMMITTED" { verify_graph(txn, fixed).await?; return fixed.stored.receipt(); diff --git a/src/jupiter/storage/native_snapshot_session.rs b/src/jupiter/storage/native_snapshot_session.rs index fb8d9134..da15a716 100644 --- a/src/jupiter/storage/native_snapshot_session.rs +++ b/src/jupiter/storage/native_snapshot_session.rs @@ -366,9 +366,11 @@ mod routes; #[path = "native_snapshot_metadata_routes.rs"] mod metadata_routes; -pub(crate) use metadata_routes::MetadataRouteRequest; #[cfg(test)] pub(crate) use metadata_routes::with_metadata_read_barriers; +pub(crate) use metadata_routes::{ + MetadataRouteRequest, PersistedMetadataReadWork, PersistedMetadataRouteBatch, +}; const SESSION_SQL: &str = "SELECT s.snapshot_id,s.canonical_descriptor,s.commit_oid,s.root_tree_oid, (SELECT storage_uuid FROM mst2_metadata_storage_scope WHERE singleton=1) AS authority_storage_uuid, @@ -665,7 +667,7 @@ fn forbidden() -> SnapshotError { "snapshot serving state or authorization epoch changed", ) } -fn install_error(error: MetadataInstallError) -> SnapshotError { +pub(crate) fn install_error(error: MetadataInstallError) -> SnapshotError { match error { MetadataInstallError::Rejected(error) if error.code == SnapshotErrorCode::Internal => { tracing::error!(%error,"native metadata installation unavailable"); @@ -690,6 +692,44 @@ fn install_error(error: MetadataInstallError) -> SnapshotError { } impl super::Storage { + pub(crate) async fn snapshot_metadata_family( + &self, + identity: &str, + lease: bool, + ) -> Result, SnapshotError> + { + use super::base_storage::StorageConnector; + super::qualified_metadata_family::select_snapshot_family( + self.mono_storage().get_connection(), + identity, + lease, + ) + .await + } + + pub(crate) async fn snapshot_metadata_routes( + &self, + context: &SnapshotContext, + requests: &[MetadataRouteRequest<'_>], + ) -> Result { + if self + .snapshot_metadata_family(&context.lease_id, true) + .await? + == Some(super::qualified_metadata_family::SnapshotMetadataFamily::Rooted) + { + return self + .rooted_qualified_metadata_writer() + .await + .map_err(internal)? + .metadata_routes(context, requests) + .await; + } + self.snapshot_sessions() + .await + .metadata_routes(context, requests) + .await + } + pub(crate) async fn snapshot_sessions(&self) -> &PostgresNativeSessionRepository { use super::base_storage::StorageConnector; self.native_snapshot_sessions @@ -714,6 +754,16 @@ impl super::Storage { let instance = uuid::Uuid::parse_str(instance) .map_err(|_| not_ready("invalid native instance"))? .to_string(); + if self.snapshot_metadata_family(lease, true).await? + == Some(super::qualified_metadata_family::SnapshotMetadataFamily::Rooted) + { + return self + .rooted_qualified_metadata_writer() + .await + .map_err(internal)? + .context(sid, lease, &instance) + .await; + } self.snapshot_sessions() .await .context(sid, lease, &instance) @@ -740,6 +790,16 @@ impl super::Storage { let instance = uuid::Uuid::parse_str(instance) .map_err(|_| not_ready("invalid native instance"))? .to_string(); + if self.snapshot_metadata_family(lease, true).await? + == Some(super::qualified_metadata_family::SnapshotMetadataFamily::Rooted) + { + return self + .rooted_qualified_metadata_writer() + .await + .map_err(internal)? + .renew(lease, seconds, &instance) + .await; + } self.snapshot_sessions() .await .renew(lease, seconds, &instance) @@ -751,6 +811,16 @@ impl super::Storage { pub(crate) async fn snapshot_release(&self, lease: &str) -> Result { if self.config().mst2.publication_enabled { + if self.snapshot_metadata_family(lease, true).await? + == Some(super::qualified_metadata_family::SnapshotMetadataFamily::Rooted) + { + return self + .rooted_qualified_metadata_writer() + .await + .map_err(internal)? + .release(lease) + .await; + } self.snapshot_sessions().await.release(lease).await } else { Ok(crate::ceres::snapshot::runtime::runtime().release_lease(lease)) diff --git a/src/jupiter/storage/qualified_family_catalog.sql b/src/jupiter/storage/qualified_family_catalog.sql new file mode 100644 index 00000000..acb8cce9 --- /dev/null +++ b/src/jupiter/storage/qualified_family_catalog.sql @@ -0,0 +1,57 @@ +WITH selected_namespaces AS ( + SELECT oid,nspname,nspowner,nspacl FROM pg_catalog.pg_namespace + WHERE oid IN ($CORE_OID$,$Q_OID$) +), selected_relations AS ( + SELECT c.oid,c.relnamespace,c.relname,c.relkind,c.relpersistence,c.relowner,c.relacl, + c.relrowsecurity,c.relforcerowsecurity,c.reloptions,c.relam,c.relhasrules,c.relispartition + FROM pg_catalog.pg_class c + WHERE c.relnamespace=$Q_OID$ + OR (c.relnamespace=$CORE_OID$ AND (c.relname IN ('mst2_metadata_namespace','mst2_qualified_family_policy', + 'mega_tree','mst2_rooted_source_tree_revision','mst2_verified_object','mst2_retention_node','mst2_retention_gc_op', + 'mega_commit','mega_refs','mst2_native_head','mst2_native_publication','mst2_publication','mst2_publication_outbox', + 'mst2_snapshot_storage_route','mst2_lease_storage_route','mst2_generic_session_storage_binding', + 'mst2_snapshot_context','mst2_snapshot_lease') + OR c.oid IN (SELECT i.indexrelid FROM pg_catalog.pg_index i JOIN pg_catalog.pg_class parent ON parent.oid=i.indrelid + WHERE parent.relnamespace=$CORE_OID$ AND parent.relname IN ('mst2_metadata_namespace','mst2_qualified_family_policy', + 'mega_tree','mst2_rooted_source_tree_revision','mst2_verified_object','mst2_retention_node','mst2_retention_gc_op', + 'mega_commit','mega_refs','mst2_native_head','mst2_native_publication','mst2_publication','mst2_publication_outbox', + 'mst2_snapshot_storage_route','mst2_lease_storage_route','mst2_generic_session_storage_binding', + 'mst2_snapshot_context','mst2_snapshot_lease')))) +), selected_triggers AS ( + SELECT t.* FROM pg_catalog.pg_trigger t + WHERE (t.tgrelid IN (SELECT oid FROM selected_relations) + OR EXISTS(SELECT 1 FROM pg_catalog.pg_constraint x + WHERE x.oid=t.tgconstraint AND x.conrelid IN (SELECT oid FROM pg_catalog.pg_class WHERE relnamespace=$Q_OID$))) + AND NOT ($Q_OID$=0 AND $EXEMPT_Q_OID$<>0 AND t.tgisinternal + AND EXISTS(SELECT 1 FROM pg_catalog.pg_constraint fk + JOIN pg_catalog.pg_class source ON source.oid=fk.conrelid + JOIN pg_catalog.pg_class target ON target.oid=fk.confrelid + WHERE fk.oid=t.tgconstraint AND fk.contype='f' AND fk.connamespace=$EXEMPT_Q_OID$ + AND source.relnamespace=$EXEMPT_Q_OID$ AND target.relnamespace=$CORE_OID$ + AND t.tgrelid=fk.confrelid AND t.tgconstrrelid=fk.conrelid)) +), objects AS ( + SELECT 'namespace' AS kind,oid::text AS key,pg_catalog.to_jsonb(n) AS value FROM selected_namespaces n + UNION ALL SELECT 'relation',oid::text,pg_catalog.to_jsonb(c) FROM selected_relations c + UNION ALL SELECT 'column',a.attrelid||':'||a.attnum,pg_catalog.to_jsonb(a) + FROM pg_catalog.pg_attribute a JOIN selected_relations c ON c.oid=a.attrelid WHERE a.attnum>0 + UNION ALL SELECT 'default',d.oid::text,pg_catalog.to_jsonb(d) + FROM pg_catalog.pg_attrdef d JOIN selected_relations c ON c.oid=d.adrelid + UNION ALL SELECT 'constraint',x.oid::text,pg_catalog.to_jsonb(x) + FROM pg_catalog.pg_constraint x JOIN selected_relations c ON c.oid=x.conrelid + UNION ALL SELECT 'index',i.indexrelid::text,pg_catalog.to_jsonb(i) + FROM pg_catalog.pg_index i JOIN selected_relations c ON c.oid=i.indrelid + UNION ALL SELECT 'trigger',t.oid::text,pg_catalog.to_jsonb(t) + FROM selected_triggers t + UNION ALL SELECT 'function',p.oid::text,pg_catalog.to_jsonb(p) + FROM pg_catalog.pg_proc p WHERE p.pronamespace=$Q_OID$ + OR (p.pronamespace=$CORE_OID$ AND (pg_catalog.left(p.proname,11)='mst2_route_' + OR p.proname='mst2_metadata_has_generic_overlap')) + OR EXISTS(SELECT 1 FROM selected_triggers t WHERE t.tgfoid=p.oid) + UNION ALL SELECT 'policy',p.oid::text,pg_catalog.to_jsonb(p) + FROM pg_catalog.pg_policy p JOIN selected_relations c ON c.oid=p.polrelid + UNION ALL SELECT 'rule',r.oid::text,pg_catalog.to_jsonb(r) + FROM pg_catalog.pg_rewrite r JOIN selected_relations c ON c.oid=r.ev_class +) +SELECT pg_catalog.sha256(pg_catalog.convert_to( + pg_catalog.jsonb_agg(pg_catalog.jsonb_build_array(kind,key,value) ORDER BY kind,key)::text,'UTF8')) AS fingerprint +FROM objects diff --git a/src/jupiter/storage/qualified_family_shape.sql b/src/jupiter/storage/qualified_family_shape.sql new file mode 100644 index 00000000..4269fe74 --- /dev/null +++ b/src/jupiter/storage/qualified_family_shape.sql @@ -0,0 +1,65 @@ +WITH q AS ( + SELECT n.oid,n.nspname FROM pg_catalog.pg_namespace n WHERE n.oid=q_oid +), relations AS ( + SELECT c.* FROM pg_catalog.pg_class c WHERE c.relnamespace=q_oid +), objects AS ( + SELECT 'relation' AS kind,c.relname AS key,jsonb_build_array(c.relname,c.relkind,c.relpersistence,c.relowner,c.relacl, + c.relrowsecurity,c.relforcerowsecurity,c.reloptions,c.relispartition,a.amname) AS value + FROM relations c LEFT JOIN pg_catalog.pg_am a ON a.oid=c.relam + UNION ALL SELECT 'column',c.relname||':'||a.attnum,jsonb_build_array(c.relname,a.attnum,a.attname, + CASE WHEN tn.oid=q_oid THEN '$Q_SCHEMA$' ELSE tn.nspname END,t.typname,a.attnotnull,a.attisdropped,a.attidentity,a.attgenerated,a.attislocal,a.attinhcount, + a.atttypmod,a.attndims,a.attacl,a.attlen,a.attbyval,a.attalign,a.attstorage,a.attcompression, + CASE WHEN cn.oid=q_oid THEN '$Q_SCHEMA$' ELSE cn.nspname END,co.collname, + pg_catalog.replace(pg_catalog.pg_get_expr(d.adbin,d.adrelid),(SELECT nspname FROM q),'$Q_SCHEMA$')) + FROM pg_catalog.pg_attribute a JOIN relations c ON c.oid=a.attrelid + JOIN pg_catalog.pg_type t ON t.oid=a.atttypid JOIN pg_catalog.pg_namespace tn ON tn.oid=t.typnamespace + LEFT JOIN pg_catalog.pg_collation co ON co.oid=a.attcollation LEFT JOIN pg_catalog.pg_namespace cn ON cn.oid=co.collnamespace + LEFT JOIN pg_catalog.pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum WHERE a.attnum>0 + UNION ALL SELECT 'constraint',c.relname||':'||x.conname,jsonb_build_array(c.relname,x.conname,x.contype, + pg_catalog.replace(pg_catalog.pg_get_constraintdef(x.oid,true),(SELECT nspname FROM q),'$Q_SCHEMA$'), + x.condeferrable,x.condeferred,x.convalidated,x.conislocal,x.coninhcount,x.connoinherit) + FROM pg_catalog.pg_constraint x JOIN relations c ON c.oid=x.conrelid + UNION ALL SELECT 'index',c.relname||':'||ic.relname,jsonb_build_array(c.relname,ic.relname, + pg_catalog.replace(pg_catalog.pg_get_indexdef(i.indexrelid),(SELECT nspname FROM q),'$Q_SCHEMA$'), + i.indisunique,i.indisprimary,i.indisexclusion,i.indimmediate,i.indisvalid,i.indisready,i.indislive, + i.indnullsnotdistinct,i.indisclustered,i.indisreplident) + FROM pg_catalog.pg_index i JOIN relations c ON c.oid=i.indrelid JOIN relations ic ON ic.oid=i.indexrelid + UNION ALL SELECT 'function',p.proname||':'||pg_catalog.replace(pg_catalog.pg_get_function_identity_arguments(p.oid),(SELECT nspname FROM q),'$Q_SCHEMA$'),jsonb_build_array( + p.proname,pg_catalog.replace(pg_catalog.pg_get_function_identity_arguments(p.oid),(SELECT nspname FROM q),'$Q_SCHEMA$'), + pg_catalog.replace(pg_catalog.pg_get_function_result(p.oid),(SELECT nspname FROM q),'$Q_SCHEMA$'),l.lanname, + p.proowner,p.proacl,p.prokind,p.provolatile,p.proisstrict,p.prosecdef,p.proleakproof,p.proparallel, + p.procost,p.prorows,p.probin,p.pronargdefaults, + pg_catalog.replace(pg_catalog.pg_get_expr(p.proargdefaults,0),(SELECT nspname FROM q),'$Q_SCHEMA$'), + pg_catalog.replace(pg_catalog.replace(pg_catalog.replace(pg_catalog.replace(p.prosrc, + pg_catalog.quote_literal((SELECT nspname FROM q)),pg_catalog.quote_literal('$Q_SCHEMA$')), + pg_catalog.quote_literal(q_oid::text),pg_catalog.quote_literal('$Q_OID$')), + pg_catalog.quote_literal(n_uuid::text),pg_catalog.quote_literal('$NAMESPACE_UUID$')), + pg_catalog.quote_literal(s_uuid),pg_catalog.quote_literal('$STORAGE_UUID$')), + pg_catalog.replace(pg_catalog.array_to_string(p.proconfig,E'\n'),(SELECT nspname FROM q),'$Q_SCHEMA$')) + FROM pg_catalog.pg_proc p JOIN pg_catalog.pg_language l ON l.oid=p.prolang WHERE p.pronamespace=q_oid + UNION ALL SELECT 'trigger',c.relname||':'||p.proname||':'||t.tgtype||':'||coalesce(x.conname,t.tgname),jsonb_build_array( + CASE WHEN c.relnamespace=q_oid THEN '$Q_SCHEMA$' ELSE cns.nspname END,c.relname, + CASE WHEN t.tgisinternal THEN NULL ELSE t.tgname END,t.tgtype,t.tgenabled,t.tgisinternal, + CASE WHEN pn.oid=q_oid THEN '$Q_SCHEMA$' ELSE pn.nspname END,p.proname, + pg_catalog.replace(pg_catalog.pg_get_function_identity_arguments(p.oid),(SELECT nspname FROM q),'$Q_SCHEMA$'), + t.tgdeferrable,t.tginitdeferred,t.tgnargs, + pg_catalog.replace(encode(t.tgargs,'hex'),encode(pg_catalog.convert_to((SELECT nspname FROM q),'UTF8'),'hex'), + encode(pg_catalog.convert_to('$Q_SCHEMA$','UTF8'),'hex')),t.tgattr::text, + pg_catalog.replace(pg_catalog.pg_get_expr(t.tgqual,t.tgrelid),(SELECT nspname FROM q),'$Q_SCHEMA$'),t.tgoldtable,t.tgnewtable,x.conname, + CASE WHEN rn.oid=q_oid THEN '$Q_SCHEMA$' ELSE rn.nspname END,rc.relname, + CASE WHEN t.tgparentid=0 THEN NULL ELSE 'unexpected parent trigger' END) + FROM pg_catalog.pg_trigger t JOIN pg_catalog.pg_class c ON c.oid=t.tgrelid + JOIN pg_catalog.pg_namespace cns ON cns.oid=c.relnamespace + JOIN pg_catalog.pg_proc p ON p.oid=t.tgfoid JOIN pg_catalog.pg_namespace pn ON pn.oid=p.pronamespace + LEFT JOIN pg_catalog.pg_constraint x ON x.oid=t.tgconstraint + LEFT JOIN pg_catalog.pg_class rc ON rc.oid=t.tgconstrrelid LEFT JOIN pg_catalog.pg_namespace rn ON rn.oid=rc.relnamespace + WHERE c.relnamespace=q_oid OR EXISTS(SELECT 1 FROM pg_catalog.pg_constraint fk + WHERE fk.oid=t.tgconstraint AND fk.conrelid IN (SELECT oid FROM relations)) + UNION ALL SELECT 'rule',c.relname||':'||r.rulename,pg_catalog.to_jsonb(r) + FROM pg_catalog.pg_rewrite r JOIN relations c ON c.oid=r.ev_class + UNION ALL SELECT 'policy',c.relname||':'||p.polname,pg_catalog.to_jsonb(p) + FROM pg_catalog.pg_policy p JOIN relations c ON c.oid=p.polrelid +) +SELECT pg_catalog.sha256(pg_catalog.convert_to( + pg_catalog.jsonb_agg(pg_catalog.jsonb_build_array(kind,key,value) ORDER BY kind,key,value::text)::text,'UTF8')) AS fingerprint +FROM objects diff --git a/src/jupiter/storage/qualified_metadata_anchors.sql b/src/jupiter/storage/qualified_metadata_anchors.sql new file mode 100644 index 00000000..08b3d7ea --- /dev/null +++ b/src/jupiter/storage/qualified_metadata_anchors.sql @@ -0,0 +1,426 @@ +CREATE TABLE mst2_metadata_source_root_attestation ( + attestation_id uuid PRIMARY KEY,namespace_uuid uuid NOT NULL REFERENCES mst2_metadata_family_identity(namespace_uuid), + origin_prepare_id text NOT NULL REFERENCES mst2_metadata_prepare(prepare_id), + tagged_tree_oid text NOT NULL CHECK(tagged_tree_oid ~ '^(sha1:[0-9a-f]{40}|sha256:[0-9a-f]{64}|blake3:[0-9a-f]{64})$'), + source_profile jsonb NOT NULL,profile_digest bytea NOT NULL CHECK(octet_length(profile_digest)=32), + source_body_digest bytea NOT NULL CHECK(octet_length(source_body_digest)=32), + source_revision uuid NOT NULL, + root_page bytea NOT NULL,root_generation bigint NOT NULL, + root_certificate_digest bytea NOT NULL CHECK(octet_length(root_certificate_digest)=32), + source_proof jsonb NOT NULL,attestation_digest bytea NOT NULL CHECK(octet_length(attestation_digest)=32), + FOREIGN KEY(root_page,root_generation,root_certificate_digest) + REFERENCES mst2_metadata_page_certificate(page_id,generation,certificate_digest), + UNIQUE(attestation_id,root_page,root_generation,attestation_digest) +); +CREATE INDEX mst2_metadata_source_root_lookup ON mst2_metadata_source_root_attestation(profile_digest,tagged_tree_oid,root_page,root_generation); +CREATE TABLE mst2_metadata_prepare_reuse_root ( + prepare_id text NOT NULL REFERENCES mst2_metadata_prepare(prepare_id),root_page bytea NOT NULL, + root_generation bigint NOT NULL,attestation_id uuid NOT NULL,attestation_digest bytea NOT NULL, + PRIMARY KEY(prepare_id,root_page), + FOREIGN KEY(attestation_id,root_page,root_generation,attestation_digest) + REFERENCES mst2_metadata_source_root_attestation(attestation_id,root_page,root_generation,attestation_digest) +); +CREATE INDEX mst2_metadata_prepare_reuse_root_page ON mst2_metadata_prepare_reuse_root(root_page,root_generation,prepare_id); +CREATE TABLE mst2_metadata_reuse_index ( + profile_digest bytea NOT NULL,tagged_tree_oid text NOT NULL,attestation_id uuid NOT NULL, + root_page bytea NOT NULL,root_generation bigint NOT NULL,attestation_digest bytea NOT NULL, + PRIMARY KEY(profile_digest,tagged_tree_oid), + FOREIGN KEY(attestation_id,root_page,root_generation,attestation_digest) + REFERENCES mst2_metadata_source_root_attestation(attestation_id,root_page,root_generation,attestation_digest) +); +CREATE INDEX mst2_metadata_reuse_index_page ON mst2_metadata_reuse_index(root_page,root_generation); + +ALTER TABLE mst2_qualified_session_incarnation ADD COLUMN canonical_descriptor bytea NOT NULL, + ADD COLUMN attestation_id uuid NOT NULL,ADD COLUMN attestation_digest bytea NOT NULL, + ADD COLUMN state text NOT NULL CHECK(state IN ('READY','RETIRED')), + ADD FOREIGN KEY(attestation_id,metadata_root,root_generation,attestation_digest) + REFERENCES mst2_metadata_source_root_attestation(attestation_id,root_page,root_generation,attestation_digest); +ALTER TABLE mst2_qualified_lease_binding ADD COLUMN namespace_uuid uuid NOT NULL, + ADD COLUMN authorization_epoch bigint NOT NULL,ADD COLUMN publication_sequence bigint NOT NULL, + ADD COLUMN writer_epoch bigint NOT NULL,ADD COLUMN certificate_receipt_id bigint NOT NULL, + ADD COLUMN expires_at_unix bigint NOT NULL,ADD COLUMN state text NOT NULL CHECK(state IN ('ACTIVE','RELEASED','EXPIRED')), + ADD COLUMN lease_epoch bigint NOT NULL CHECK(lease_epoch>0); +CREATE INDEX mst2_qualified_lease_active_incarnation ON mst2_qualified_lease_binding(snapshot_id,session_incarnation,state,lease_id); +CREATE TABLE mst2_metadata_reader_operation ( + operation_id uuid PRIMARY KEY,lease_id text NOT NULL REFERENCES mst2_qualified_lease_binding(lease_id), + snapshot_id text NOT NULL,session_incarnation uuid NOT NULL,root_page bytea NOT NULL,root_generation bigint NOT NULL, + lease_epoch bigint NOT NULL CHECK(lease_epoch>0),hard_deadline_unix bigint NOT NULL, + state text NOT NULL CHECK(state IN ('ACTIVE','FINISHED','EXPIRED')), + FOREIGN KEY(snapshot_id,session_incarnation) REFERENCES mst2_qualified_session_incarnation(snapshot_id,session_incarnation) +); +CREATE INDEX mst2_metadata_reader_active_lease ON mst2_metadata_reader_operation(lease_id,state,operation_id); +CREATE INDEX mst2_metadata_reader_active_deadline ON mst2_metadata_reader_operation(hard_deadline_unix,operation_id) WHERE state='ACTIVE'; +CREATE TABLE mst2_metadata_root_anchor ( + anchor_id uuid PRIMARY KEY,anchor_kind text NOT NULL CHECK(anchor_kind IN ('PREPARE','REUSE','SESSION','LEASE','REQUEST','READER')), + owner_key text NOT NULL CHECK(octet_length(owner_key) BETWEEN 1 AND 512), + root_page bytea NOT NULL,root_generation bigint NOT NULL,root_certificate_digest bytea NOT NULL, + prepare_id text REFERENCES mst2_metadata_prepare(prepare_id), + snapshot_id text,session_incarnation uuid,lease_id text REFERENCES mst2_qualified_lease_binding(lease_id), + reader_operation_id uuid REFERENCES mst2_metadata_reader_operation(operation_id), + UNIQUE(anchor_kind,owner_key,root_page,root_generation), + FOREIGN KEY(root_page,root_generation) REFERENCES mst2_metadata_graph_node(page_id,generation), + FOREIGN KEY(root_page,root_generation,root_certificate_digest) + REFERENCES mst2_metadata_page_certificate(page_id,generation,certificate_digest), + FOREIGN KEY(snapshot_id,session_incarnation) REFERENCES mst2_qualified_session_incarnation(snapshot_id,session_incarnation) +); +CREATE INDEX mst2_metadata_root_anchor_page ON mst2_metadata_root_anchor(root_page,root_generation,anchor_kind,owner_key); +CREATE INDEX mst2_metadata_root_anchor_prepare ON mst2_metadata_root_anchor(prepare_id,anchor_kind,anchor_id); +CREATE INDEX mst2_metadata_root_anchor_lease ON mst2_metadata_root_anchor(lease_id,anchor_kind,anchor_id); + +CREATE FUNCTION mst2_metadata_session_covers_prepare(pid text) RETURNS boolean LANGUAGE sql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ + SELECT EXISTS(SELECT 1 FROM mst2_metadata_prepare q + JOIN mst2_metadata_current cur ON cur.page_id=q.metadata_root + JOIN mst2_metadata_page_certificate certificate USING(page_id,generation) + JOIN mst2_qualified_session_incarnation session ON session.metadata_root=q.metadata_root + AND session.root_generation=cur.generation + JOIN mst2_metadata_source_root_attestation source ON source.attestation_id=session.attestation_id + AND source.root_page=session.metadata_root AND source.root_generation=session.root_generation + AND source.attestation_digest=session.attestation_digest + AND source.root_certificate_digest=certificate.certificate_digest + JOIN mst2_metadata_root_anchor anchor ON anchor.snapshot_id=session.snapshot_id + AND anchor.session_incarnation=session.session_incarnation AND anchor.anchor_kind='SESSION' + AND anchor.owner_key=session.snapshot_id||':'||session.session_incarnation::text + AND anchor.root_page=session.metadata_root AND anchor.root_generation=session.root_generation + AND anchor.root_certificate_digest=certificate.certificate_digest + WHERE q.prepare_id=pid AND q.plan_kind='ROOTED' AND q.state='COMMITTED' AND session.state='READY' + AND session.namespace_uuid=(SELECT namespace_uuid FROM mst2_metadata_family_identity WHERE singleton=1) + AND convert_from(session.source_profile,'UTF8')::jsonb=$CORE_SCHEMA$.mst2_route_profile( + q.source_domain,q.tagged_root_tree_oid,q.scope,q.schema_version,q.metadata_codec, + q.materialization_policy,q.fs_semantics,q.access_projection,q.verification_revision,q.projection_revision)) +$$; + +CREATE FUNCTION mst2_metadata_native_profile(pid text) RETURNS jsonb LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE q record; +BEGIN + SELECT source_domain,tagged_root_tree_oid,schema_version,metadata_codec,materialization_policy, + fs_semantics,access_projection,verification_revision,projection_revision + INTO q FROM mst2_metadata_prepare WHERE prepare_id=pid AND state IN ('PREPARING','COMMITTED') + AND graph_domain='qualified-v1' AND mst2_metadata_scope_matches(primary_scope); + IF NOT FOUND OR q.source_domain<>'native-git' OR q.schema_version<>2 OR q.metadata_codec<>1 + OR q.materialization_policy<>1 OR q.fs_semantics<>1 OR q.access_projection<>0 + OR q.verification_revision<>2 OR q.projection_revision<>1 + OR q.tagged_root_tree_oid !~ '^(sha1:[0-9a-f]{40}|sha256:[0-9a-f]{64}|blake3:[0-9a-f]{64})$' THEN + RAISE EXCEPTION 'source attestation requires the exact current native metadata profile'; + END IF; + RETURN jsonb_build_object('source_domain',q.source_domain,'hash_kind',split_part(q.tagged_root_tree_oid,':',1), + 'schema_version',q.schema_version,'metadata_codec',q.metadata_codec,'materialization_policy',q.materialization_policy, + 'fs_semantics',q.fs_semantics,'access_projection',q.access_projection, + 'verification_revision',q.verification_revision,'projection_revision',q.projection_revision); +END $$; + +CREATE FUNCTION mst2_metadata_decode_git_tree(b bytea,hash_kind text) RETURNS jsonb +LANGUAGE plpgsql IMMUTABLE STRICT SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE width integer; p integer:=0; start integer; mode text; name bytea; oid bytea; kind integer; + result jsonb[]:=ARRAY[]::jsonb[]; sorted jsonb; count integer:=0; +BEGIN + width:=CASE hash_kind WHEN 'sha1' THEN 20 WHEN 'sha256' THEN 32 WHEN 'blake3' THEN 32 ELSE 0 END; + IF width=0 OR octet_length(b)>67108864 THEN RAISE EXCEPTION 'source Git tree kind or byte budget is invalid'; END IF; + WHILE p32 AND p-start<6 LOOP p:=p+1; END LOOP; + IF p>=octet_length(b) OR get_byte(b,p)<>32 THEN RAISE EXCEPTION 'source Git tree mode is malformed'; END IF; + mode:=convert_from(substring(b FROM start+1 FOR p-start),'UTF8'); p:=p+1; start:=p; + WHILE p0 AND p-start<=255 LOOP p:=p+1; END LOOP; + IF p>=octet_length(b) OR get_byte(b,p)<>0 THEN RAISE EXCEPTION 'source Git tree name is malformed'; END IF; + name:=substring(b FROM start+1 FOR p-start); PERFORM mst2_metadata_valid_name(name); p:=p+1; + IF p>octet_length(b)-width THEN RAISE EXCEPTION 'source Git tree object identity is truncated'; END IF; + oid:=substring(b FROM p+1 FOR width); p:=p+width; + kind:=CASE mode WHEN '40000' THEN 4 WHEN '100644' THEN 1 WHEN '100664' THEN 1 WHEN '100640' THEN 1 + WHEN '100755' THEN 2 WHEN '120000' THEN 3 ELSE 0 END; + IF kind=0 THEN RAISE EXCEPTION 'source Git tree contains an unsupported entry'; END IF; + result:=array_append(result,jsonb_build_object('kind',kind,'name',encode(name,'hex'), + 'oid',hash_kind||':'||encode(oid,'hex'))); count:=count+1; + IF count>131072 THEN RAISE EXCEPTION 'source Git tree entry budget exceeded'; END IF; + END LOOP; + IF EXISTS(SELECT 1 FROM unnest(result) AS rows(value) GROUP BY value->>'name' HAVING count(*)>1) THEN + RAISE EXCEPTION 'source Git tree has duplicate names'; + END IF; + SELECT coalesce(jsonb_agg(value ORDER BY decode(value->>'name','hex')),'[]'::jsonb) INTO sorted + FROM unnest(result) AS rows(value); + RETURN sorted; +END $$; + +CREATE FUNCTION mst2_metadata_compute_source_proof(pid text,tree_oid text,p bytea,g bigint) RETURNS jsonb +LANGUAGE plpgsql VOLATILE SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE profile jsonb:=mst2_metadata_native_profile(pid); profile_hash bytea; body bytea; decoded jsonb; + item jsonb; mapped jsonb; entry_rows jsonb[]:=ARRAY[]::jsonb[]; entries jsonb; child_root bytea; + child_binding record; source_entries jsonb; reference_rows jsonb[]:=ARRAY[]::jsonb[]; + reference jsonb; fact record; built jsonb; proof jsonb; source_revision uuid; body_digest bytea; +BEGIN + IF split_part(tree_oid,':',1) IS DISTINCT FROM profile->>'hash_kind' + OR tree_oid !~ '^(sha1:[0-9a-f]{40}|sha256:[0-9a-f]{64}|blake3:[0-9a-f]{64})$' THEN + RAISE EXCEPTION 'source tree attestation crossed its tagged hash kind'; + END IF; + profile_hash:=sha256(convert_to('mega.mst2.native-profile.v1','UTF8')||decode('00','hex')||convert_to(profile::text,'UTF8')); + SELECT sub_trees INTO body FROM $CORE_SCHEMA$.mega_tree WHERE tree_id=split_part(tree_oid,':',2); + IF NOT FOUND THEN RAISE EXCEPTION 'source Git tree is missing from its captured core'; END IF; + body_digest:=sha256(body); + source_revision:=$CORE_SCHEMA$.mst2_route_capture_source_tree(split_part(tree_oid,':',2),body_digest); + decoded:=mst2_metadata_decode_git_tree(body,profile->>'hash_kind'); + FOR item IN SELECT value FROM jsonb_array_elements(decoded) LOOP + mapped:=jsonb_build_object('kind',(item->>'kind')::integer,'name',item->>'name'); + reference:=jsonb_build_object('kind',(item->>'kind')::integer,'git_oid',item->>'oid'); + IF (item->>'kind')::integer=4 THEN + SELECT a.root_page,a.root_generation,a.root_certificate_digest INTO child_binding FROM mst2_metadata_source_root_attestation a + JOIN mst2_metadata_current cur ON cur.page_id=a.root_page AND cur.generation=a.root_generation + JOIN mst2_metadata_lifetime life USING(page_id,generation) + JOIN mst2_metadata_graph_node node USING(page_id,generation) + JOIN mst2_metadata_prepare origin ON origin.prepare_id=a.origin_prepare_id + JOIN $CORE_SCHEMA$.mega_tree child_tree ON child_tree.tree_id=split_part(a.tagged_tree_oid,':',2) + WHERE a.tagged_tree_oid=item->>'oid' AND a.profile_digest=profile_hash AND a.source_profile=profile + AND a.namespace_uuid=(SELECT namespace_uuid FROM mst2_metadata_family_identity WHERE singleton=1) + AND node.state='LIVE' AND node.certificate_digest=a.root_certificate_digest + AND $CORE_SCHEMA$.mst2_route_source_tree_matches(split_part(a.tagged_tree_oid,':',2),a.source_revision,a.source_body_digest) + AND (life.state='LIVE' AND origin.state='COMMITTED' OR life.state='RESERVED' AND origin.prepare_id=pid AND origin.state='PREPARING') + AND NOT EXISTS(SELECT 1 FROM mst2_metadata_gc_op gc WHERE gc.page_id=a.root_page AND gc.generation=a.root_generation) + ORDER BY a.attestation_id LIMIT 1; + IF NOT FOUND THEN RAISE EXCEPTION 'source directory lacks its exact-profile certified child root'; END IF; + child_root:=child_binding.root_page; + mapped:=mapped||jsonb_build_object('child',encode(child_root,'hex')); + reference:=reference||jsonb_build_object('child_root',encode(child_root,'hex'), + 'child_generation',child_binding.root_generation,'child_certificate',encode(child_binding.root_certificate_digest,'hex')); + ELSE + SELECT size,raw_sha256 INTO fact FROM $CORE_SCHEMA$.mst2_verified_object + WHERE storage_domain='git' AND git_oid=split_part(item->>'oid',':',2) AND object_kind='blob' + AND state='VERIFIED' AND verification_version=2 FOR SHARE NOWAIT; + IF NOT FOUND OR fact.size<0 OR fact.size>8796093022208 OR octet_length(fact.raw_sha256)<>32 + OR (item->>'kind')::integer=3 AND fact.size NOT BETWEEN 1 AND 4095 THEN + RAISE EXCEPTION 'source file lacks its valid current verified-object fact'; + END IF; + mapped:=mapped||jsonb_build_object('size',fact.size,'content_id',encode(fact.raw_sha256,'hex')); + reference:=reference||jsonb_build_object('size',fact.size,'content_digest',encode(fact.raw_sha256,'hex')); + END IF; + reference_rows:=array_append(reference_rows,reference||jsonb_build_object('name',item->>'name')); + entry_rows:=array_append(entry_rows,mapped); + END LOOP; + entries:=to_jsonb(entry_rows); + SELECT coalesce(jsonb_object_agg(value->>'name',value-'name'),'{}'::jsonb) INTO source_entries + FROM unnest(reference_rows) input(value); + built:=mst2_metadata_build_map(entries); + IF decode(built->>'page_id','hex')<>p OR NOT EXISTS(SELECT 1 FROM mst2_metadata_page_certificate c + JOIN mst2_metadata_current cur USING(page_id,generation) JOIN mst2_metadata_graph_node n USING(page_id,generation) + WHERE c.page_id=p AND c.generation=g AND n.state='LIVE' AND n.certificate_digest=c.certificate_digest) THEN + RAISE EXCEPTION 'source projection differs from its independently canonical certified root'; + END IF; + proof:=jsonb_build_object('namespace',(SELECT namespace_uuid::text FROM mst2_metadata_family_identity WHERE singleton=1), + 'source_profile',profile,'profile_digest',encode(profile_hash,'hex'),'tagged_tree_oid',tree_oid, + 'source_body_digest',encode(body_digest,'hex'),'source_revision',source_revision::text,'root_page',encode(p,'hex'),'root_generation',g, + 'root_certificate',(SELECT encode(certificate_digest,'hex') FROM mst2_metadata_page_certificate WHERE page_id=p AND generation=g), + 'source_entry_count',jsonb_array_length(entries),'source_entries',source_entries,'source_work_units',built->'source_work_units', + 'encoded_map_digest',encode(sha256(convert_to(entries::text,'UTF8')),'hex')); + RETURN proof||jsonb_build_object('attestation',encode(sha256(convert_to('mega.mst2.source-root.v1','UTF8') + ||decode('00','hex')||convert_to(proof::text,'UTF8')),'hex')); +END $$; + +CREATE FUNCTION mst2_metadata_source_attestation_guard() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE proof jsonb; +BEGIN + IF TG_OP<>'INSERT' THEN RAISE EXCEPTION 'source root attestation history is immutable'; END IF; + proof:=mst2_metadata_compute_source_proof(NEW.origin_prepare_id,NEW.tagged_tree_oid,NEW.root_page,NEW.root_generation); + IF NEW.source_revision IS NULL THEN NEW.source_revision:=(proof->>'source_revision')::uuid; END IF; + IF NEW.namespace_uuid::text IS DISTINCT FROM proof->>'namespace' OR NEW.source_profile IS DISTINCT FROM proof->'source_profile' + OR NEW.profile_digest IS DISTINCT FROM decode(proof->>'profile_digest','hex') + OR NEW.source_body_digest IS DISTINCT FROM decode(proof->>'source_body_digest','hex') + OR NEW.source_revision IS DISTINCT FROM (proof->>'source_revision')::uuid + OR NEW.root_certificate_digest IS DISTINCT FROM decode(proof->>'root_certificate','hex') + OR NEW.attestation_digest IS DISTINCT FROM decode(proof->>'attestation','hex') OR NEW.source_proof IS DISTINCT FROM proof THEN + RAISE EXCEPTION 'source attestation was not independently derived from the captured core and canonical root'; + END IF; + RETURN NEW; +END $$; +CREATE TRIGGER mst2_metadata_source_attestation_guard BEFORE INSERT OR UPDATE OR DELETE ON mst2_metadata_source_root_attestation + FOR EACH ROW EXECUTE FUNCTION mst2_metadata_source_attestation_guard(); + +CREATE FUNCTION mst2_metadata_source_attestation_committed() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +BEGIN + IF NOT EXISTS(SELECT 1 FROM mst2_metadata_prepare WHERE prepare_id=NEW.origin_prepare_id AND state='COMMITTED') THEN + RAISE EXCEPTION 'source attestation cannot commit without its definitive finalized preparation'; + END IF; + IF NOT $CORE_SCHEMA$.mst2_route_source_tree_matches(split_part(NEW.tagged_tree_oid,':',2),NEW.source_revision,NEW.source_body_digest) + OR NEW.source_profile IS DISTINCT FROM mst2_metadata_native_profile(NEW.origin_prepare_id) THEN + RAISE EXCEPTION 'source attestation cannot commit after its exact captured source body or profile changed'; + END IF; + RETURN NULL; +END $$; +CREATE CONSTRAINT TRIGGER mst2_metadata_source_attestation_committed AFTER INSERT ON mst2_metadata_source_root_attestation + DEFERRABLE INITIALLY DEFERRED FOR EACH ROW EXECUTE FUNCTION mst2_metadata_source_attestation_committed(); + +CREATE FUNCTION mst2_metadata_reuse_root_guard() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE profile jsonb; profile_hash bytea; +BEGIN + IF TG_OP<>'INSERT' THEN RAISE EXCEPTION 'rooted reuse membership history is immutable'; END IF; + profile:=mst2_metadata_native_profile(NEW.prepare_id); + profile_hash:=sha256(convert_to('mega.mst2.native-profile.v1','UTF8')||decode('00','hex')||convert_to(profile::text,'UTF8')); + IF NOT EXISTS(SELECT 1 FROM mst2_metadata_prepare q,mst2_metadata_source_root_attestation a + JOIN mst2_metadata_current cur ON cur.page_id=a.root_page AND cur.generation=a.root_generation + JOIN mst2_metadata_lifetime life USING(page_id,generation) + JOIN mst2_metadata_graph_node node USING(page_id,generation) + JOIN mst2_metadata_prepare origin ON origin.prepare_id=a.origin_prepare_id + JOIN $CORE_SCHEMA$.mega_tree tree ON tree.tree_id=split_part(a.tagged_tree_oid,':',2) + WHERE q.prepare_id=NEW.prepare_id AND q.state='PREPARING' AND a.attestation_id=NEW.attestation_id + AND a.root_page=NEW.root_page AND a.root_generation=NEW.root_generation AND a.attestation_digest=NEW.attestation_digest + AND a.profile_digest=profile_hash AND a.source_profile=profile + AND $CORE_SCHEMA$.mst2_route_source_tree_matches(split_part(a.tagged_tree_oid,':',2),a.source_revision,a.source_body_digest) + AND origin.state='COMMITTED' AND life.state='LIVE' AND life.graph_domain='qualified-v1' + AND node.state='LIVE' AND node.certificate_digest=a.root_certificate_digest + AND NOT EXISTS(SELECT 1 FROM mst2_metadata_gc_op gc WHERE gc.page_id=a.root_page AND gc.generation=a.root_generation)) THEN + RAISE EXCEPTION 'rooted reuse membership lacks its exact finalized source and current lifetime'; + END IF; + IF (SELECT count(*) FROM mst2_metadata_prepare_reuse_root WHERE prepare_id=NEW.prepare_id)>=4096 THEN + RAISE EXCEPTION 'rooted reuse boundary budget exceeded'; + END IF; + RETURN NEW; +END $$; +CREATE TRIGGER mst2_metadata_reuse_root_guard BEFORE INSERT OR UPDATE OR DELETE ON mst2_metadata_prepare_reuse_root + FOR EACH ROW EXECUTE FUNCTION mst2_metadata_reuse_root_guard(); + +CREATE FUNCTION mst2_metadata_reuse_index_guard() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +BEGIN + IF TG_OP='UPDATE' THEN RAISE EXCEPTION 'rooted reuse index cannot retarget an exact lifetime'; END IF; + IF TG_OP='DELETE' THEN + IF NOT EXISTS(SELECT 1 FROM mst2_metadata_gc_op WHERE page_id=OLD.root_page AND generation=OLD.root_generation AND state='PENDING') THEN + RAISE EXCEPTION 'rooted reuse index retirement requires its exact GC claim'; + END IF; + RETURN OLD; + END IF; + IF NOT EXISTS(SELECT 1 FROM mst2_metadata_source_root_attestation a + JOIN mst2_metadata_prepare q ON q.prepare_id=a.origin_prepare_id + JOIN mst2_metadata_current cur ON cur.page_id=a.root_page AND cur.generation=a.root_generation + JOIN mst2_metadata_lifetime life USING(page_id,generation) + JOIN mst2_metadata_graph_node node USING(page_id,generation) + WHERE a.attestation_id=NEW.attestation_id AND a.profile_digest=NEW.profile_digest AND a.tagged_tree_oid=NEW.tagged_tree_oid + AND a.root_page=NEW.root_page AND a.root_generation=NEW.root_generation AND a.attestation_digest=NEW.attestation_digest + AND q.state='COMMITTED' AND life.state='LIVE' AND node.state='LIVE' AND node.certificate_digest=a.root_certificate_digest + AND NOT EXISTS(SELECT 1 FROM mst2_metadata_gc_op gc WHERE gc.page_id=a.root_page AND gc.generation=a.root_generation)) THEN + RAISE EXCEPTION 'rooted reuse index is not bound to its definitive exact current source proof'; + END IF; + RETURN NEW; +END $$; +CREATE TRIGGER mst2_metadata_reuse_index_guard BEFORE INSERT OR UPDATE OR DELETE ON mst2_metadata_reuse_index + FOR EACH ROW EXECUTE FUNCTION mst2_metadata_reuse_index_guard(); + +CREATE FUNCTION mst2_metadata_root_anchor_guard() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +BEGIN + IF TG_OP='UPDATE' THEN RAISE EXCEPTION 'rooted anchors cannot retarget immutable owner or root identities'; END IF; + IF TG_OP='DELETE' THEN + IF OLD.anchor_kind IN ('PREPARE','REUSE') THEN + IF NOT EXISTS(SELECT 1 FROM mst2_metadata_prepare WHERE prepare_id=OLD.prepare_id AND state IN ('PREPARING','COMMITTED','ABORTED')) THEN + RAISE EXCEPTION 'temporary anchor owner history is missing'; + END IF; + ELSIF OLD.anchor_kind='SESSION' THEN + IF NOT EXISTS(SELECT 1 FROM mst2_qualified_session_incarnation s + WHERE s.snapshot_id=OLD.snapshot_id AND s.session_incarnation=OLD.session_incarnation AND s.state='RETIRED') + OR EXISTS(SELECT 1 FROM mst2_qualified_lease_binding l WHERE l.snapshot_id=OLD.snapshot_id + AND l.session_incarnation=OLD.session_incarnation AND l.state='ACTIVE') THEN + RAISE EXCEPTION 'session root still has its active incarnation or leases'; + END IF; + ELSIF OLD.anchor_kind='LEASE' THEN + IF NOT EXISTS(SELECT 1 FROM mst2_qualified_lease_binding WHERE lease_id=OLD.lease_id AND state IN ('RELEASED','EXPIRED')) + OR EXISTS(SELECT 1 FROM mst2_metadata_reader_operation WHERE lease_id=OLD.lease_id AND state='ACTIVE') THEN + RAISE EXCEPTION 'lease root still has its active lease or readers'; + END IF; + ELSE + IF NOT EXISTS(SELECT 1 FROM mst2_metadata_reader_operation r WHERE r.operation_id=OLD.reader_operation_id + AND (r.state='FINISHED' OR r.state='EXPIRED' AND r.hard_deadline_unix<=floor(extract(epoch FROM clock_timestamp()))::bigint)) THEN + RAISE EXCEPTION 'reader root still has an active operation'; + END IF; + END IF; + RETURN OLD; + END IF; + IF NOT EXISTS(SELECT 1 FROM mst2_metadata_graph_node node + JOIN mst2_metadata_current cur USING(page_id,generation) + JOIN mst2_metadata_lifetime life USING(page_id,generation) + JOIN mst2_metadata_page_certificate proof USING(page_id,generation) + WHERE node.page_id=NEW.root_page AND node.generation=NEW.root_generation AND node.state='LIVE' + AND node.certificate_digest=NEW.root_certificate_digest AND proof.certificate_digest=NEW.root_certificate_digest + AND life.state IN ('RESERVED','LIVE') AND life.graph_domain='qualified-v1' + AND NOT EXISTS(SELECT 1 FROM mst2_metadata_gc_op gc WHERE gc.page_id=node.page_id AND gc.generation=node.generation)) THEN + RAISE EXCEPTION 'rooted anchor does not protect its exact canonical current graph'; + END IF; + IF NEW.anchor_kind IN ('PREPARE','REUSE') THEN + IF NEW.owner_key IS DISTINCT FROM NEW.prepare_id OR NEW.snapshot_id IS NOT NULL OR NEW.session_incarnation IS NOT NULL + OR NEW.lease_id IS NOT NULL OR NEW.reader_operation_id IS NOT NULL + OR NOT EXISTS(SELECT 1 FROM mst2_metadata_prepare WHERE prepare_id=NEW.prepare_id + AND state IN ('PREPARING','COMMITTED') AND coverage_retired_at IS NULL) + OR NEW.anchor_kind='PREPARE' AND NOT (EXISTS(SELECT 1 FROM mst2_metadata_prepare_page + WHERE prepare_id=NEW.prepare_id AND page_id=NEW.root_page AND generation=NEW.root_generation) + OR EXISTS(SELECT 1 FROM mst2_metadata_prepare q JOIN mst2_metadata_prepare_reuse_root r USING(prepare_id) + WHERE q.prepare_id=NEW.prepare_id AND q.plan_kind='ROOTED' AND q.metadata_root=NEW.root_page + AND r.root_page=NEW.root_page AND r.root_generation=NEW.root_generation)) + OR NEW.anchor_kind='REUSE' AND NOT EXISTS(SELECT 1 FROM mst2_metadata_prepare_reuse_root + WHERE prepare_id=NEW.prepare_id AND root_page=NEW.root_page AND root_generation=NEW.root_generation) THEN + RAISE EXCEPTION 'temporary anchor differs from its immutable delta or reused-root owner'; + END IF; + ELSIF NEW.anchor_kind='SESSION' THEN + IF NEW.owner_key IS DISTINCT FROM NEW.snapshot_id||':'||NEW.session_incarnation::text OR NEW.prepare_id IS NOT NULL + OR NEW.lease_id IS NOT NULL OR NEW.reader_operation_id IS NOT NULL + OR NOT EXISTS(SELECT 1 FROM mst2_qualified_session_incarnation s WHERE s.snapshot_id=NEW.snapshot_id + AND s.session_incarnation=NEW.session_incarnation AND s.metadata_root=NEW.root_page AND s.root_generation=NEW.root_generation AND s.state='READY') THEN + RAISE EXCEPTION 'session anchor differs from its exact ready incarnation'; + END IF; + ELSIF NEW.anchor_kind='LEASE' THEN + IF NEW.owner_key IS DISTINCT FROM NEW.lease_id OR NEW.prepare_id IS NOT NULL OR NEW.reader_operation_id IS NOT NULL + OR NOT EXISTS(SELECT 1 FROM mst2_qualified_lease_binding l WHERE l.lease_id=NEW.lease_id + AND l.snapshot_id=NEW.snapshot_id AND l.session_incarnation=NEW.session_incarnation + AND l.metadata_root=NEW.root_page AND l.root_generation=NEW.root_generation AND l.state='ACTIVE' + AND l.expires_at_unix>floor(extract(epoch FROM clock_timestamp()))::bigint) THEN + RAISE EXCEPTION 'lease anchor differs from its exact active lease'; + END IF; + ELSE + IF NEW.owner_key IS DISTINCT FROM NEW.reader_operation_id::text OR NEW.prepare_id IS NOT NULL + OR NOT EXISTS(SELECT 1 FROM mst2_metadata_reader_operation r JOIN mst2_qualified_lease_binding l USING(lease_id) + WHERE r.operation_id=NEW.reader_operation_id AND r.lease_id=NEW.lease_id AND r.snapshot_id=NEW.snapshot_id + AND r.session_incarnation=NEW.session_incarnation AND r.root_page=NEW.root_page AND r.root_generation=NEW.root_generation + AND r.state='ACTIVE' AND l.state='ACTIVE' AND l.lease_epoch=r.lease_epoch + AND r.hard_deadline_unix<=l.expires_at_unix AND r.hard_deadline_unix>floor(extract(epoch FROM clock_timestamp()))::bigint) THEN + RAISE EXCEPTION 'reader anchor differs from its exact active lease operation'; + END IF; + END IF; + RETURN NEW; +END $$; +CREATE TRIGGER mst2_metadata_root_anchor_guard BEFORE INSERT OR UPDATE OR DELETE ON mst2_metadata_root_anchor + FOR EACH ROW EXECUTE FUNCTION mst2_metadata_root_anchor_guard(); + +CREATE FUNCTION mst2_metadata_reuse_root_protected() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +BEGIN + IF EXISTS(SELECT 1 FROM mst2_metadata_prepare WHERE prepare_id=NEW.prepare_id + AND state IN ('PREPARING','COMMITTED') AND coverage_retired_at IS NULL) + AND NOT EXISTS(SELECT 1 FROM mst2_metadata_root_anchor WHERE anchor_kind='REUSE' AND prepare_id=NEW.prepare_id + AND root_page=NEW.root_page AND root_generation=NEW.root_generation) THEN + RAISE EXCEPTION 'reused boundary must commit with continuously owned root protection'; + END IF; + RETURN NULL; +END $$; +CREATE CONSTRAINT TRIGGER mst2_metadata_reuse_root_protected AFTER INSERT ON mst2_metadata_prepare_reuse_root + DEFERRABLE INITIALLY DEFERRED FOR EACH ROW EXECUTE FUNCTION mst2_metadata_reuse_root_protected(); + +CREATE FUNCTION mst2_metadata_temporary_anchor_continuity() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE q mst2_metadata_prepare%ROWTYPE; +BEGIN + IF OLD.anchor_kind NOT IN ('PREPARE','REUSE') THEN RETURN NULL; END IF; + SELECT * INTO STRICT q FROM mst2_metadata_prepare WHERE prepare_id=OLD.prepare_id; + IF q.state='ABORTED' THEN RETURN NULL; END IF; + IF q.coverage_retired_at IS NULL THEN + IF NOT EXISTS(SELECT 1 FROM mst2_metadata_root_anchor a WHERE a.anchor_kind=OLD.anchor_kind + AND a.owner_key=OLD.owner_key AND a.prepare_id=OLD.prepare_id + AND a.root_page=OLD.root_page AND a.root_generation=OLD.root_generation + AND a.root_certificate_digest=OLD.root_certificate_digest) THEN + RAISE EXCEPTION 'active preparation cannot lose its continuously owned canonical root'; + END IF; + ELSIF q.state<>'COMMITTED' OR NOT (mst2_metadata_session_covers_prepare(q.prepare_id) + OR mst2_metadata_orphan_prepare_retired(q.prepare_id)) THEN + RAISE EXCEPTION 'retired preparation coverage requires a definitive independent session root'; + END IF; + RETURN NULL; +END $$; +CREATE CONSTRAINT TRIGGER mst2_metadata_temporary_anchor_continuity AFTER DELETE ON mst2_metadata_root_anchor + DEFERRABLE INITIALLY DEFERRED FOR EACH ROW EXECUTE FUNCTION mst2_metadata_temporary_anchor_continuity(); diff --git a/src/jupiter/storage/qualified_metadata_canonical.sql b/src/jupiter/storage/qualified_metadata_canonical.sql new file mode 100644 index 00000000..2ce0c283 --- /dev/null +++ b/src/jupiter/storage/qualified_metadata_canonical.sql @@ -0,0 +1,227 @@ +CREATE FUNCTION mst2_metadata_read_le(b bytea,p integer,w integer) RETURNS numeric +LANGUAGE plpgsql IMMUTABLE STRICT SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE v numeric:=0; power numeric:=1; i integer; +BEGIN + IF w NOT IN (2,4,8) OR p<0 OR p>octet_length(b)-w THEN + RAISE EXCEPTION 'MTP2 integer is out of bounds'; + END IF; + FOR i IN 0..w-1 LOOP + v:=v+get_byte(b,p+i)*power; power:=power*256; + END LOOP; + RETURN v; +END $$; + +CREATE FUNCTION mst2_metadata_write_le(v numeric,w integer) RETURNS bytea +LANGUAGE plpgsql IMMUTABLE STRICT SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE result bytea; i integer; +BEGIN + IF w NOT IN (2,4,8) OR v<0 OR v<>trunc(v) OR v>=power(256::numeric,w) THEN + RAISE EXCEPTION 'MTP2 output integer is out of range'; + END IF; + result:=decode(repeat('00',w),'hex'); + FOR i IN 0..w-1 LOOP result:=set_byte(result,i,mod(v,256)::integer); v:=trunc(v/256); END LOOP; + RETURN result; +END $$; + +CREATE FUNCTION mst2_metadata_encode_entry(e jsonb) RETURNS bytea +LANGUAGE plpgsql IMMUTABLE STRICT SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE kind integer:=(e->>'kind')::integer; name bytea:=decode(e->>'name','hex'); target bytea; result bytea; +BEGIN + IF kind IS NULL OR kind NOT BETWEEN 1 AND 4 OR name IS NULL THEN RAISE EXCEPTION 'MTP2 output entry is incomplete'; END IF; + PERFORM mst2_metadata_valid_name(name); + result:=set_byte(decode('00','hex'),0,kind)||mst2_metadata_write_le(octet_length(name),2)||name; + IF kind=4 THEN + target:=decode(e->>'child','hex'); + IF target IS NULL OR octet_length(target)<>32 OR target=decode(repeat('00',32),'hex') THEN + RAISE EXCEPTION 'MTP2 output directory reference is invalid'; + END IF; + ELSE + target:=decode(e->>'content_id','hex'); + IF target IS NULL OR octet_length(target)<>32 OR e->>'size' IS NULL THEN + RAISE EXCEPTION 'MTP2 output file reference is invalid'; + END IF; + result:=result||mst2_metadata_write_le((e->>'size')::numeric,8); + END IF; + RETURN result||target; +END $$; + +CREATE FUNCTION mst2_metadata_build_map(items jsonb,budget bigint DEFAULT 67108864) RETURNS jsonb +LANGUAGE plpgsql IMMUTABLE STRICT SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE n integer; item jsonb; previous bytea; name bytea; minimum bytea; maximum bytea; + raw bytea:=''::bytea; result bytea; prefix bytea; terminal jsonb; + partition record; grouped jsonb:='[]'::jsonb; child jsonb; + visits bigint:=0; encoded_bytes bigint:=0; pages bigint:=1; total_bytes bigint; payload bytea; children integer:=0; +BEGIN + IF jsonb_typeof(items)<>'array' OR budget NOT BETWEEN 1 AND 67108864 THEN + RAISE EXCEPTION 'MTP2 canonical map input or work budget is invalid'; + END IF; + n:=jsonb_array_length(items); + IF n>131072 THEN RAISE EXCEPTION 'MTP2 canonical map exceeds its entry budget'; END IF; + FOR item IN SELECT value FROM jsonb_array_elements(items) LOOP + name:=decode(item->>'name','hex'); + IF previous IS NOT NULL AND previous>=name THEN RAISE EXCEPTION 'MTP2 build names are not strictly byte ordered'; END IF; + previous:=name; maximum:=name; IF minimum IS NULL THEN minimum:=name; END IF; + payload:=mst2_metadata_encode_entry(item); visits:=visits+1+octet_length(payload); + IF visits>budget THEN RAISE EXCEPTION 'MTP2 canonical source work budget exceeded'; END IF; + encoded_bytes:=encoded_bytes+octet_length(payload); + IF n<=128 AND 20+encoded_bytes<=16384 THEN raw:=raw||payload; END IF; + END LOOP; + IF n<=128 AND 20+encoded_bytes<=16384 THEN + result:=decode('4d5450320000','hex')||mst2_metadata_write_le(n,2)||mst2_metadata_write_le(n,8) + ||mst2_metadata_write_le(octet_length(raw),4)||raw; + RETURN jsonb_build_object('bytes',encode(result,'hex'),'page_id',encode(sha256( + convert_to('mega.mst2.metapage','UTF8')||decode('00','hex')||result),'hex'), + 'source_work_units',visits,'pages',pages,'metadata_bytes',octet_length(result)); + END IF; + prefix:=mst2_metadata_lcp(minimum,maximum); + SELECT value INTO terminal FROM jsonb_array_elements(items) WHERE decode(value->>'name','hex')=prefix; + visits:=visits+2*n; + IF visits>budget THEN RAISE EXCEPTION 'MTP2 canonical source grouping work budget exceeded'; END IF; + payload:=mst2_metadata_write_le(octet_length(prefix),2)||prefix; + IF terminal IS NULL THEN payload:=payload||decode('00','hex'); + ELSE payload:=payload||decode('01','hex')||mst2_metadata_encode_entry(terminal); END IF; + total_bytes:=0; + FOR partition IN SELECT get_byte(decode(value->>'name','hex'),octet_length(prefix)) AS label, + jsonb_agg(value ORDER BY decode(value->>'name','hex')) AS members + FROM jsonb_array_elements(items) WHERE octet_length(decode(value->>'name','hex'))>octet_length(prefix) + GROUP BY get_byte(decode(value->>'name','hex'),octet_length(prefix)) ORDER BY label LOOP + grouped:=partition.members; + IF visits>=budget THEN RAISE EXCEPTION 'MTP2 canonical source work budget exceeded'; END IF; + child:=mst2_metadata_build_map(grouped,budget-visits); visits:=visits+(child->>'source_work_units')::bigint; + pages:=pages+(child->>'pages')::bigint; total_bytes:=total_bytes+(child->>'metadata_bytes')::bigint; + IF pages>4096 OR total_bytes>67108864 THEN RAISE EXCEPTION 'MTP2 canonical source encoding exceeds metadata budget'; END IF; + payload:=payload||set_byte(decode('00','hex'),0,partition.label)||mst2_metadata_write_le(jsonb_array_length(grouped),8) + ||decode(child->>'page_id','hex'); children:=children+1; + END LOOP; + IF children+(terminal IS NOT NULL)::integer<2 OR 20+octet_length(payload)>16384 THEN + RAISE EXCEPTION 'MTP2 canonical branch shape or size is invalid'; + END IF; + result:=decode('4d5450320100','hex')||mst2_metadata_write_le(children,2)||mst2_metadata_write_le(n,8) + ||mst2_metadata_write_le(octet_length(payload),4)||payload; + total_bytes:=total_bytes+octet_length(result); + IF total_bytes>67108864 THEN RAISE EXCEPTION 'MTP2 canonical source encoding exceeds metadata byte budget'; END IF; + RETURN jsonb_build_object('bytes',encode(result,'hex'),'page_id',encode(sha256( + convert_to('mega.mst2.metapage','UTF8')||decode('00','hex')||result),'hex'), + 'source_work_units',visits,'pages',pages,'metadata_bytes',total_bytes); +END $$; + +CREATE FUNCTION mst2_metadata_valid_name(n bytea) RETURNS boolean +LANGUAGE plpgsql IMMUTABLE STRICT SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE i integer; +BEGIN + IF octet_length(n) NOT BETWEEN 1 AND 255 OR n IN (decode('2e','hex'),decode('2e2e','hex')) THEN + RAISE EXCEPTION 'MTP2 name has an invalid length or dot component'; + END IF; + FOR i IN 0..octet_length(n)-1 LOOP + IF get_byte(n,i) IN (0,47) THEN RAISE EXCEPTION 'MTP2 name contains NUL or slash'; END IF; + END LOOP; + PERFORM convert_from(n,'UTF8'); + RETURN true; +END $$; + +CREATE FUNCTION mst2_metadata_decode_entry(b bytea,p integer) RETURNS jsonb +LANGUAGE plpgsql IMMUTABLE STRICT SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE start integer:=p; kind integer; n integer; name bytea; target bytea; size numeric; +BEGIN + IF p<0 OR p>=octet_length(b) THEN RAISE EXCEPTION 'MTP2 entry kind is truncated'; END IF; + kind:=get_byte(b,p); p:=p+1; + IF kind NOT BETWEEN 1 AND 4 THEN RAISE EXCEPTION 'MTP2 entry kind is invalid'; END IF; + n:=mst2_metadata_read_le(b,p,2)::integer; p:=p+2; + IF n>octet_length(b)-p THEN RAISE EXCEPTION 'MTP2 entry name is truncated'; END IF; + name:=substring(b FROM p+1 FOR n); p:=p+n; + PERFORM mst2_metadata_valid_name(name); + IF kind=4 THEN + IF p>octet_length(b)-32 THEN RAISE EXCEPTION 'MTP2 directory reference is truncated'; END IF; + target:=substring(b FROM p+1 FOR 32); p:=p+32; + IF target=decode(repeat('00',32),'hex') THEN RAISE EXCEPTION 'MTP2 empty directory has a zero reference'; END IF; + RETURN jsonb_build_object('end',p,'kind',kind,'name',encode(name,'hex'), + 'encoded_bytes',p-start,'child',encode(target,'hex')); + END IF; + size:=mst2_metadata_read_le(b,p,8); p:=p+8; + IF p>octet_length(b)-32 THEN RAISE EXCEPTION 'MTP2 file content reference is truncated'; END IF; + target:=substring(b FROM p+1 FOR 32); p:=p+32; + RETURN jsonb_build_object('end',p,'kind',kind,'name',encode(name,'hex'), + 'encoded_bytes',p-start,'size',size,'content_id',encode(target,'hex')); +END $$; + +CREATE FUNCTION mst2_metadata_decode_local(b bytea) RETURNS jsonb +LANGUAGE plpgsql IMMUTABLE STRICT SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE kind integer; n integer; count numeric; declared numeric; p integer:=20; plen integer; + i integer; label integer; previous_label integer:=-1; previous_name bytea; name bytea; prefix bytea; + terminal integer:=0; item jsonb; entries jsonb:='[]'::jsonb; children jsonb:='[]'::jsonb; + refs jsonb:='[]'::jsonb; child bytea; child_count numeric; entry_bytes bigint:=0; +BEGIN + IF octet_length(b) NOT BETWEEN 20 AND 16384 THEN RAISE EXCEPTION 'MTP2 page length is invalid'; END IF; + IF substring(b FROM 1 FOR 4)<>decode('4d545032','hex') OR get_byte(b,5)<>0 THEN + RAISE EXCEPTION 'MTP2 magic or flags are invalid'; + END IF; + kind:=get_byte(b,4); n:=mst2_metadata_read_le(b,6,2)::integer; + count:=mst2_metadata_read_le(b,8,8); + IF count>9223372036854775807 OR mst2_metadata_read_le(b,16,4)<>octet_length(b)-20 THEN + RAISE EXCEPTION 'MTP2 header count or payload length is invalid'; + END IF; + IF kind=0 THEN + IF n>128 OR count<>n THEN RAISE EXCEPTION 'MTP2 leaf count is invalid'; END IF; + IF n>0 THEN + FOR i IN 1..n LOOP + item:=mst2_metadata_decode_entry(b,p); p:=(item->>'end')::integer; + name:=decode(item->>'name','hex'); + IF previous_name IS NOT NULL AND previous_name>=name THEN + RAISE EXCEPTION 'MTP2 leaf names are not strictly byte ordered'; + END IF; + previous_name:=name; entries:=entries||jsonb_build_array(item-'end'); + entry_bytes:=entry_bytes+(item->>'encoded_bytes')::bigint; + IF (item->>'kind')::integer=4 THEN + refs:=refs||jsonb_build_array(jsonb_build_object('kind','DIRECTORY','name',item->>'name','child',item->>'child')); + END IF; + END LOOP; + END IF; + ELSIF kind=1 THEN + plen:=mst2_metadata_read_le(b,p,2)::integer; p:=p+2; + IF plen>octet_length(b)-p THEN RAISE EXCEPTION 'MTP2 branch prefix is truncated'; END IF; + prefix:=substring(b FROM p+1 FOR plen); p:=p+plen; + IF p>=octet_length(b) THEN RAISE EXCEPTION 'MTP2 terminal flag is truncated'; END IF; + terminal:=get_byte(b,p); p:=p+1; + IF terminal NOT IN (0,1) THEN RAISE EXCEPTION 'MTP2 terminal flag is invalid'; END IF; + IF terminal=1 THEN + item:=mst2_metadata_decode_entry(b,p); p:=(item->>'end')::integer; + IF decode(item->>'name','hex')<>prefix THEN RAISE EXCEPTION 'MTP2 terminal name differs from prefix'; END IF; + entries:=entries||jsonb_build_array(item-'end'); entry_bytes:=(item->>'encoded_bytes')::bigint; + IF (item->>'kind')::integer=4 THEN + refs:=refs||jsonb_build_array(jsonb_build_object('kind','DIRECTORY','name',item->>'name','child',item->>'child')); + END IF; + END IF; + IF n>256 OR n+terminal<2 THEN RAISE EXCEPTION 'MTP2 branch group count is invalid'; END IF; + declared:=terminal; + IF n>0 THEN + FOR i IN 1..n LOOP + IF p>=octet_length(b) THEN RAISE EXCEPTION 'MTP2 branch child label is truncated'; END IF; + label:=get_byte(b,p); p:=p+1; + IF label<=previous_label THEN RAISE EXCEPTION 'MTP2 branch labels are not strictly ordered'; END IF; + previous_label:=label; child_count:=mst2_metadata_read_le(b,p,8); p:=p+8; + IF child_count NOT BETWEEN 1 AND 9223372036854775807 THEN RAISE EXCEPTION 'MTP2 child count is invalid'; END IF; + IF p>octet_length(b)-32 THEN RAISE EXCEPTION 'MTP2 branch child digest is truncated'; END IF; + child:=substring(b FROM p+1 FOR 32); p:=p+32; + declared:=declared+child_count; + IF declared>9223372036854775807 THEN RAISE EXCEPTION 'MTP2 branch count overflows'; END IF; + item:=jsonb_build_object('kind','RADIX','label',label,'count',child_count,'child',encode(child,'hex')); + children:=children||jsonb_build_array(item); refs:=refs||jsonb_build_array(item); + END LOOP; + END IF; + IF declared<>count THEN RAISE EXCEPTION 'MTP2 branch header count differs from children'; END IF; + ELSE + RAISE EXCEPTION 'MTP2 page kind is invalid'; + END IF; + IF p<>octet_length(b) THEN RAISE EXCEPTION 'MTP2 page has trailing bytes'; END IF; + RETURN jsonb_build_object('kind',kind,'count',count,'prefix',encode(prefix,'hex'), + 'entries',entries,'children',children,'refs',refs,'direct_entry_bytes',entry_bytes, + 'page_id',encode(sha256(convert_to('mega.mst2.metapage','UTF8')||decode('00','hex')||b),'hex')); +END $$; + +CREATE FUNCTION mst2_metadata_lcp(a bytea,b bytea) RETURNS bytea +LANGUAGE plpgsql IMMUTABLE STRICT SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE n integer:=least(octet_length(a),octet_length(b)); i integer:=0; +BEGIN + WHILE i Value { + Value::Array( + entries + .iter() + .map(|entry| { + json!({"kind":1,"name":hex::encode(&entry.name),"size":u64::MAX, + "content_id":hex::encode([17;32])}) + }) + .collect(), + ) +} + +fn files(names: impl IntoIterator) -> Vec { + let mut entries: Vec<_> = names + .into_iter() + .map(|name| Entry::file(EntryKind::Regular, name.as_bytes(), u64::MAX, [17; 32])) + .collect(); + entries.sort_by(|left, right| left.name.cmp(&right.name)); + entries +} + +#[tokio::test] +async fn database_canonical_builder_matches_pinned_codec_at_split_and_name_boundaries() { + let (_config, _core, _namespace, q, _guard) = fixture().await; + let terminal_names = + std::iter::once("a".to_owned()).chain((0..129).map(|index| format!("a{index:03}"))); + let cases = [ + Vec::new(), + files((0..128).map(|index| format!("f{index:03}"))), + files((0..129).map(|index| format!("f{index:03}"))), + files((0..128).map(|index| format!("{}{index:03}", "x".repeat(197)))), + files(terminal_names), + files((0..160).map(|index| format!("目录{index:03}"))), + ]; + for entries in cases { + let bytes = Page::build(&entries).unwrap(); + let proof: Value = q + .query_one_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + "SELECT mst2_metadata_build_map($1::jsonb) AS proof", + [encoded_entries(&entries).to_string().into()], + )) + .await + .unwrap() + .unwrap() + .try_get("", "proof") + .unwrap(); + assert_eq!(proof["bytes"].as_str().unwrap(), hex::encode(&bytes)); + assert_eq!( + proof["page_id"].as_str().unwrap(), + hex::encode(page_id(&bytes)) + ); + let decoded: Value = q + .query_one_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + "SELECT mst2_metadata_decode_local($1) AS decoded", + [bytes.into()], + )) + .await + .unwrap() + .unwrap() + .try_get("", "decoded") + .unwrap(); + assert_eq!(decoded["count"].as_u64().unwrap(), entries.len() as u64); + } +} + +#[tokio::test] +async fn database_parser_rejects_nonexact_bytes_and_invalid_names() { + let (_config, _core, _namespace, q, _guard) = fixture().await; + let bytes = Page::build(&files(["file".to_owned()])).unwrap(); + let mut trailing = bytes.clone(); + trailing.push(0); + let mut wrong_length = bytes.clone(); + wrong_length[16] = wrong_length[16].wrapping_add(1); + let mut flags = bytes.clone(); + flags[5] = 1; + let mut wrong_count = bytes.clone(); + wrong_count[8] = 2; + let mut invalid_utf8 = bytes.clone(); + invalid_utf8[23] = 255; + let mut slash = bytes.clone(); + slash[23] = b'/'; + for malformed in [ + trailing, + wrong_length, + flags, + wrong_count, + invalid_utf8, + slash, + ] { + assert!( + q.query_one_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + "SELECT mst2_metadata_decode_local($1)", + [malformed.into()], + )) + .await + .is_err() + ); + } + let mut entries = encoded_entries(&files(["one".to_owned(), "two".to_owned()])); + entries.as_array_mut().unwrap().reverse(); + assert!( + q.query_one_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + "SELECT mst2_metadata_build_map($1::jsonb)", + [entries.to_string().into()], + )) + .await + .is_err() + ); +} + +#[tokio::test] +async fn exact_typed_certificates_keep_shared_directory_occurrences_and_dedup_physical_edges() { + let (config, core, _namespace, q, _guard) = fixture().await; + let writer = ShadowQualifiedMetadataWriter::open(&core, &config) + .await + .unwrap(); + let pages = prepared(); + let receipt = write(&writer, "certified-shared-directory", &pages).await; + assert_eq!( + count(&q, "SELECT count(*) FROM mst2_metadata_page_certificate").await, + 2 + ); + assert_eq!( + count( + &q, + "SELECT count(*) FROM mst2_metadata_verified_ref WHERE reference_kind='DIRECTORY'" + ) + .await, + 2 + ); + assert_eq!( + count(&q, "SELECT count(*) FROM mst2_metadata_graph_edge").await, + 1 + ); + assert_eq!( + count( + &q, + "SELECT max(rank)::bigint FROM mst2_metadata_page_certificate" + ) + .await, + 1 + ); + assert_eq!(writer.finalize(receipt.intent()).await.unwrap(), receipt); + for table in [ + "mst2_metadata_page_certificate", + "mst2_metadata_verified_ref", + "mst2_metadata_source_root_attestation", + "mst2_metadata_prepare_reuse_root", + "mst2_metadata_reuse_index", + "mst2_metadata_root_anchor", + "mst2_metadata_reader_operation", + ] { + assert!( + q.execute_unprepared(&format!("TRUNCATE {table} CASCADE")) + .await + .is_err() + ); + } + for sql in [ + "UPDATE mst2_metadata_page_certificate SET rank=rank+1", + "DELETE FROM mst2_metadata_verified_ref", + "UPDATE mst2_metadata_verified_ref SET reference_ordinal=reference_ordinal+1", + ] { + assert!(q.execute_unprepared(sql).await.is_err()); + } +} + +#[tokio::test] +async fn raw_sql_cannot_certify_a_leafable_branch_from_valid_certified_children() { + let (config, core, namespace, q, _guard) = fixture().await; + let writer = ShadowQualifiedMetadataWriter::open(&core, &config) + .await + .unwrap(); + let receipt = write(&writer, "raw-canonical-oracle", &prepared()).await; + let child = Page::build(&[Entry::file(EntryKind::Regular, b"file", 3, [42; 32])]).unwrap(); + let mut payload = vec![1, 0, b'f', 1, 1, 1, 0, b'f']; + payload.extend_from_slice(&3_u64.to_le_bytes()); + payload.extend_from_slice(&[17; 32]); + payload.push(b'i'); + payload.extend_from_slice(&1_u64.to_le_bytes()); + payload.extend_from_slice(&page_id(&child)); + let mut branch = b"MTP2\x01\x00".to_vec(); + branch.extend_from_slice(&1_u16.to_le_bytes()); + branch.extend_from_slice(&2_u64.to_le_bytes()); + branch.extend_from_slice(&(payload.len() as u32).to_le_bytes()); + branch.extend_from_slice(&payload); + let root = page_id(&branch); + let pid = uuid::Uuid::new_v4().to_string(); + let txn = q + .begin_with_config(Some(IsolationLevel::ReadCommitted), None) + .await + .unwrap(); + namespace.enter(&txn).await.unwrap(); + txn.execute_raw(Statement::from_sql_and_values(DbBackend::Postgres, + "INSERT INTO mst2_metadata_prepare SELECT (jsonb_populate_record(NULL::mst2_metadata_prepare, + to_jsonb(q)||jsonb_build_object('prepare_id',$1::text,'operation_id','raw-leafable-branch','metadata_root',$2::bytea, + 'state','PREPARING','committed_at',NULL,'node_count',2,'edge_count',1,'total_bytes',$3::bigint))).* + FROM mst2_metadata_prepare q WHERE q.prepare_id=$4", + [pid.clone().into(),root.to_vec().into(),((branch.len()+child.len()) as i64).into(),receipt.intent().prepare_id().into()], + )).await.unwrap(); + txn.execute_raw(Statement::from_sql_and_values(DbBackend::Postgres, + "INSERT INTO mst2_metadata_lifetime(page_id,node_id,generation,state,metadata_codec,expected_size,graph_domain) + VALUES($1,'page:sha256:'||encode($1,'hex'),1,'RESERVED',1,$2,'qualified-v1')", + [root.to_vec().into(),(branch.len() as i32).into()], + )).await.unwrap(); + txn.execute_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + "INSERT INTO mst2_metadata_current VALUES($1,1)", + [root.to_vec().into()], + )) + .await + .unwrap(); + txn.execute_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + "INSERT INTO mst2_metadata_prepare_page SELECT $1,$2,1,$3 + UNION ALL SELECT $1,page_id,generation,expected_size + FROM mst2_metadata_prepare_page WHERE prepare_id=$4 AND page_id=$5", + [ + pid.clone().into(), + root.to_vec().into(), + (branch.len() as i32).into(), + receipt.intent().prepare_id().into(), + page_id(&child).to_vec().into(), + ], + )) + .await + .unwrap(); + txn.execute_raw(Statement::from_sql_and_values(DbBackend::Postgres, + "INSERT INTO mst2_metadata_payload(page_id,generation,metadata_codec,byte_size,payload) VALUES($1,1,1,$2,$3)", + [root.to_vec().into(),(branch.len() as i32).into(),branch.into()], + )).await.unwrap(); + let error = txn + .query_one_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + "SELECT mst2_metadata_certify_page($1,$2,1)", + [pid.into(), root.to_vec().into()], + )) + .await + .unwrap_err(); + assert!(error.to_string().contains("leafable"), "{error}"); + txn.rollback().await.unwrap(); +} + +#[tokio::test] +async fn temporary_root_cannot_disappear_in_a_later_transaction() { + let (config, core, namespace, q, _guard) = fixture().await; + let writer = ShadowQualifiedMetadataWriter::open(&core, &config) + .await + .unwrap(); + let receipt = write(&writer, "continuous-owned-root", &prepared()).await; + let txn = q + .begin_with_config(Some(IsolationLevel::ReadCommitted), None) + .await + .unwrap(); + namespace.enter(&txn).await.unwrap(); + txn.execute_raw(Statement::from_sql_and_values(DbBackend::Postgres, + "INSERT INTO mst2_metadata_root_anchor(anchor_id,anchor_kind,owner_key,root_page,root_generation,root_certificate_digest,prepare_id) + SELECT $1::uuid,'PREPARE',$2,c.page_id,c.generation,c.certificate_digest,$2 + FROM mst2_metadata_page_certificate c JOIN mst2_metadata_prepare q ON q.metadata_root=c.page_id + WHERE q.prepare_id=$2", + [uuid::Uuid::new_v4().to_string().into(),receipt.intent().prepare_id().into()], + )).await.unwrap(); + txn.commit().await.unwrap(); + let error = q + .execute_unprepared("DELETE FROM mst2_metadata_root_anchor WHERE anchor_kind='PREPARE'") + .await + .unwrap_err(); + assert!( + error + .to_string() + .contains("continuously owned canonical root"), + "{error}" + ); + assert_eq!( + count(&q, "SELECT count(*) FROM mst2_metadata_root_anchor").await, + 1 + ); +} + +#[tokio::test] +async fn source_root_is_derived_from_current_core_body_and_verified_blob_facts() { + let (config, core, namespace, q, _guard) = fixture().await; + let entries = [Entry::file(EntryKind::Regular, b"file", 3, [17; 32])]; + let page = Page::build(&entries).unwrap(); + let mut builder = MetadataDagBuilder::new(MetadataDagLimits::default()); + builder.add_directory(&page, &entries).unwrap(); + let prepared = PreparedNativeMetadataRetention::test_installation( + Arc::new(builder.finish(page_id(&page)).unwrap()), + "/", + ); + let mut body = b"100644 file\0".to_vec(); + body.extend_from_slice(&[187; 20]); + core.execute_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + "INSERT INTO mega_tree(id,tree_id,sub_trees,size,created_at,pack_id,pack_offset,commit_id) + VALUES(1,$1,$2,0,now(),'fixture',0,'fixture')", + ["a".repeat(40).into(), body.clone().into()], + )) + .await + .unwrap(); + core.execute_raw(Statement::from_sql_and_values(DbBackend::Postgres, + "INSERT INTO mst2_verified_object(storage_domain,git_oid,object_kind,raw_sha256,size,verification_version,state,created_at) + VALUES('git',$1,'blob',$2,3,2,'VERIFIED',now())", + ["b".repeat(40).into(),vec![17_u8;32].into()], + )).await.unwrap(); + let writer = ShadowQualifiedMetadataWriter::open(&core, &config) + .await + .unwrap(); + let receipt = write(&writer, "bound-real-core-source", &prepared).await; + let aid = uuid::Uuid::new_v4().to_string(); + let txn = q + .begin_with_config(Some(IsolationLevel::ReadCommitted), None) + .await + .unwrap(); + namespace.enter(&txn).await.unwrap(); + txn.execute_raw(Statement::from_sql_and_values(DbBackend::Postgres, + "INSERT INTO mst2_metadata_source_root_attestation(attestation_id,namespace_uuid,origin_prepare_id, + tagged_tree_oid,source_profile,profile_digest,source_body_digest,root_page,root_generation, + root_certificate_digest,source_proof,attestation_digest) + SELECT $1::uuid,(proof->>'namespace')::uuid,$2,$3,proof->'source_profile',decode(proof->>'profile_digest','hex'), + decode(proof->>'source_body_digest','hex'),$4,1,decode(proof->>'root_certificate','hex'),proof, + decode(proof->>'attestation','hex') FROM (SELECT mst2_metadata_compute_source_proof($2,$3,$4,1) AS proof) input", + [aid.into(),receipt.intent().prepare_id().into(),prepared.fixed_root_tree_oid().into(),page_id(&page).to_vec().into()], + )).await.unwrap(); + txn.commit().await.unwrap(); + assert_eq!( + count( + &q, + "SELECT count(*) FROM mst2_metadata_source_root_attestation" + ) + .await, + 1 + ); + body[7] = b'F'; + core.execute_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + "UPDATE mega_tree SET sub_trees=$1 WHERE tree_id=$2", + [body.into(), "a".repeat(40).into()], + )) + .await + .unwrap(); + let error = q + .query_one_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + "SELECT mst2_metadata_compute_source_proof($1,$2,$3,1)", + [ + receipt.intent().prepare_id().into(), + prepared.fixed_root_tree_oid().into(), + page_id(&page).to_vec().into(), + ], + )) + .await + .unwrap_err(); + assert!( + error + .to_string() + .contains("independently canonical certified root"), + "{error}" + ); +} + +pub(super) async fn seeded_rooted_plan( + core: &DatabaseConnection, + tree_char: char, + name: &str, + id: i64, +) -> (RootedMetadataInstallPlan, MetadataPagePayload) { + let entries = [Entry::file( + EntryKind::Regular, + name.as_bytes(), + 3, + [17; 32], + )]; + let bytes = Page::build(&entries).unwrap(); + let page = page_id(&bytes); + let mut builder = MetadataDagBuilder::new(MetadataDagLimits::default()); + builder.add_directory(&bytes, &entries).unwrap(); + let prepared = PreparedNativeMetadataRetention::test_installation( + Arc::new(builder.finish(page).unwrap()), + "/", + ); + let mut identity = prepared.install_plan().unwrap().identity; + identity.tagged_root_tree_oid = format!("sha1:{}", tree_char.to_string().repeat(40)); + let mut body = format!("100644 {name}\0").into_bytes(); + body.extend_from_slice(&[187; 20]); + core.execute_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + "INSERT INTO mega_tree(id,tree_id,sub_trees,size,created_at,pack_id,pack_offset,commit_id) + VALUES($1,$2,$3,0,now(),'fixture',0,'fixture')", + [ + id.into(), + tree_char.to_string().repeat(40).into(), + body.into(), + ], + )) + .await + .unwrap(); + core.execute_raw(Statement::from_sql_and_values(DbBackend::Postgres, + "INSERT INTO mst2_verified_object(storage_domain,git_oid,object_kind,raw_sha256,size,verification_version,state,created_at) + VALUES('git',$1,'blob',$2,3,2,'VERIFIED',now()) ON CONFLICT(storage_domain,git_oid,object_kind) DO NOTHING", + ["b".repeat(40).into(),vec![17_u8;32].into()], + )).await.unwrap(); + let source_roots = BTreeMap::from([(identity.tagged_root_tree_oid.clone(), page)]); + let plan = RootedMetadataInstallPlan::new( + identity, + page, + BTreeMap::from([(page, bytes.len() as u64)]), + BTreeSet::new(), + BTreeMap::new(), + source_roots, + ) + .unwrap(); + ( + plan, + MetadataPagePayload { + id: page, + size: bytes.len() as u64, + bytes, + }, + ) +} + +#[tokio::test] +async fn actual_rooted_cold_and_zero_delta_reuse_keep_one_canonical_graph() { + let (config, core, _namespace, q, _guard) = fixture().await; + let (cold, payload) = seeded_rooted_plan(&core, 'a', "file", 1).await; + let writer = RootedQualifiedMetadataRepository::open(&core, &config) + .await + .unwrap(); + let intent = writer + .begin_intent("actual-rooted-cold", &cold) + .await + .unwrap(); + writer.install_pages(&intent, &[payload]).await.unwrap(); + let receipt = writer.finalize(&intent).await.unwrap(); + assert_eq!(receipt.metadata_root(), cold.root); + assert_eq!( + writer.recover("actual-rooted-cold", &cold).await.unwrap(), + Some(receipt.clone()) + ); + let proof=q.query_one_raw(Statement::from_string(DbBackend::Postgres, + "SELECT a.attestation_id::text,a.attestation_digest,a.root_certificate_digest FROM mst2_metadata_source_root_attestation a" + )).await.unwrap().unwrap(); + let reused = RootedReuseRoot { + generation: intent.root_generation(), + attestation_id: uuid::Uuid::parse_str( + &proof.try_get::("", "attestation_id").unwrap(), + ) + .unwrap(), + attestation_digest: proof + .try_get::>("", "attestation_digest") + .unwrap() + .try_into() + .unwrap(), + certificate_digest: proof + .try_get::>("", "root_certificate_digest") + .unwrap() + .try_into() + .unwrap(), + }; + let warm = RootedMetadataInstallPlan::new( + cold.identity.clone(), + cold.root, + BTreeMap::new(), + BTreeSet::new(), + BTreeMap::from([(cold.root, reused)]), + cold.source_roots.clone(), + ) + .unwrap(); + let warm_intent = writer + .begin_intent("actual-zero-delta-root", &warm) + .await + .unwrap(); + assert_eq!( + writer.finalize(&warm_intent).await.unwrap().metadata_root(), + cold.root + ); + assert_eq!( + count(&q, "SELECT count(*) FROM mst2_metadata_payload").await, + 1 + ); + assert_eq!( + count(&q, "SELECT count(*) FROM mst2_metadata_page_certificate").await, + 1 + ); + assert_eq!( + count( + &q, + "SELECT count(*) FROM mst2_metadata_source_root_attestation" + ) + .await, + 1 + ); + assert_eq!(count(&q,"SELECT count(*) FROM mst2_metadata_prepare WHERE plan_kind='ROOTED' AND state='COMMITTED' AND node_count=0").await,1); + let error = q + .execute_unprepared("DELETE FROM mst2_metadata_root_anchor WHERE anchor_kind='REUSE'") + .await + .unwrap_err(); + assert!( + error + .to_string() + .contains("continuously owned canonical root"), + "{error}" + ); +} + +#[tokio::test] +async fn changed_ancestor_installs_only_delta_and_accepts_independently_attested_source_aliases() { + let (config, core, _namespace, q, _guard) = fixture().await; + let writer = RootedQualifiedMetadataRepository::open(&core, &config) + .await + .unwrap(); + let (first, payload) = seeded_rooted_plan(&core, 'a', "file", 1).await; + let first_intent = writer.begin_intent("alias-first", &first).await.unwrap(); + writer + .install_pages(&first_intent, &[payload]) + .await + .unwrap(); + writer.finalize(&first_intent).await.unwrap(); + let (alias, payload) = seeded_rooted_plan(&core, 'c', "file", 2).await; + assert_eq!(alias.root, first.root); + let alias_intent = writer.begin_intent("alias-second", &alias).await.unwrap(); + writer + .install_pages(&alias_intent, &[payload]) + .await + .unwrap(); + writer.finalize(&alias_intent).await.unwrap(); + let first_hint = writer + .lookup_reuse(&first.identity.tagged_root_tree_oid, &first.identity) + .await + .unwrap() + .unwrap(); + let alias_hint = writer + .lookup_reuse(&alias.identity.tagged_root_tree_oid, &alias.identity) + .await + .unwrap() + .unwrap(); + assert_eq!(first_hint.page_id, alias_hint.page_id); + assert_eq!( + first_hint.proof.certificate_digest, + alias_hint.proof.certificate_digest + ); + assert_ne!( + first_hint.proof.attestation_id, + alias_hint.proof.attestation_id + ); + + let entries = [ + Entry::dir(b"one", first.root), + Entry::dir(b"two", first.root), + ]; + let bytes = Page::build(&entries).unwrap(); + let root = page_id(&bytes); + let mut body = b"40000 one\0".to_vec(); + body.extend_from_slice(&[0xaa; 20]); + body.extend_from_slice(b"40000 two\0"); + body.extend_from_slice(&[0xcc; 20]); + core.execute_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + "INSERT INTO mega_tree(id,tree_id,sub_trees,size,created_at,pack_id,pack_offset,commit_id) + VALUES(3,$1,$2,0,now(),'fixture',0,'fixture')", + ["d".repeat(40).into(), body.into()], + )) + .await + .unwrap(); + let mut identity = first.identity.clone(); + identity.tagged_root_tree_oid = format!("sha1:{}", "d".repeat(40)); + let plan = RootedMetadataInstallPlan::new( + identity.clone(), + root, + BTreeMap::from([(root, bytes.len() as u64)]), + BTreeSet::from([(root, first.root)]), + BTreeMap::from([(first.root, first_hint.proof)]), + BTreeMap::from([ + (identity.tagged_root_tree_oid, root), + (first.identity.tagged_root_tree_oid.clone(), first.root), + (alias.identity.tagged_root_tree_oid.clone(), first.root), + ]), + ) + .unwrap(); + let intent = writer + .begin_intent("changed-ancestor-aliases", &plan) + .await + .unwrap(); + writer + .install_pages( + &intent, + &[MetadataPagePayload { + id: root, + size: bytes.len() as u64, + bytes, + }], + ) + .await + .unwrap(); + let receipt = writer.finalize(&intent).await.unwrap(); + assert_eq!(receipt.metadata_root(), root); + assert_eq!( + count(&q, "SELECT count(*) FROM mst2_metadata_payload").await, + 2 + ); + assert_eq!( + count(&q, "SELECT count(*) FROM mst2_metadata_graph_edge").await, + 1 + ); + assert_eq!( + count( + &q, + "SELECT count(*) FROM mst2_metadata_verified_ref WHERE reference_kind='DIRECTORY'" + ) + .await, + 2 + ); + assert_eq!( + count( + &q, + "SELECT count(*) FROM mst2_metadata_source_root_attestation" + ) + .await, + 3 + ); + let row = q.query_one_raw(Statement::from_sql_and_values(DbBackend::Postgres, + "SELECT node_count,edge_count,(SELECT count(*) FROM mst2_metadata_prepare_reuse_root r + WHERE r.prepare_id=q.prepare_id)::bigint AS reuse_count FROM mst2_metadata_prepare q WHERE prepare_id=$1", + [intent.prepare_id().into()])).await.unwrap().unwrap(); + assert_eq!(row.try_get::("", "node_count").unwrap(), 1); + assert_eq!(row.try_get::("", "edge_count").unwrap(), 1); + assert_eq!(row.try_get::("", "reuse_count").unwrap(), 1); + assert_eq!( + writer + .recover("changed-ancestor-aliases", &plan) + .await + .unwrap(), + Some(receipt) + ); +} + +#[tokio::test] +async fn late_raw_delta_membership_is_rejected_after_intent_commit() { + let (config, core, _namespace, q, _guard) = fixture().await; + let (plan, payload) = seeded_rooted_plan(&core, 'a', "file", 1).await; + let (donor, donor_payload) = seeded_rooted_plan(&core, 'c', "other", 2).await; + let writer = RootedQualifiedMetadataRepository::open(&core, &config) + .await + .unwrap(); + let donor_intent = writer + .begin_intent("rooted-extra-donor", &donor) + .await + .unwrap(); + writer + .install_pages(&donor_intent, &[donor_payload.clone()]) + .await + .unwrap(); + writer.finalize(&donor_intent).await.unwrap(); + let intent = writer + .begin_intent("rooted-bounded-intent", &plan) + .await + .unwrap(); + writer.install_pages(&intent, &[payload]).await.unwrap(); + let error=q.execute_raw(Statement::from_sql_and_values(DbBackend::Postgres, + "INSERT INTO mst2_metadata_prepare_page(prepare_id,page_id,generation,expected_size) VALUES($1,$2,1,$3)", + [intent.prepare_id().into(),donor.root.to_vec().into(),(donor_payload.size as i32).into()], + )).await.unwrap_err(); + assert!( + error + .to_string() + .contains("missing or extra exact delta/reuse"), + "{error}" + ); + assert_eq!( + writer.finalize(&intent).await.unwrap().metadata_root(), + plan.root + ); +} diff --git a/src/jupiter/storage/qualified_metadata_certificates.sql b/src/jupiter/storage/qualified_metadata_certificates.sql new file mode 100644 index 00000000..6ddd7e66 --- /dev/null +++ b/src/jupiter/storage/qualified_metadata_certificates.sql @@ -0,0 +1,293 @@ +CREATE TABLE mst2_metadata_page_certificate ( + page_id bytea NOT NULL,generation bigint NOT NULL,certificate_digest bytea NOT NULL CHECK(octet_length(certificate_digest)=32), + origin_prepare_id text NOT NULL REFERENCES mst2_metadata_prepare(prepare_id), + namespace_uuid uuid NOT NULL REFERENCES mst2_metadata_family_identity(namespace_uuid), + proof_revision integer NOT NULL CHECK(proof_revision=1),metadata_codec smallint NOT NULL CHECK(metadata_codec=1), + byte_size integer NOT NULL CHECK(byte_size BETWEEN 20 AND 16384), + rank integer NOT NULL CHECK(rank BETWEEN 0 AND 4095), + map_entry_count bigint NOT NULL CHECK(map_entry_count BETWEEN 0 AND 131072), + map_encoded_entry_bytes bigint NOT NULL CHECK(map_encoded_entry_bytes BETWEEN 0 AND 67108864), + min_name bytea,max_name bytea, + relative_path_bytes integer NOT NULL CHECK(relative_path_bytes BETWEEN 0 AND 4096), + relative_components integer NOT NULL CHECK(relative_components BETWEEN 0 AND 256), + closure_nodes_upper integer NOT NULL CHECK(closure_nodes_upper BETWEEN 1 AND 4096), + closure_edges_upper integer NOT NULL CHECK(closure_edges_upper BETWEEN 0 AND 16384), + closure_bytes_upper bigint NOT NULL CHECK(closure_bytes_upper BETWEEN 20 AND 67108864), + closure_entries_upper bigint NOT NULL CHECK(closure_entries_upper BETWEEN 0 AND 131072), + canonical_proof jsonb NOT NULL, + PRIMARY KEY(page_id,generation),UNIQUE(page_id,generation,certificate_digest), + FOREIGN KEY(page_id,generation) REFERENCES mst2_metadata_lifetime(page_id,generation), + CHECK((map_entry_count=0)=(min_name IS NULL AND max_name IS NULL)), + CHECK(map_entry_count=0 OR (min_name IS NOT NULL AND max_name IS NOT NULL AND min_name<=max_name)) +); +CREATE TABLE mst2_metadata_verified_ref ( + parent_page bytea NOT NULL,parent_generation bigint NOT NULL, + reference_ordinal integer NOT NULL CHECK(reference_ordinal BETWEEN 0 AND 256), + reference_kind text NOT NULL CHECK(reference_kind IN ('DIRECTORY','RADIX')), + name bytea,label integer,advertised_count bigint, + child_page bytea NOT NULL,child_generation bigint NOT NULL, + child_certificate_digest bytea NOT NULL CHECK(octet_length(child_certificate_digest)=32), + PRIMARY KEY(parent_page,parent_generation,reference_ordinal), + FOREIGN KEY(parent_page,parent_generation) REFERENCES mst2_metadata_page_certificate(page_id,generation), + FOREIGN KEY(child_page,child_generation,child_certificate_digest) + REFERENCES mst2_metadata_page_certificate(page_id,generation,certificate_digest), + CHECK((reference_kind='DIRECTORY' AND name IS NOT NULL AND label IS NULL AND advertised_count IS NULL) + OR (reference_kind='RADIX' AND name IS NULL AND label IS NOT NULL AND advertised_count IS NOT NULL + AND label BETWEEN 0 AND 255 AND advertised_count>0)) +); +CREATE INDEX mst2_metadata_verified_ref_child ON mst2_metadata_verified_ref(child_page,child_generation,parent_page,parent_generation); +ALTER TABLE mst2_metadata_graph_node ADD COLUMN certificate_digest bytea NOT NULL, + ADD FOREIGN KEY(page_id,generation,certificate_digest) + REFERENCES mst2_metadata_page_certificate(page_id,generation,certificate_digest); + +CREATE FUNCTION mst2_metadata_child_certificate(p bytea,g bigint,pid text) RETURNS jsonb +LANGUAGE plpgsql VOLATILE SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE c mst2_metadata_page_certificate%ROWTYPE; +BEGIN + SELECT proof.* INTO c FROM mst2_metadata_page_certificate proof + JOIN mst2_metadata_current cur USING(page_id,generation) + JOIN mst2_metadata_lifetime life USING(page_id,generation) + JOIN mst2_metadata_graph_node node USING(page_id,generation) + JOIN mst2_metadata_payload body USING(page_id,generation) + WHERE proof.page_id=p AND proof.generation=g AND life.graph_domain='qualified-v1' + AND node.state='LIVE' AND node.certificate_digest=proof.certificate_digest + AND node.metadata_codec=proof.metadata_codec AND body.metadata_codec=proof.metadata_codec + AND body.byte_size=proof.byte_size AND node.bytes=proof.byte_size AND life.expected_size=proof.byte_size + AND (life.state='LIVE' OR (life.state='RESERVED' AND proof.origin_prepare_id=pid + AND EXISTS(SELECT 1 FROM mst2_metadata_prepare q WHERE q.prepare_id=pid AND q.state='PREPARING'))) + AND NOT EXISTS(SELECT 1 FROM mst2_metadata_gc_op op WHERE op.page_id=p AND op.generation=g); + IF NOT FOUND THEN RAISE EXCEPTION 'MTP2 child lacks its exact certified current graph'; END IF; + RETURN jsonb_build_object('page',encode(c.page_id,'hex'),'generation',c.generation, + 'certificate',encode(c.certificate_digest,'hex'),'rank',c.rank,'map_entry_count',c.map_entry_count, + 'map_encoded_entry_bytes',c.map_encoded_entry_bytes,'min_name',encode(c.min_name,'hex'), + 'max_name',encode(c.max_name,'hex'),'relative_path_bytes',c.relative_path_bytes, + 'relative_components',c.relative_components,'closure_nodes_upper',c.closure_nodes_upper, + 'closure_edges_upper',c.closure_edges_upper,'closure_bytes_upper',c.closure_bytes_upper, + 'closure_entries_upper',c.closure_entries_upper); +END $$; + +CREATE FUNCTION mst2_metadata_compute_certificate(p bytea,g bigint,pid text) RETURNS jsonb +LANGUAGE plpgsql VOLATILE SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE body mst2_metadata_payload%ROWTYPE; q mst2_metadata_prepare%ROWTYPE; decoded jsonb; + ref jsonb; entry jsonb; child jsonb; bound jsonb; bound_refs jsonb:='[]'::jsonb; + seen jsonb:='[]'::jsonb; children jsonb:='{}'::jsonb; child_page bytea; child_generation bigint; + key text; name bytea; child_min bytea; child_max bytea; prefix bytea; minimum bytea; maximum bytea; + map_count bigint:=0; map_bytes bigint:=0; path_bytes integer:=0; components integer:=0; rank integer:=0; + nodes bigint:=1; edges bigint:=0; bytes bigint; entries bigint; proof jsonb; +BEGIN + SELECT * INTO q FROM mst2_metadata_prepare WHERE prepare_id=pid AND state='PREPARING' + AND graph_domain='qualified-v1' AND mst2_metadata_scope_matches(primary_scope); + IF NOT FOUND THEN RAISE EXCEPTION 'MTP2 certification needs an active exact preparation'; END IF; + SELECT b.* INTO body FROM mst2_metadata_payload b JOIN mst2_metadata_current cur USING(page_id,generation) + JOIN mst2_metadata_lifetime life USING(page_id,generation) + JOIN mst2_metadata_prepare_page member USING(page_id,generation) + WHERE b.page_id=p AND b.generation=g AND member.prepare_id=pid AND life.state='RESERVED' + AND life.graph_domain='qualified-v1' AND life.metadata_codec=q.metadata_codec + AND b.metadata_codec=q.metadata_codec AND b.byte_size=member.expected_size AND b.byte_size=life.expected_size + AND NOT EXISTS(SELECT 1 FROM mst2_metadata_gc_op op WHERE op.page_id=p AND op.generation=g); + IF NOT FOUND THEN RAISE EXCEPTION 'MTP2 certification crossed its durable payload lifetime'; END IF; + decoded:=mst2_metadata_decode_local(body.payload); + IF decode(decoded->>'page_id','hex')<>p THEN RAISE EXCEPTION 'MTP2 durable payload digest differs from its page'; END IF; + map_count:=jsonb_array_length(decoded->'entries'); map_bytes:=(decoded->>'direct_entry_bytes')::bigint; + bytes:=body.byte_size; entries:=map_count; + FOR ref IN SELECT value FROM jsonb_array_elements(decoded->'refs') LOOP + child_page:=decode(ref->>'child','hex'); + SELECT member.generation INTO child_generation FROM mst2_metadata_prepare_page member + WHERE member.prepare_id=pid AND member.page_id=child_page; + IF NOT FOUND THEN + SELECT reused.root_generation INTO child_generation FROM mst2_metadata_prepare_reuse_root reused + JOIN mst2_metadata_root_anchor anchor ON anchor.prepare_id=reused.prepare_id + AND anchor.anchor_kind='REUSE' AND anchor.owner_key=pid + AND anchor.root_page=reused.root_page AND anchor.root_generation=reused.root_generation + WHERE reused.prepare_id=pid AND reused.root_page=child_page; + IF NOT FOUND THEN RAISE EXCEPTION 'MTP2 reference is outside exact delta or reused-root membership'; END IF; + END IF; + child:=mst2_metadata_child_certificate(child_page,child_generation,pid); + bound:=ref||jsonb_build_object('generation',child_generation,'certificate',child->>'certificate'); + bound_refs:=bound_refs||jsonb_build_array(bound); + key:=encode(child_page,'hex')||':'||child_generation; + children:=children||jsonb_build_object(encode(child_page,'hex'),child); + IF NOT seen ? key THEN + seen:=seen||jsonb_build_array(key); rank:=greatest(rank,(child->>'rank')::integer+1); + nodes:=nodes+(child->>'closure_nodes_upper')::bigint; + edges:=edges+1+(child->>'closure_edges_upper')::bigint; + bytes:=bytes+(child->>'closure_bytes_upper')::bigint; + entries:=entries+(child->>'closure_entries_upper')::bigint; + END IF; + IF ref->>'kind'='RADIX' THEN + child_min:=decode(child->>'min_name','hex'); child_max:=decode(child->>'max_name','hex'); + prefix:=decode(decoded->>'prefix','hex'); + IF child_min IS NULL OR child_max IS NULL OR (child->>'map_entry_count')::bigint<>(ref->>'count')::bigint + OR octet_length(child_min)<=octet_length(prefix) OR octet_length(child_max)<=octet_length(prefix) + OR substring(child_min FROM 1 FOR octet_length(prefix))<>prefix + OR substring(child_max FROM 1 FOR octet_length(prefix))<>prefix + OR get_byte(child_min,octet_length(prefix))<>(ref->>'label')::integer + OR get_byte(child_max,octet_length(prefix))<>(ref->>'label')::integer THEN + RAISE EXCEPTION 'MTP2 radix child count or name partition differs from its canonical certificate'; + END IF; + IF minimum IS NULL OR child_minmaximum THEN maximum:=child_max; END IF; + map_count:=map_count+(child->>'map_entry_count')::bigint; + map_bytes:=map_bytes+(child->>'map_encoded_entry_bytes')::bigint; + path_bytes:=greatest(path_bytes,(child->>'relative_path_bytes')::integer); + components:=greatest(components,(child->>'relative_components')::integer); + END IF; + IF rank>4095 OR nodes>4096 OR edges>16384 OR bytes>67108864 OR entries>131072 THEN + RAISE EXCEPTION 'MTP2 certified closure upper bound exceeds its fixed budget'; + END IF; + END LOOP; + FOR entry IN SELECT value FROM jsonb_array_elements(decoded->'entries') LOOP + name:=decode(entry->>'name','hex'); + IF minimum IS NULL OR namemaximum THEN maximum:=name; END IF; + IF (entry->>'kind')::integer=4 THEN + child:=children->(entry->>'child'); + path_bytes:=greatest(path_bytes,1+octet_length(name)+(child->>'relative_path_bytes')::integer); + components:=greatest(components,1+(child->>'relative_components')::integer); + ELSE + path_bytes:=greatest(path_bytes,1+octet_length(name)); components:=greatest(components,1); + END IF; + END LOOP; + IF map_count<>(decoded->>'count')::bigint OR map_count>131072 OR map_bytes>67108864 + OR path_bytes>4096 OR components>256 THEN RAISE EXCEPTION 'MTP2 canonical map or path budget is invalid'; END IF; + IF (decoded->>'kind')::integer=1 AND ( + (map_count<=128 AND 20+map_bytes<=16384) + OR decode(decoded->>'prefix','hex') IS DISTINCT FROM mst2_metadata_lcp(minimum,maximum)) THEN + RAISE EXCEPTION 'MTP2 branch is leafable or its prefix is not the true canonical LCP'; + END IF; + proof:=jsonb_build_object('namespace',(SELECT namespace_uuid::text FROM mst2_metadata_family_identity WHERE singleton=1), + 'page',encode(p,'hex'),'generation',g,'codec',body.metadata_codec,'byte_size',body.byte_size, + 'proof_revision',1,'page_kind',(decoded->>'kind')::integer,'references',bound_refs,'rank',rank, + 'map_entry_count',map_count,'map_encoded_entry_bytes',map_bytes,'min_name',encode(minimum,'hex'), + 'max_name',encode(maximum,'hex'),'relative_path_bytes',path_bytes,'relative_components',components, + 'closure_nodes_upper',nodes,'closure_edges_upper',edges,'closure_bytes_upper',bytes,'closure_entries_upper',entries); + RETURN proof||jsonb_build_object('certificate',encode(sha256(convert_to('mega.mst2.canonical-proof.v1','UTF8') + ||decode('00','hex')||convert_to(proof::text,'UTF8')),'hex')); +END $$; + +CREATE FUNCTION mst2_metadata_certificate_guard() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE proof jsonb; +BEGIN + IF TG_OP<>'INSERT' THEN RAISE EXCEPTION 'MTP2 canonical certificate history is immutable'; END IF; + proof:=mst2_metadata_compute_certificate(NEW.page_id,NEW.generation,NEW.origin_prepare_id); + IF NEW.certificate_digest<>decode(proof->>'certificate','hex') OR NEW.canonical_proof<>proof + OR NEW.namespace_uuid::text<>proof->>'namespace' OR NEW.proof_revision<>1 OR NEW.metadata_codec<>(proof->>'codec')::smallint + OR NEW.byte_size<>(proof->>'byte_size')::integer OR NEW.rank<>(proof->>'rank')::integer + OR NEW.map_entry_count<>(proof->>'map_entry_count')::bigint + OR NEW.map_encoded_entry_bytes<>(proof->>'map_encoded_entry_bytes')::bigint + OR NEW.min_name IS DISTINCT FROM decode(proof->>'min_name','hex') + OR NEW.max_name IS DISTINCT FROM decode(proof->>'max_name','hex') + OR NEW.relative_path_bytes<>(proof->>'relative_path_bytes')::integer + OR NEW.relative_components<>(proof->>'relative_components')::integer + OR NEW.closure_nodes_upper<>(proof->>'closure_nodes_upper')::integer + OR NEW.closure_edges_upper<>(proof->>'closure_edges_upper')::integer + OR NEW.closure_bytes_upper<>(proof->>'closure_bytes_upper')::bigint + OR NEW.closure_entries_upper<>(proof->>'closure_entries_upper')::bigint THEN + RAISE EXCEPTION 'MTP2 canonical certificate was not derived from its durable bytes and exact child proofs'; + END IF; + RETURN NEW; +END $$; +CREATE TRIGGER mst2_metadata_certificate_guard BEFORE INSERT OR UPDATE OR DELETE ON mst2_metadata_page_certificate + FOR EACH ROW EXECUTE FUNCTION mst2_metadata_certificate_guard(); + +CREATE FUNCTION mst2_metadata_verified_ref_guard() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE ref jsonb; +BEGIN + IF TG_OP<>'INSERT' THEN RAISE EXCEPTION 'MTP2 canonical reference history is immutable'; END IF; + SELECT canonical_proof->'references'->NEW.reference_ordinal INTO ref FROM mst2_metadata_page_certificate + WHERE page_id=NEW.parent_page AND generation=NEW.parent_generation; + IF ref IS NULL OR NEW.reference_kind<>ref->>'kind' OR NEW.child_page<>decode(ref->>'child','hex') + OR NEW.child_generation<>(ref->>'generation')::bigint + OR NEW.child_certificate_digest<>decode(ref->>'certificate','hex') + OR NEW.name IS DISTINCT FROM decode(ref->>'name','hex') + OR NEW.label IS DISTINCT FROM (ref->>'label')::integer + OR NEW.advertised_count IS DISTINCT FROM (ref->>'count')::bigint THEN + RAISE EXCEPTION 'MTP2 canonical reference differs from its independently derived occurrence'; + END IF; + RETURN NEW; +END $$; +CREATE TRIGGER mst2_metadata_verified_ref_guard BEFORE INSERT OR UPDATE OR DELETE ON mst2_metadata_verified_ref + FOR EACH ROW EXECUTE FUNCTION mst2_metadata_verified_ref_guard(); + +CREATE FUNCTION mst2_metadata_certificate_complete() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE p bytea; g bigint; proof jsonb; +BEGIN + IF TG_TABLE_NAME IN ('mst2_metadata_verified_ref','mst2_metadata_graph_edge') THEN p:=NEW.parent_page; g:=NEW.parent_generation; + ELSE p:=NEW.page_id; g:=NEW.generation; END IF; + SELECT canonical_proof INTO proof FROM mst2_metadata_page_certificate WHERE page_id=p AND generation=g; + IF proof IS NULL OR (SELECT count(*) FROM mst2_metadata_verified_ref WHERE parent_page=p AND parent_generation=g) + <>jsonb_array_length(proof->'references') + OR NOT EXISTS(SELECT 1 FROM mst2_metadata_graph_node n WHERE n.page_id=p AND n.generation=g + AND n.state='LIVE' AND n.certificate_digest=decode(proof->>'certificate','hex')) + OR EXISTS((SELECT child_page,child_generation FROM mst2_metadata_verified_ref WHERE parent_page=p AND parent_generation=g) + EXCEPT (SELECT child_page,child_generation FROM mst2_metadata_graph_edge WHERE parent_page=p AND parent_generation=g)) + OR EXISTS((SELECT child_page,child_generation FROM mst2_metadata_graph_edge WHERE parent_page=p AND parent_generation=g) + EXCEPT (SELECT child_page,child_generation FROM mst2_metadata_verified_ref WHERE parent_page=p AND parent_generation=g)) THEN + RAISE EXCEPTION 'MTP2 canonical certificate must commit with its complete exact graph and typed references'; + END IF; + RETURN NULL; +END $$; +CREATE CONSTRAINT TRIGGER mst2_metadata_certificate_complete AFTER INSERT ON mst2_metadata_page_certificate + DEFERRABLE INITIALLY DEFERRED FOR EACH ROW EXECUTE FUNCTION mst2_metadata_certificate_complete(); +CREATE CONSTRAINT TRIGGER mst2_metadata_verified_refs_complete AFTER INSERT ON mst2_metadata_verified_ref + DEFERRABLE INITIALLY DEFERRED FOR EACH ROW EXECUTE FUNCTION mst2_metadata_certificate_complete(); +CREATE CONSTRAINT TRIGGER mst2_metadata_graph_refs_complete AFTER INSERT ON mst2_metadata_graph_edge + DEFERRABLE INITIALLY DEFERRED FOR EACH ROW EXECUTE FUNCTION mst2_metadata_certificate_complete(); + +CREATE FUNCTION mst2_metadata_certify_page(pid text,p bytea,g bigint) RETURNS bytea LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE proof jsonb; existing bytea; +BEGIN + SELECT c.certificate_digest INTO existing FROM mst2_metadata_page_certificate c + JOIN mst2_metadata_current cur USING(page_id,generation) JOIN mst2_metadata_lifetime life USING(page_id,generation) + JOIN mst2_metadata_graph_node n USING(page_id,generation) JOIN mst2_metadata_prepare_page member USING(page_id,generation) + JOIN mst2_metadata_prepare q USING(prepare_id) + WHERE c.page_id=p AND c.generation=g AND member.prepare_id=pid AND q.state='PREPARING' + AND mst2_metadata_scope_matches(q.primary_scope) AND life.state='LIVE' AND life.graph_domain='qualified-v1' + AND n.state='LIVE' AND n.certificate_digest=c.certificate_digest AND n.bytes=member.expected_size + AND n.metadata_codec=q.metadata_codec + AND NOT EXISTS(SELECT 1 FROM mst2_metadata_gc_op op WHERE op.page_id=p AND op.generation=g); + IF FOUND THEN RETURN existing; END IF; + proof:=mst2_metadata_compute_certificate(p,g,pid); + INSERT INTO mst2_metadata_page_certificate(page_id,generation,certificate_digest,origin_prepare_id,namespace_uuid, + proof_revision,metadata_codec,byte_size,rank,map_entry_count,map_encoded_entry_bytes,min_name,max_name, + relative_path_bytes,relative_components,closure_nodes_upper,closure_edges_upper,closure_bytes_upper,closure_entries_upper,canonical_proof) + VALUES(p,g,decode(proof->>'certificate','hex'),pid,(proof->>'namespace')::uuid,1,(proof->>'codec')::smallint, + (proof->>'byte_size')::integer,(proof->>'rank')::integer,(proof->>'map_entry_count')::bigint, + (proof->>'map_encoded_entry_bytes')::bigint,decode(proof->>'min_name','hex'),decode(proof->>'max_name','hex'), + (proof->>'relative_path_bytes')::integer,(proof->>'relative_components')::integer, + (proof->>'closure_nodes_upper')::integer,(proof->>'closure_edges_upper')::integer, + (proof->>'closure_bytes_upper')::bigint,(proof->>'closure_entries_upper')::bigint,proof); + INSERT INTO mst2_metadata_verified_ref(parent_page,parent_generation,reference_ordinal,reference_kind, + name,label,advertised_count,child_page,child_generation,child_certificate_digest) + SELECT p,g,ordinality::integer-1,value->>'kind',decode(value->>'name','hex'),(value->>'label')::integer, + (value->>'count')::bigint,decode(value->>'child','hex'),(value->>'generation')::bigint,decode(value->>'certificate','hex') + FROM jsonb_array_elements(proof->'references') WITH ORDINALITY refs(value,ordinality); + INSERT INTO mst2_metadata_graph_node(page_id,generation,state,metadata_codec,bytes,incoming_refs,certificate_digest) + VALUES(p,g,'LIVE',(proof->>'codec')::smallint,(proof->>'byte_size')::integer,0,decode(proof->>'certificate','hex')); + INSERT INTO mst2_metadata_graph_edge(parent_page,parent_generation,child_page,child_generation) + SELECT DISTINCT p,g,child_page,child_generation FROM mst2_metadata_verified_ref WHERE parent_page=p AND parent_generation=g; + RETURN decode(proof->>'certificate','hex'); +END $$; + +CREATE FUNCTION mst2_metadata_certify_batch(pid text,members jsonb) RETURNS integer LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE member jsonb; certified integer:=0; +BEGIN + IF members IS NULL OR jsonb_typeof(members)<>'array' OR jsonb_array_length(members) NOT BETWEEN 1 AND 4096 THEN + RAISE EXCEPTION 'canonical certification batch exceeds its fixed member budget'; + END IF; + -- Array order is supplied by the independently validated cold DAG. Each + -- invocation still proves its durable bytes and already certified children. + FOR member IN SELECT value FROM jsonb_array_elements(members) WITH ORDINALITY AS input(value,ordinal) + ORDER BY ordinal LOOP + IF jsonb_typeof(member)<>'object' OR member->>'page' !~ '^[0-9a-f]{64}$' + OR member->>'generation' IS NULL THEN RAISE EXCEPTION 'canonical certification batch member is malformed'; END IF; + PERFORM mst2_metadata_certify_page(pid,decode(member->>'page','hex'),(member->>'generation')::bigint); + certified:=certified+1; + END LOOP; + RETURN certified; +END $$; diff --git a/src/jupiter/storage/qualified_metadata_family.rs b/src/jupiter/storage/qualified_metadata_family.rs new file mode 100644 index 00000000..52594dd3 --- /dev/null +++ b/src/jupiter/storage/qualified_metadata_family.rs @@ -0,0 +1,624 @@ +//! Captured physical Q provisioning and the sealed rooted production repository. + +use mst2_codec::metapage::{HEADER_LEN, PAGE_MAX_BYTES}; +use sea_orm::{ + ConnectionTrait, DatabaseConnection, DatabaseTransaction, DbBackend, IsolationLevel, Statement, + TransactionTrait, +}; +use sha2::{Digest, Sha256}; +use url::Url; + +#[cfg(test)] +use super::native_metadata_install::generations::{ + GenerationMetadataReceipt, GenerationPrepareIntent, + qualified::PostgresQualifiedMetadataRepository, +}; +use super::{init::postgres_connection, native_metadata_install::MetadataInstallError}; +#[cfg(test)] +use crate::ceres::snapshot::pages::PreparedNativeMetadataRetention; +use crate::{ + ceres::snapshot::{error::SnapshotError, retention_dag::MetadataPagePayload}, + common::errors::MegaError, + config::DbConfig, +}; + +const FAMILY: &str = "v3-rooted-qualified-1"; +const FAMILY_SQL: &str = include_str!("qualified_metadata_family.sql"); +const CANONICAL_SQL: &str = include_str!("qualified_metadata_canonical.sql"); +const CERTIFICATES_SQL: &str = include_str!("qualified_metadata_certificates.sql"); +const ANCHORS_SQL: &str = include_str!("qualified_metadata_anchors.sql"); +const SOURCE_READ_SQL: &str = include_str!("qualified_metadata_source_read.sql"); +const SOURCE_REVISION_SQL: &str = include_str!("qualified_source_revision.sql"); +const ROOTED_SQL: &str = include_str!("qualified_metadata_rooted.sql"); +const SERVING_SQL: &str = include_str!("qualified_metadata_serving.sql"); +const GC_SQL: &str = include_str!("qualified_metadata_gc.sql"); + +#[path = "qualified_metadata_rooted.rs"] +mod rooted; +pub(crate) use rooted::{ + RootedDirectoryWindow, RootedLookupBatch, RootedLookupStatus, RootedQualifiedMetadataRepository, +}; +#[cfg(test)] +pub(crate) use rooted::{ + RootedPrepareIntent, with_rooted_reader_barriers, with_rooted_source_fact_barriers, + with_rooted_source_temporary_shadow, +}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum SnapshotMetadataFamily { + Generic, + Rooted, +} + +pub(crate) async fn select_snapshot_family( + connection: &DatabaseConnection, + identity: &str, + lease: bool, +) -> Result, SnapshotError> { + let result = async { + let txn = connection + .begin_with_config(Some(IsolationLevel::ReadCommitted), None) + .await?; + let (schema, _) = captured_core(&txn).await?; + let function = if lease { + "mst2_route_family_for_lease" + } else { + "mst2_route_family_for_snapshot" + }; + let rows = txn + .query_all_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + format!( + "SELECT namespace_uuid::text,graph_domain FROM {}.{function}($1,$2)", + identifier(&schema) + ), + [identity.into(), schema.into()], + )) + .await?; + let family = match rows.as_slice() { + [] => None, + [row] => Some(match row.try_get::("", "graph_domain")?.as_str() { + "generic-v1" => SnapshotMetadataFamily::Generic, + "qualified-v1" => SnapshotMetadataFamily::Rooted, + _ => { + return Err(rejected( + "snapshot route selected an unsupported physical family", + )); + } + }), + _ => { + return Err(rejected( + "snapshot route selected multiple physical families", + )); + } + }; + txn.commit().await?; + Ok::<_, MegaError>(family) + } + .await; + result.map_err(|error| { + if let MegaError::Db(db_error) = &error + && rooted::is_lock_unavailable(db_error) + { + return SnapshotError::new( + crate::ceres::snapshot::error::SnapshotErrorCode::TemporaryUnavailable, + "fixed source is being updated; retry the operation", + ); + } + SnapshotError::new( + crate::ceres::snapshot::error::SnapshotErrorCode::IntegrityError, + error.to_string(), + ) + }) +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct VerifiedQualifiedNamespace { + core_schema: String, + core_oid: i64, + schema: String, + schema_oid: i64, + namespace_uuid: String, + storage_uuid: String, + catalog_fingerprint: Vec, +} + +fn identifier(value: &str) -> String { + format!("\"{}\"", value.replace('"', "\"\"")) +} +fn literal(value: &str) -> String { + format!("'{}'", value.replace('\'', "''")) +} +pub(crate) fn implementation_fingerprint() -> Vec { + let mut hash = Sha256::new(); + hash.update(b"mega.mst2.rooted-qualified-implementation.v1\0"); + // Length-prefix every source component so source selection and proof + // revisions cannot change while retaining an accepted physical stamp. + for (name, bytes) in [ + ("family", FAMILY.as_bytes()), + ("family-ddl", FAMILY_SQL.as_bytes()), + ("canonical-proof-revision-1", CANONICAL_SQL.as_bytes()), + ("indexed-source-read-revision-1", SOURCE_READ_SQL.as_bytes()), + ("captured-source-revision-1", SOURCE_REVISION_SQL.as_bytes()), + ("rooted-collector-revision-1", GC_SQL.as_bytes()), + ("typed-certificate-revision-1", CERTIFICATES_SQL.as_bytes()), + ( + "source-attestation-and-anchor-revision-1", + ANCHORS_SQL.as_bytes(), + ), + ("rooted-plan-and-bindings-revision-1", ROOTED_SQL.as_bytes()), + ( + "rooted-session-and-reader-revision-1", + SERVING_SQL.as_bytes(), + ), + ( + "rooted-physical-route-revision-1", + include_bytes!("../migration/m20261008_000200_rooted_routes.sql").as_slice(), + ), + ( + "authority-catalog-selector", + include_bytes!("qualified_family_catalog.sql").as_slice(), + ), + ( + "normalized-family-shape", + include_bytes!("qualified_family_shape.sql").as_slice(), + ), + ( + "initial-core-registration", + include_bytes!("../migration/m20261008_000200_rooted_qualified_family.sql").as_slice(), + ), + ] { + hash.update((name.len() as u64).to_le_bytes()); + hash.update(name.as_bytes()); + hash.update((bytes.len() as u64).to_le_bytes()); + hash.update(bytes); + } + hash.finalize().to_vec() +} +pub(crate) fn render_family( + core_schema: &str, + core_oid: i64, + q_schema: &str, + q_oid: i64, + n_uuid: &str, + s_uuid: &str, +) -> String { + FAMILY_SQL + .replace("$CANONICAL_SQL$", CANONICAL_SQL) + .replace("$CERTIFICATES_SQL$", CERTIFICATES_SQL) + .replace("$ANCHORS_SQL$", ANCHORS_SQL) + .replace("$SOURCE_READ_SQL$", SOURCE_READ_SQL) + .replace("$ROOTED_SQL$", ROOTED_SQL) + .replace("$SERVING_SQL$", SERVING_SQL) + .replace("$GC_SQL$", GC_SQL) + .replace("$CORE_SCHEMA$", &identifier(core_schema)) + .replace("$CORE_LITERAL$", &literal(core_schema)) + .replace("$Q_SCHEMA$", &identifier(q_schema)) + .replace("$Q_LITERAL$", &literal(q_schema)) + .replace("$CORE_OID$", &core_oid.to_string()) + .replace("$Q_OID$", &q_oid.to_string()) + .replace("$NAMESPACE_UUID$", n_uuid) + .replace("$STORAGE_UUID$", s_uuid) + .replace( + "$IMPLEMENTATION_SHA$", + &hex::encode(implementation_fingerprint()), + ) + .replace("$HEADER_LEN$", &HEADER_LEN.to_string()) + .replace("$PAGE_MAX_BYTES$", &PAGE_MAX_BYTES.to_string()) +} +fn rejected(message: &str) -> MegaError { + MegaError::Other(message.into()) +} + +async fn captured_core(connection: &C) -> Result<(String, i64), MegaError> { + let row = connection.query_one_raw(Statement::from_string(DbBackend::Postgres, + "SELECT current_schema() AS schema,n.oid::bigint AS oid FROM pg_catalog.pg_namespace n WHERE n.nspname=current_schema()")) + .await?.ok_or_else(|| rejected("qualified family core schema is missing"))?; + let schema: String = row.try_get("", "schema")?; + let oid: i64 = row.try_get("", "oid")?; + let valid = connection + .query_one_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + format!( + "SELECT {}.mst2_route_scope_valid($1) AS valid", + identifier(&schema) + ), + [schema.clone().into()], + )) + .await? + .ok_or_else(|| rejected("qualified family core identity is missing"))? + .try_get::("", "valid")?; + if !valid { + return Err(rejected( + "qualified family requires its registered primary core schema", + )); + } + Ok((schema, oid)) +} + +async fn catalog( + connection: &C, + core_oid: i64, + q_oid: i64, +) -> Result, MegaError> { + catalog_with_exemption(connection, core_oid, q_oid, 0).await +} + +async fn catalog_with_exemption( + connection: &C, + core_oid: i64, + q_oid: i64, + exempt_q_oid: i64, +) -> Result, MegaError> { + // Inspect the actual catalogs directly. A replaced helper function cannot + // turn a bad physical structure into a fresh accepted fingerprint. + let sql = include_str!("qualified_family_catalog.sql") + .replace("$CORE_OID$", "$1::bigint::oid") + .replace("$Q_OID$", "$2::bigint::oid") + .replace("$EXEMPT_Q_OID$", "$3::bigint::oid"); + let row = connection + .query_one_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + sql, + [core_oid.into(), q_oid.into(), exempt_q_oid.into()], + )) + .await? + .ok_or_else(|| rejected("qualified family catalog is missing"))?; + Ok(row.try_get("", "fingerprint")?) +} + +async fn registered( + connection: &C, + core: &(String, i64), +) -> Result, MegaError> { + let policy=connection.query_one_raw(Statement::from_string(DbBackend::Postgres,format!( + "SELECT implementation_fingerprint,expected_shape,authority_catalog FROM {}.mst2_qualified_family_policy WHERE singleton=1",identifier(&core.0)))) + .await?.ok_or_else(||rejected("qualified trusted family policy is missing"))?; + if policy.try_get::>("", "implementation_fingerprint")? != implementation_fingerprint() + { + return Err(rejected("qualified trusted core authority catalog changed")); + } + let rows=connection.query_all_raw(Statement::from_string(DbBackend::Postgres,format!( + "SELECT n.namespace_uuid::text,n.metadata_schema,n.metadata_schema_oid::bigint,n.metadata_storage_uuid, + n.implementation_fingerprint,n.catalog_fingerprint, + n.family_identity,n.admission_state,n.collector_state,n.storage_uuid AS core_storage_uuid, + n.core_schema,n.core_schema_oid::bigint, + (SELECT count(*) FROM {c}.mst2_metadata_namespace)::bigint AS namespace_count, + EXISTS(SELECT 1 FROM pg_catalog.pg_namespace p WHERE p.oid=n.metadata_schema_oid AND p.nspname=n.metadata_schema) AS schema_present, + {c}.mst2_route_scope_valid({core_literal}) AS core_valid + FROM {c}.mst2_metadata_namespace n WHERE n.graph_domain='qualified-v1'",c=identifier(&core.0),core_literal=literal(&core.0)))) + .await?; + if rows.is_empty() { + if policy.try_get::>("", "authority_catalog")? + != catalog(connection, core.1, 0).await? + { + return Err(rejected("qualified trusted core authority catalog changed")); + } + return Ok(None); + } + if rows.len() != 1 { + return Err(rejected( + "qualified family registry exceeds its one-Q hard limit", + )); + } + let row = &rows[0]; + let schema: String = row.try_get("", "metadata_schema")?; + let namespace_uuid: String = row.try_get("", "namespace_uuid")?; + let storage_uuid: String = row.try_get("", "metadata_storage_uuid")?; + let schema_oid: i64 = row.try_get("", "metadata_schema_oid")?; + let expected_schema = format!("mst2q_{}", namespace_uuid.replace('-', "")); + let namespace_identity = uuid::Uuid::parse_str(&namespace_uuid) + .map_err(|_| rejected("qualified namespace UUID is invalid"))?; + let storage_identity = uuid::Uuid::parse_str(&storage_uuid) + .map_err(|_| rejected("qualified storage UUID is invalid"))?; + if schema != expected_schema + || namespace_identity.get_version_num() != 4 + || namespace_identity.get_variant() != uuid::Variant::RFC4122 + || namespace_identity.to_string() != namespace_uuid + || storage_identity.get_version_num() != 4 + || storage_identity.get_variant() != uuid::Variant::RFC4122 + || storage_identity.to_string() != storage_uuid + || storage_uuid == row.try_get::("", "core_storage_uuid")? + || row.try_get::("", "core_schema")? != core.0 + || row.try_get::("", "core_schema_oid")? != core.1 + || row.try_get::("", "family_identity")? != FAMILY + || row.try_get::("", "admission_state")? != "ROOTED_Q_ADMITTED" + || row.try_get::("", "collector_state")? != "ENABLED" + || row.try_get::("", "namespace_count")? != 2 + || !row.try_get::("", "schema_present")? + || !row.try_get::("", "core_valid")? + || row.try_get::>("", "implementation_fingerprint")? != implementation_fingerprint() + { + return Err(rejected( + "qualified family registry and physical identity disagree", + )); + } + // A single registered, physically present Q identity is the only permitted + // source of core-side RI triggers omitted from the core authority stamp. + // The complete catalog and trusted shape below still bind all of them. + if policy.try_get::>("", "authority_catalog")? + != catalog_with_exemption(connection, core.1, 0, schema_oid).await? + { + return Err(rejected("qualified trusted core authority catalog changed")); + } + let fingerprint: Vec = row.try_get("", "catalog_fingerprint")?; + if fingerprint.len() != 32 || catalog(connection, core.1, schema_oid).await? != fingerprint { + return Err(rejected( + "qualified family actual catalog fingerprint changed", + )); + } + let shape: Vec = connection + .query_one_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + format!( + "SELECT {}.mst2_route_family_shape($1::bigint::oid,$2::uuid,$3) AS fingerprint", + identifier(&core.0) + ), + [ + schema_oid.into(), + namespace_uuid.clone().into(), + storage_uuid.clone().into(), + ], + )) + .await? + .ok_or_else(|| rejected("qualified family actual shape is missing"))? + .try_get("", "fingerprint")?; + if shape != policy.try_get::>("", "expected_shape")? { + return Err(rejected( + "qualified namespace does not have the trusted complete physical family shape", + )); + } + let stamp=connection.query_one_raw(Statement::from_sql_and_values(DbBackend::Postgres,format!( + "SELECT EXISTS(SELECT 1 FROM {q}.mst2_metadata_family_identity i JOIN {q}.mst2_metadata_storage_scope s USING(singleton) + WHERE i.singleton=1 AND i.namespace_uuid=$1::uuid AND i.storage_uuid=$2 AND s.storage_uuid=$2 + AND i.core_schema_oid=$3::bigint::oid AND i.metadata_schema_oid=$4::bigint::oid + AND i.family_identity=$5 AND i.implementation_fingerprint=$6) AS valid",q=identifier(&schema)), + [namespace_uuid.clone().into(),storage_uuid.clone().into(),core.1.into(),schema_oid.into(),FAMILY.into(),implementation_fingerprint().into()])) + .await?.ok_or_else(||rejected("qualified family physical stamp is missing"))?; + if !stamp.try_get::("", "valid")? { + return Err(rejected("qualified family physical stamp changed")); + } + Ok(Some(VerifiedQualifiedNamespace { + core_schema: core.0.clone(), + core_oid: core.1, + schema, + schema_oid, + namespace_uuid, + storage_uuid, + catalog_fingerprint: fingerprint, + })) +} + +/// Production bootstrap calls this after core migrations, before returning a +/// writable app connection. Existing registrations are verified, never reset. +pub(crate) async fn provision_or_verify_rooted_qualified_family( + connection: &DatabaseConnection, +) -> Result { + let core = captured_core(connection).await?; + for attempt in 0..2 { + let txn = connection + .begin_with_config(Some(IsolationLevel::ReadCommitted), None) + .await?; + if let Some(namespace) = registered(&txn, &core).await? { + txn.execute_unprepared(&format!( + "SELECT {}.mst2_route_enter({})", + identifier(&core.0), + literal(&core.0) + )) + .await?; + let locked = registered(&txn, &core) + .await? + .ok_or_else(|| rejected("qualified registration disappeared"))?; + if locked != namespace { + return Err(rejected("qualified registration changed during bootstrap")); + } + txn.commit().await?; + return Ok(locked); + } + let candidate = uuid::Uuid::new_v4().to_string(); + let schema = format!("mst2q_{}", candidate.replace('-', "")); + let enter = txn + .execute_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + format!( + "SELECT {}.mst2_route_family_candidate_enter($1,$2::uuid,$3)", + identifier(&core.0) + ), + [ + core.0.clone().into(), + candidate.clone().into(), + schema.clone().into(), + ], + )) + .await; + if let Err(error) = enter { + txn.rollback().await?; + if attempt == 0 && error.to_string().contains("lost bootstrap serialization") { + continue; + } + return Err(error.into()); + } + txn.execute_unprepared(&format!("CREATE SCHEMA {}", identifier(&schema))) + .await?; + let row = txn + .query_one_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + "SELECT oid::bigint AS oid FROM pg_catalog.pg_namespace WHERE nspname=$1", + [schema.clone().into()], + )) + .await? + .ok_or_else(|| rejected("qualified provisioning schema was not created"))?; + let schema_oid: i64 = row.try_get("", "oid")?; + let storage_uuid = uuid::Uuid::new_v4().to_string(); + let sql = render_family( + &core.0, + core.1, + &schema, + schema_oid, + &candidate, + &storage_uuid, + ); + txn.execute_unprepared(&sql).await?; + let fingerprint = catalog(&txn, core.1, schema_oid).await?; + txn.execute_unprepared(&format!( + "SET LOCAL search_path={},pg_catalog,pg_temp", + identifier(&core.0) + )) + .await?; + txn.execute_raw(Statement::from_sql_and_values(DbBackend::Postgres,format!( + "INSERT INTO {c}.mst2_metadata_namespace(singleton,namespace_uuid,core_schema,core_schema_oid, + database_name,database_oid,storage_uuid,server_address,server_port,mono_lock_key2,metadata_schema, + metadata_schema_oid,family_identity,graph_domain,admission_state,collector_state, + metadata_storage_uuid,implementation_fingerprint,catalog_fingerprint) + SELECT NULL,$1::uuid,g.core_schema,g.core_schema_oid,g.database_name,g.database_oid,g.storage_uuid, + g.server_address,g.server_port,g.mono_lock_key2,$2,$3::bigint::oid,$4,'qualified-v1','ROOTED_Q_ADMITTED','ENABLED',$5,$6,$7 + FROM {c}.mst2_metadata_namespace g WHERE singleton=1",c=identifier(&core.0)), + [candidate.into(),schema.into(),schema_oid.into(),FAMILY.into(),storage_uuid.into(),implementation_fingerprint().into(),fingerprint.into()])).await?; + let namespace = registered(&txn, &core) + .await? + .ok_or_else(|| rejected("qualified provisioning did not register its family"))?; + txn.commit().await?; + return Ok(namespace); + } + Err(rejected( + "qualified provisioning could not serialize bootstrap", + )) +} + +impl VerifiedQualifiedNamespace { + pub(crate) async fn enter(&self, txn: &DatabaseTransaction) -> Result<(), SnapshotError> { + let result: Result<(), MegaError> = async { + let actual = txn + .query_one_raw(Statement::from_string( + DbBackend::Postgres, + "SELECT current_schema() AS schema", + )) + .await? + .ok_or_else(|| rejected("qualified writer schema is missing"))? + .try_get::("", "schema")?; + if actual != self.schema { + return Err(rejected( + "qualified writer escaped its physical pool schema", + )); + } + txn.execute_unprepared(&format!( + "SELECT {}.mst2_route_enter({})", + identifier(&self.core_schema), + literal(&self.core_schema) + )) + .await?; + if registered(txn, &(self.core_schema.clone(), self.core_oid)) + .await? + .as_ref() + != Some(self) + { + return Err(rejected("qualified writer physical namespace changed")); + } + Ok(()) + } + .await; + result.map_err(|e| { + if let MegaError::Db(error) = &e + && rooted::is_lock_unavailable(error) + { + return SnapshotError::new( + crate::ceres::snapshot::error::SnapshotErrorCode::TemporaryUnavailable, + "qualified source is being updated; retry the operation", + ); + } + SnapshotError::new( + crate::ceres::snapshot::error::SnapshotErrorCode::SnapshotNotReady, + e.to_string(), + ) + }) + } +} + +fn pool_url(db_url: &str, namespace: &VerifiedQualifiedNamespace) -> Result { + let mut url = Url::parse(db_url) + .map_err(|_| rejected("qualified writer requires a valid PostgreSQL URL"))?; + let mut options = Vec::new(); + let mut others = Vec::new(); + for (key, value) in url.query_pairs() { + if key == "options" { + options.push(value.into_owned()); + } else { + others.push((key.into_owned(), value.into_owned())); + } + } + // Only server-generated lowercase schema names can reach this option. + options.push(format!( + "-csearch_path={},pg_catalog,pg_temp", + namespace.schema + )); + url.set_query(None); + { + let mut pairs = url.query_pairs_mut(); + for (key, value) in others { + pairs.append_pair(&key, &value); + } + pairs.append_pair("options", &options.join(" ")); + } + Ok(url.to_string()) +} + +/// The adapter has no Deref or unsealed connection accessor. It permits only +/// test-only cold preparation writes, without a production factory. +#[cfg(test)] +pub(crate) struct ShadowQualifiedMetadataWriter { + repository: PostgresQualifiedMetadataRepository, +} +#[cfg(test)] +impl ShadowQualifiedMetadataWriter { + pub(crate) async fn open( + core: &DatabaseConnection, + config: &DbConfig, + ) -> Result { + let captured = captured_core(core).await?; + let namespace = registered(core, &captured) + .await? + .ok_or_else(|| rejected("qualified shadow family is not provisioned at bootstrap"))?; + let mut q_config = config.clone(); + q_config.db_url = pool_url(&config.db_url, &namespace)?; + q_config.max_connection = q_config.max_connection.clamp(1, 2); + q_config.min_connection = 1; + let connection = postgres_connection(&q_config).await?; + let txn = connection + .begin_with_config(Some(IsolationLevel::ReadCommitted), None) + .await?; + namespace + .enter(&txn) + .await + .map_err(|e| rejected(&e.to_string()))?; + txn.commit().await?; + let repository = + PostgresQualifiedMetadataRepository::registered_shadow(connection, namespace) + .await + .map_err(|e| rejected(&e.to_string()))?; + Ok(Self { repository }) + } + pub(crate) async fn begin_intent( + &self, + operation_id: &str, + prepared: &PreparedNativeMetadataRetention, + ) -> Result { + self.repository.begin_intent(operation_id, prepared).await + } + pub(crate) async fn install_pages( + &self, + intent: &GenerationPrepareIntent, + payloads: &[MetadataPagePayload], + ) -> Result<(), MetadataInstallError> { + self.repository.install_pages(intent, payloads).await + } + pub(crate) async fn finalize( + &self, + intent: &GenerationPrepareIntent, + ) -> Result { + self.repository.finalize(intent).await + } +} + +#[cfg(test)] +#[path = "qualified_metadata_family_tests.rs"] +mod tests; diff --git a/src/jupiter/storage/qualified_metadata_family.sql b/src/jupiter/storage/qualified_metadata_family.sql new file mode 100644 index 00000000..853834e5 --- /dev/null +++ b/src/jupiter/storage/qualified_metadata_family.sql @@ -0,0 +1,491 @@ +-- Dedicated physical v3 Q family with exact generations and owned roots. +SET LOCAL search_path=$Q_SCHEMA$,pg_catalog,pg_temp; +CREATE TABLE mst2_metadata_storage_scope ( + singleton smallint PRIMARY KEY CHECK(singleton=1),storage_uuid text NOT NULL UNIQUE +); +INSERT INTO mst2_metadata_storage_scope VALUES(1,'$STORAGE_UUID$'); +CREATE TABLE mst2_metadata_family_identity ( + singleton smallint PRIMARY KEY CHECK(singleton=1),namespace_uuid uuid NOT NULL UNIQUE, + storage_uuid text NOT NULL UNIQUE,core_schema_oid oid NOT NULL,metadata_schema_oid oid NOT NULL, + family_identity text NOT NULL CHECK(family_identity='v3-rooted-qualified-1'), + implementation_fingerprint bytea NOT NULL CHECK(octet_length(implementation_fingerprint)=32) +); +INSERT INTO mst2_metadata_family_identity VALUES(1,'$NAMESPACE_UUID$'::uuid,'$STORAGE_UUID$', + $CORE_OID$,$Q_OID$,'v3-rooted-qualified-1',decode('$IMPLEMENTATION_SHA$','hex')); +CREATE TABLE mst2_metadata_lifetime ( + page_id bytea NOT NULL CHECK(octet_length(page_id)=32), + node_id text NOT NULL CHECK(node_id='page:sha256:'||encode(page_id,'hex')), + generation bigint NOT NULL CHECK(generation>0), + state text NOT NULL CHECK(state IN ('RESERVED','LIVE','DELETING','REMOVED')), + metadata_codec smallint NOT NULL CHECK(metadata_codec=1), + expected_size integer NOT NULL CHECK(expected_size BETWEEN $HEADER_LEN$ AND $PAGE_MAX_BYTES$), + graph_domain text NOT NULL CHECK(graph_domain='qualified-v1'),PRIMARY KEY(page_id,generation) +); +CREATE INDEX idx_mst2_metadata_lifetime_state_page ON mst2_metadata_lifetime(state,page_id); +CREATE INDEX idx_mst2_metadata_lifetime_node_generation ON mst2_metadata_lifetime(node_id,generation); +CREATE TABLE mst2_metadata_current ( + page_id bytea PRIMARY KEY CHECK(octet_length(page_id)=32),generation bigint NOT NULL CHECK(generation>0), + FOREIGN KEY(page_id,generation) REFERENCES mst2_metadata_lifetime(page_id,generation) +); +CREATE TABLE mst2_metadata_payload ( + page_id bytea PRIMARY KEY CHECK(octet_length(page_id)=32),generation bigint NOT NULL CHECK(generation>0), + metadata_codec smallint NOT NULL CHECK(metadata_codec=1), + byte_size integer NOT NULL CHECK(byte_size BETWEEN $HEADER_LEN$ AND $PAGE_MAX_BYTES$), + payload bytea NOT NULL CHECK(octet_length(payload)=byte_size),created_at timestamptz NOT NULL DEFAULT now(), + FOREIGN KEY(page_id,generation) REFERENCES mst2_metadata_lifetime(page_id,generation) +); +CREATE TABLE mst2_metadata_prepare ( + prepare_id text PRIMARY KEY CHECK(prepare_id ~ '^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$'), + operation_id text NOT NULL UNIQUE CHECK(octet_length(operation_id) BETWEEN 1 AND 255), + manifest_digest bytea NOT NULL CHECK(octet_length(manifest_digest)=32), + canonical_plan bytea NOT NULL CHECK(octet_length(canonical_plan)<=2097152), + source_domain text NOT NULL CHECK(source_domain='native-git'),tagged_root_tree_oid text NOT NULL, + plan_kind text NOT NULL DEFAULT 'COLD' CHECK(plan_kind IN ('COLD','ROOTED')), + bindings_revision bigint NOT NULL DEFAULT 0 CHECK(bindings_revision>=0), + scope text NOT NULL CHECK(octet_length(scope)<=4096),schema_version smallint NOT NULL, + metadata_codec smallint NOT NULL CHECK(metadata_codec=1),materialization_policy smallint NOT NULL, + fs_semantics smallint NOT NULL,access_projection smallint NOT NULL,verification_revision integer NOT NULL, + projection_revision smallint NOT NULL,metadata_root bytea NOT NULL CHECK(octet_length(metadata_root)=32), + node_count integer NOT NULL CHECK(node_count BETWEEN 0 AND 4096), + edge_count integer NOT NULL CHECK(edge_count BETWEEN 0 AND 16384), + total_bytes bigint NOT NULL CHECK(total_bytes BETWEEN 0 AND 67108864), + state text NOT NULL CHECK(state IN ('PREPARING','COMMITTED','ABORTED')), + created_at timestamptz NOT NULL DEFAULT now(),committed_at timestamptz,aborted_at timestamptz, + coverage_retired_at timestamptz, + canonical_bindings bytea NOT NULL CHECK(octet_length(canonical_bindings) BETWEEN 12 AND 196620), + bindings_digest bytea NOT NULL CHECK(octet_length(bindings_digest)=32), + primary_scope bytea NOT NULL CHECK(octet_length(primary_scope) BETWEEN 1 AND 16384), + storage_seal bytea NOT NULL CHECK(octet_length(storage_seal)=32), + graph_domain text NOT NULL CHECK(graph_domain='qualified-v1'),UNIQUE(prepare_id,storage_seal), + CHECK((state='COMMITTED')=(committed_at IS NOT NULL) AND (state='ABORTED')=(aborted_at IS NOT NULL) + AND (coverage_retired_at IS NULL OR state='COMMITTED')) +); +CREATE TABLE mst2_metadata_prepare_page ( + prepare_id text NOT NULL REFERENCES mst2_metadata_prepare(prepare_id), + page_id bytea NOT NULL CHECK(octet_length(page_id)=32),generation bigint NOT NULL CHECK(generation>0), + expected_size integer NOT NULL CHECK(expected_size BETWEEN $HEADER_LEN$ AND $PAGE_MAX_BYTES$), + PRIMARY KEY(prepare_id,page_id),UNIQUE(prepare_id,page_id,generation), + FOREIGN KEY(page_id,generation) REFERENCES mst2_metadata_lifetime(page_id,generation) +); +CREATE INDEX idx_mst2_metadata_prepare_page_lifetime ON mst2_metadata_prepare_page(page_id,generation,prepare_id); +CREATE TABLE mst2_metadata_graph_node ( + page_id bytea NOT NULL CHECK (octet_length(page_id)=32), + generation bigint NOT NULL CHECK (generation>0), + state text NOT NULL CHECK (state IN ('LIVE','DELETING')), + metadata_codec smallint NOT NULL CHECK (metadata_codec=1), + bytes bigint NOT NULL CHECK (bytes BETWEEN $HEADER_LEN$ AND $PAGE_MAX_BYTES$), + incoming_refs bigint NOT NULL DEFAULT 0 CHECK (incoming_refs>=0), + PRIMARY KEY(page_id,generation), + FOREIGN KEY(page_id,generation) REFERENCES mst2_metadata_lifetime(page_id,generation) +); +CREATE INDEX idx_mst2_metadata_graph_node_gc ON mst2_metadata_graph_node(state,incoming_refs,page_id,generation); +CREATE TABLE mst2_metadata_graph_edge ( + parent_page bytea NOT NULL, + parent_generation bigint NOT NULL, + child_page bytea NOT NULL, + child_generation bigint NOT NULL, + PRIMARY KEY(parent_page,parent_generation,child_page,child_generation), + FOREIGN KEY(parent_page,parent_generation) REFERENCES mst2_metadata_graph_node(page_id,generation), + FOREIGN KEY(child_page,child_generation) REFERENCES mst2_metadata_graph_node(page_id,generation), + CHECK (parent_page<>child_page OR parent_generation<>child_generation) +); +CREATE INDEX idx_mst2_metadata_graph_edge_child ON mst2_metadata_graph_edge(child_page,child_generation,parent_page,parent_generation); +CREATE TABLE mst2_metadata_graph_root ( + prepare_id text NOT NULL, + storage_seal bytea NOT NULL CHECK (octet_length(storage_seal)=32), + page_id bytea NOT NULL, + generation bigint NOT NULL, + PRIMARY KEY(prepare_id,page_id,generation), + FOREIGN KEY(prepare_id,storage_seal) REFERENCES mst2_metadata_prepare(prepare_id,storage_seal), + FOREIGN KEY(prepare_id,page_id,generation) REFERENCES mst2_metadata_prepare_page(prepare_id,page_id,generation), + FOREIGN KEY(page_id,generation) REFERENCES mst2_metadata_graph_node(page_id,generation) +); +CREATE INDEX idx_mst2_metadata_graph_root_page ON mst2_metadata_graph_root(page_id,generation,prepare_id); +CREATE TABLE mst2_metadata_gc_op ( + operation_id uuid PRIMARY KEY, + page_id bytea NOT NULL CHECK (octet_length(page_id)=32), + generation bigint NOT NULL CHECK (generation>0), + primary_scope bytea NOT NULL CHECK (octet_length(primary_scope) BETWEEN 1 AND 16384), + graph_domain text NOT NULL CHECK (graph_domain='qualified-v1'), + metadata_codec smallint NOT NULL CHECK (metadata_codec=1), + expected_size integer NOT NULL CHECK (expected_size BETWEEN $HEADER_LEN$ AND $PAGE_MAX_BYTES$), + graph_present boolean NOT NULL, + had_payload boolean NOT NULL, + payload_delete_xid bigint, + state text NOT NULL CHECK (state IN ('PENDING','APPLIED')), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + completed_at timestamptz, + UNIQUE(page_id,generation), + FOREIGN KEY(page_id,generation) REFERENCES mst2_metadata_lifetime(page_id,generation), + CHECK ((state='APPLIED')=(completed_at IS NOT NULL)) +); +CREATE INDEX idx_mst2_metadata_gc_op_pending ON mst2_metadata_gc_op(created_at,operation_id) WHERE state='PENDING'; + +-- Incarnations are historical; SID alone is never a unique physical binding. +CREATE TABLE mst2_qualified_session_incarnation ( + snapshot_id text NOT NULL,session_incarnation uuid NOT NULL,namespace_uuid uuid NOT NULL, + prepare_id text NOT NULL,storage_seal bytea NOT NULL CHECK(octet_length(storage_seal)=32), + metadata_root bytea NOT NULL CHECK(octet_length(metadata_root)=32),root_generation bigint NOT NULL CHECK(root_generation>0), + source_profile bytea NOT NULL,instance_id text NOT NULL,commit_oid text NOT NULL,root_tree_oid text NOT NULL, + authorization_epoch bigint NOT NULL,publication_sequence bigint NOT NULL,writer_epoch bigint NOT NULL, + certificate_receipt_id bigint NOT NULL,PRIMARY KEY(snapshot_id,session_incarnation), + UNIQUE(snapshot_id,session_incarnation,prepare_id,storage_seal,metadata_root,root_generation), + FOREIGN KEY(snapshot_id,namespace_uuid) REFERENCES $CORE_SCHEMA$.mst2_snapshot_storage_route(snapshot_id,namespace_uuid), + FOREIGN KEY(prepare_id,storage_seal) REFERENCES mst2_metadata_prepare(prepare_id,storage_seal) +); +CREATE TABLE mst2_qualified_lease_binding ( + lease_id text PRIMARY KEY,snapshot_id text NOT NULL,session_incarnation uuid NOT NULL, + prepare_id text NOT NULL,storage_seal bytea NOT NULL,metadata_root bytea NOT NULL,root_generation bigint NOT NULL, + FOREIGN KEY(snapshot_id,session_incarnation,prepare_id,storage_seal,metadata_root,root_generation) + REFERENCES mst2_qualified_session_incarnation(snapshot_id,session_incarnation,prepare_id,storage_seal,metadata_root,root_generation) +); + +$CANONICAL_SQL$ +$CERTIFICATES_SQL$ +$ANCHORS_SQL$ +$SOURCE_READ_SQL$ +$ROOTED_SQL$ + +CREATE FUNCTION mst2_metadata_closed() RETURNS trigger LANGUAGE plpgsql +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +BEGIN RAISE EXCEPTION 'qualified session serving and collector are closed'; END $$; +CREATE FUNCTION mst2_metadata_gc_apply(id uuid) RETURNS void LANGUAGE plpgsql +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +BEGIN RAISE EXCEPTION 'qualified collector is closed'; END $$; +CREATE FUNCTION mst2_metadata_gc_finish(id uuid) RETURNS void LANGUAGE plpgsql +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +BEGIN RAISE EXCEPTION 'qualified collector is closed'; END $$; +CREATE FUNCTION mst2_metadata_immutable() RETURNS trigger LANGUAGE plpgsql +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +BEGIN RAISE EXCEPTION 'qualified family identity and history are immutable'; END $$; + +-- Observe the caller before entering fixed trusted functions. Core fallback +-- never enters the pool search path; all cross-family authority is explicit. +CREATE FUNCTION mst2_metadata_dml_barrier() RETURNS trigger LANGUAGE plpgsql VOLATILE AS $$ +BEGIN + IF pg_catalog.current_schema() IS DISTINCT FROM $Q_LITERAL$ OR TG_TABLE_SCHEMA IS DISTINCT FROM $Q_LITERAL$ + OR NOT EXISTS(SELECT 1 FROM pg_catalog.pg_class WHERE oid=TG_RELID AND relnamespace='$Q_OID$'::oid) + OR NOT EXISTS(SELECT 1 FROM pg_catalog.pg_namespace WHERE oid='$Q_OID$'::oid AND nspname=$Q_LITERAL$) + OR pg_catalog.pg_is_in_recovery() OR pg_catalog.current_setting('transaction_isolation')<>'read committed' THEN + RAISE EXCEPTION 'qualified mutation requires its captured primary family and READ COMMITTED'; + END IF; + PERFORM $CORE_SCHEMA$.mst2_route_enter($CORE_LITERAL$); + IF NOT EXISTS(SELECT 1 FROM $CORE_SCHEMA$.mst2_metadata_namespace n + WHERE n.namespace_uuid='$NAMESPACE_UUID$'::uuid AND n.metadata_schema=$Q_LITERAL$ + AND n.metadata_schema_oid='$Q_OID$'::oid AND n.core_schema_oid=$CORE_OID$ + AND n.metadata_storage_uuid='$STORAGE_UUID$' AND n.admission_state='ROOTED_Q_ADMITTED' + AND n.collector_state='ENABLED' AND n.implementation_fingerprint=pg_catalog.decode('$IMPLEMENTATION_SHA$','hex') + AND n.catalog_fingerprint=$CORE_SCHEMA$.mst2_route_family_catalog($CORE_OID$,'$Q_OID$'::oid)) THEN + RAISE EXCEPTION 'qualified physical family catalog fingerprint is unavailable'; + END IF; + RETURN NULL; +END $$; + + +CREATE FUNCTION mst2_metadata_scope_matches(s bytea) RETURNS boolean LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE actual jsonb; +BEGIN + IF pg_is_in_recovery() OR current_setting('transaction_isolation')<>'read committed' THEN RETURN false; END IF; + SELECT jsonb_build_array(x.storage_uuid,current_database(),d.oid::bigint,$Q_LITERAL$,'$Q_OID$'::bigint, + inet_server_addr()::text,inet_server_port()) INTO actual FROM mst2_metadata_storage_scope x + JOIN pg_database d ON d.datname=current_database() WHERE x.singleton=1; + RETURN actual IS NOT NULL AND convert_from(s,'UTF8')::jsonb=actual; +EXCEPTION WHEN OTHERS THEN RETURN false; +END $$; +CREATE FUNCTION mst2_metadata_has_generic_overlap(p bytea) RETURNS boolean LANGUAGE sql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ SELECT false $$; + +CREATE FUNCTION mst2_metadata_lifetime_guard() RETURNS trigger LANGUAGE plpgsql +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +BEGIN + IF TG_OP='DELETE' THEN RAISE EXCEPTION 'metadata lifetime watermark cannot be deleted'; END IF; + IF TG_OP='INSERT' THEN + IF NEW.generation<>1 OR NEW.state<>'RESERVED' OR EXISTS(SELECT 1 FROM mst2_metadata_current WHERE page_id=NEW.page_id) THEN + RAISE EXCEPTION 'qualified initial lifetime cannot adopt old history; collector is closed'; + END IF; + RETURN NEW; + END IF; + IF (to_jsonb(NEW)-'state') IS DISTINCT FROM (to_jsonb(OLD)-'state') THEN + RAISE EXCEPTION 'metadata lifetime identity is immutable'; + END IF; + IF NEW.state=OLD.state THEN RETURN NEW; END IF; + IF OLD.state='RESERVED' AND NEW.state='LIVE' THEN + IF NOT EXISTS(SELECT 1 FROM mst2_metadata_graph_node n JOIN mst2_metadata_payload b USING(page_id,generation) + WHERE n.page_id=OLD.page_id AND n.generation=OLD.generation AND n.state='LIVE' + AND n.metadata_codec=OLD.metadata_codec AND n.bytes=OLD.expected_size + AND b.metadata_codec=OLD.metadata_codec AND b.byte_size=OLD.expected_size) + OR NOT (EXISTS(SELECT 1 FROM mst2_metadata_graph_root r WHERE r.page_id=OLD.page_id AND r.generation=OLD.generation) + OR EXISTS(SELECT 1 FROM mst2_metadata_prepare_page member JOIN mst2_metadata_prepare q USING(prepare_id) + JOIN mst2_metadata_root_anchor anchor ON anchor.prepare_id=q.prepare_id AND anchor.anchor_kind='PREPARE' + AND anchor.owner_key=q.prepare_id AND anchor.root_page=q.metadata_root + WHERE member.page_id=OLD.page_id AND member.generation=OLD.generation AND q.plan_kind='ROOTED' + AND q.state='COMMITTED' AND q.coverage_retired_at IS NULL)) THEN + RAISE EXCEPTION 'qualified LIVE transition needs its graph payload and prepare root'; + END IF; + RETURN NEW; + END IF; + RAISE EXCEPTION 'qualified lifetime collection is closed'; +END $$; +CREATE TRIGGER mst2_metadata_lifetime_guard BEFORE INSERT OR UPDATE OR DELETE ON mst2_metadata_lifetime + FOR EACH ROW EXECUTE FUNCTION mst2_metadata_lifetime_guard(); +CREATE FUNCTION mst2_metadata_current_guard() RETURNS trigger LANGUAGE plpgsql +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +BEGIN + IF TG_OP<>'INSERT' THEN RAISE EXCEPTION 'qualified current collection is closed'; END IF; + IF NEW.generation<>1 OR EXISTS(SELECT 1 FROM mst2_metadata_lifetime WHERE page_id=NEW.page_id AND generation<>1) + OR EXISTS(SELECT 1 FROM mst2_metadata_payload WHERE page_id=NEW.page_id) + OR EXISTS(SELECT 1 FROM mst2_metadata_graph_node WHERE page_id=NEW.page_id) THEN + RAISE EXCEPTION 'initial qualified current cannot reset or adopt old history'; + END IF; + RETURN NEW; +END $$; +CREATE TRIGGER mst2_metadata_current_guard BEFORE INSERT OR UPDATE OR DELETE ON mst2_metadata_current + FOR EACH ROW EXECUTE FUNCTION mst2_metadata_current_guard(); +CREATE FUNCTION mst2_metadata_current_protected() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +BEGIN + IF EXISTS(SELECT 1 FROM mst2_metadata_lifetime WHERE page_id=NEW.page_id AND generation=NEW.generation AND graph_domain='qualified-v1') + AND NOT EXISTS(SELECT 1 FROM mst2_metadata_prepare_page m JOIN mst2_metadata_prepare q USING(prepare_id) + JOIN mst2_metadata_current c ON c.page_id=m.page_id AND c.generation=m.generation + WHERE m.page_id=NEW.page_id AND m.generation=NEW.generation AND q.graph_domain='qualified-v1' + AND q.storage_seal IS NOT NULL AND (q.state='PREPARING' OR (q.state='COMMITTED' AND + (q.coverage_retired_at IS NULL OR mst2_metadata_session_covers_prepare(q.prepare_id))))) THEN + RAISE EXCEPTION 'qualified current change must commit with fresh prepare protection'; + END IF; + RETURN NULL; +END $$; +CREATE CONSTRAINT TRIGGER mst2_metadata_current_protected AFTER INSERT OR UPDATE ON mst2_metadata_current + DEFERRABLE INITIALLY DEFERRED FOR EACH ROW EXECUTE FUNCTION mst2_metadata_current_protected(); + +CREATE FUNCTION mst2_metadata_prepare_guard() RETURNS trigger LANGUAGE plpgsql +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +BEGIN + IF TG_OP='DELETE' THEN RAISE EXCEPTION 'qualified preparation history is immutable'; END IF; + IF TG_OP='INSERT' THEN + IF NEW.state<>'PREPARING' OR NEW.committed_at IS NOT NULL OR NEW.aborted_at IS NOT NULL + OR NEW.coverage_retired_at IS NOT NULL OR NEW.bindings_revision<>0 OR NOT mst2_metadata_scope_matches(NEW.primary_scope) + OR NEW.manifest_digest IS DISTINCT FROM sha256(NEW.canonical_plan) + OR NEW.bindings_digest IS DISTINCT FROM sha256(NEW.canonical_bindings) THEN + RAISE EXCEPTION 'qualified prepare needs its actual fixed primary plan and bindings'; + END IF; + IF NEW.plan_kind='ROOTED' THEN PERFORM mst2_metadata_rooted_manifest(NEW); + ELSIF NEW.node_count=0 OR octet_length(NEW.canonical_bindings)<60 THEN + RAISE EXCEPTION 'cold preparation requires its nonempty full closure'; + END IF; + RETURN NEW; + END IF; + IF (to_jsonb(NEW)-ARRAY['state','committed_at','aborted_at','coverage_retired_at','bindings_revision']) IS DISTINCT FROM + (to_jsonb(OLD)-ARRAY['state','committed_at','aborted_at','coverage_retired_at','bindings_revision']) THEN + RAISE EXCEPTION 'qualified preparation complete identity is immutable'; + END IF; + IF NEW.bindings_revision<>OLD.bindings_revision AND (OLD.plan_kind<>'ROOTED' OR OLD.state<>'PREPARING' + OR NEW.state<>'PREPARING' OR NEW.bindings_revision<>OLD.bindings_revision+1) THEN + RAISE EXCEPTION 'rooted membership revision must advance its exact active preparation'; + END IF; + IF (OLD.state IN ('COMMITTED','ABORTED') AND NEW.state IS DISTINCT FROM OLD.state) + OR (OLD.committed_at IS NOT NULL AND NEW.committed_at IS DISTINCT FROM OLD.committed_at) + OR (OLD.aborted_at IS NOT NULL AND NEW.aborted_at IS DISTINCT FROM OLD.aborted_at) + OR (OLD.coverage_retired_at IS NOT NULL AND NEW.coverage_retired_at IS DISTINCT FROM OLD.coverage_retired_at) THEN + RAISE EXCEPTION 'qualified terminal receipt cannot be revived or rewritten'; + END IF; + IF OLD.state='PREPARING' AND NEW.state='COMMITTED' THEN + IF NEW.plan_kind='ROOTED' THEN + PERFORM mst2_metadata_rooted_finalize_proof(NEW.prepare_id); + ELSE + IF (SELECT count(*) FROM mst2_metadata_prepare_page WHERE prepare_id=NEW.prepare_id)<>NEW.node_count + OR (SELECT coalesce(sum(expected_size),0) FROM mst2_metadata_prepare_page WHERE prepare_id=NEW.prepare_id)<>NEW.total_bytes + OR NOT EXISTS(SELECT 1 FROM mst2_metadata_prepare_page WHERE prepare_id=NEW.prepare_id AND page_id=NEW.metadata_root) + OR EXISTS(SELECT 1 FROM mst2_metadata_prepare_page m + LEFT JOIN mst2_metadata_current c USING(page_id,generation) + LEFT JOIN mst2_metadata_lifetime l USING(page_id,generation) + LEFT JOIN mst2_metadata_payload b USING(page_id,generation) + LEFT JOIN mst2_metadata_graph_node n USING(page_id,generation) + LEFT JOIN mst2_metadata_graph_root r ON r.prepare_id=m.prepare_id AND r.page_id=m.page_id AND r.generation=m.generation + WHERE m.prepare_id=NEW.prepare_id AND (c.page_id IS NULL OR l.page_id IS NULL OR b.page_id IS NULL OR n.page_id IS NULL + OR l.state NOT IN ('RESERVED','LIVE') OR n.state<>'LIVE' OR l.metadata_codec<>NEW.metadata_codec + OR b.metadata_codec<>NEW.metadata_codec OR n.metadata_codec<>NEW.metadata_codec + OR l.expected_size<>m.expected_size OR b.byte_size<>m.expected_size OR n.bytes<>m.expected_size + OR r.storage_seal IS DISTINCT FROM NEW.storage_seal + OR n.incoming_refs<>(SELECT count(*) FROM mst2_metadata_graph_edge WHERE child_page=m.page_id AND child_generation=m.generation))) + OR (SELECT count(*) FROM mst2_metadata_prepare_page m JOIN mst2_metadata_graph_edge e + ON e.parent_page=m.page_id AND e.parent_generation=m.generation WHERE m.prepare_id=NEW.prepare_id)<>NEW.edge_count THEN + RAISE EXCEPTION 'qualified COMMITTED transition lacks its complete exact graph payload and root coverage'; + END IF; + END IF; + END IF; + IF NEW.state='ABORTED' THEN + IF EXISTS(SELECT 1 FROM mst2_metadata_graph_root WHERE prepare_id=NEW.prepare_id) + OR EXISTS(SELECT 1 FROM mst2_qualified_session_incarnation WHERE prepare_id=NEW.prepare_id) THEN + RAISE EXCEPTION 'qualified terminal transition still has exact coverage'; + END IF; + ELSIF NEW.coverage_retired_at IS NOT NULL AND OLD.coverage_retired_at IS NULL THEN + IF NEW.plan_kind<>'ROOTED' THEN + IF EXISTS(SELECT 1 FROM mst2_metadata_graph_root WHERE prepare_id=NEW.prepare_id) + OR EXISTS(SELECT 1 FROM mst2_qualified_session_incarnation WHERE prepare_id=NEW.prepare_id) THEN + RAISE EXCEPTION 'cold terminal transition still has exact coverage'; + END IF; + ELSIF NEW.state<>'COMMITTED' OR EXISTS(SELECT 1 FROM mst2_metadata_graph_root WHERE prepare_id=NEW.prepare_id) + OR NOT (mst2_metadata_session_covers_prepare(NEW.prepare_id) + OR mst2_metadata_orphan_prepare_eligible(NEW.prepare_id)) THEN + RAISE EXCEPTION 'rooted coverage retirement needs its definitive independent ready session root'; + END IF; + END IF; + RETURN NEW; +END $$; +CREATE TRIGGER mst2_metadata_prepare_guard BEFORE INSERT OR UPDATE OR DELETE ON mst2_metadata_prepare + FOR EACH ROW EXECUTE FUNCTION mst2_metadata_prepare_guard(); +CREATE FUNCTION mst2_metadata_generation_mapping_guard() RETURNS trigger LANGUAGE plpgsql +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +BEGIN + IF TG_OP<>'INSERT' THEN RAISE EXCEPTION 'metadata generation mappings are immutable'; END IF; + IF NOT EXISTS(SELECT 1 FROM mst2_metadata_prepare q JOIN mst2_metadata_current c ON c.page_id=NEW.page_id AND c.generation=NEW.generation + JOIN mst2_metadata_lifetime l USING(page_id,generation) WHERE q.prepare_id=NEW.prepare_id AND q.state='PREPARING' + AND q.graph_domain='qualified-v1' AND q.storage_seal IS NOT NULL AND mst2_metadata_scope_matches(q.primary_scope) + AND l.state IN ('RESERVED','LIVE') AND l.expected_size=NEW.expected_size AND l.metadata_codec=q.metadata_codec) THEN + RAISE EXCEPTION 'qualified mapping cannot cross domain/current/state fence'; + END IF; + RETURN NEW; +END $$; +CREATE TRIGGER mst2_metadata_generation_mapping_guard BEFORE INSERT OR UPDATE OR DELETE ON mst2_metadata_prepare_page + FOR EACH ROW EXECUTE FUNCTION mst2_metadata_generation_mapping_guard(); +CREATE FUNCTION mst2_metadata_payload_fenced() RETURNS trigger LANGUAGE plpgsql +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +BEGIN + IF TG_OP<>'INSERT' THEN RAISE EXCEPTION 'qualified payload mutation and collector are closed'; END IF; + IF NOT EXISTS(SELECT 1 FROM mst2_metadata_current c JOIN mst2_metadata_lifetime l USING(page_id,generation) + JOIN mst2_metadata_prepare_page m USING(page_id,generation) JOIN mst2_metadata_prepare q USING(prepare_id) + WHERE c.page_id=NEW.page_id AND c.generation=NEW.generation AND l.state IN ('RESERVED','LIVE') + AND l.metadata_codec=NEW.metadata_codec AND l.expected_size=NEW.byte_size AND q.state='PREPARING' + AND q.graph_domain='qualified-v1' AND mst2_metadata_scope_matches(q.primary_scope)) THEN + RAISE EXCEPTION 'qualified payload INSERT crossed its exact active generation'; + END IF; + RETURN NEW; +END $$; +CREATE TRIGGER mst2_metadata_payload_fenced BEFORE INSERT OR UPDATE OR DELETE ON mst2_metadata_payload + FOR EACH ROW EXECUTE FUNCTION mst2_metadata_payload_fenced(); +CREATE FUNCTION mst2_metadata_graph_node_guard() RETURNS trigger LANGUAGE plpgsql +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE l mst2_metadata_lifetime%ROWTYPE; +BEGIN + IF TG_OP='DELETE' THEN RAISE EXCEPTION 'qualified graph collection is closed'; END IF; + SELECT l0.* INTO l FROM mst2_metadata_current c JOIN mst2_metadata_lifetime l0 USING(page_id,generation) + WHERE c.page_id=NEW.page_id AND c.generation=NEW.generation; + IF NOT FOUND OR l.graph_domain<>'qualified-v1' OR l.metadata_codec<>NEW.metadata_codec OR l.expected_size<>NEW.bytes + OR NEW.state<>'LIVE' OR l.state NOT IN ('RESERVED','LIVE') + OR NEW.incoming_refs<>(SELECT count(*) FROM mst2_metadata_graph_edge WHERE child_page=NEW.page_id AND child_generation=NEW.generation) THEN + RAISE EXCEPTION 'qualified node does not match exact lifetime and actual counters'; + END IF; + IF TG_OP='INSERT' AND NOT EXISTS(SELECT 1 FROM mst2_metadata_prepare_page m JOIN mst2_metadata_prepare q USING(prepare_id) + WHERE m.page_id=NEW.page_id AND m.generation=NEW.generation AND q.state='PREPARING') THEN + RAISE EXCEPTION 'qualified node creation needs active fixed preparation'; + END IF; + IF NOT EXISTS(SELECT 1 FROM mst2_metadata_page_certificate c WHERE c.page_id=NEW.page_id AND c.generation=NEW.generation + AND c.certificate_digest=NEW.certificate_digest AND c.metadata_codec=NEW.metadata_codec AND c.byte_size=NEW.bytes) THEN + RAISE EXCEPTION 'qualified graph node lacks its exact independently canonical certificate'; + END IF; + IF TG_OP='UPDATE' AND (to_jsonb(NEW)-'incoming_refs') IS DISTINCT FROM (to_jsonb(OLD)-'incoming_refs') THEN + RAISE EXCEPTION 'qualified node identity is immutable'; + END IF; + RETURN NEW; +END $$; +CREATE TRIGGER mst2_metadata_graph_node_guard BEFORE INSERT OR UPDATE OR DELETE ON mst2_metadata_graph_node + FOR EACH ROW EXECUTE FUNCTION mst2_metadata_graph_node_guard(); +CREATE FUNCTION mst2_metadata_graph_root_guard() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +BEGIN + IF TG_OP='UPDATE' THEN RAISE EXCEPTION 'qualified roots cannot be retargeted'; END IF; + IF TG_OP='DELETE' THEN + IF EXISTS(SELECT 1 FROM mst2_qualified_session_incarnation WHERE prepare_id=OLD.prepare_id) THEN + RAISE EXCEPTION 'qualified root still has historical incarnation coverage'; + END IF; + RETURN OLD; + END IF; + IF NOT EXISTS(SELECT 1 FROM mst2_metadata_prepare q JOIN mst2_metadata_current c ON c.page_id=NEW.page_id AND c.generation=NEW.generation + JOIN mst2_metadata_lifetime l USING(page_id,generation) JOIN mst2_metadata_graph_node n USING(page_id,generation) + WHERE q.prepare_id=NEW.prepare_id AND q.storage_seal=NEW.storage_seal AND q.state='PREPARING' + AND q.graph_domain='qualified-v1' AND l.graph_domain='qualified-v1' AND l.state IN ('RESERVED','LIVE') + AND n.state='LIVE' AND mst2_metadata_scope_matches(q.primary_scope)) THEN + RAISE EXCEPTION 'qualified root requires its active exact sealed preparation'; + END IF; + RETURN NEW; +END $$; +CREATE TRIGGER mst2_metadata_graph_root_guard BEFORE INSERT OR UPDATE OR DELETE ON mst2_metadata_graph_root + FOR EACH ROW EXECUTE FUNCTION mst2_metadata_graph_root_guard(); + +CREATE FUNCTION mst2_metadata_graph_edge_guard() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE op mst2_metadata_gc_op%ROWTYPE; +BEGIN + IF TG_OP='UPDATE' THEN RAISE EXCEPTION 'qualified edge identity is immutable'; END IF; + IF TG_OP='DELETE' THEN RAISE EXCEPTION 'qualified edge collection is closed'; END IF; + IF NOT EXISTS(SELECT 1 FROM mst2_metadata_verified_ref ref + JOIN mst2_metadata_page_certificate parent ON parent.page_id=ref.parent_page AND parent.generation=ref.parent_generation + JOIN mst2_metadata_page_certificate child ON child.page_id=ref.child_page AND child.generation=ref.child_generation + WHERE ref.parent_page=NEW.parent_page AND ref.parent_generation=NEW.parent_generation + AND ref.child_page=NEW.child_page AND ref.child_generation=NEW.child_generation + AND ref.child_certificate_digest=child.certificate_digest AND parent.rank>child.rank) THEN + RAISE EXCEPTION 'qualified edge differs from its exact canonical reference or strict certified rank'; + END IF; + IF (SELECT count(*) FROM mst2_metadata_graph_node n JOIN mst2_metadata_current c USING(page_id,generation) + JOIN mst2_metadata_lifetime l USING(page_id,generation) + WHERE ((n.page_id=NEW.parent_page AND n.generation=NEW.parent_generation) + OR (n.page_id=NEW.child_page AND n.generation=NEW.child_generation)) + AND n.state='LIVE' AND l.state IN ('RESERVED','LIVE') AND l.graph_domain='qualified-v1')<>2 + OR NOT EXISTS(SELECT 1 FROM mst2_metadata_prepare_page a JOIN mst2_metadata_prepare q USING(prepare_id) + WHERE a.page_id=NEW.parent_page AND a.generation=NEW.parent_generation AND q.state='PREPARING' + AND q.graph_domain='qualified-v1' AND mst2_metadata_scope_matches(q.primary_scope) + AND (EXISTS(SELECT 1 FROM mst2_metadata_prepare_page b WHERE b.prepare_id=a.prepare_id + AND b.page_id=NEW.child_page AND b.generation=NEW.child_generation) + OR EXISTS(SELECT 1 FROM mst2_metadata_prepare_reuse_root r JOIN mst2_metadata_root_anchor anchor + ON anchor.prepare_id=r.prepare_id AND anchor.anchor_kind='REUSE' AND anchor.owner_key=r.prepare_id + AND anchor.root_page=r.root_page AND anchor.root_generation=r.root_generation + WHERE r.prepare_id=a.prepare_id AND r.root_page=NEW.child_page AND r.root_generation=NEW.child_generation))) THEN + RAISE EXCEPTION 'edge creation requires active exact qualified endpoints'; + END IF; + RETURN NEW; +END $$; +CREATE TRIGGER mst2_metadata_graph_edge_guard BEFORE INSERT OR UPDATE OR DELETE ON mst2_metadata_graph_edge + FOR EACH ROW EXECUTE FUNCTION mst2_metadata_graph_edge_guard(); + +CREATE FUNCTION mst2_metadata_edges_added() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +BEGIN + IF (SELECT count(*) FROM added_edges)>16384 THEN RAISE EXCEPTION 'qualified edge batch exceeds limit'; END IF; + IF NOT EXISTS(SELECT 1 FROM added_edges) THEN RETURN NULL; END IF; + IF EXISTS(SELECT 1 FROM mst2_metadata_graph_node n JOIN ( + SELECT child_page,child_generation,count(*) AS delta FROM added_edges GROUP BY child_page,child_generation + ) d ON d.child_page=n.page_id AND d.child_generation=n.generation + WHERE n.incoming_refs+d.delta<>(SELECT count(*) FROM mst2_metadata_graph_edge x WHERE x.child_page=n.page_id AND x.child_generation=n.generation)) THEN + RAISE EXCEPTION 'qualified insertion found preexisting counter drift'; + END IF; + UPDATE mst2_metadata_graph_node n SET incoming_refs=n.incoming_refs+d.delta FROM ( + SELECT child_page,child_generation,count(*) AS delta FROM added_edges GROUP BY child_page,child_generation + ) d WHERE n.page_id=d.child_page AND n.generation=d.child_generation; + RETURN NULL; +END $$; +CREATE TRIGGER mst2_metadata_edges_added AFTER INSERT ON mst2_metadata_graph_edge + REFERENCING NEW TABLE AS added_edges FOR EACH STATEMENT EXECUTE FUNCTION mst2_metadata_edges_added(); + +DO $$ DECLARE t text; BEGIN + FOREACH t IN ARRAY ARRAY['mst2_metadata_storage_scope','mst2_metadata_family_identity', + 'mst2_metadata_lifetime','mst2_metadata_current','mst2_metadata_payload','mst2_metadata_prepare', + 'mst2_metadata_prepare_page','mst2_metadata_graph_node','mst2_metadata_graph_edge','mst2_metadata_graph_root', + 'mst2_metadata_gc_op','mst2_qualified_session_incarnation','mst2_qualified_lease_binding', + 'mst2_metadata_page_certificate','mst2_metadata_verified_ref','mst2_metadata_source_root_attestation', + 'mst2_metadata_prepare_reuse_root','mst2_metadata_reuse_index','mst2_metadata_reader_operation','mst2_metadata_root_anchor', + 'mst2_metadata_source_entry_reference','mst2_metadata_scope_source_reference'] LOOP + EXECUTE format('CREATE TRIGGER mst2_00_family_barrier BEFORE INSERT OR UPDATE OR DELETE ON %I FOR EACH STATEMENT EXECUTE FUNCTION mst2_metadata_dml_barrier()',t); + EXECUTE format('CREATE TRIGGER mst2_metadata_truncate_guard BEFORE TRUNCATE ON %I FOR EACH STATEMENT EXECUTE FUNCTION mst2_metadata_immutable()',t); + END LOOP; + FOREACH t IN ARRAY ARRAY['mst2_metadata_storage_scope','mst2_metadata_family_identity'] LOOP + EXECUTE format('CREATE TRIGGER mst2_metadata_identity_immutable BEFORE INSERT OR UPDATE OR DELETE ON %I FOR EACH STATEMENT EXECUTE FUNCTION mst2_metadata_immutable()',t); + END LOOP; + FOREACH t IN ARRAY ARRAY['mst2_metadata_gc_op','mst2_qualified_session_incarnation','mst2_qualified_lease_binding', + 'mst2_metadata_reader_operation'] LOOP + EXECUTE format('CREATE TRIGGER mst2_01_family_closed BEFORE INSERT OR UPDATE OR DELETE ON %I FOR EACH STATEMENT EXECUTE FUNCTION mst2_metadata_closed()',t); + END LOOP; +END $$; + +$SERVING_SQL$ +$GC_SQL$ diff --git a/src/jupiter/storage/qualified_metadata_family_tests.rs b/src/jupiter/storage/qualified_metadata_family_tests.rs new file mode 100644 index 00000000..54106a11 --- /dev/null +++ b/src/jupiter/storage/qualified_metadata_family_tests.rs @@ -0,0 +1,855 @@ +use std::{sync::Arc, time::Duration}; + +use mst2_codec::metapage::{Entry, EntryKind, Page, page_id}; +use sea_orm::Database; +use sea_orm_migration::MigratorTrait; + +use super::*; +use crate::{ + ceres::snapshot::retention_dag::{MetadataDagBuilder, MetadataDagLimits}, + jupiter::{ + migration::Migrator, + tests::{TestSchemaGuard, test_db_config}, + }, +}; + +#[path = "qualified_metadata_canonical_tests.rs"] +mod canonical_tests; +#[path = "qualified_metadata_gc_tests.rs"] +mod gc_tests; +#[path = "qualified_metadata_source_revision_tests.rs"] +mod source_revision_tests; + +fn prepared() -> PreparedNativeMetadataRetention { + let entries = [Entry::file(EntryKind::Regular, b"file", 3, [42; 32])]; + let child = Page::build(&entries).unwrap(); + let root_entries = [ + Entry::dir(b"one", page_id(&child)), + Entry::dir(b"two", page_id(&child)), + ]; + let root = Page::build(&root_entries).unwrap(); + let mut builder = MetadataDagBuilder::new(MetadataDagLimits::default()); + builder.add_directory(&child, &entries).unwrap(); + builder.add_directory(&root, &root_entries).unwrap(); + PreparedNativeMetadataRetention::test_installation( + Arc::new(builder.finish(page_id(&root)).unwrap()), + "/", + ) +} + +async fn fixture() -> ( + DbConfig, + DatabaseConnection, + VerifiedQualifiedNamespace, + DatabaseConnection, + TestSchemaGuard, +) { + let temp = tempfile::tempdir().unwrap(); + let (mut config, guard) = test_db_config(temp.path()).await; + config.max_connection = 2; + config.min_connection = 1; + // Exercise the actual production bootstrap, not a test-only DDL shortcut. + let core = super::super::init::database_connection(&config) + .await + .unwrap(); + let namespace = provision_or_verify_rooted_qualified_family(&core) + .await + .unwrap(); + let mut q_config = config.clone(); + q_config.db_url = pool_url(&config.db_url, &namespace).unwrap(); + q_config.max_connection = 1; + let q = postgres_connection(&q_config).await.unwrap(); + (config, core, namespace, q, guard) +} + +async fn count(db: &C, sql: &str) -> i64 { + db.query_one_raw(Statement::from_string(DbBackend::Postgres, sql)) + .await + .unwrap() + .unwrap() + .try_get_by_index(0) + .unwrap() +} +async fn write( + writer: &ShadowQualifiedMetadataWriter, + operation: &str, + pages: &PreparedNativeMetadataRetention, +) -> GenerationMetadataReceipt { + let intent = writer.begin_intent(operation, pages).await.unwrap(); + for batch in pages.dag().payloads().chunks(64) { + writer.install_pages(&intent, batch).await.unwrap(); + } + writer.finalize(&intent).await.unwrap() +} + +#[tokio::test] +async fn physical_q_shadow_writer_is_distinct_and_restart_reuses_its_exact_catalog() { + let (config, core, namespace, q, _guard) = fixture().await; + let pages = prepared(); + let generic=super::super::native_metadata_install::generations::PostgresMetadataGenerationRepository::new(core.clone()).await.unwrap(); + let g = generic.begin_intent("same-pages-g", &pages).await.unwrap(); + for batch in pages.dag().payloads().chunks(64) { + generic.install_pages(&g, batch).await.unwrap(); + } + let g_receipt = generic.finalize(&g).await.unwrap(); + let assembly_dir = tempfile::tempdir().unwrap(); + let mut app_config = crate::config::testing::isolated_config(assembly_dir.path()); + app_config.database = config.clone(); + let storage = super::super::Storage::new_with_connection( + Arc::new(app_config), + Arc::new(core.clone()), + super::super::object_storage::mock_object_storage(), + ) + .await + .unwrap(); + let writer = storage.shadow_qualified_metadata_writer().await.unwrap(); + let clone = storage.clone(); + assert!(std::ptr::eq( + writer, + clone.shadow_qualified_metadata_writer().await.unwrap() + )); + let q_receipt = write(writer, "same-pages-q", &pages).await; + assert_eq!(g_receipt.metadata_root(), q_receipt.metadata_root()); + assert_ne!(g.prepare_id(), q_receipt.intent().prepare_id()); + assert_eq!( + count(&core, "SELECT count(*) FROM mst2_metadata_payload").await, + 2 + ); + assert_eq!( + count( + &q, + "SELECT count(*) FROM mst2_metadata_payload WHERE generation=1" + ) + .await, + 2 + ); + assert_eq!(count(&q,"SELECT count(*) FROM mst2_metadata_prepare WHERE graph_domain='qualified-v1' AND state='COMMITTED'").await,1); + assert_eq!( + count( + &q, + "SELECT count(*) FROM mst2_qualified_session_incarnation" + ) + .await, + 0 + ); + assert_eq!( + count(&q, "SELECT count(*) FROM mst2_qualified_lease_binding").await, + 0 + ); + assert_eq!(count(&q,"SELECT count(*) FROM pg_catalog.pg_class WHERE relnamespace=(SELECT oid FROM pg_catalog.pg_namespace WHERE nspname=current_schema()) AND relkind='r'").await,22); + assert_eq!(count(&q,"SELECT count(*) FROM pg_catalog.pg_class WHERE relnamespace=(SELECT oid FROM pg_catalog.pg_namespace WHERE nspname=current_schema()) AND relname IN ('mst2_retention_node','mst2_retention_edge','mst2_snapshot_context','mst2_snapshot_lease','mega_refs','git_repo','seaql_migrations')").await,0); + let q_scope = q + .query_one_raw(Statement::from_string( + DbBackend::Postgres, + "SELECT primary_scope,storage_seal FROM mst2_metadata_prepare", + )) + .await + .unwrap() + .unwrap(); + let g_scope = core + .query_one_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + "SELECT primary_scope,storage_seal FROM mst2_metadata_prepare WHERE prepare_id=$1", + [g.prepare_id().into()], + )) + .await + .unwrap() + .unwrap(); + assert_ne!( + q_scope.try_get::>("", "primary_scope").unwrap(), + g_scope.try_get::>("", "primary_scope").unwrap() + ); + assert_ne!( + q_scope.try_get::>("", "storage_seal").unwrap(), + g_scope.try_get::>("", "storage_seal").unwrap() + ); + let restarted = super::super::init::database_connection(&config) + .await + .unwrap(); + assert_eq!( + provision_or_verify_rooted_qualified_family(&restarted) + .await + .unwrap(), + namespace + ); + let fresh = ShadowQualifiedMetadataWriter::open(&restarted, &config) + .await + .unwrap(); + assert_eq!(fresh.finalize(q_receipt.intent()).await.unwrap(), q_receipt); + assert_eq!( + catalog(&q, namespace.core_oid, namespace.schema_oid) + .await + .unwrap(), + namespace.catalog_fingerprint + ); +} + +#[tokio::test] +async fn q_catalog_trigger_function_and_index_tamper_are_never_refreshed() { + for tamper in [ + "ALTER TABLE {q}.mst2_metadata_payload DISABLE TRIGGER mst2_metadata_payload_fenced", + "CREATE OR REPLACE FUNCTION {q}.mst2_metadata_has_generic_overlap(p bytea) RETURNS boolean LANGUAGE sql AS 'SELECT true'", + "DROP INDEX {q}.idx_mst2_metadata_graph_edge_child", + "CREATE RULE forged_skip AS ON INSERT TO {q}.mst2_metadata_prepare DO INSTEAD NOTHING", + ] { + let (config, core, namespace, q, _guard) = fixture().await; + let writer = ShadowQualifiedMetadataWriter::open(&core, &config) + .await + .unwrap(); + core.execute_unprepared(&tamper.replace("{q}", &identifier(&namespace.schema))) + .await + .unwrap(); + assert!(writer.begin_intent("tampered", &prepared()).await.is_err()); + let error = provision_or_verify_rooted_qualified_family(&core) + .await + .unwrap_err(); + assert!( + error.to_string().contains("catalog fingerprint changed"), + "{error}" + ); + assert_eq!( + count(&q, "SELECT count(*) FROM mst2_metadata_prepare").await, + 0 + ); + assert_eq!( + count(&core, "SELECT count(*) FROM mst2_metadata_namespace").await, + 2 + ); + let stored:Vec=core.query_one_raw(Statement::from_string(DbBackend::Postgres, + "SELECT catalog_fingerprint FROM mst2_metadata_namespace WHERE graph_domain='qualified-v1'")) + .await.unwrap().unwrap().try_get_by_index(0).unwrap(); + assert_eq!(stored, namespace.catalog_fingerprint); + } +} + +#[tokio::test] +async fn q_schema_rename_fails_closed_without_reprovisioning_or_g_fallback() { + let (config, core, namespace, _q, _guard) = fixture().await; + let writer = ShadowQualifiedMetadataWriter::open(&core, &config) + .await + .unwrap(); + let renamed = format!("{}_renamed", namespace.schema); + core.execute_unprepared(&format!( + "ALTER SCHEMA {} RENAME TO {}", + identifier(&namespace.schema), + identifier(&renamed) + )) + .await + .unwrap(); + assert!(writer.begin_intent("renamed", &prepared()).await.is_err()); + assert!( + provision_or_verify_rooted_qualified_family(&core) + .await + .is_err() + ); + assert_eq!( + count(&core, "SELECT count(*) FROM mst2_metadata_namespace").await, + 2 + ); + assert_eq!( + count(&core, "SELECT count(*) FROM mst2_metadata_prepare").await, + 0 + ); + assert_eq!( + count( + &core, + &format!( + "SELECT count(*) FROM {}.mst2_metadata_prepare", + identifier(&renamed) + ) + ) + .await, + 0 + ); +} + +#[tokio::test] +async fn q_actual_schema_rr_temp_shadow_and_admitted_ledgers_keep_their_guards() { + let (_config, core, namespace, q, _guard) = fixture().await; + let bad = core + .execute_unprepared(&format!( + "INSERT INTO {}.mst2_metadata_gc_op SELECT * FROM {}.mst2_metadata_gc_op WHERE false", + identifier(&namespace.schema), + identifier(&namespace.schema) + )) + .await + .unwrap_err(); + assert!(bad.to_string().contains("captured primary family"), "{bad}"); + let rr = q + .begin_with_config(Some(IsolationLevel::RepeatableRead), None) + .await + .unwrap(); + let error = rr + .execute_unprepared( + "INSERT INTO mst2_metadata_gc_op SELECT * FROM mst2_metadata_gc_op WHERE false", + ) + .await + .unwrap_err(); + assert!(error.to_string().contains("READ COMMITTED"), "{error}"); + rr.rollback().await.unwrap(); + for statement in [ + "SELECT mst2_metadata_gc_apply(gen_random_uuid())", + "SELECT mst2_metadata_gc_finish(gen_random_uuid())", + ] { + let error = q.execute_unprepared(statement).await.unwrap_err(); + assert!( + error.to_string().contains("query returned no rows"), + "{error}" + ); + } + assert_eq!( + count(&q, "SELECT count(*) FROM mst2_metadata_gc_op").await, + 0 + ); + for table in [ + "mst2_qualified_session_incarnation", + "mst2_qualified_lease_binding", + ] { + assert!( + q.execute_unprepared(&format!("INSERT INTO {table} DEFAULT VALUES")) + .await + .is_err() + ); + assert_eq!(count(&q, &format!("SELECT count(*) FROM {table}")).await, 0); + } + q.execute_unprepared("CREATE TEMP TABLE mst2_metadata_prepare (prepare_id text)") + .await + .unwrap(); + let repository = + PostgresQualifiedMetadataRepository::registered_shadow(q.clone(), namespace.clone()) + .await + .unwrap(); + repository + .begin_intent("temp-shadow", &prepared()) + .await + .unwrap(); + assert_eq!( + count( + &q, + &format!( + "SELECT count(*) FROM {}.mst2_metadata_prepare", + identifier(&namespace.schema) + ) + ) + .await, + 1 + ); + assert_eq!( + count(&q, "SELECT count(*) FROM pg_temp.mst2_metadata_prepare").await, + 0 + ); + let duplicate=core.execute_unprepared("INSERT INTO mst2_metadata_namespace SELECT * FROM mst2_metadata_namespace WHERE graph_domain='qualified-v1'").await.unwrap_err(); + assert!( + duplicate + .to_string() + .contains("exact admitted rooted family scope"), + "{duplicate}" + ); + assert_eq!( + count(&core, "SELECT count(*) FROM mst2_metadata_namespace").await, + 2 + ); + let pk:String=q.query_one_raw(Statement::from_string(DbBackend::Postgres, + "SELECT conkey::text FROM pg_catalog.pg_constraint WHERE conrelid='mst2_qualified_session_incarnation'::regclass AND contype='p'")) + .await.unwrap().unwrap().try_get_by_index(0).unwrap(); + assert_eq!( + pk, "{1,2}", + "same SID can have future distinct incarnations" + ); +} + +#[tokio::test] +async fn provisioning_locks_candidate_and_registered_union_in_uuid_order() { + let temp = tempfile::tempdir().unwrap(); + let (config, _guard) = test_db_config(temp.path()).await; + let core = Database::connect(config.db_url.clone()).await.unwrap(); + Migrator::up(&core, None).await.unwrap(); + let other = Database::connect(config.db_url).await.unwrap(); + let captured = captured_core(&core).await.unwrap(); + let candidate = "00000000-0000-4000-8000-000000000001"; + let schema = format!("mst2q_{}", candidate.replace('-', "")); + let held = core.begin().await.unwrap(); + held.execute_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + "SELECT pg_catalog.pg_advisory_xact_lock(1296717362,pg_catalog.hashtext($1))", + [schema.clone().into()], + )) + .await + .unwrap(); + let (core_name, q_name) = (captured.0.clone(), schema.clone()); + let waiter = tokio::spawn(async move { + let txn = other.begin().await.unwrap(); + txn.execute_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + format!( + "SELECT {}.mst2_route_family_candidate_enter($1,$2::uuid,$3)", + identifier(&core_name) + ), + [core_name.into(), candidate.into(), q_name.into()], + )) + .await + .unwrap(); + txn.rollback().await.unwrap(); + }); + let deadline = tokio::time::Instant::now() + Duration::from_secs(4); + loop { + let row=held.query_one_raw(Statement::from_sql_and_values(DbBackend::Postgres, + "SELECT EXISTS(SELECT 1 FROM pg_catalog.pg_locks w WHERE w.locktype='advisory' AND NOT w.granted + AND w.classid=1296717362::oid AND w.objid=pg_catalog.hashtext($1)::oid AND w.objsubid=2 + AND w.database=(SELECT oid FROM pg_catalog.pg_database WHERE datname=current_database()) + AND NOT EXISTS(SELECT 1 FROM pg_catalog.pg_locks g WHERE g.pid=w.pid AND g.locktype='advisory' + AND g.classid=1296717362::oid AND g.objid=pg_catalog.hashtext($2)::oid AND g.objsubid=2 AND g.granted)) AS sorted_wait", + [schema.clone().into(),captured.0.clone().into()])).await.unwrap().unwrap(); + if row.try_get::("", "sorted_wait").unwrap() { + break; + } + assert!( + tokio::time::Instant::now() < deadline, + "candidate must be locked before G when its UUID sorts first" + ); + tokio::task::yield_now().await; + } + held.rollback().await.unwrap(); + waiter.await.unwrap(); + assert_eq!( + count(&core, "SELECT count(*) FROM mst2_metadata_namespace").await, + 1 + ); +} + +#[tokio::test] +async fn q_incomplete_commit_null_binding_and_unsealed_plan_are_rejected() { + let (config, core, namespace, q, _guard) = fixture().await; + let writer = ShadowQualifiedMetadataWriter::open(&core, &config) + .await + .unwrap(); + let intent = writer + .begin_intent("fixed-incomplete", &prepared()) + .await + .unwrap(); + let error=q.execute_raw(Statement::from_sql_and_values(DbBackend::Postgres, + "UPDATE mst2_metadata_prepare SET state='COMMITTED',committed_at=clock_timestamp() WHERE prepare_id=$1",[intent.prepare_id().into()])).await.unwrap_err(); + assert!( + error.to_string().contains("complete exact graph payload"), + "{error}" + ); + for mutation in [ + "graph_domain='generic-v1'", + "canonical_plan=canonical_plan||decode('ff','hex')", + "storage_seal=NULL", + "primary_scope=NULL", + ] { + let error = q + .execute_unprepared(&format!("UPDATE mst2_metadata_prepare SET {mutation}")) + .await + .unwrap_err(); + assert!( + error.to_string().contains("complete identity is immutable"), + "{error}" + ); + } + for batch in prepared().dag().payloads().chunks(64) { + writer.install_pages(&intent, batch).await.unwrap(); + } + writer.finalize(&intent).await.unwrap(); + assert_eq!( + count( + &q, + "SELECT count(*) FROM mst2_metadata_prepare WHERE state='COMMITTED'" + ) + .await, + 1 + ); + assert_eq!( + catalog(&q, namespace.core_oid, namespace.schema_oid) + .await + .unwrap(), + namespace.catalog_fingerprint + ); +} + +#[tokio::test] +async fn first_registration_cannot_self_sign_a_minimal_or_half_installed_family() { + for complete in [false, true] { + let temp = tempfile::tempdir().unwrap(); + let (config, _guard) = test_db_config(temp.path()).await; + let core = Database::connect(config.db_url).await.unwrap(); + Migrator::up(&core, None).await.unwrap(); + let captured = captured_core(&core).await.unwrap(); + let candidate = uuid::Uuid::new_v4().to_string(); + let schema = format!("mst2q_{}", candidate.replace('-', "")); + let storage = uuid::Uuid::new_v4().to_string(); + let txn = core.begin().await.unwrap(); + txn.execute_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + format!( + "SELECT {}.mst2_route_family_candidate_enter($1,$2::uuid,$3)", + identifier(&captured.0) + ), + [ + captured.0.clone().into(), + candidate.clone().into(), + schema.clone().into(), + ], + )) + .await + .unwrap(); + txn.execute_unprepared(&format!("CREATE SCHEMA {}", identifier(&schema))) + .await + .unwrap(); + let oid: i64 = txn + .query_one_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + "SELECT oid::bigint FROM pg_catalog.pg_namespace WHERE nspname=$1", + [schema.clone().into()], + )) + .await + .unwrap() + .unwrap() + .try_get_by_index(0) + .unwrap(); + if complete { + txn.execute_unprepared(&render_family( + &captured.0, + captured.1, + &schema, + oid, + &candidate, + &storage, + )) + .await + .unwrap(); + } else { + txn.execute_unprepared(&format!( + "CREATE TABLE {q}.mst2_metadata_storage_scope(singleton integer,storage_uuid text); + INSERT INTO {q}.mst2_metadata_storage_scope VALUES(1,{storage}); + CREATE TABLE {q}.mst2_metadata_family_identity(singleton integer,namespace_uuid uuid,storage_uuid text, + core_schema_oid oid,metadata_schema_oid oid,family_identity text,implementation_fingerprint bytea); + INSERT INTO {q}.mst2_metadata_family_identity VALUES(1,{candidate}::uuid,{storage},{core_oid},{oid},'{family}',decode('{implementation}','hex'))", + q=identifier(&schema),storage=literal(&storage),candidate=literal(&candidate),core_oid=captured.1, + family=FAMILY,implementation=hex::encode(implementation_fingerprint()))).await.unwrap(); + } + let fingerprint = if complete { + vec![0xff; 32] + } else { + catalog(&txn, captured.1, oid).await.unwrap() + }; + txn.execute_unprepared(&format!( + "SET LOCAL search_path={},pg_catalog,pg_temp", + identifier(&captured.0) + )) + .await + .unwrap(); + let error=txn.execute_raw(Statement::from_sql_and_values(DbBackend::Postgres,format!( + "INSERT INTO {c}.mst2_metadata_namespace(singleton,namespace_uuid,core_schema,core_schema_oid,database_name, + database_oid,storage_uuid,server_address,server_port,mono_lock_key2,metadata_schema,metadata_schema_oid, + family_identity,graph_domain,admission_state,collector_state,metadata_storage_uuid,implementation_fingerprint,catalog_fingerprint) + SELECT NULL,$1::uuid,g.core_schema,g.core_schema_oid,g.database_name,g.database_oid,g.storage_uuid,g.server_address, + g.server_port,g.mono_lock_key2,$2,$3::bigint::oid,$4,'qualified-v1','ROOTED_Q_ADMITTED','ENABLED',$5,$6,$7 + FROM {c}.mst2_metadata_namespace g WHERE singleton=1",c=identifier(&captured.0)), + [candidate.into(),schema.clone().into(),oid.into(),FAMILY.into(),storage.into(),implementation_fingerprint().into(),fingerprint.into()])).await.unwrap_err(); + let expected = if complete { + "fingerprint disagrees" + } else { + "trusted complete physical family shape" + }; + assert!(error.to_string().contains(expected), "{error}"); + txn.rollback().await.unwrap(); + assert_eq!( + count(&core, "SELECT count(*) FROM mst2_metadata_namespace").await, + 1 + ); + assert_eq!( + count( + &core, + &format!( + "SELECT count(*) FROM pg_catalog.pg_namespace WHERE nspname={}", + literal(&schema) + ) + ) + .await, + 0 + ); + assert_eq!( + count(&core, "SELECT count(*) FROM mst2_qualified_family_policy").await, + 1 + ); + } +} + +#[tokio::test] +async fn simultaneous_production_bootstraps_reuse_one_physical_q_namespace() { + let temp = tempfile::tempdir().unwrap(); + let (config, _guard) = test_db_config(temp.path()).await; + let core = postgres_connection(&config).await.unwrap(); + Migrator::up(&core, None).await.unwrap(); + let (left, right) = tokio::join!( + super::super::init::database_connection(&config), + super::super::init::database_connection(&config) + ); + let left = left.unwrap(); + let right = right.unwrap(); + let a = provision_or_verify_rooted_qualified_family(&left) + .await + .unwrap(); + let b = provision_or_verify_rooted_qualified_family(&right) + .await + .unwrap(); + assert_eq!(a, b); + assert_eq!( + count(&core, "SELECT count(*) FROM mst2_metadata_namespace").await, + 2 + ); + assert_eq!( + count( + &core, + &format!( + "SELECT count(*) FROM pg_catalog.pg_namespace WHERE nspname={}", + literal(&a.schema) + ) + ) + .await, + 1 + ); +} + +#[tokio::test] +async fn trusted_shape_policy_and_core_validation_functions_are_immutable_or_fail_closed() { + let (config, core, namespace, _q, _guard) = fixture().await; + for sql in [ + "UPDATE mst2_qualified_family_policy SET expected_shape=decode(repeat('ff',32),'hex')", + "DELETE FROM mst2_qualified_family_policy", + "TRUNCATE mst2_qualified_family_policy", + ] { + let error = core.execute_unprepared(sql).await.unwrap_err(); + assert!(error.to_string().contains("immutable"), "{error}"); + } + let writer = ShadowQualifiedMetadataWriter::open(&core, &config) + .await + .unwrap(); + core.execute_unprepared(&format!( + "CREATE OR REPLACE FUNCTION {}.mst2_route_family_shape(q_oid oid,n_uuid uuid,s_uuid text) + RETURNS bytea LANGUAGE sql AS 'SELECT decode(repeat(''ff'',32),''hex'')'", + identifier(&namespace.core_schema) + )) + .await + .unwrap(); + let error = provision_or_verify_rooted_qualified_family(&core) + .await + .unwrap_err(); + assert!( + error + .to_string() + .contains("trusted core authority catalog changed"), + "{error}" + ); + assert!( + writer + .begin_intent("bad-core-validator", &prepared()) + .await + .is_err() + ); + assert_eq!( + count(&core, "SELECT count(*) FROM mst2_metadata_namespace").await, + 2 + ); +} + +#[tokio::test] +async fn random_q_templates_have_one_trusted_shape_and_changed_composite_signature_is_rejected() { + let temp = tempfile::tempdir().unwrap(); + let (config, _guard) = test_db_config(temp.path()).await; + let core = postgres_connection(&config).await.unwrap(); + Migrator::up(&core, None).await.unwrap(); + let captured = captured_core(&core).await.unwrap(); + let expected: Vec = core + .query_one_raw(Statement::from_string( + DbBackend::Postgres, + "SELECT expected_shape FROM mst2_qualified_family_policy WHERE singleton=1", + )) + .await + .unwrap() + .unwrap() + .try_get_by_index(0) + .unwrap(); + let authority = catalog(&core, captured.1, 0).await.unwrap(); + let mut schemas = Vec::new(); + for _ in 0..2 { + let namespace = uuid::Uuid::new_v4().to_string(); + let schema = format!("mst2q_{}", namespace.replace('-', "")); + let storage = uuid::Uuid::new_v4().to_string(); + let txn = core.begin().await.unwrap(); + txn.execute_unprepared(&format!("CREATE SCHEMA {}", identifier(&schema))) + .await + .unwrap(); + let oid = count( + &txn, + &format!( + "SELECT oid::bigint FROM pg_catalog.pg_namespace WHERE nspname={}", + literal(&schema), + ), + ) + .await; + txn.execute_unprepared(&render_family( + &captured.0, + captured.1, + &schema, + oid, + &namespace, + &storage, + )) + .await + .unwrap(); + let shape = || { + Statement::from_sql_and_values( + DbBackend::Postgres, + format!( + "SELECT {}.mst2_route_family_shape($1::bigint::oid,$2::uuid,$3)", + identifier(&captured.0), + ), + [oid.into(), namespace.clone().into(), storage.clone().into()], + ) + }; + let actual: Vec = txn + .query_one_raw(shape()) + .await + .unwrap() + .unwrap() + .try_get_by_index(0) + .unwrap(); + assert_eq!( + actual, expected, + "random template must retain its trusted structural signature" + ); + assert_eq!( + catalog_with_exemption(&txn, captured.1, 0, oid) + .await + .unwrap(), + authority + ); + assert_ne!( + catalog(&txn, captured.1, 0).await.unwrap(), + authority, + "an unregistered candidate is never implicitly exempted" + ); + txn.execute_unprepared(&format!( + "DROP FUNCTION {q}.mst2_metadata_rooted_manifest({q}.mst2_metadata_prepare); + CREATE FUNCTION {q}.mst2_metadata_rooted_manifest(q text) RETURNS jsonb + LANGUAGE sql IMMUTABLE STRICT AS 'SELECT to_jsonb(q)'", + q = identifier(&schema), + )) + .await + .unwrap(); + let altered: Vec = txn + .query_one_raw(shape()) + .await + .unwrap() + .unwrap() + .try_get_by_index(0) + .unwrap(); + assert_ne!( + altered, expected, + "normalization must preserve the composite argument type" + ); + txn.rollback().await.unwrap(); + schemas.push(schema); + } + assert_ne!(schemas[0], schemas[1]); + assert_eq!(catalog(&core, captured.1, 0).await.unwrap(), authority); + assert!(registered(&core, &captured).await.unwrap().is_none()); +} + +#[tokio::test] +async fn fresh_q_core_authority_exempts_only_exact_q_ri_and_restart_keeps_full_catalog_binding() { + for tamper in ["fk", "ri", "external"] { + let (config, core, namespace, _q, _guard) = fixture().await; + let captured = (namespace.core_schema.clone(), namespace.core_oid); + let authority: Vec = core + .query_one_raw(Statement::from_string( + DbBackend::Postgres, + "SELECT authority_catalog FROM mst2_qualified_family_policy WHERE singleton=1", + )) + .await + .unwrap() + .unwrap() + .try_get_by_index(0) + .unwrap(); + assert_eq!( + catalog_with_exemption(&core, captured.1, 0, namespace.schema_oid) + .await + .unwrap(), + authority + ); + assert_ne!(catalog(&core, captured.1, 0).await.unwrap(), authority); + for _ in 0..2 { + assert_eq!( + registered(&core, &captured).await.unwrap(), + Some(namespace.clone()) + ); + } + let restarted = postgres_connection(&config).await.unwrap(); + assert_eq!( + provision_or_verify_rooted_qualified_family(&restarted) + .await + .unwrap(), + namespace + ); + let txn = core.begin().await.unwrap(); + if tamper == "external" { + let external = format!("hostile_{}", uuid::Uuid::new_v4().simple()); + txn.execute_unprepared(&format!( + "CREATE SCHEMA {external}; CREATE TABLE {external}.forged_route(snapshot_id text,namespace_uuid uuid, + FOREIGN KEY(snapshot_id,namespace_uuid) REFERENCES {c}.mst2_snapshot_storage_route(snapshot_id,namespace_uuid))", + external=identifier(&external),c=identifier(&captured.0), + )).await.unwrap(); + assert_ne!( + catalog_with_exemption(&txn, captured.1, 0, namespace.schema_oid) + .await + .unwrap(), + authority, + "another namespace's internal RI triggers remain part of core authority" + ); + } else { + let row=txn.query_one_raw(Statement::from_sql_and_values(DbBackend::Postgres, + "SELECT fk.conname,t.tgname FROM pg_catalog.pg_constraint fk + JOIN pg_catalog.pg_trigger t ON t.tgconstraint=fk.oid AND t.tgrelid=fk.confrelid AND t.tgisinternal + WHERE fk.connamespace=$1::bigint::oid AND fk.confrelid=$2::regclass AND fk.contype='f' LIMIT 1", + [namespace.schema_oid.into(),format!("{}.mst2_snapshot_storage_route",identifier(&captured.0)).into()])) + .await.unwrap().unwrap(); + let conname: String = row.try_get("", "conname").unwrap(); + let trigger: String = row.try_get("", "tgname").unwrap(); + let sql = if tamper == "fk" { + format!( + "ALTER TABLE {}.mst2_qualified_session_incarnation DROP CONSTRAINT {}", + identifier(&namespace.schema), + identifier(&conname) + ) + } else { + format!( + "ALTER TABLE {}.mst2_snapshot_storage_route DISABLE TRIGGER {}", + identifier(&captured.0), + identifier(&trigger) + ) + }; + txn.execute_unprepared(&sql).await.unwrap(); + assert_eq!( + catalog_with_exemption(&txn, captured.1, 0, namespace.schema_oid) + .await + .unwrap(), + authority + ); + assert_ne!( + catalog(&txn, captured.1, namespace.schema_oid) + .await + .unwrap(), + namespace.catalog_fingerprint + ); + } + assert!( + registered(&txn, &captured).await.is_err(), + "{tamper} tampering cannot refresh admission" + ); + txn.rollback().await.unwrap(); + assert_eq!(registered(&core, &captured).await.unwrap(), Some(namespace)); + } +} diff --git a/src/jupiter/storage/qualified_metadata_gc.rs b/src/jupiter/storage/qualified_metadata_gc.rs new file mode 100644 index 00000000..394d9335 --- /dev/null +++ b/src/jupiter/storage/qualified_metadata_gc.rs @@ -0,0 +1,540 @@ +//! Bounded rooted maintenance. Local retry state is a hint, never GC authority. + +use super::*; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum GcPhase { + Claim, + Apply, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct GcSeal { + operation_id: String, + page_id: [u8; 32], + generation: i64, + primary_scope: Vec, + metadata_codec: i16, + expected_size: i32, + graph_present: bool, + had_payload: bool, + certificate_digest: Option<[u8; 32]>, +} + +#[derive(Debug, Clone)] +struct RetryOperation { + seal: GcSeal, + phase: GcPhase, +} + +#[derive(Debug, Default)] +pub(super) struct RootedMaintenanceState { + retry: Option, + cursor: Option<([u8; 32], i64)>, +} + +#[derive(Debug, Default, Clone, serde::Serialize)] +pub(crate) struct RootedMaintenanceWork { + pub collector_enabled: bool, + pub examined: u16, + pub lifetimes_examined: u16, + pub owners_examined: u16, + pub pending_replayed: u16, + pub uncertain_receipts_observed: u16, + pub absent_claims_observed: u16, + pub gc_claimed: u16, + pub gc_applied: u16, + pub payload_pages_removed: u16, + pub payload_bytes_removed: u64, + pub readers_expired: u16, + pub leases_expired: u16, + pub prepares_aborted: u16, + pub handovers_retired: u16, + pub orphans_retired: u16, +} + +struct GcRecord { + seal: GcSeal, + applied: bool, +} + +fn uncertain(operation: &RetryOperation) -> SnapshotError { + SnapshotError::new( + crate::ceres::snapshot::error::SnapshotErrorCode::TemporaryUnavailable, + format!( + "qualified GC outcome unknown for operation {} generation {} phase {:?}; retry the sealed operation on the same primary", + operation.seal.operation_id, operation.seal.generation, operation.phase + ), + ) +} + +fn gc_record(row: &QueryResult) -> Result { + let operation_id: String = row.try_get("", "operation_id").map_err(internal)?; + let id = uuid::Uuid::parse_str(&operation_id).map_err(internal)?; + let certificate: Option> = row.try_get("", "certificate_digest").map_err(internal)?; + let seal = GcSeal { + operation_id, + page_id: digest_column(row, "page_id")?, + generation: row.try_get("", "generation").map_err(internal)?, + primary_scope: row.try_get("", "primary_scope").map_err(internal)?, + metadata_codec: row.try_get("", "metadata_codec").map_err(internal)?, + expected_size: row.try_get("", "expected_size").map_err(internal)?, + graph_present: row.try_get("", "graph_present").map_err(internal)?, + had_payload: row.try_get("", "had_payload").map_err(internal)?, + certificate_digest: certificate + .map(|bytes| bytes.as_slice().try_into().map_err(internal)) + .transpose()?, + }; + if id.get_version_num() != 4 + || id.to_string() != seal.operation_id + || seal.generation <= 0 + || seal.metadata_codec != 1 + || seal.expected_size <= 0 + || seal.graph_present != seal.certificate_digest.is_some() + || row + .try_get::("", "graph_domain") + .map_err(internal)? + != "qualified-v1" + { + return Err(integrity( + "qualified GC immutable operation profile is invalid", + )); + } + let state: String = row.try_get("", "state").map_err(internal)?; + let applied = match state.as_str() { + "PENDING" => false, + "APPLIED" => true, + _ => return Err(integrity("qualified GC operation has an unknown state")), + }; + Ok(GcRecord { seal, applied }) +} + +async fn load_gc( + db: &C, + operation: &str, +) -> Result, SnapshotError> { + db.query_one_raw(sql( + "SELECT operation_id::text,page_id,generation,primary_scope,graph_domain,metadata_codec, + expected_size,graph_present,had_payload,certificate_digest,state + FROM mst2_metadata_gc_op WHERE operation_id=$1::uuid", + [operation.into()], + )) + .await + .map_err(internal)? + .as_ref() + .map(gc_record) + .transpose() +} + +impl RootedQualifiedMetadataRepository { + pub(crate) fn start_maintenance(repository: &std::sync::Arc) { + let repository = std::sync::Arc::downgrade(repository); + tokio::spawn(async move { + let mut interval = tokio::time::interval(std::time::Duration::from_secs(30)); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + interval.tick().await; + loop { + interval.tick().await; + let Some(repository) = repository.upgrade() else { + break; + }; + match tokio::time::timeout( + std::time::Duration::from_secs(15), + repository.maintenance_tick(64), + ) + .await + { + Ok(Ok(_)) => {} + Ok(Err(error)) => { + tracing::warn!(code=?error.code, "rooted metadata maintenance failed; durable roots and sealed operations remain recoverable") + } + Err(_) => tracing::warn!( + "rooted metadata maintenance reached its deadline; retrying durable state on the next tick" + ), + } + } + }); + } + + async fn collector_enabled(&self) -> Result { + let txn = self.transaction().await?; + let result = async { + let row = txn + .query_one_raw(sql("SELECT mst2_metadata_gc_enabled() AS enabled", [])) + .await + .map_err(internal)? + .ok_or_else(|| integrity("qualified collector policy is missing"))?; + row.try_get("", "enabled").map_err(internal) + } + .await; + let _ = txn.rollback().await; + result + } + + fn require_gc_seal(&self, seal: &GcSeal) -> Result<(), SnapshotError> { + if seal.primary_scope != self.primary_scope { + return Err(integrity( + "qualified GC seal belongs to another captured primary", + )); + } + Ok(()) + } + + async fn prove_gc_record( + &self, + txn: &DatabaseTransaction, + record: &GcRecord, + ) -> Result<(), SnapshotError> { + self.require_gc_seal(&record.seal)?; + let seal = &record.seal; + txn.query_one_raw(sql( + "SELECT mst2_metadata_gc_proof($1,$2,$3,$4,$5,$6)", + [ + seal.page_id.to_vec().into(), + seal.generation.into(), + if record.applied { "APPLIED" } else { "PENDING" }.into(), + (!record.applied && seal.graph_present).into(), + (!record.applied && seal.had_payload).into(), + seal.certificate_digest.map(|value| value.to_vec()).into(), + ], + )) + .await + .map_err(internal)?; + Ok(()) + } + + async fn observe_gc_retry( + &self, + operation: &RetryOperation, + ) -> Result, SnapshotError> { + self.require_gc_seal(&operation.seal)?; + // transaction() acquires the same-primary route completion barrier. + // A failed barrier or read never proves that an uncertain claim is absent. + let txn = self.transaction().await.map_err(|_| uncertain(operation))?; + let result = async { + let record = load_gc(&txn, &operation.seal.operation_id).await?; + if let Some(record) = &record { + if record.seal != operation.seal { + return Err(integrity( + "qualified GC retry retargeted its immutable sealed identity", + )); + } + self.prove_gc_record(&txn, record).await?; + } + Ok(record) + } + .await; + txn.rollback().await.map_err(|_| uncertain(operation))?; + result.map_err(|error: SnapshotError| { + if error.code == crate::ceres::snapshot::error::SnapshotErrorCode::Internal { + uncertain(operation) + } else { + error + } + }) + } + + async fn claim_gc_candidate( + &self, + candidate: GcSeal, + state: &mut RootedMaintenanceState, + ) -> Result { + self.require_gc_seal(&candidate)?; + let txn = self.transaction().await?; + let result = async { + txn.query_one_raw(sql( + "SELECT mst2_metadata_gc_claim($1,$2,$3,$4::uuid)", + [ + candidate.page_id.to_vec().into(), + candidate.generation.into(), + candidate.primary_scope.clone().into(), + candidate.operation_id.clone().into(), + ], + )) + .await + .map_err(internal)?; + let record = load_gc(&txn, &candidate.operation_id) + .await? + .ok_or_else(|| integrity("qualified claimed GC operation disappeared"))?; + if record.seal != candidate || record.applied { + return Err(integrity( + "qualified claim differs from its exact captured candidate", + )); + } + self.prove_gc_record(&txn, &record).await?; + Ok(record.seal) + } + .await; + let seal = match result { + Ok(seal) => seal, + Err(error) => { + let _ = txn.rollback().await; + return Err(error); + } + }; + let retry = RetryOperation { + seal: seal.clone(), + phase: GcPhase::Claim, + }; + // Retain the hint before COMMIT so task cancellation cannot lose an + // already transmitted commit whose durable outcome is still unknown. + state.retry = Some(retry.clone()); + txn.commit().await.map_err(|_| uncertain(&retry))?; + state.retry = None; + Ok(seal) + } + + async fn apply_gc_seal( + &self, + seal: &GcSeal, + state: &mut RootedMaintenanceState, + ) -> Result { + self.require_gc_seal(seal)?; + let txn = self.transaction().await?; + let result = async { + let record = load_gc(&txn, &seal.operation_id) + .await? + .ok_or_else(|| integrity("qualified committed GC operation is missing"))?; + if &record.seal != seal { + return Err(integrity( + "qualified apply retargeted its exact immutable GC operation", + )); + } + self.prove_gc_record(&txn, &record).await?; + if record.applied { + return Ok(false); + } + txn.query_one_raw(sql( + "SELECT mst2_metadata_gc_apply($1::uuid)", + [seal.operation_id.clone().into()], + )) + .await + .map_err(internal)?; + let applied = load_gc(&txn, &seal.operation_id) + .await? + .ok_or_else(|| integrity("qualified GC apply lost its durable receipt"))?; + if &applied.seal != seal || !applied.applied { + return Err(integrity( + "qualified GC apply did not preserve its sealed APPLIED identity", + )); + } + self.prove_gc_record(&txn, &applied).await?; + Ok(true) + } + .await; + let changed = match result { + Ok(changed) => changed, + Err(error) => { + let _ = txn.rollback().await; + return Err(error); + } + }; + let retry = RetryOperation { + seal: seal.clone(), + phase: GcPhase::Apply, + }; + state.retry = Some(retry.clone()); + txn.commit().await.map_err(|_| uncertain(&retry))?; + state.retry = None; + Ok(changed) + } + + async fn pending_gc_seals(&self, limit: u16) -> Result, SnapshotError> { + let txn = self.transaction().await?; + let result = async { + let rows = txn.query_all_raw(sql( + "SELECT operation_id::text,page_id,generation,primary_scope,graph_domain,metadata_codec, + expected_size,graph_present,had_payload,certificate_digest,state + FROM mst2_metadata_gc_op WHERE state='PENDING' ORDER BY created_at,operation_id LIMIT $1", + [i64::from(limit).into()], + )).await.map_err(internal)?; + rows.iter().map(|row| { + let record=gc_record(row)?; + self.require_gc_seal(&record.seal)?; + Ok(record.seal) + }).collect() + }.await; + let _ = txn.rollback().await; + result + } + + async fn cleanup_gc_owners( + &self, + limit: u16, + work: &mut RootedMaintenanceWork, + ) -> Result<(), SnapshotError> { + let txn = self.transaction().await?; + let result = async { + txn.query_one_raw(sql( + "SELECT * FROM mst2_metadata_gc_owner_cleanup($1::integer)", + [i32::from(limit).into()], + )) + .await + .map_err(internal)? + .ok_or_else(|| integrity("qualified owner cleanup lost its bounded counters")) + } + .await; + let row = sessions::finish(txn, result).await?; + let count = |name: &str| -> Result { + u16::try_from(row.try_get::("", name).map_err(internal)?).map_err(internal) + }; + let examined = count("examined")?; + let readers = count("readers_expired")?; + let leases = count("leases_expired")?; + let prepares = count("prepares_aborted")?; + let handovers = count("handovers_retired")?; + let orphans = count("orphans_retired")?; + if examined > limit || readers + leases + prepares + handovers + orphans > examined { + return Err(integrity( + "qualified owner cleanup counters exceed their shared budget", + )); + } + work.examined += examined; + work.owners_examined += examined; + work.readers_expired += readers; + work.leases_expired += leases; + work.prepares_aborted += prepares; + work.handovers_retired += handovers; + work.orphans_retired += orphans; + Ok(()) + } + + async fn gc_candidates( + &self, + after: Option<([u8; 32], i64)>, + limit: u16, + ) -> Result, SnapshotError> { + let (page, generation) = after.map(|(p, g)| (p.to_vec(), g)).unwrap_or_default(); + let txn = self.transaction().await?; + let result = async { + let rows = txn.query_all_raw(sql( + "SELECT l.page_id,l.generation,l.metadata_codec,l.expected_size, + n.page_id IS NOT NULL AS graph_present,n.certificate_digest, + EXISTS(SELECT 1 FROM mst2_metadata_payload b WHERE b.page_id=l.page_id AND b.generation=l.generation) AS had_payload, + NOT EXISTS(SELECT 1 FROM mst2_metadata_root_anchor a WHERE a.root_page=l.page_id AND a.root_generation=l.generation) + AND NOT EXISTS(SELECT 1 FROM mst2_metadata_graph_root r WHERE r.page_id=l.page_id AND r.generation=l.generation) + AND NOT EXISTS(SELECT 1 FROM mst2_metadata_graph_edge e WHERE e.child_page=l.page_id AND e.child_generation=l.generation) + AND NOT EXISTS(SELECT 1 FROM mst2_metadata_prepare_page m JOIN mst2_metadata_prepare q USING(prepare_id) + WHERE m.page_id=l.page_id AND m.generation=l.generation + AND (q.state='PREPARING' OR q.state='COMMITTED' AND q.coverage_retired_at IS NULL)) + AND NOT EXISTS(SELECT 1 FROM mst2_metadata_prepare_reuse_root r JOIN mst2_metadata_prepare q USING(prepare_id) + WHERE r.root_page=l.page_id AND r.root_generation=l.generation + AND (q.state='PREPARING' OR q.state='COMMITTED' AND q.coverage_retired_at IS NULL)) + AND (l.state='LIVE' OR EXISTS(SELECT 1 FROM mst2_metadata_prepare_page m JOIN mst2_metadata_prepare q USING(prepare_id) + WHERE m.page_id=l.page_id AND m.generation=l.generation AND q.state='ABORTED' AND q.storage_seal IS NOT NULL) + AND NOT EXISTS(SELECT 1 FROM mst2_metadata_prepare_page m JOIN mst2_metadata_prepare q USING(prepare_id) + WHERE m.page_id=l.page_id AND m.generation=l.generation AND q.state<>'ABORTED')) AS eligible + FROM mst2_metadata_lifetime l JOIN mst2_metadata_current c USING(page_id,generation) + LEFT JOIN mst2_metadata_graph_node n USING(page_id,generation) + WHERE l.state IN ('RESERVED','LIVE') AND (l.page_id,l.generation)>($1,$2) + ORDER BY l.page_id,l.generation LIMIT $3", + [page.into(),generation.into(),i64::from(limit).into()], + )).await.map_err(internal)?; + rows.iter().map(|row| { + let certificate:Option>=row.try_get("","certificate_digest").map_err(internal)?; + Ok((GcSeal { + operation_id:uuid::Uuid::new_v4().to_string(), + page_id:digest_column(row,"page_id")?, + generation:row.try_get("","generation").map_err(internal)?, + primary_scope:self.primary_scope.clone(), + metadata_codec:row.try_get("","metadata_codec").map_err(internal)?, + expected_size:row.try_get("","expected_size").map_err(internal)?, + graph_present:row.try_get("","graph_present").map_err(internal)?, + had_payload:row.try_get("","had_payload").map_err(internal)?, + certificate_digest:certificate.map(|bytes|bytes.as_slice().try_into().map_err(internal)).transpose()?, + },row.try_get("","eligible").map_err(internal)?)) + }).collect() + }.await; + let _ = txn.rollback().await; + result + } + + /// A shared owner/page work budget; SQL proof and capacity scans are separately bounded. + pub(crate) async fn maintenance_tick( + &self, + limit: u16, + ) -> Result { + if !(1..=64).contains(&limit) { + return Err(integrity("qualified maintenance limit must be 1..=64")); + } + let mut state = self.maintenance_state.lock().await; + let mut work = RootedMaintenanceWork { + collector_enabled: self.collector_enabled().await?, + ..Default::default() + }; + if let Some(retry) = state.retry.clone() { + work.examined += 1; + work.lifetimes_examined += 1; + match self.observe_gc_retry(&retry).await? { + Some(record) if record.applied => { + state.retry = None; + work.uncertain_receipts_observed += 1; + } + Some(record) if work.collector_enabled => { + let changed = self.apply_gc_seal(&record.seal, &mut state).await?; + work.pending_replayed += 1; + work.record_applied(&record.seal, changed)?; + } + Some(_) => {} + None if retry.phase == GcPhase::Claim => { + state.retry = None; + work.absent_claims_observed += 1; + } + None => { + return Err(integrity( + "qualified committed GC apply lost its sealed operation history", + )); + } + } + } + if work.collector_enabled && work.examined < limit { + for seal in self.pending_gc_seals(limit - work.examined).await? { + work.examined += 1; + work.lifetimes_examined += 1; + let changed = self.apply_gc_seal(&seal, &mut state).await?; + work.pending_replayed += 1; + work.record_applied(&seal, changed)?; + } + } + if work.examined < limit { + let remaining = limit - work.examined; + let owner_budget = if work.collector_enabled { + remaining.div_ceil(2) + } else { + remaining + }; + self.cleanup_gc_owners(owner_budget, &mut work).await?; + } + if work.collector_enabled && work.examined < limit { + let remaining = limit - work.examined; + let candidates = self.gc_candidates(state.cursor, remaining).await?; + let reached_end = candidates.len() < usize::from(remaining); + for (candidate, eligible) in candidates { + work.examined += 1; + work.lifetimes_examined += 1; + state.cursor = Some((candidate.page_id, candidate.generation)); + if eligible { + let seal = self.claim_gc_candidate(candidate, &mut state).await?; + work.gc_claimed += 1; + let changed = self.apply_gc_seal(&seal, &mut state).await?; + work.record_applied(&seal, changed)?; + } + } + if reached_end { + state.cursor = None; + } + } + Ok(work) + } +} + +impl RootedMaintenanceWork { + fn record_applied(&mut self, seal: &GcSeal, changed: bool) -> Result<(), SnapshotError> { + self.gc_applied += 1; + if changed && seal.had_payload { + self.payload_pages_removed += 1; + self.payload_bytes_removed += u64::try_from(seal.expected_size).map_err(internal)?; + } + Ok(()) + } +} diff --git a/src/jupiter/storage/qualified_metadata_gc.sql b/src/jupiter/storage/qualified_metadata_gc.sql new file mode 100644 index 00000000..13b41998 --- /dev/null +++ b/src/jupiter/storage/qualified_metadata_gc.sql @@ -0,0 +1,574 @@ +-- Historical certificates and source/session records do not own live graph rows. +ALTER TABLE mst2_metadata_gc_op ADD COLUMN certificate_digest bytea, + ADD CONSTRAINT mst2_gc_certificate_binding CHECK( + (graph_present AND octet_length(certificate_digest)=32 OR NOT graph_present AND certificate_digest IS NULL) IS TRUE); +ALTER TABLE mst2_metadata_prepare ADD COLUMN orphan_expires_at timestamptz NOT NULL; +CREATE INDEX mst2_metadata_prepare_orphan_scan ON mst2_metadata_prepare(orphan_expires_at,prepare_id) + WHERE state='PREPARING' OR state='COMMITTED' AND coverage_retired_at IS NULL; +CREATE INDEX mst2_metadata_current_active_scan ON mst2_metadata_lifetime(page_id,generation) + WHERE state IN ('RESERVED','LIVE'); + +CREATE FUNCTION mst2_metadata_prepare_expiry_guard() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +BEGIN + IF TG_OP='INSERT' THEN NEW.created_at:=clock_timestamp(); NEW.orphan_expires_at:=NEW.created_at+interval '3600 seconds'; + ELSIF NEW.created_at IS DISTINCT FROM OLD.created_at OR NEW.orphan_expires_at IS DISTINCT FROM OLD.orphan_expires_at THEN + RAISE EXCEPTION 'qualified preparation expiry is immutable database evidence'; END IF; + RETURN NEW; +END $$; +CREATE TRIGGER mst2_metadata_prepare_00_expiry_guard BEFORE INSERT OR UPDATE ON mst2_metadata_prepare + FOR EACH ROW EXECUTE FUNCTION mst2_metadata_prepare_expiry_guard(); + +CREATE FUNCTION mst2_metadata_orphan_prepare_eligible(pid text) RETURNS boolean LANGUAGE sql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ + SELECT EXISTS(SELECT 1 FROM mst2_metadata_prepare q WHERE q.prepare_id=$1 AND q.state='COMMITTED' + AND q.plan_kind='ROOTED' AND q.graph_domain='qualified-v1' AND mst2_metadata_scope_matches(q.primary_scope) + AND q.orphan_expires_at=q.created_at+interval '3600 seconds' AND q.orphan_expires_at<=clock_timestamp() + AND NOT EXISTS(SELECT 1 FROM mst2_qualified_session_incarnation s WHERE s.state='READY' + AND (s.prepare_id=q.prepare_id OR s.namespace_uuid='$NAMESPACE_UUID$'::uuid AND s.metadata_root=q.metadata_root + AND s.source_profile=convert_to($CORE_SCHEMA$.mst2_route_profile(q.source_domain,q.tagged_root_tree_oid,q.scope, + q.schema_version,q.metadata_codec,q.materialization_policy,q.fs_semantics,q.access_projection, + q.verification_revision,q.projection_revision)::text,'UTF8') + AND s.root_generation IN (SELECT generation FROM mst2_metadata_prepare_page m + WHERE m.prepare_id=q.prepare_id AND m.page_id=q.metadata_root UNION ALL + SELECT root_generation FROM mst2_metadata_prepare_reuse_root r WHERE r.prepare_id=q.prepare_id AND r.root_page=q.metadata_root))) + AND NOT EXISTS(SELECT 1 FROM mst2_qualified_lease_binding l WHERE l.prepare_id=q.prepare_id AND l.state='ACTIVE') + AND NOT EXISTS(SELECT 1 FROM mst2_metadata_reader_operation r JOIN mst2_qualified_lease_binding l USING(lease_id) + WHERE l.prepare_id=q.prepare_id AND r.state='ACTIVE')) +$$; +CREATE FUNCTION mst2_metadata_orphan_prepare_retired(pid text) RETURNS boolean LANGUAGE sql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ + SELECT mst2_metadata_orphan_prepare_eligible($1) AND EXISTS(SELECT 1 FROM mst2_metadata_prepare q + WHERE q.prepare_id=$1 AND q.coverage_retired_at>=q.orphan_expires_at) + AND NOT EXISTS(SELECT 1 FROM mst2_metadata_root_anchor WHERE prepare_id=$1 AND anchor_kind IN ('PREPARE','REUSE')) +$$; + +CREATE FUNCTION mst2_metadata_gc_enabled() RETURNS boolean LANGUAGE sql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ + SELECT NOT pg_is_in_recovery() AND current_setting('transaction_isolation')='read committed' + AND EXISTS(SELECT 1 FROM $CORE_SCHEMA$.mst2_metadata_namespace n WHERE n.namespace_uuid='$NAMESPACE_UUID$'::uuid + AND n.graph_domain='qualified-v1' AND n.metadata_schema=$Q_LITERAL$ AND n.metadata_schema_oid='$Q_OID$'::oid + AND n.core_schema_oid=$CORE_OID$ AND n.metadata_storage_uuid='$STORAGE_UUID$' + AND n.admission_state='ROOTED_Q_ADMITTED' AND n.collector_state='ENABLED' + AND n.implementation_fingerprint=decode('$IMPLEMENTATION_SHA$','hex') + AND n.catalog_fingerprint=$CORE_SCHEMA$.mst2_route_family_catalog($CORE_OID$,'$Q_OID$'::oid)) +$$; +CREATE FUNCTION mst2_metadata_gc_enter() RETURNS void LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +BEGIN + PERFORM $CORE_SCHEMA$.mst2_route_enter($CORE_LITERAL$); + IF NOT mst2_metadata_gc_enabled() THEN RAISE EXCEPTION 'qualified collector is not independently admitted'; END IF; +END $$; + +CREATE FUNCTION mst2_metadata_assert_uncovered(p bytea,g bigint) RETURNS void LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +BEGIN + IF EXISTS(SELECT 1 FROM mst2_metadata_root_anchor WHERE root_page=p AND root_generation=g) + OR EXISTS(SELECT 1 FROM mst2_metadata_graph_root WHERE page_id=p AND generation=g) + OR EXISTS(SELECT 1 FROM mst2_metadata_graph_edge WHERE child_page=p AND child_generation=g) + OR EXISTS(SELECT 1 FROM mst2_metadata_prepare_page m JOIN mst2_metadata_prepare q USING(prepare_id) + WHERE m.page_id=p AND m.generation=g AND (q.state='PREPARING' OR q.state='COMMITTED' AND q.coverage_retired_at IS NULL)) + OR EXISTS(SELECT 1 FROM mst2_metadata_prepare_reuse_root r JOIN mst2_metadata_prepare q USING(prepare_id) + WHERE r.root_page=p AND r.root_generation=g AND (q.state='PREPARING' OR q.state='COMMITTED' AND q.coverage_retired_at IS NULL)) THEN + RAISE EXCEPTION 'qualified current lifetime still has exact owned coverage or incoming edges'; END IF; +END $$; + +CREATE FUNCTION mst2_metadata_gc_proof(p bytea,g bigint,stage text,graph_present boolean,payload_present boolean,c bytea) +RETURNS void LANGUAGE plpgsql VOLATILE SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE life mst2_metadata_lifetime%ROWTYPE; node mst2_metadata_graph_node%ROWTYPE; + body mst2_metadata_payload%ROWTYPE; certificate mst2_metadata_page_certificate%ROWTYPE; actual boolean; +BEGIN + IF p IS NULL OR octet_length(p)<>32 OR g IS NULL OR g<=0 OR stage IS NULL + OR graph_present IS NULL OR payload_present IS NULL OR graph_present AND c IS NULL THEN + RAISE EXCEPTION 'qualified GC proof has an incomplete exact identity'; END IF; + SELECT * INTO life FROM mst2_metadata_lifetime WHERE page_id=p AND generation=g FOR UPDATE; + IF NOT FOUND OR life.graph_domain<>'qualified-v1' OR life.metadata_codec<>1 OR stage NOT IN ('CLAIM','PENDING','APPLIED') + OR stage='CLAIM' AND life.state NOT IN ('RESERVED','LIVE') OR stage='PENDING' AND life.state<>'DELETING' + OR stage='APPLIED' AND life.state<>'REMOVED' THEN RAISE EXCEPTION 'qualified GC stage is not its exact historical lifetime'; END IF; + IF stage<>'APPLIED' AND NOT EXISTS(SELECT 1 FROM mst2_metadata_current WHERE page_id=p AND generation=g) THEN + RAISE EXCEPTION 'qualified GC does not name its exact current generation'; END IF; + PERFORM mst2_metadata_assert_uncovered(p,g); + SELECT * INTO node FROM mst2_metadata_graph_node WHERE page_id=p AND generation=g FOR UPDATE; + actual:=FOUND; + IF actual IS DISTINCT FROM graph_present OR actual AND (node.incoming_refs<>0 OR node.metadata_codec<>life.metadata_codec + OR node.bytes<>life.expected_size OR node.certificate_digest IS DISTINCT FROM c + OR stage='CLAIM' AND node.state<>'LIVE' OR stage='PENDING' AND node.state<>'DELETING') THEN + RAISE EXCEPTION 'qualified GC graph identity or actual counter changed'; END IF; + SELECT * INTO body FROM mst2_metadata_payload WHERE page_id=p AND generation=g FOR UPDATE; + actual:=FOUND; + IF actual IS DISTINCT FROM payload_present OR actual AND (body.metadata_codec<>life.metadata_codec + OR body.byte_size<>life.expected_size OR octet_length(body.payload)<>body.byte_size + OR sha256(convert_to('mega.mst2.metapage','UTF8')||decode('00','hex')||body.payload)<>p) THEN + RAISE EXCEPTION 'qualified GC durable bytes differ from their exact lifetime and page digest'; END IF; + IF payload_present THEN PERFORM mst2_metadata_decode_local(body.payload); END IF; + IF c IS NOT NULL THEN + SELECT * INTO certificate FROM mst2_metadata_page_certificate WHERE page_id=p AND generation=g AND certificate_digest=c; + IF NOT FOUND OR certificate.namespace_uuid<>'$NAMESPACE_UUID$'::uuid OR certificate.metadata_codec<>life.metadata_codec + OR certificate.byte_size<>life.expected_size OR certificate.proof_revision<>1 + OR (SELECT count(*) FROM (SELECT 1 FROM mst2_metadata_verified_ref WHERE parent_page=p AND parent_generation=g LIMIT 258) refs) + <>jsonb_array_length(certificate.canonical_proof->'references') THEN + RAISE EXCEPTION 'qualified GC lost its immutable canonical certificate and typed reference evidence'; END IF; + ELSIF graph_present OR EXISTS(SELECT 1 FROM mst2_metadata_page_certificate WHERE page_id=p AND generation=g) THEN + RAISE EXCEPTION 'qualified GC cannot omit a certified graph identity'; END IF; + IF graph_present THEN + IF (SELECT count(*) FROM (SELECT 1 FROM mst2_metadata_graph_edge WHERE parent_page=p AND parent_generation=g LIMIT 258) edges)>257 THEN + RAISE EXCEPTION 'qualified canonical node has too many physical outgoing edges'; END IF; + IF payload_present AND (EXISTS((SELECT child_page,child_generation FROM mst2_metadata_verified_ref WHERE parent_page=p AND parent_generation=g) + EXCEPT (SELECT child_page,child_generation FROM mst2_metadata_graph_edge WHERE parent_page=p AND parent_generation=g)) + OR EXISTS((SELECT child_page,child_generation FROM mst2_metadata_graph_edge WHERE parent_page=p AND parent_generation=g) + EXCEPT (SELECT child_page,child_generation FROM mst2_metadata_verified_ref WHERE parent_page=p AND parent_generation=g))) THEN + RAISE EXCEPTION 'qualified GC physical outgoing edges differ from exact typed occurrences'; END IF; + END IF; + IF stage='CLAIM' AND life.state='LIVE' AND NOT (graph_present AND payload_present AND c IS NOT NULL) THEN + RAISE EXCEPTION 'LIVE metadata corruption cannot be collected'; END IF; + IF stage='CLAIM' AND graph_present AND NOT payload_present THEN + RAISE EXCEPTION 'certified RESERVED graph corruption cannot be collected without its durable bytes'; END IF; + IF stage='CLAIM' AND life.state='RESERVED' AND (NOT EXISTS(SELECT 1 FROM mst2_metadata_prepare_page m + JOIN mst2_metadata_prepare q USING(prepare_id) WHERE m.page_id=p AND m.generation=g AND m.expected_size=life.expected_size + AND q.graph_domain='qualified-v1' AND q.state='ABORTED' AND q.storage_seal IS NOT NULL) + OR EXISTS(SELECT 1 FROM mst2_metadata_prepare_page m JOIN mst2_metadata_prepare q USING(prepare_id) + WHERE m.page_id=p AND m.generation=g AND q.state<>'ABORTED')) THEN + RAISE EXCEPTION 'RESERVED metadata requires an explicitly aborted exact origin'; END IF; + IF stage='APPLIED' AND (EXISTS(SELECT 1 FROM mst2_metadata_graph_edge WHERE parent_page=p AND parent_generation=g) + OR EXISTS(SELECT 1 FROM mst2_metadata_reuse_index WHERE root_page=p AND root_generation=g)) THEN + RAISE EXCEPTION 'APPLIED qualified receipt still has old physical edges or reuse hints'; END IF; +END $$; + +CREATE FUNCTION mst2_metadata_gc_op_guard() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE life mst2_metadata_lifetime%ROWTYPE; +BEGIN + IF TG_OP='DELETE' THEN RAISE EXCEPTION 'qualified GC operation history is immutable'; END IF; + PERFORM mst2_metadata_gc_enter(); + IF NOT mst2_metadata_scope_matches(NEW.primary_scope) THEN RAISE EXCEPTION 'qualified GC left its captured primary scope'; END IF; + IF TG_OP='INSERT' THEN + IF NEW.state<>'PENDING' OR NEW.completed_at IS NOT NULL OR NEW.payload_delete_xid IS NOT NULL + OR substr(NEW.operation_id::text,15,1)<>'4' OR substr(NEW.operation_id::text,20,1) NOT IN ('8','9','a','b') THEN + RAISE EXCEPTION 'qualified GC must begin with a fresh UUID and unmodified PENDING proof'; END IF; + PERFORM mst2_metadata_gc_proof(NEW.page_id,NEW.generation,'CLAIM',NEW.graph_present,NEW.had_payload,NEW.certificate_digest); + SELECT * INTO STRICT life FROM mst2_metadata_lifetime WHERE page_id=NEW.page_id AND generation=NEW.generation; + IF ROW(NEW.graph_domain,NEW.metadata_codec,NEW.expected_size) IS DISTINCT FROM + ROW(life.graph_domain,life.metadata_codec,life.expected_size) THEN RAISE EXCEPTION 'qualified GC copied profile is incorrect'; END IF; + NEW.created_at:=clock_timestamp(); + ELSE + IF (to_jsonb(NEW)-ARRAY['state','completed_at','payload_delete_xid']) IS DISTINCT FROM + (to_jsonb(OLD)-ARRAY['state','completed_at','payload_delete_xid']) THEN RAISE EXCEPTION 'qualified GC operation cannot retarget immutable evidence'; END IF; + IF OLD.state='APPLIED' AND NEW IS DISTINCT FROM OLD THEN RAISE EXCEPTION 'qualified GC receipt cannot revive or change'; END IF; + IF NEW.payload_delete_xid IS DISTINCT FROM OLD.payload_delete_xid THEN + IF OLD.state<>'PENDING' OR NEW.state<>'PENDING' OR NOT OLD.had_payload OR OLD.payload_delete_xid IS NOT NULL THEN + RAISE EXCEPTION 'qualified payload deletion has no fresh same-transaction phase'; END IF; + PERFORM mst2_metadata_gc_proof(OLD.page_id,OLD.generation,'PENDING',OLD.graph_present,true,OLD.certificate_digest); + NEW.payload_delete_xid:=txid_current(); + END IF; + IF OLD.state='PENDING' AND NEW.state='APPLIED' THEN + IF OLD.had_payload AND OLD.payload_delete_xid IS DISTINCT FROM txid_current() THEN + RAISE EXCEPTION 'qualified GC completion has no same-transaction payload removal'; END IF; + PERFORM mst2_metadata_gc_proof(OLD.page_id,OLD.generation,'APPLIED',false,false,OLD.certificate_digest); + NEW.completed_at:=clock_timestamp(); + ELSIF NEW.state IS DISTINCT FROM OLD.state OR NEW.completed_at IS DISTINCT FROM OLD.completed_at THEN + RAISE EXCEPTION 'qualified GC state transition is invalid'; END IF; + END IF; + RETURN NEW; +END $$; +CREATE TRIGGER mst2_metadata_gc_op_guard BEFORE INSERT OR UPDATE OR DELETE ON mst2_metadata_gc_op + FOR EACH ROW EXECUTE FUNCTION mst2_metadata_gc_op_guard(); + +CREATE FUNCTION mst2_metadata_gc_complete() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE op mst2_metadata_gc_op%ROWTYPE; +BEGIN + SELECT * INTO STRICT op FROM mst2_metadata_gc_op WHERE operation_id=NEW.operation_id; + IF op.state='PENDING' THEN + IF op.payload_delete_xid IS NOT NULL THEN RAISE EXCEPTION 'qualified payload deletion phase cannot escape its atomic apply transaction'; END IF; + PERFORM mst2_metadata_gc_proof(op.page_id,op.generation,'PENDING',op.graph_present,op.had_payload,op.certificate_digest); + ELSE PERFORM mst2_metadata_gc_proof(op.page_id,op.generation,'APPLIED',false,false,op.certificate_digest); END IF; + RETURN NULL; +END $$; +CREATE CONSTRAINT TRIGGER mst2_metadata_gc_complete AFTER INSERT OR UPDATE ON mst2_metadata_gc_op + DEFERRABLE INITIALLY DEFERRED FOR EACH ROW EXECUTE FUNCTION mst2_metadata_gc_complete(); + +CREATE OR REPLACE FUNCTION mst2_metadata_lifetime_guard() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE current_root mst2_metadata_current%ROWTYPE; op mst2_metadata_gc_op%ROWTYPE; previous mst2_metadata_lifetime%ROWTYPE; +BEGIN + IF TG_OP='DELETE' THEN RAISE EXCEPTION 'qualified lifetime watermarks are immutable'; END IF; + IF TG_OP='INSERT' THEN + IF NEW.state<>'RESERVED' THEN RAISE EXCEPTION 'qualified lifetime must begin RESERVED'; END IF; + SELECT * INTO current_root FROM mst2_metadata_current WHERE page_id=NEW.page_id; + IF FOUND THEN + SELECT * INTO previous FROM mst2_metadata_lifetime WHERE page_id=NEW.page_id AND generation=current_root.generation; + SELECT * INTO op FROM mst2_metadata_gc_op WHERE page_id=NEW.page_id AND generation=current_root.generation AND state='APPLIED'; + IF NOT FOUND OR previous.state<>'REMOVED' OR current_root.generation=9223372036854775807 + OR NEW.generation<>current_root.generation+1 OR NEW.metadata_codec<>op.metadata_codec OR NEW.expected_size<>op.expected_size + OR NOT mst2_metadata_scope_matches(op.primary_scope) THEN RAISE EXCEPTION 'qualified reservation needs its exact next-generation APPLIED predecessor'; END IF; + PERFORM mst2_metadata_gc_proof(NEW.page_id,current_root.generation,'APPLIED',false,false,op.certificate_digest); + ELSIF NEW.generation<>1 OR EXISTS(SELECT 1 FROM mst2_metadata_lifetime WHERE page_id=NEW.page_id) + OR EXISTS(SELECT 1 FROM mst2_metadata_payload WHERE page_id=NEW.page_id) + OR EXISTS(SELECT 1 FROM mst2_metadata_graph_node WHERE page_id=NEW.page_id) THEN + RAISE EXCEPTION 'qualified initial reservation cannot adopt history'; END IF; + RETURN NEW; + END IF; + IF (to_jsonb(NEW)-'state') IS DISTINCT FROM (to_jsonb(OLD)-'state') THEN RAISE EXCEPTION 'qualified lifetime identity is immutable'; END IF; + IF NEW.state=OLD.state THEN RETURN NEW; END IF; + IF OLD.state='RESERVED' AND NEW.state='LIVE' THEN + IF NOT EXISTS(SELECT 1 FROM mst2_metadata_graph_node node JOIN mst2_metadata_payload body USING(page_id,generation) + JOIN mst2_metadata_page_certificate certificate USING(page_id,generation) + WHERE node.page_id=OLD.page_id AND node.generation=OLD.generation AND node.state='LIVE' + AND node.certificate_digest=certificate.certificate_digest AND node.metadata_codec=OLD.metadata_codec + AND body.metadata_codec=OLD.metadata_codec AND body.byte_size=OLD.expected_size AND node.bytes=OLD.expected_size) + OR NOT (EXISTS(SELECT 1 FROM mst2_metadata_graph_root root JOIN mst2_metadata_prepare q USING(prepare_id) + WHERE root.page_id=OLD.page_id AND root.generation=OLD.generation AND q.plan_kind='COLD' + AND q.state='COMMITTED' AND root.storage_seal=q.storage_seal AND q.coverage_retired_at IS NULL) + OR EXISTS(SELECT 1 FROM mst2_metadata_prepare_page m JOIN mst2_metadata_prepare q USING(prepare_id) + JOIN mst2_metadata_root_anchor anchor ON anchor.prepare_id=q.prepare_id AND anchor.anchor_kind='PREPARE' + AND anchor.root_page=q.metadata_root AND anchor.owner_key=q.prepare_id + WHERE m.page_id=OLD.page_id AND m.generation=OLD.generation AND q.plan_kind='ROOTED' + AND q.state='COMMITTED' AND q.coverage_retired_at IS NULL)) THEN + RAISE EXCEPTION 'qualified LIVE transition lacks its certified graph and exact owned preparation'; END IF; + RETURN NEW; + END IF; + PERFORM mst2_metadata_gc_enter(); + SELECT * INTO op FROM mst2_metadata_gc_op WHERE page_id=OLD.page_id AND generation=OLD.generation AND state='PENDING'; + IF NOT FOUND OR NOT mst2_metadata_scope_matches(op.primary_scope) THEN RAISE EXCEPTION 'qualified lifecycle transition lacks its exact pending operation'; END IF; + IF OLD.state IN ('RESERVED','LIVE') AND NEW.state='DELETING' THEN + PERFORM mst2_metadata_gc_proof(OLD.page_id,OLD.generation,'CLAIM',op.graph_present,op.had_payload,op.certificate_digest); + ELSIF OLD.state='DELETING' AND NEW.state='REMOVED' THEN + IF op.had_payload AND op.payload_delete_xid IS DISTINCT FROM txid_current() THEN RAISE EXCEPTION 'qualified removal has no atomic payload deletion'; END IF; + PERFORM mst2_metadata_gc_proof(OLD.page_id,OLD.generation,'PENDING',false,false,op.certificate_digest); + IF EXISTS(SELECT 1 FROM mst2_metadata_graph_edge WHERE parent_page=OLD.page_id AND parent_generation=OLD.generation) + OR EXISTS(SELECT 1 FROM mst2_metadata_reuse_index WHERE root_page=OLD.page_id AND root_generation=OLD.generation) THEN + RAISE EXCEPTION 'qualified removal still has old physical references'; END IF; + ELSE RAISE EXCEPTION 'qualified lifecycle transition is invalid'; END IF; + RETURN NEW; +END $$; + +CREATE OR REPLACE FUNCTION mst2_metadata_current_guard() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE op mst2_metadata_gc_op%ROWTYPE; life mst2_metadata_lifetime%ROWTYPE; +BEGIN + IF TG_OP='DELETE' THEN RAISE EXCEPTION 'qualified current watermark cannot be deleted'; END IF; + IF TG_OP='INSERT' THEN + IF NEW.generation<>1 OR EXISTS(SELECT 1 FROM mst2_metadata_lifetime WHERE page_id=NEW.page_id AND generation<>1) + OR EXISTS(SELECT 1 FROM mst2_metadata_payload WHERE page_id=NEW.page_id) + OR EXISTS(SELECT 1 FROM mst2_metadata_graph_node WHERE page_id=NEW.page_id) THEN + RAISE EXCEPTION 'qualified initial current cannot adopt history'; END IF; + RETURN NEW; + END IF; + IF NEW.page_id<>OLD.page_id OR OLD.generation=9223372036854775807 OR NEW.generation<>OLD.generation+1 THEN + RAISE EXCEPTION 'qualified current requires exact next-generation CAS'; END IF; + SELECT * INTO op FROM mst2_metadata_gc_op WHERE page_id=OLD.page_id AND generation=OLD.generation AND state='APPLIED'; + IF NOT FOUND OR NOT mst2_metadata_scope_matches(op.primary_scope) + OR EXISTS(SELECT 1 FROM mst2_metadata_payload WHERE page_id=OLD.page_id) + OR EXISTS(SELECT 1 FROM mst2_metadata_graph_node WHERE page_id=OLD.page_id AND generation=OLD.generation) THEN + RAISE EXCEPTION 'qualified current has no definitive old-generation removal'; END IF; + PERFORM mst2_metadata_gc_proof(OLD.page_id,OLD.generation,'APPLIED',false,false,op.certificate_digest); + SELECT * INTO life FROM mst2_metadata_lifetime WHERE page_id=NEW.page_id AND generation=NEW.generation; + IF NOT FOUND OR life.state<>'RESERVED' OR life.graph_domain<>'qualified-v1' + OR life.metadata_codec<>op.metadata_codec OR life.expected_size<>op.expected_size THEN + RAISE EXCEPTION 'qualified fresh current differs from its immutable page profile'; END IF; + RETURN NEW; +END $$; + +CREATE FUNCTION mst2_metadata_capacity() RETURNS TABLE(resident_pages bigint,resident_bytes bigint) LANGUAGE sql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ + SELECT count(*)::bigint,coalesce(sum(byte_size),0)::bigint FROM (SELECT byte_size FROM mst2_metadata_payload LIMIT 16385) actual +$$; +CREATE FUNCTION mst2_metadata_capacity_guard() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE pages bigint; bytes bigint; actual bigint; +BEGIN + IF TG_TABLE_NAME='mst2_metadata_payload' THEN + SELECT resident_pages,resident_bytes INTO pages,bytes FROM mst2_metadata_capacity(); + IF pages>16384 OR bytes>268435456 THEN RAISE EXCEPTION 'qualified resident payload capacity is exceeded'; END IF; + ELSIF TG_TABLE_NAME='mst2_metadata_source_entry_reference' THEN + SELECT count(*) INTO actual FROM (SELECT 1 FROM mst2_metadata_source_entry_reference LIMIT 262145) bounded; + IF actual>262144 THEN RAISE EXCEPTION 'qualified retained source dictionary capacity is exceeded'; END IF; + RETURN NULL; + ELSIF TG_TABLE_NAME='mst2_qualified_session_incarnation' THEN + SELECT count(*) INTO actual FROM (SELECT 1 FROM mst2_qualified_session_incarnation WHERE state='READY' LIMIT 4097) bounded; + ELSIF TG_TABLE_NAME='mst2_qualified_lease_binding' THEN + SELECT count(*) INTO actual FROM (SELECT 1 FROM mst2_qualified_lease_binding WHERE state='ACTIVE' LIMIT 4097) bounded; + ELSE SELECT count(*) INTO actual FROM (SELECT 1 FROM mst2_metadata_reader_operation WHERE state='ACTIVE' LIMIT 4097) bounded; END IF; + IF actual>4096 THEN RAISE EXCEPTION 'qualified active serving capacity is exceeded'; END IF; + RETURN NULL; +END $$; +DO $$ DECLARE name text; BEGIN + FOREACH name IN ARRAY ARRAY['mst2_metadata_payload','mst2_qualified_session_incarnation', + 'mst2_qualified_lease_binding','mst2_metadata_reader_operation','mst2_metadata_source_entry_reference'] LOOP + EXECUTE format('CREATE TRIGGER mst2_metadata_capacity_guard AFTER INSERT OR UPDATE ON %I FOR EACH STATEMENT EXECUTE FUNCTION mst2_metadata_capacity_guard()',name); + END LOOP; +END $$; + +-- Append-only identity and replay evidence remains bounded independently of +-- resident payloads. The admission check reads actual stored rows, never hints. +CREATE FUNCTION mst2_metadata_history_capacity_guard() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE actual bigint; bytes bigint; row_limit integer; size_expression text; +BEGIN + IF TG_TABLE_NAME IN ('mst2_metadata_prepare_page','mst2_metadata_prepare_reuse_root','mst2_metadata_verified_ref') THEN + row_limit:=262144; size_expression:='0'; + ELSIF TG_TABLE_NAME='mst2_metadata_prepare' THEN + row_limit:=16384; + size_expression:='pg_column_size(canonical_plan)::bigint+pg_column_size(canonical_bindings)+pg_column_size(primary_scope)'; + ELSIF TG_TABLE_NAME='mst2_metadata_source_root_attestation' THEN + row_limit:=16384; size_expression:='pg_column_size(source_proof)::bigint+pg_column_size(source_profile)'; + ELSIF TG_TABLE_NAME='mst2_metadata_page_certificate' THEN + row_limit:=16384; size_expression:='pg_column_size(canonical_proof)::bigint'; + ELSIF TG_TABLE_NAME='mst2_metadata_scope_source_reference' THEN + row_limit:=16384; size_expression:='pg_column_size(ancestor_revisions)::bigint'; + ELSIF TG_TABLE_NAME='mst2_qualified_session_incarnation' THEN + row_limit:=65536; size_expression:='pg_column_size(canonical_descriptor)::bigint+pg_column_size(source_profile)'; + ELSIF TG_TABLE_NAME IN ('mst2_metadata_lifetime','mst2_metadata_current','mst2_metadata_gc_op', + 'mst2_qualified_lease_binding','mst2_metadata_reader_operation') THEN + row_limit:=65536; size_expression:='0'; + ELSE RAISE EXCEPTION 'qualified history quota has an unknown physical relation'; END IF; + EXECUTE format('SELECT count(*),coalesce(sum(size),0) FROM (SELECT %s AS size FROM %I.%I LIMIT %s) actual', + size_expression,TG_TABLE_SCHEMA,TG_TABLE_NAME,row_limit+1) INTO actual,bytes; + IF actual>row_limit OR bytes>268435456 THEN + RAISE EXCEPTION 'qualified immutable history capacity is exceeded for %',TG_TABLE_NAME; END IF; + RETURN NULL; +END $$; +DO $$ DECLARE name text; BEGIN + FOREACH name IN ARRAY ARRAY['mst2_metadata_prepare','mst2_metadata_prepare_page','mst2_metadata_prepare_reuse_root', + 'mst2_metadata_verified_ref','mst2_metadata_source_root_attestation','mst2_metadata_page_certificate','mst2_metadata_scope_source_reference', + 'mst2_qualified_session_incarnation','mst2_metadata_lifetime','mst2_metadata_current','mst2_metadata_gc_op', + 'mst2_qualified_lease_binding','mst2_metadata_reader_operation'] LOOP + EXECUTE format('CREATE TRIGGER mst2_metadata_history_capacity_guard AFTER INSERT ON %I FOR EACH STATEMENT EXECUTE FUNCTION mst2_metadata_history_capacity_guard()',name); + END LOOP; +END $$; + +CREATE FUNCTION mst2_metadata_gc_claim(p bytea,g bigint,scope bytea,id uuid) RETURNS uuid LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE life mst2_metadata_lifetime%ROWTYPE; node mst2_metadata_graph_node%ROWTYPE; op mst2_metadata_gc_op%ROWTYPE; + has_graph boolean; has_body boolean; +BEGIN + PERFORM mst2_metadata_gc_enter(); + IF NOT mst2_metadata_scope_matches(scope) THEN RAISE EXCEPTION 'qualified claim scope differs from its captured primary'; END IF; + SELECT * INTO op FROM mst2_metadata_gc_op WHERE operation_id=id; + IF FOUND THEN + IF op.page_id IS DISTINCT FROM p OR op.generation<>g OR op.primary_scope IS DISTINCT FROM scope THEN + RAISE EXCEPTION 'qualified GC operation cannot be reused for a different lifetime'; END IF; + RETURN id; + END IF; + SELECT * INTO STRICT life FROM mst2_metadata_lifetime WHERE page_id=p AND generation=g; + SELECT * INTO node FROM mst2_metadata_graph_node WHERE page_id=p AND generation=g; + has_graph:=FOUND; has_body:=EXISTS(SELECT 1 FROM mst2_metadata_payload WHERE page_id=p AND generation=g); + INSERT INTO mst2_metadata_gc_op(operation_id,page_id,generation,primary_scope,graph_domain,metadata_codec, + expected_size,graph_present,had_payload,certificate_digest,state) + VALUES(id,p,g,scope,'qualified-v1',life.metadata_codec,life.expected_size,has_graph,has_body, + CASE WHEN has_graph THEN node.certificate_digest ELSE NULL END,'PENDING'); + UPDATE mst2_metadata_lifetime SET state='DELETING' WHERE page_id=p AND generation=g; + IF has_graph THEN UPDATE mst2_metadata_graph_node SET state='DELETING' WHERE page_id=p AND generation=g; END IF; + RETURN id; +END $$; + +CREATE OR REPLACE FUNCTION mst2_metadata_gc_apply(id uuid) RETURNS void LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE op mst2_metadata_gc_op%ROWTYPE; +BEGIN + PERFORM mst2_metadata_gc_enter(); + SELECT * INTO STRICT op FROM mst2_metadata_gc_op WHERE operation_id=id FOR UPDATE; + IF NOT mst2_metadata_scope_matches(op.primary_scope) THEN RAISE EXCEPTION 'qualified GC replay left its captured primary'; END IF; + IF op.state='APPLIED' THEN + PERFORM mst2_metadata_gc_proof(op.page_id,op.generation,'APPLIED',false,false,op.certificate_digest); RETURN; + END IF; + PERFORM mst2_metadata_gc_proof(op.page_id,op.generation,'PENDING',op.graph_present,op.had_payload,op.certificate_digest); + IF op.had_payload THEN + UPDATE mst2_metadata_gc_op SET payload_delete_xid=txid_current() WHERE operation_id=id; + DELETE FROM mst2_metadata_payload WHERE page_id=op.page_id AND generation=op.generation; + IF NOT FOUND THEN RAISE EXCEPTION 'qualified atomic payload deletion lost its exact row'; END IF; + END IF; + DELETE FROM mst2_metadata_graph_edge WHERE parent_page=op.page_id AND parent_generation=op.generation; + DELETE FROM mst2_metadata_reuse_index WHERE root_page=op.page_id AND root_generation=op.generation; + DELETE FROM mst2_metadata_graph_node WHERE page_id=op.page_id AND generation=op.generation; + UPDATE mst2_metadata_lifetime SET state='REMOVED' WHERE page_id=op.page_id AND generation=op.generation AND state='DELETING'; + IF NOT FOUND THEN RAISE EXCEPTION 'qualified atomic removal lost its exact lifecycle'; END IF; + UPDATE mst2_metadata_gc_op SET state='APPLIED' WHERE operation_id=id; +END $$; +CREATE OR REPLACE FUNCTION mst2_metadata_gc_finish(id uuid) RETURNS void LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ BEGIN PERFORM mst2_metadata_gc_apply(id); END $$; + +-- Additional graph/payload guards follow; no history relation is collected. +CREATE OR REPLACE FUNCTION mst2_metadata_payload_fenced() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE op mst2_metadata_gc_op%ROWTYPE; +BEGIN + IF TG_OP='UPDATE' THEN RAISE EXCEPTION 'qualified immutable payload cannot be updated'; END IF; + IF TG_OP='DELETE' THEN + PERFORM mst2_metadata_gc_enter(); + SELECT * INTO op FROM mst2_metadata_gc_op WHERE page_id=OLD.page_id AND generation=OLD.generation AND state='PENDING'; + IF NOT FOUND OR NOT op.had_payload OR op.payload_delete_xid IS DISTINCT FROM txid_current() + OR NOT mst2_metadata_scope_matches(op.primary_scope) OR op.expected_size<>OLD.byte_size OR op.metadata_codec<>OLD.metadata_codec THEN + RAISE EXCEPTION 'qualified payload removal needs its exact same-transaction pending evidence'; END IF; + PERFORM mst2_metadata_gc_proof(OLD.page_id,OLD.generation,'PENDING',op.graph_present,true,op.certificate_digest); + RETURN OLD; + END IF; + IF NOT EXISTS(SELECT 1 FROM mst2_metadata_current c JOIN mst2_metadata_lifetime l USING(page_id,generation) + JOIN mst2_metadata_prepare_page m USING(page_id,generation) JOIN mst2_metadata_prepare q USING(prepare_id) + WHERE c.page_id=NEW.page_id AND c.generation=NEW.generation AND l.state IN ('RESERVED','LIVE') + AND l.metadata_codec=NEW.metadata_codec AND l.expected_size=NEW.byte_size AND q.state='PREPARING' + AND q.graph_domain='qualified-v1' AND mst2_metadata_scope_matches(q.primary_scope)) THEN + RAISE EXCEPTION 'qualified payload INSERT crossed its exact active generation'; END IF; + RETURN NEW; +END $$; + +CREATE OR REPLACE FUNCTION mst2_metadata_graph_node_guard() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE life mst2_metadata_lifetime%ROWTYPE; op mst2_metadata_gc_op%ROWTYPE; +BEGIN + IF TG_OP='DELETE' THEN + PERFORM mst2_metadata_gc_enter(); + SELECT * INTO op FROM mst2_metadata_gc_op WHERE page_id=OLD.page_id AND generation=OLD.generation AND state='PENDING'; + IF NOT FOUND OR NOT op.graph_present OR op.certificate_digest IS DISTINCT FROM OLD.certificate_digest + OR OLD.state<>'DELETING' OR NOT mst2_metadata_scope_matches(op.primary_scope) + OR op.had_payload AND op.payload_delete_xid IS DISTINCT FROM txid_current() + OR EXISTS(SELECT 1 FROM mst2_metadata_graph_edge WHERE parent_page=OLD.page_id AND parent_generation=OLD.generation) THEN + RAISE EXCEPTION 'qualified graph removal lacks its atomic byte removal and empty outgoing edges'; END IF; + PERFORM mst2_metadata_gc_proof(OLD.page_id,OLD.generation,'PENDING',true,false,op.certificate_digest); + RETURN OLD; + END IF; + SELECT l.* INTO life FROM mst2_metadata_current c JOIN mst2_metadata_lifetime l USING(page_id,generation) + WHERE c.page_id=NEW.page_id AND c.generation=NEW.generation; + IF NOT FOUND OR life.graph_domain<>'qualified-v1' OR life.metadata_codec<>NEW.metadata_codec OR life.expected_size<>NEW.bytes + OR NEW.incoming_refs<>(SELECT count(*) FROM mst2_metadata_graph_edge WHERE child_page=NEW.page_id AND child_generation=NEW.generation) + OR NOT EXISTS(SELECT 1 FROM mst2_metadata_page_certificate certificate WHERE certificate.page_id=NEW.page_id + AND certificate.generation=NEW.generation AND certificate.certificate_digest=NEW.certificate_digest + AND certificate.metadata_codec=NEW.metadata_codec AND certificate.byte_size=NEW.bytes) THEN + RAISE EXCEPTION 'qualified graph identity, certificate or physical counter changed'; END IF; + IF TG_OP='INSERT' THEN + IF NEW.state<>'LIVE' OR life.state NOT IN ('RESERVED','LIVE') + OR NOT EXISTS(SELECT 1 FROM mst2_metadata_prepare_page m JOIN mst2_metadata_prepare q USING(prepare_id) + WHERE m.page_id=NEW.page_id AND m.generation=NEW.generation AND q.state='PREPARING') THEN + RAISE EXCEPTION 'qualified graph creation requires an active exact preparation'; END IF; + ELSE + IF (to_jsonb(NEW)-ARRAY['state','incoming_refs']) IS DISTINCT FROM (to_jsonb(OLD)-ARRAY['state','incoming_refs']) THEN + RAISE EXCEPTION 'qualified graph identity is immutable'; END IF; + IF NEW.state<>OLD.state THEN + PERFORM mst2_metadata_gc_enter(); + SELECT * INTO op FROM mst2_metadata_gc_op WHERE page_id=OLD.page_id AND generation=OLD.generation AND state='PENDING'; + IF NOT FOUND OR OLD.state<>'LIVE' OR NEW.state<>'DELETING' OR life.state<>'DELETING' OR NEW.incoming_refs<>0 + OR op.certificate_digest IS DISTINCT FROM NEW.certificate_digest OR NOT mst2_metadata_scope_matches(op.primary_scope) THEN + RAISE EXCEPTION 'qualified graph state change has no exact pending claim'; END IF; + PERFORM mst2_metadata_assert_uncovered(OLD.page_id,OLD.generation); + ELSIF NEW.state='LIVE' AND life.state NOT IN ('RESERVED','LIVE') OR NEW.state='DELETING' AND life.state<>'DELETING' THEN + RAISE EXCEPTION 'qualified graph state differs from its current lifecycle'; END IF; + END IF; + RETURN NEW; +END $$; + +CREATE OR REPLACE FUNCTION mst2_metadata_graph_edge_guard() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE op mst2_metadata_gc_op%ROWTYPE; +BEGIN + IF TG_OP='UPDATE' THEN RAISE EXCEPTION 'qualified edge identity is immutable'; END IF; + IF TG_OP='DELETE' THEN + PERFORM mst2_metadata_gc_enter(); + SELECT * INTO op FROM mst2_metadata_gc_op WHERE page_id=OLD.parent_page AND generation=OLD.parent_generation AND state='PENDING'; + IF NOT FOUND OR NOT op.graph_present OR NOT mst2_metadata_scope_matches(op.primary_scope) + OR op.had_payload AND op.payload_delete_xid IS DISTINCT FROM txid_current() + OR EXISTS(SELECT 1 FROM mst2_metadata_payload WHERE page_id=OLD.parent_page AND generation=OLD.parent_generation) + OR NOT EXISTS(SELECT 1 FROM mst2_metadata_graph_node node JOIN mst2_metadata_current cur USING(page_id,generation) + JOIN mst2_metadata_lifetime life USING(page_id,generation) + WHERE node.page_id=OLD.parent_page AND node.generation=OLD.parent_generation AND node.state='DELETING' + AND life.state='DELETING' AND node.incoming_refs=0 AND node.certificate_digest=op.certificate_digest) THEN + RAISE EXCEPTION 'qualified edge removal is outside its exact atomic graph deletion phase'; END IF; + PERFORM mst2_metadata_assert_uncovered(OLD.parent_page,OLD.parent_generation); + RETURN OLD; + END IF; + IF NOT EXISTS(SELECT 1 FROM mst2_metadata_verified_ref ref + JOIN mst2_metadata_page_certificate parent ON parent.page_id=ref.parent_page AND parent.generation=ref.parent_generation + JOIN mst2_metadata_page_certificate child ON child.page_id=ref.child_page AND child.generation=ref.child_generation + WHERE ref.parent_page=NEW.parent_page AND ref.parent_generation=NEW.parent_generation + AND ref.child_page=NEW.child_page AND ref.child_generation=NEW.child_generation + AND ref.child_certificate_digest=child.certificate_digest AND parent.rank>child.rank) THEN + RAISE EXCEPTION 'qualified edge differs from its exact canonical reference or strict certified rank'; END IF; + IF (SELECT count(*) FROM mst2_metadata_graph_node n JOIN mst2_metadata_current c USING(page_id,generation) + JOIN mst2_metadata_lifetime l USING(page_id,generation) + WHERE ((n.page_id=NEW.parent_page AND n.generation=NEW.parent_generation) + OR (n.page_id=NEW.child_page AND n.generation=NEW.child_generation)) + AND n.state='LIVE' AND l.state IN ('RESERVED','LIVE') AND l.graph_domain='qualified-v1')<>2 + OR NOT EXISTS(SELECT 1 FROM mst2_metadata_prepare_page a JOIN mst2_metadata_prepare q USING(prepare_id) + WHERE a.page_id=NEW.parent_page AND a.generation=NEW.parent_generation AND q.state='PREPARING' + AND q.graph_domain='qualified-v1' AND mst2_metadata_scope_matches(q.primary_scope) + AND (EXISTS(SELECT 1 FROM mst2_metadata_prepare_page b WHERE b.prepare_id=a.prepare_id + AND b.page_id=NEW.child_page AND b.generation=NEW.child_generation) + OR EXISTS(SELECT 1 FROM mst2_metadata_prepare_reuse_root r JOIN mst2_metadata_root_anchor anchor + ON anchor.prepare_id=r.prepare_id AND anchor.anchor_kind='REUSE' AND anchor.owner_key=r.prepare_id + AND anchor.root_page=r.root_page AND anchor.root_generation=r.root_generation + WHERE r.prepare_id=a.prepare_id AND r.root_page=NEW.child_page AND r.root_generation=NEW.child_generation))) THEN + RAISE EXCEPTION 'edge creation requires active exact qualified endpoints'; END IF; + RETURN NEW; +END $$; + +CREATE FUNCTION mst2_metadata_edges_removed() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +BEGIN + IF (SELECT count(*) FROM removed_edges)>257 THEN RAISE EXCEPTION 'qualified GC edge deletion exceeds one canonical node'; END IF; + IF EXISTS(SELECT 1 FROM mst2_metadata_graph_node node JOIN (SELECT child_page,child_generation,count(*) AS delta + FROM removed_edges GROUP BY child_page,child_generation) removed + ON removed.child_page=node.page_id AND removed.child_generation=node.generation + WHERE node.incoming_refs-removed.delta<>(SELECT count(*) FROM mst2_metadata_graph_edge edge + WHERE edge.child_page=node.page_id AND edge.child_generation=node.generation)) THEN + RAISE EXCEPTION 'qualified removal discovered physical incoming counter drift'; END IF; + UPDATE mst2_metadata_graph_node node SET incoming_refs=node.incoming_refs-removed.delta + FROM (SELECT child_page,child_generation,count(*) AS delta FROM removed_edges GROUP BY child_page,child_generation) removed + WHERE node.page_id=removed.child_page AND node.generation=removed.child_generation; + RETURN NULL; +END $$; +CREATE TRIGGER mst2_metadata_edges_removed AFTER DELETE ON mst2_metadata_graph_edge + REFERENCING OLD TABLE AS removed_edges FOR EACH STATEMENT EXECUTE FUNCTION mst2_metadata_edges_removed(); + +CREATE FUNCTION mst2_metadata_gc_owner_cleanup(maximum integer) +RETURNS TABLE(examined bigint,readers_expired bigint,leases_expired bigint,prepares_aborted bigint, + handovers_retired bigint,orphans_retired bigint) +LANGUAGE plpgsql VOLATILE SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE bound integer:=maximum; item record; q mst2_metadata_prepare%ROWTYPE; now_unix bigint; +BEGIN + PERFORM $CORE_SCHEMA$.mst2_route_enter($CORE_LITERAL$); + IF maximum IS NULL OR maximum NOT BETWEEN 0 AND 64 THEN RAISE EXCEPTION 'qualified owner cleanup budget must be 0..=64'; END IF; + examined:=0; readers_expired:=0; leases_expired:=0; prepares_aborted:=0; handovers_retired:=0; orphans_retired:=0; + now_unix:=floor(extract(epoch FROM clock_timestamp()))::bigint; + FOR item IN SELECT * FROM ( + (SELECT 'READER'::text AS kind,operation_id::text AS owner,lease_id,hard_deadline_unix AS deadline + FROM mst2_metadata_reader_operation WHERE state='ACTIVE' AND hard_deadline_unix<=now_unix + ORDER BY hard_deadline_unix,operation_id LIMIT bound) + UNION ALL + (SELECT 'LEASE'::text,lease_id,lease_id,expires_at_unix FROM mst2_qualified_lease_binding + WHERE state='ACTIVE' AND expires_at_unix<=now_unix ORDER BY expires_at_unix,lease_id LIMIT bound) + UNION ALL + (SELECT 'PREPARE'::text,prepare_id,NULL::text,floor(extract(epoch FROM orphan_expires_at))::bigint + FROM mst2_metadata_prepare WHERE (state='PREPARING' OR state='COMMITTED' AND coverage_retired_at IS NULL) + AND orphan_expires_at<=clock_timestamp() ORDER BY orphan_expires_at,prepare_id LIMIT bound) + ) expired ORDER BY deadline,kind,owner LIMIT bound LOOP + examined:=examined+1; + IF item.kind='READER' THEN + UPDATE mst2_metadata_reader_operation SET state='EXPIRED' WHERE operation_id=item.owner::uuid AND state='ACTIVE'; + IF NOT FOUND THEN RAISE EXCEPTION 'qualified expired reader changed behind its mutation barrier'; END IF; + readers_expired:=readers_expired+1; + DELETE FROM mst2_metadata_root_anchor WHERE reader_operation_id=item.owner::uuid AND anchor_kind IN ('REQUEST','READER'); + PERFORM mst2_metadata_cleanup_lease(item.lease_id); + ELSIF item.kind='LEASE' THEN + UPDATE mst2_qualified_lease_binding SET state='EXPIRED',lease_epoch=lease_epoch+1 WHERE lease_id=item.owner AND state='ACTIVE'; + IF NOT FOUND THEN RAISE EXCEPTION 'qualified expired lease changed behind its mutation barrier'; END IF; + leases_expired:=leases_expired+1; PERFORM mst2_metadata_cleanup_lease(item.lease_id); + ELSE + SELECT * INTO STRICT q FROM mst2_metadata_prepare WHERE prepare_id=item.owner; + IF q.state='PREPARING' THEN + DELETE FROM mst2_metadata_graph_root WHERE prepare_id=q.prepare_id; + UPDATE mst2_metadata_prepare SET state='ABORTED',aborted_at=clock_timestamp() WHERE prepare_id=q.prepare_id; + DELETE FROM mst2_metadata_root_anchor WHERE prepare_id=q.prepare_id AND anchor_kind IN ('PREPARE','REUSE'); + prepares_aborted:=prepares_aborted+1; + ELSIF mst2_metadata_session_covers_prepare(q.prepare_id) THEN + UPDATE mst2_metadata_prepare SET coverage_retired_at=clock_timestamp() WHERE prepare_id=q.prepare_id; + DELETE FROM mst2_metadata_root_anchor WHERE prepare_id=q.prepare_id AND anchor_kind IN ('PREPARE','REUSE'); + handovers_retired:=handovers_retired+1; + ELSIF mst2_metadata_orphan_prepare_eligible(q.prepare_id) THEN + UPDATE mst2_metadata_prepare SET coverage_retired_at=clock_timestamp() WHERE prepare_id=q.prepare_id; + DELETE FROM mst2_metadata_root_anchor WHERE prepare_id=q.prepare_id AND anchor_kind IN ('PREPARE','REUSE'); + orphans_retired:=orphans_retired+1; + END IF; + END IF; + END LOOP; + RETURN NEXT; +END $$; +DROP TRIGGER mst2_01_family_closed ON mst2_metadata_gc_op; diff --git a/src/jupiter/storage/qualified_metadata_gc_tests.rs b/src/jupiter/storage/qualified_metadata_gc_tests.rs new file mode 100644 index 00000000..346254bf --- /dev/null +++ b/src/jupiter/storage/qualified_metadata_gc_tests.rs @@ -0,0 +1,456 @@ +use std::collections::{BTreeMap, BTreeSet}; + +use super::{canonical_tests::seeded_rooted_plan, *}; +use crate::ceres::snapshot::{ + rooted_metadata_install::RootedMetadataInstallPlan, + rooted_metadata_projection::RootedReuseLookup, +}; + +async fn write_rooted( + writer: &RootedQualifiedMetadataRepository, + operation: &str, + plan: &RootedMetadataInstallPlan, + payload: MetadataPagePayload, +) -> RootedPrepareIntent { + let intent = writer.begin_intent(operation, plan).await.unwrap(); + writer.install_pages(&intent, &[payload]).await.unwrap(); + writer.finalize(&intent).await.unwrap(); + intent +} + +// Fault injection only: expire database evidence in the isolated test schema, +// restore the original guards/catalog, then use normal production maintenance. +async fn age_orphan(q: &DatabaseConnection, namespace: &VerifiedQualifiedNamespace, prepare: &str) { + let txn = q.begin().await.unwrap(); + for trigger in [ + "mst2_00_family_barrier", + "mst2_metadata_prepare_guard", + "mst2_metadata_prepare_00_expiry_guard", + ] { + txn.execute_unprepared(&format!( + "ALTER TABLE mst2_metadata_prepare DISABLE TRIGGER {trigger}" + )) + .await + .unwrap(); + } + txn.execute_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + "UPDATE mst2_metadata_prepare SET created_at=clock_timestamp()-interval '3601 seconds', + orphan_expires_at=clock_timestamp()-interval '1 second' WHERE prepare_id=$1", + [prepare.into()], + )) + .await + .unwrap(); + // clock_timestamp() calls differ; preserve the exact database TTL relation. + txn.execute_raw(Statement::from_sql_and_values(DbBackend::Postgres, + "UPDATE mst2_metadata_prepare SET orphan_expires_at=created_at+interval '3600 seconds' WHERE prepare_id=$1", + [prepare.into()])).await.unwrap(); + for trigger in [ + "mst2_00_family_barrier", + "mst2_metadata_prepare_guard", + "mst2_metadata_prepare_00_expiry_guard", + ] { + txn.execute_unprepared(&format!( + "ALTER TABLE mst2_metadata_prepare ENABLE TRIGGER {trigger}" + )) + .await + .unwrap(); + } + txn.commit().await.unwrap(); + assert_eq!( + catalog(q, namespace.core_oid, namespace.schema_oid) + .await + .unwrap(), + namespace.catalog_fingerprint + ); +} + +async fn claim( + q: &DatabaseConnection, + intent: &RootedPrepareIntent, +) -> Result { + let operation = uuid::Uuid::new_v4().to_string(); + q.execute_raw(Statement::from_sql_and_values(DbBackend::Postgres, + "SELECT mst2_metadata_gc_claim($1,$2,(SELECT primary_scope FROM mst2_metadata_prepare WHERE prepare_id=$3),$4::uuid)", + [intent.metadata_root().to_vec().into(),intent.root_generation().into(),intent.prepare_id().into(),operation.clone().into()] + )).await?; + Ok(operation) +} + +#[tokio::test] +async fn admitted_orphan_gc_preserves_replay_identity_and_advances_exact_generation() { + let (config, core, namespace, q, _guard) = fixture().await; + let (plan, payload) = seeded_rooted_plan(&core, 'a', "file", 1).await; + let writer = RootedQualifiedMetadataRepository::open(&core, &config) + .await + .unwrap(); + let first = write_rooted(&writer, "gc-generation-one", &plan, payload.clone()).await; + assert_eq!( + count(&q, "SELECT mst2_metadata_gc_enabled()::bigint").await, + 1 + ); + assert!( + claim(&q, &first) + .await + .unwrap_err() + .to_string() + .contains("owned coverage") + ); + assert_eq!( + count(&q, "SELECT count(*) FROM mst2_metadata_gc_op").await, + 0 + ); + age_orphan(&q, &namespace, first.prepare_id()).await; + let work = writer.maintenance_tick(64).await.unwrap(); + assert!(work.collector_enabled); + assert!(work.examined <= 64); + assert_eq!(work.orphans_retired, 1); + assert_eq!(work.payload_pages_removed, 1); + assert_eq!(work.payload_bytes_removed, payload.size); + assert_eq!( + count(&q, "SELECT count(*) FROM mst2_metadata_payload").await, + 0 + ); + assert_eq!( + count(&q, "SELECT count(*) FROM mst2_metadata_graph_node").await, + 0 + ); + assert_eq!( + count(&q, "SELECT count(*) FROM mst2_metadata_page_certificate").await, + 1 + ); + assert_eq!( + count( + &q, + "SELECT count(*) FROM mst2_metadata_source_root_attestation" + ) + .await, + 1 + ); + assert_eq!( + count( + &q, + "SELECT count(*) FROM mst2_metadata_gc_op WHERE state='APPLIED'" + ) + .await, + 1 + ); + let operation: String = q + .query_one_raw(Statement::from_string( + DbBackend::Postgres, + "SELECT operation_id::text FROM mst2_metadata_gc_op", + )) + .await + .unwrap() + .unwrap() + .try_get_by_index(0) + .unwrap(); + let fresh = write_rooted(&writer, "gc-generation-two", &plan, payload.clone()).await; + assert_eq!(fresh.root_generation(), first.root_generation() + 1); + assert!(writer.recover("gc-generation-one", &plan).await.is_err()); + for _ in 0..2 { + q.execute_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + "SELECT mst2_metadata_gc_apply($1::uuid)", + [operation.clone().into()], + )) + .await + .unwrap(); + } + assert_eq!( + count( + &q, + "SELECT count(*) FROM mst2_metadata_payload WHERE generation=2" + ) + .await, + 1 + ); + assert_eq!( + count( + &q, + "SELECT count(*) FROM mst2_metadata_lifetime WHERE generation=1 AND state='REMOVED'" + ) + .await, + 1 + ); + assert_eq!( + count( + &q, + "SELECT count(*) FROM mst2_metadata_lifetime WHERE generation=2 AND state='LIVE'" + ) + .await, + 1 + ); + assert_eq!( + count(&q, "SELECT count(*) FROM mst2_metadata_page_certificate").await, + 2 + ); + assert_eq!( + count( + &q, + "SELECT count(*) FROM mst2_metadata_current WHERE generation=2" + ) + .await, + 1 + ); +} + +#[tokio::test] +async fn pending_gc_survives_rebuild_and_payload_stamp_cannot_commit_alone() { + let (config, core, namespace, q, _guard) = fixture().await; + let (plan, payload) = seeded_rooted_plan(&core, 'a', "file", 1).await; + let writer = RootedQualifiedMetadataRepository::open(&core, &config) + .await + .unwrap(); + let intent = write_rooted(&writer, "gc-pending", &plan, payload).await; + age_orphan(&q, &namespace, intent.prepare_id()).await; + assert_eq!(writer.maintenance_tick(1).await.unwrap().orphans_retired, 1); + let operation = claim(&q, &intent).await.unwrap(); + assert_eq!( + count( + &q, + "SELECT count(*) FROM mst2_metadata_gc_op WHERE state='PENDING'" + ) + .await, + 1 + ); + let txn = q.begin().await.unwrap(); + txn.execute_raw(Statement::from_sql_and_values(DbBackend::Postgres, + "UPDATE mst2_metadata_gc_op SET payload_delete_xid=txid_current() WHERE operation_id=$1::uuid", + [operation.into()])).await.unwrap(); + let error = txn.commit().await.unwrap_err(); + assert!( + error.to_string().contains("cannot escape its atomic apply"), + "{error}" + ); + assert_eq!( + count(&q, "SELECT count(*) FROM mst2_metadata_payload").await, + 1 + ); + assert_eq!(count(&q,"SELECT count(*) FROM mst2_metadata_gc_op WHERE state='PENDING' AND payload_delete_xid IS NULL").await,1); + drop(writer); + let rebuilt = RootedQualifiedMetadataRepository::open(&core, &config) + .await + .unwrap(); + let work = rebuilt.maintenance_tick(1).await.unwrap(); + assert_eq!(work.examined, 1); + assert_eq!(work.pending_replayed, 1); + assert_eq!(work.gc_applied, 1); + assert_eq!(work.payload_pages_removed, 1); + assert_eq!( + count(&q, "SELECT count(*) FROM mst2_metadata_payload").await, + 0 + ); + assert_eq!( + count( + &q, + "SELECT count(*) FROM mst2_metadata_gc_op WHERE state='APPLIED'" + ) + .await, + 1 + ); +} + +#[tokio::test] +async fn actual_incoming_edge_blocks_child_gc_until_parent_removal() { + let (config, core, namespace, q, _guard) = fixture().await; + let (child, payload) = seeded_rooted_plan(&core, 'a', "file", 1).await; + let writer = RootedQualifiedMetadataRepository::open(&core, &config) + .await + .unwrap(); + let child_intent = write_rooted(&writer, "gc-edge-child", &child, payload).await; + let hint = writer + .lookup_reuse(&child.identity.tagged_root_tree_oid, &child.identity) + .await + .unwrap() + .unwrap(); + let entries = [ + Entry::dir(b"one", child.root), + Entry::dir(b"two", child.root), + ]; + let bytes = Page::build(&entries).unwrap(); + let root = page_id(&bytes); + let mut body = b"40000 one\0".to_vec(); + body.extend_from_slice(&[0xaa; 20]); + body.extend_from_slice(b"40000 two\0"); + body.extend_from_slice(&[0xaa; 20]); + core.execute_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + "INSERT INTO mega_tree(id,tree_id,sub_trees,size,created_at,pack_id,pack_offset,commit_id) + VALUES(2,$1,$2,0,now(),'fixture',0,'fixture')", + ["d".repeat(40).into(), body.into()], + )) + .await + .unwrap(); + let mut identity = child.identity.clone(); + identity.tagged_root_tree_oid = format!("sha1:{}", "d".repeat(40)); + let parent = RootedMetadataInstallPlan::new( + identity.clone(), + root, + BTreeMap::from([(root, bytes.len() as u64)]), + BTreeSet::from([(root, child.root)]), + BTreeMap::from([(child.root, hint.proof)]), + BTreeMap::from([ + (identity.tagged_root_tree_oid, root), + (child.identity.tagged_root_tree_oid.clone(), child.root), + ]), + ) + .unwrap(); + let parent_intent = write_rooted( + &writer, + "gc-edge-parent", + &parent, + MetadataPagePayload { + id: root, + size: bytes.len() as u64, + bytes, + }, + ) + .await; + age_orphan(&q, &namespace, child_intent.prepare_id()).await; + age_orphan(&q, &namespace, parent_intent.prepare_id()).await; + for _ in 0..2 { + assert_eq!(writer.maintenance_tick(1).await.unwrap().orphans_retired, 1); + } + assert_eq!( + count(&q, "SELECT count(*) FROM mst2_metadata_root_anchor").await, + 0 + ); + assert_eq!( + count(&q, "SELECT count(*) FROM mst2_metadata_graph_edge").await, + 1 + ); + assert_eq!( + count( + &q, + "SELECT sum(incoming_refs)::bigint FROM mst2_metadata_graph_node" + ) + .await, + 1 + ); + assert!( + claim(&q, &child_intent) + .await + .unwrap_err() + .to_string() + .contains("incoming edges") + ); + let operation = claim(&q, &parent_intent).await.unwrap(); + q.execute_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + "SELECT mst2_metadata_gc_apply($1::uuid)", + [operation.into()], + )) + .await + .unwrap(); + assert_eq!( + count(&q, "SELECT count(*) FROM mst2_metadata_graph_edge").await, + 0 + ); + assert_eq!( + count( + &q, + "SELECT sum(incoming_refs)::bigint FROM mst2_metadata_graph_node" + ) + .await, + 0 + ); + let operation = claim(&q, &child_intent).await.unwrap(); + q.execute_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + "SELECT mst2_metadata_gc_apply($1::uuid)", + [operation.into()], + )) + .await + .unwrap(); + assert_eq!( + count(&q, "SELECT count(*) FROM mst2_metadata_payload").await, + 0 + ); + assert_eq!( + count(&q, "SELECT count(*) FROM mst2_metadata_verified_ref").await, + 2 + ); +} + +#[tokio::test] +async fn restored_guards_refuse_corrupt_payload_size_counter_and_current_bindings() { + for case in 0..4 { + let (config, core, namespace, q, _guard) = fixture().await; + let (plan, payload) = seeded_rooted_plan(&core, 'a', "file", 1).await; + let writer = RootedQualifiedMetadataRepository::open(&core, &config) + .await + .unwrap(); + let intent = write_rooted(&writer, "gc-corruption", &plan, payload).await; + age_orphan(&q, &namespace, intent.prepare_id()).await; + assert_eq!(writer.maintenance_tick(1).await.unwrap().orphans_retired, 1); + let (table, guard, statement) = match case { + 0 => ( + "mst2_metadata_payload", + "mst2_metadata_payload_fenced", + "UPDATE mst2_metadata_payload SET payload=set_byte(payload,20,255)", + ), + 1 => ( + "mst2_metadata_graph_node", + "mst2_metadata_graph_node_guard", + "UPDATE mst2_metadata_graph_node SET bytes=bytes+1", + ), + 2 => ( + "mst2_metadata_graph_node", + "mst2_metadata_graph_node_guard", + "UPDATE mst2_metadata_graph_node SET incoming_refs=1", + ), + _ => ( + "mst2_metadata_current", + "mst2_metadata_current_guard", + "DELETE FROM mst2_metadata_current", + ), + }; + let txn = q.begin().await.unwrap(); + for trigger in ["mst2_00_family_barrier", guard] { + txn.execute_unprepared(&format!("ALTER TABLE {table} DISABLE TRIGGER {trigger}")) + .await + .unwrap(); + } + txn.execute_unprepared(statement).await.unwrap(); + for trigger in ["mst2_00_family_barrier", guard] { + txn.execute_unprepared(&format!("ALTER TABLE {table} ENABLE TRIGGER {trigger}")) + .await + .unwrap(); + } + txn.commit().await.unwrap(); + assert_eq!( + catalog(&q, namespace.core_oid, namespace.schema_oid) + .await + .unwrap(), + namespace.catalog_fingerprint + ); + let error = claim(&q, &intent).await.unwrap_err(); + assert!( + error.to_string().contains(if case == 0 { + "durable bytes" + } else if case < 3 { + "actual counter" + } else { + "current generation" + }), + "{error}" + ); + assert_eq!( + count(&q, "SELECT count(*) FROM mst2_metadata_gc_op").await, + 0 + ); + assert_eq!( + count(&q, "SELECT count(*) FROM mst2_metadata_payload").await, + 1 + ); + assert_eq!( + count( + &q, + "SELECT count(*) FROM mst2_metadata_lifetime WHERE state='LIVE'" + ) + .await, + 1 + ); + } +} diff --git a/src/jupiter/storage/qualified_metadata_reader.rs b/src/jupiter/storage/qualified_metadata_reader.rs new file mode 100644 index 00000000..63ccbdcd --- /dev/null +++ b/src/jupiter/storage/qualified_metadata_reader.rs @@ -0,0 +1,961 @@ +//! Persisted routes traverse exact certified occurrences, including reused roots. + +use std::collections::{BTreeSet, HashMap}; + +use mst2_codec::metapage::{HEADER_LEN, PAGE_MAX_BYTES}; +use serde::Deserialize; + +use super::*; +use crate::{ + ceres::snapshot::{ + retention_dag::MetadataDagLimits, runtime::SnapshotContext, + view::validate_scope_relative_path, + }, + jupiter::storage::native_snapshot_session::{ + MetadataRouteRequest, PersistedMetadataReadWork, PersistedMetadataRouteBatch, + }, +}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct Binding { + generation: i64, + certificate: [u8; 32], +} + +#[cfg(test)] +tokio::task_local! { + static READER_ADMISSION_BARRIERS: (Arc, Arc); + static SOURCE_FACT_BARRIERS: (Arc, Arc); + pub(super) static READER_TEMP_SOURCE_SHADOW: bool; +} + +#[cfg(test)] +pub(crate) async fn with_rooted_source_fact_barriers( + admitted: Arc, + resume: Arc, + future: F, +) -> F::Output { + SOURCE_FACT_BARRIERS.scope((admitted, resume), future).await +} + +#[cfg(test)] +pub(crate) async fn with_rooted_source_temporary_shadow( + future: F, +) -> F::Output { + READER_TEMP_SOURCE_SHADOW.scope(true, future).await +} + +#[cfg(test)] +pub(crate) async fn with_rooted_reader_barriers( + admitted: Arc, + resume: Arc, + future: F, +) -> F::Output { + READER_ADMISSION_BARRIERS + .scope((admitted, resume), future) + .await +} + +#[derive(Debug, Deserialize)] +struct Reference { + kind: String, + child: String, + generation: i64, + certificate: String, + name: Option, + label: Option, + count: Option, +} + +fn digest_hex(value: &str) -> Result<[u8; 32], SnapshotError> { + hex::decode(value) + .map_err(internal)? + .as_slice() + .try_into() + .map_err(internal) +} +fn limit(message: &str) -> SnapshotError { + SnapshotError::new( + crate::ceres::snapshot::error::SnapshotErrorCode::LimitExceeded, + message, + ) +} +fn absent(message: &str) -> SnapshotError { + SnapshotError::new( + crate::ceres::snapshot::error::SnapshotErrorCode::PathNotFound, + message, + ) +} + +struct StoredPage { + bytes: Vec, + page: Page, + entries: u64, +} + +pub(crate) struct RootedDirectoryWindow { + pub directory_root: [u8; 32], + pub entry_count: u64, + pub entries: Vec, + pub has_more: bool, + pub proof_pages: Vec<([u8; 32], Vec)>, + pub ancestors: Vec<(String, [u8; 32])>, +} + +pub(crate) enum RootedLookupStatus { + Directory([u8; 32]), + File { + entry: mst2_codec::metapage::Entry, + git_oid: String, + }, + Absent, + NotDirectory { + symlink: bool, + }, +} +pub(crate) struct RootedLookupBatch { + pub results: Vec, + pub proof_pages: Vec<([u8; 32], Vec)>, +} +struct Reader<'a> { + txn: &'a DatabaseTransaction, + operation: uuid::Uuid, + bindings: HashMap<[u8; 32], Binding>, + cache: HashMap<[u8; 32], StoredPage>, + work: PersistedMetadataReadWork, + limits: MetadataDagLimits, + directories: BTreeMap, + source_ids: BTreeMap, + file_oids: HashMap<(uuid::Uuid, Vec), String>, +} + +impl RootedQualifiedMetadataRepository { + pub(crate) async fn fixed_path_metadata( + &self, + pinned: &SnapshotContext, + path: &str, + ) -> Result { + validate_scope_relative_path(path)?; + let absolute = if pinned.built.descriptor.scope == "/" { + path.to_owned() + } else if path == "/" { + pinned.built.descriptor.scope.clone() + } else { + format!("{}{}", pinned.built.descriptor.scope, path) + }; + validate_scope_relative_path(&absolute)?; + let (operation, binding, source) = self.admit_reader(pinned).await?; + let result = async { + let txn = self.read_transaction().await?; + let root = pinned.built.descriptor.metadata_root; + let mut reader = Reader::new(&txn, operation, root, binding, source); + let result = reader.lookup(root, path).await; + sessions::finish(txn, result).await + } + .await; + self.finish_reader(operation, result).await + } + + pub(crate) async fn directory_window( + &self, + pinned: &SnapshotContext, + path: &str, + after: Option<&str>, + count: usize, + ) -> Result { + validate_scope_relative_path(path)?; + if !(1..=256).contains(&count) { + return Err(limit("directory limit must be 1..256")); + } + let absolute = if pinned.built.descriptor.scope == "/" { + path.to_owned() + } else if path == "/" { + pinned.built.descriptor.scope.clone() + } else { + format!("{}{}", pinned.built.descriptor.scope, path) + }; + validate_scope_relative_path(&absolute)?; + let (operation, binding, source) = self.admit_reader(pinned).await?; + let result = async { + let txn = self.read_transaction().await?; + let read = async { + let root = pinned.built.descriptor.metadata_root; + let mut reader = Reader::new(&txn, operation, root, binding, source); + let directory = reader.directory(root, path).await?; + reader.load(directory).await?; + let total = reader.cache[&directory].entries; + let mut entries = Vec::with_capacity(count + 1); + reader + .range(directory, after.map(str::as_bytes), count + 1, &mut entries) + .await?; + let has_more = entries.len() > count; + entries.truncate(count); + reader + .source_entries(reader.source_ids[path], directory, &entries) + .await?; + Ok(RootedDirectoryWindow { + directory_root: directory, + entry_count: total, + entries, + has_more, + proof_pages: reader.proofs()?, + ancestors: reader.directories.into_iter().collect(), + }) + } + .await; + sessions::finish(txn, read).await + } + .await; + self.finish_reader(operation, result).await + } + + pub(crate) async fn lookup_metadata( + &self, + pinned: &SnapshotContext, + paths: &[String], + ) -> Result { + if paths.len() > 128 { + return Err(limit("at most 128 paths per lookup")); + } + for path in paths { + validate_scope_relative_path(path)?; + let absolute = if pinned.built.descriptor.scope == "/" { + path.clone() + } else if path == "/" { + pinned.built.descriptor.scope.clone() + } else { + format!("{}{}", pinned.built.descriptor.scope, path) + }; + validate_scope_relative_path(&absolute)?; + } + let (operation, binding, source) = self.admit_reader(pinned).await?; + let result = async { + let txn = self.read_transaction().await?; + let read = async { + let root = pinned.built.descriptor.metadata_root; + let mut reader = Reader::new(&txn, operation, root, binding, source); + let mut results = Vec::with_capacity(paths.len()); + for path in paths { + results.push(reader.lookup(root, path).await?); + } + Ok(RootedLookupBatch { + results, + proof_pages: reader.proofs()?, + }) + } + .await; + sessions::finish(txn, read).await + } + .await; + self.finish_reader(operation, result).await + } + pub(crate) async fn metadata_routes( + &self, + pinned: &SnapshotContext, + requests: &[MetadataRouteRequest<'_>], + ) -> Result { + if requests.is_empty() || requests.len() > 64 { + return Err(limit("items must hold 1..64 entries")); + } + for request in requests { + validate_scope_relative_path(request.directory_path)?; + let scope = &pinned.built.descriptor.scope; + let absolute = if scope == "/" { + request.directory_path.to_owned() + } else if request.directory_path == "/" { + scope.clone() + } else { + format!("{scope}{}", request.directory_path) + }; + validate_scope_relative_path(&absolute)?; + } + // Acquire REQUEST and READER ownership atomically, then release the + // mutation barrier before fetching payloads. Cleanup never precedes + // ownership of the returned byte buffers. + let (operation, binding, source) = self.admit_reader(pinned).await?; + let result = self + .read_routes(pinned, requests, operation, binding, source) + .await; + self.finish_reader(operation, result).await + } + + async fn admit_reader( + &self, + pinned: &SnapshotContext, + ) -> Result<(uuid::Uuid, Binding, uuid::Uuid), SnapshotError> { + let txn = self.transaction().await?; + let admitted = async { + let row = self + .session_row( + &txn, + &pinned.built.snapshot_id, + &pinned.lease_id, + &pinned.built.instance_id, + ) + .await?; + let current = sessions::context( + &row, + &pinned.built.snapshot_id, + &pinned.lease_id, + &pinned.built.instance_id, + )?; + if current.built.descriptor != pinned.built.descriptor + || current.commit_oid != pinned.commit_oid + || current.root_tree_oid != pinned.root_tree_oid + || current.authorization_epoch != pinned.authorization_epoch + { + return Err(integrity( + "qualified fixed session changed during reader admission", + )); + } + let reader = txn + .query_one_raw(sql( + "SELECT operation_id::text,root_generation,certificate_digest + FROM mst2_metadata_begin_reader($1,$2,$3)", + [ + pinned.built.snapshot_id.clone().into(), + pinned.lease_id.clone().into(), + pinned.built.instance_id.clone().into(), + ], + )) + .await + .map_err(database_error)? + .ok_or_else(|| unavailable("qualified reader admission returned no owned root"))?; + Ok(( + uuid::Uuid::parse_str( + &reader + .try_get::("", "operation_id") + .map_err(internal)?, + ) + .map_err(internal)?, + Binding { + generation: reader.try_get("", "root_generation").map_err(internal)?, + certificate: digest_column(&reader, "certificate_digest")?, + }, + row.try_get("", "attestation_id").map_err(internal)?, + )) + } + .await; + let owned = sessions::finish(txn, admitted).await?; + #[cfg(test)] + if let Ok((admitted, resume)) = READER_ADMISSION_BARRIERS.try_with(Clone::clone) { + admitted.wait().await; + resume.wait().await; + } + Ok(owned) + } + + async fn finish_reader( + &self, + operation: uuid::Uuid, + result: Result, + ) -> Result { + let cleanup = self.transaction().await; + let cleanup = match cleanup { + Ok(txn) => { + let finished = txn + .execute_raw(sql( + "SELECT mst2_metadata_finish_reader($1::uuid)", + [operation.to_string().into()], + )) + .await + .map_err(internal) + .map(|_| ()); + sessions::finish(txn, finished).await + } + Err(error) => Err(error), + }; + // A failed cleanup leaves durable roots until the bounded hard deadline. + // Report the failure instead of pretending that a protected operation + // has been definitively finished. + match (result, cleanup) { + (Err(error), _) => Err(error), + (Ok(_), Err(error)) => Err(error), + (Ok(value), Ok(())) => Ok(value), + } + } + + async fn read_routes( + &self, + pinned: &SnapshotContext, + requests: &[MetadataRouteRequest<'_>], + operation: uuid::Uuid, + binding: Binding, + source: uuid::Uuid, + ) -> Result { + let txn = self.read_transaction().await?; + let result = async { + let root = pinned.built.descriptor.metadata_root; + let mut reader = Reader::new(&txn, operation, root, binding, source); + let mut seen = BTreeSet::new(); + let mut pages = Vec::new(); + for request in requests { + let directory = reader.directory(root, request.directory_path).await?; + let route = reader.route(directory, request.route).await?; + let reached = route + .last() + .ok_or_else(|| integrity("qualified metadata route is empty"))?; + if let Some(expected) = request.expected_digest + && expected != format!("sha256:{}", hex::encode(reached)) + { + return Err(SnapshotError::new( + crate::ceres::snapshot::error::SnapshotErrorCode::DigestMismatch, + "qualified metadata route does not reach expected_digest", + )); + } + for page in route { + if seen.insert(page) { + pages.push((page, reader.cache[&page].bytes.clone())); + } + } + } + Ok(PersistedMetadataRouteBatch { + pages, + work: reader.work, + }) + } + .await; + sessions::finish(txn, result).await + } +} + +impl Reader<'_> { + fn new( + txn: &DatabaseTransaction, + operation: uuid::Uuid, + root: [u8; 32], + binding: Binding, + source: uuid::Uuid, + ) -> Reader<'_> { + Reader { + txn, + operation, + bindings: HashMap::from([(root, binding)]), + cache: HashMap::new(), + work: PersistedMetadataReadWork::default(), + limits: MetadataDagLimits::default(), + directories: BTreeMap::from([("/".into(), root)]), + source_ids: BTreeMap::from([("/".into(), source)]), + file_oids: HashMap::new(), + } + } + + async fn source_entries( + &mut self, + source: uuid::Uuid, + directory: [u8; 32], + entries: &[mst2_codec::metapage::Entry], + ) -> Result, uuid::Uuid>, SnapshotError> { + let binding = self + .bindings + .get(&directory) + .ok_or_else(|| integrity("qualified source directory has no certified binding"))?; + let names: Vec<_> = entries + .iter() + .map(|entry| hex::encode(&entry.name)) + .collect(); + let rows = self.txn.query_all_raw(sql("SELECT * FROM mst2_metadata_read_source_entries($1::uuid,$2::uuid,$3,$4,$5,$6::jsonb)", + [self.operation.to_string().into(), source.to_string().into(), directory.to_vec().into(), + binding.generation.into(), binding.certificate.to_vec().into(),json!(names).into()])).await.map_err(database_error)?; + #[cfg(test)] + if entries.iter().any(|entry| !entry.is_dir()) + && let Ok((admitted, resume)) = SOURCE_FACT_BARRIERS.try_with(Clone::clone) + { + admitted.wait().await; + resume.wait().await; + } + if rows.len() != entries.len() { + return Err(integrity( + "qualified source-name index differs from selected actual page entries", + )); + } + let mut children = HashMap::new(); + for row in rows { + let name: Vec = row.try_get("", "name").map_err(internal)?; + let entry = entries + .iter() + .find(|entry| entry.name == name) + .ok_or_else(|| { + integrity("qualified source-name read returned an unrequested occurrence") + })?; + let state: String = row.try_get("", "fact_state").map_err(internal)?; + match state.as_str() { + "READY" => {} + "MISSING" => { + return Err(SnapshotError::new( + crate::ceres::snapshot::error::SnapshotErrorCode::MetadataNotReady, + "selected fixed file has no current verified object metadata", + )); + } + "SOURCE_UNAVAILABLE" => { + return Err(unavailable( + "selected fixed directory has no current exact source attestation", + )); + } + _ => { + return Err(integrity( + "selected fixed file has invalid or different current verified object metadata", + )); + } + } + let kind: i16 = row.try_get("", "kind").map_err(internal)?; + if kind as u8 != entry.kind as u8 { + return Err(integrity( + "qualified selected occurrence changed its fixed filesystem kind", + )); + } + if entry.is_dir() { + let child = self.bindings.get(&entry.child_root).ok_or_else(|| { + integrity("qualified source child has no certified occurrence") + })?; + let root = digest_column(&row, "child_root")?; + if root != entry.child_root + || child.generation + != row + .try_get::("", "child_generation") + .map_err(internal)? + || child.certificate != digest_column(&row, "child_certificate_digest")? + { + return Err(integrity( + "qualified source-name directory differs from its exact certified lifetime", + )); + } + children.insert( + name.clone(), + row.try_get("", "child_attestation_id").map_err(internal)?, + ); + } else if row.try_get::("", "byte_size").map_err(internal)? as u64 != entry.size + || digest_column(&row, "content_digest")? != entry.content_id + { + return Err(integrity( + "qualified source-name file differs from its certified actual page", + )); + } + if !entry.is_dir() { + self.file_oids.insert( + (source, name), + row.try_get("", "git_oid").map_err(internal)?, + ); + } + } + Ok(children) + } + + fn proofs(&self) -> Result)>, SnapshotError> { + let bytes: usize = self.cache.values().map(|page| page.bytes.len()).sum(); + if bytes > 1_048_576 { + return Err(SnapshotError::new( + crate::ceres::snapshot::error::SnapshotErrorCode::ProofBudgetExceeded, + "qualified proof pages exceed the response budget; use metadata/pages", + )); + } + let mut pages: Vec<_> = self + .cache + .iter() + .map(|(id, page)| (*id, page.bytes.clone())) + .collect(); + pages.sort_by_key(|item| item.0); + Ok(pages) + } + + async fn range( + &mut self, + id: [u8; 32], + after: Option<&[u8]>, + count: usize, + out: &mut Vec, + ) -> Result<(), SnapshotError> { + if out.len() >= count { + return Ok(()); + } + self.load(id).await?; + match self.cache[&id].page.clone() { + Page::Leaf { entries } => { + let start = after + .map(|name| entries.partition_point(|entry| entry.name.as_slice() <= name)) + .unwrap_or(0); + out.extend(entries.into_iter().skip(start).take(count - out.len())); + } + Page::Branch { + prefix, + terminal, + children, + } => { + if let Some(entry) = terminal + && after.is_none_or(|name| entry.name.as_slice() > name) + { + out.push(entry); + } + for child in children { + if out.len() >= count { + break; + } + let mut partition = prefix.clone(); + partition.push(child.label); + if after.is_some_and(|name| { + partition.as_slice() < name && !name.starts_with(&partition) + }) { + continue; + } + self.descend(id, child.label).await?; + Box::pin(self.range(child.child_page_id, after, count, out)).await?; + } + } + } + Ok(()) + } + + async fn find_entry( + &mut self, + mut id: [u8; 32], + name: &[u8], + ) -> Result, SnapshotError> { + loop { + self.load(id).await?; + match &self.cache[&id].page { + Page::Leaf { entries } => { + return Ok(entries.iter().find(|entry| entry.name == name).cloned()); + } + Page::Branch { + prefix, + terminal, + children, + } => { + if name == prefix { + return Ok(terminal.clone()); + } + if !name.starts_with(prefix) { + return Ok(None); + } + let Some(label) = name.get(prefix.len()).copied() else { + return Ok(None); + }; + if !children.iter().any(|child| child.label == label) { + return Ok(None); + } + id = self.descend(id, label).await?; + } + } + } + } + + async fn lookup( + &mut self, + mut root: [u8; 32], + path: &str, + ) -> Result { + if path == "/" { + self.load(root).await?; + return Ok(RootedLookupStatus::Directory(root)); + } + let mut prefix = String::new(); + let mut source = self.source_ids["/"]; + let mut parts = path[1..].split('/').peekable(); + while let Some(name) = parts.next() { + let Some(entry) = self.find_entry(root, name.as_bytes()).await? else { + return Ok(RootedLookupStatus::Absent); + }; + if !entry.is_dir() { + if parts.peek().is_some() { + return Ok(RootedLookupStatus::NotDirectory { + symlink: entry.kind == mst2_codec::metapage::EntryKind::Symlink, + }); + } + self.source_entries(source, root, std::slice::from_ref(&entry)) + .await?; + let git_oid = self + .file_oids + .get(&(source, entry.name.clone())) + .cloned() + .ok_or_else(|| { + integrity("selected fixed file has no exact source OID binding") + })?; + return Ok(RootedLookupStatus::File { entry, git_oid }); + } + let children = self + .source_entries(source, root, std::slice::from_ref(&entry)) + .await?; + source = *children.get(&entry.name).ok_or_else(|| { + integrity("qualified named directory has no independently derived source child") + })?; + root = entry.child_root; + prefix.push('/'); + prefix.push_str(name); + self.directories.insert(prefix.clone(), root); + self.source_ids.insert(prefix.clone(), source); + } + self.load(root).await?; + Ok(RootedLookupStatus::Directory(root)) + } + async fn load(&mut self, id: [u8; 32]) -> Result<(), SnapshotError> { + self.work.walk_visits += 1; + if self.work.walk_visits > self.limits.prepare_entry_visits as u64 { + return Err(limit("qualified route work budget exceeded")); + } + if self.cache.contains_key(&id) { + return Ok(()); + } + if self.cache.len() >= self.limits.nodes { + return Err(limit("qualified route page budget exceeded")); + } + let binding = *self.bindings.get(&id).ok_or_else(|| { + integrity("qualified page was not reached through a certified occurrence") + })?; + self.work.page_queries += 1; + let row = self + .txn + .query_one_raw(sql( + PAGE_SQL, + [ + id.to_vec().into(), + binding.generation.into(), + binding.certificate.to_vec().into(), + self.operation.to_string().into(), + ], + )) + .await + .map_err(internal)? + .ok_or_else(|| { + unavailable("qualified certified page or reader ownership is unavailable") + })?; + let bytes: Vec = row.try_get("", "payload").map_err(internal)?; + let size: i32 = row.try_get("", "byte_size").map_err(internal)?; + if !(HEADER_LEN..=PAGE_MAX_BYTES).contains(&bytes.len()) + || bytes.len() != size as usize + || page_id(&bytes) != id + { + return Err(integrity( + "qualified durable metadata bytes differ from their certified page", + )); + } + let (page, entries) = Page::decode(&bytes).map_err(internal)?; + self.work.payload_bytes = self + .work + .payload_bytes + .checked_add(bytes.len() as u64) + .filter(|value| *value <= self.limits.payload_bytes) + .ok_or_else(|| limit("qualified route byte budget exceeded"))?; + let raw_refs: serde_json::Value = row.try_get("", "references").map_err(internal)?; + let refs: Vec = serde_json::from_value(raw_refs).map_err(internal)?; + if refs.len() > 257 { + return Err(integrity("qualified canonical reference count is invalid")); + } + let mut expected = Vec::new(); + let direct = match &page { + Page::Leaf { entries } => entries.as_slice(), + Page::Branch { + terminal, children, .. + } => { + for child in children { + expected.push(( + "RADIX", + child.child_page_id, + None, + Some(child.label), + Some(child.subtree_entries), + )); + } + terminal.as_slice() + } + }; + // Reference ordering is direct DIRECTORY occurrences followed by RADIX, + // as defined by the independent canonical parser. Compare occurrences + // as a set here; SQL validates the certificate's ordinal completeness. + for entry in direct.iter().filter(|entry| entry.is_dir()) { + expected.push(( + "DIRECTORY", + entry.child_root, + Some(hex::encode(&entry.name)), + None, + None, + )); + } + let mut actual = BTreeSet::new(); + for reference in refs { + let child = digest_hex(&reference.child)?; + let binding = Binding { + generation: reference.generation, + certificate: digest_hex(&reference.certificate)?, + }; + if binding.generation <= 0 + || self + .bindings + .get(&child) + .is_some_and(|known| *known != binding) + { + return Err(integrity( + "qualified occurrence retargeted its exact lifetime or certificate", + )); + } + self.bindings.insert(child, binding); + if !actual.insert(( + reference.kind, + child, + reference.name, + reference.label, + reference.count, + )) { + return Err(integrity("qualified typed occurrence has a duplicate")); + } + } + let expected: BTreeSet<_> = expected + .into_iter() + .map(|(kind, child, name, label, count)| (kind.to_owned(), child, name, label, count)) + .collect(); + if actual != expected { + return Err(integrity( + "qualified certificate references differ from its actual durable page", + )); + } + self.work.edge_references_checked += actual.len() as u64; + if self.work.edge_references_checked > self.limits.edges as u64 { + return Err(limit("qualified route reference budget exceeded")); + } + self.work.pages_loaded += 1; + self.cache.insert( + id, + StoredPage { + bytes, + page, + entries, + }, + ); + Ok(()) + } + + async fn descend(&mut self, parent: [u8; 32], label: u8) -> Result<[u8; 32], SnapshotError> { + self.load(parent).await?; + let (prefix, child) = match &self.cache[&parent].page { + Page::Branch { + prefix, children, .. + } => ( + prefix.clone(), + children + .iter() + .find(|child| child.label == label) + .ok_or_else(|| absent("route label is absent in fixed qualified metadata"))? + .clone(), + ), + Page::Leaf { .. } => return Err(absent("route descends past a qualified leaf")), + }; + let id = child.child_page_id; + self.load(id).await?; + let received = &self.cache[&id]; + let mut partition = prefix; + partition.push(label); + let valid = match &received.page { + Page::Leaf { entries } => entries + .iter() + .all(|entry| entry.name.starts_with(&partition)), + Page::Branch { prefix, .. } => prefix.starts_with(&partition), + }; + if !valid || received.entries != child.subtree_entries { + return Err(integrity( + "qualified radix partition differs from its exact child", + )); + } + Ok(id) + } + + async fn directory( + &mut self, + mut root: [u8; 32], + path: &str, + ) -> Result<[u8; 32], SnapshotError> { + if path == "/" { + return Ok(root); + } + let mut directory_path = String::new(); + let mut source = self.source_ids["/"]; + for name in path[1..].split('/') { + let mut id = root; + loop { + self.load(id).await?; + let entry = match &self.cache[&id].page { + Page::Leaf { entries } => entries + .iter() + .find(|entry| entry.name == name.as_bytes()) + .cloned(), + Page::Branch { + prefix, terminal, .. + } => { + if name.as_bytes() == prefix { + terminal.clone() + } else { + if !name.as_bytes().starts_with(prefix) { + return Err(absent("name is absent in qualified directory")); + } + let label = + name.as_bytes().get(prefix.len()).copied().ok_or_else(|| { + absent("name is absent in qualified directory") + })?; + id = self.descend(id, label).await?; + continue; + } + } + } + .ok_or_else(|| absent("name is absent in qualified directory"))?; + if !entry.is_dir() { + return Err(SnapshotError::new( + crate::ceres::snapshot::error::SnapshotErrorCode::NotDirectory, + "fixed qualified path is not a directory", + )); + } + let children = self + .source_entries(source, root, std::slice::from_ref(&entry)) + .await?; + source = *children.get(&entry.name).ok_or_else(|| { + integrity("qualified directory route lost its exact source-name binding") + })?; + root = entry.child_root; + directory_path.push('/'); + directory_path.push_str(name); + self.directories.insert(directory_path.clone(), root); + self.source_ids.insert(directory_path.clone(), source); + break; + } + } + Ok(root) + } + async fn route( + &mut self, + root: [u8; 32], + labels: &[u8], + ) -> Result, SnapshotError> { + self.load(root).await?; + let mut pages = vec![root]; + let mut current = root; + for label in labels { + current = self.descend(current, *label).await?; + pages.push(current); + } + Ok(pages) + } +} + +const PAGE_SQL:&str="SELECT body.payload,body.byte_size,proof.canonical_proof->'references' AS references + FROM mst2_metadata_page_certificate proof JOIN mst2_metadata_current cur USING(page_id,generation) + JOIN mst2_metadata_lifetime life USING(page_id,generation) JOIN mst2_metadata_graph_node node USING(page_id,generation) + JOIN mst2_metadata_payload body USING(page_id,generation) + WHERE proof.page_id=$1 AND proof.generation=$2 AND proof.certificate_digest=$3 + AND proof.namespace_uuid=(SELECT namespace_uuid FROM mst2_metadata_family_identity WHERE singleton=1) + AND life.state='LIVE' AND life.graph_domain='qualified-v1' AND node.state='LIVE' + AND node.certificate_digest=proof.certificate_digest AND node.bytes=proof.byte_size + AND life.expected_size=proof.byte_size AND body.byte_size=proof.byte_size AND body.metadata_codec=1 + AND octet_length(body.payload)=proof.byte_size AND NOT EXISTS(SELECT 1 FROM mst2_metadata_gc_op gc + WHERE gc.page_id=proof.page_id AND gc.generation=proof.generation) + AND EXISTS(SELECT 1 FROM mst2_metadata_reader_operation reader JOIN mst2_metadata_root_anchor anchor + ON anchor.reader_operation_id=reader.operation_id AND anchor.anchor_kind='READER' + AND anchor.root_page=reader.root_page AND anchor.root_generation=reader.root_generation + WHERE reader.operation_id=$4::uuid AND reader.state='ACTIVE' + AND reader.hard_deadline_unix>floor(extract(epoch FROM clock_timestamp()))::bigint) + AND (SELECT count(*) FROM mst2_metadata_verified_ref ref WHERE ref.parent_page=proof.page_id + AND ref.parent_generation=proof.generation)=jsonb_array_length(proof.canonical_proof->'references') + AND NOT EXISTS((SELECT ref.child_page,ref.child_generation FROM mst2_metadata_verified_ref ref + WHERE ref.parent_page=proof.page_id AND ref.parent_generation=proof.generation) EXCEPT + (SELECT edge.child_page,edge.child_generation FROM mst2_metadata_graph_edge edge + WHERE edge.parent_page=proof.page_id AND edge.parent_generation=proof.generation)) + AND NOT EXISTS((SELECT edge.child_page,edge.child_generation FROM mst2_metadata_graph_edge edge + WHERE edge.parent_page=proof.page_id AND edge.parent_generation=proof.generation) EXCEPT + (SELECT ref.child_page,ref.child_generation FROM mst2_metadata_verified_ref ref + WHERE ref.parent_page=proof.page_id AND ref.parent_generation=proof.generation))"; diff --git a/src/jupiter/storage/qualified_metadata_rooted.rs b/src/jupiter/storage/qualified_metadata_rooted.rs new file mode 100644 index 00000000..5dc2f4be --- /dev/null +++ b/src/jupiter/storage/qualified_metadata_rooted.rs @@ -0,0 +1,766 @@ +use std::{collections::BTreeMap, sync::Arc}; + +use mst2_codec::metapage::{Page, page_id}; +use sea_orm::{QueryResult, Value}; +use serde_json::json; + +use super::*; +use crate::ceres::snapshot::{ + metadata_install::MetadataInstallIdentity, + rooted_metadata_install::{RootedMetadataInstallPlan, RootedReuseRoot}, + rooted_metadata_projection::{CertifiedReusableDirectory, RootedReuseLookup}, +}; + +#[path = "qualified_metadata_session.rs"] +mod sessions; + +#[path = "qualified_metadata_gc.rs"] +mod gc; +#[path = "qualified_metadata_reader.rs"] +mod reader; +pub(crate) use reader::{RootedDirectoryWindow, RootedLookupBatch, RootedLookupStatus}; +#[cfg(test)] +pub(crate) use reader::{ + with_rooted_reader_barriers, with_rooted_source_fact_barriers, + with_rooted_source_temporary_shadow, +}; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct RootedPrepareIntent { + prepare_id: String, + operation_id: String, + plan: Arc, + bindings: BTreeMap<[u8; 32], (i64, u64)>, + manifest_digest: [u8; 32], + bindings_digest: [u8; 32], + primary_scope: Vec, + storage_seal: [u8; 32], + root_generation: i64, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct RootedMetadataReceipt { + intent: RootedPrepareIntent, + certificate_digest: [u8; 32], + attestation_id: uuid::Uuid, + attestation_digest: [u8; 32], +} + +impl RootedPrepareIntent { + #[cfg(test)] + pub(crate) fn prepare_id(&self) -> &str { + &self.prepare_id + } + #[cfg(test)] + pub(crate) fn metadata_root(&self) -> [u8; 32] { + self.plan.root + } + #[cfg(test)] + pub(crate) fn root_generation(&self) -> i64 { + self.root_generation + } +} +impl RootedMetadataReceipt { + pub(crate) fn metadata_root(&self) -> [u8; 32] { + self.intent.plan.root + } +} + +pub(crate) struct RootedQualifiedMetadataRepository { + connection: DatabaseConnection, + namespace: VerifiedQualifiedNamespace, + primary_scope: Vec, + maintenance_state: tokio::sync::Mutex, +} + +#[async_trait::async_trait] +impl RootedReuseLookup for RootedQualifiedMetadataRepository { + async fn lookup_reuse( + &self, + tree_oid: &str, + identity: &MetadataInstallIdentity, + ) -> Result, SnapshotError> { + // This is a bounded projection hint. It acquires no writer/retention + // lock and grants no installation authority: begin_intent and finalize + // independently bind the exact attestation and current lifetime. + let kind = identity + .tagged_root_tree_oid + .split_once(':') + .ok_or_else(|| integrity("rooted source identity has no tagged hash kind"))? + .0; + let profile = json!({"source_domain": identity.source_domain,"hash_kind": kind, + "schema_version": identity.schema_version,"metadata_codec": identity.metadata_codec, + "materialization_policy": identity.materialization_policy,"fs_semantics": identity.fs_semantics, + "access_projection": identity.access_projection,"verification_revision": identity.verification_revision, + "projection_revision": identity.projection_revision}); + let q = identifier(&self.namespace.schema); + let c = identifier(&self.namespace.core_schema); + let row = self.connection.query_one_raw(sql(format!( + "SELECT a.root_page,a.root_generation,a.attestation_id::text,a.attestation_digest, + p.certificate_digest,p.relative_path_bytes,p.relative_components,p.closure_nodes_upper, + p.closure_edges_upper,p.closure_bytes_upper,p.closure_entries_upper + FROM {q}.mst2_metadata_reuse_index i JOIN {q}.mst2_metadata_source_root_attestation a + ON a.attestation_id=i.attestation_id AND a.root_page=i.root_page AND a.root_generation=i.root_generation + AND a.attestation_digest=i.attestation_digest + JOIN {q}.mst2_metadata_page_certificate p ON p.page_id=a.root_page AND p.generation=a.root_generation + AND p.certificate_digest=a.root_certificate_digest + JOIN {q}.mst2_metadata_current cur ON cur.page_id=p.page_id AND cur.generation=p.generation + JOIN {q}.mst2_metadata_lifetime life ON life.page_id=p.page_id AND life.generation=p.generation + JOIN {q}.mst2_metadata_graph_node node ON node.page_id=p.page_id AND node.generation=p.generation + JOIN {q}.mst2_metadata_payload body ON body.page_id=p.page_id AND body.generation=p.generation + JOIN {q}.mst2_metadata_prepare origin ON origin.prepare_id=a.origin_prepare_id + JOIN {c}.mega_tree source ON source.tree_id=split_part(a.tagged_tree_oid,':',2) + WHERE i.tagged_tree_oid=$1 AND i.profile_digest=sha256(convert_to('mega.mst2.native-profile.v1','UTF8') + ||decode('00','hex')||convert_to($2::jsonb::text,'UTF8')) AND a.source_profile=$2::jsonb + AND a.namespace_uuid=$3::uuid AND {c}.mst2_route_source_tree_matches(split_part(a.tagged_tree_oid,':',2),a.source_revision,a.source_body_digest) + AND life.state='LIVE' AND life.graph_domain='qualified-v1' AND node.state='LIVE' + AND node.certificate_digest=p.certificate_digest AND node.bytes=p.byte_size + AND body.byte_size=p.byte_size AND body.metadata_codec=p.metadata_codec + AND life.expected_size=p.byte_size AND origin.state='COMMITTED' AND NOT pg_is_in_recovery() + AND NOT EXISTS(SELECT 1 FROM {q}.mst2_metadata_gc_op gc WHERE gc.page_id=p.page_id AND gc.generation=p.generation)" + ),[tree_oid.into(),profile.into(),self.namespace.namespace_uuid.clone().into()])) + .await.map_err(database_error)?; + let Some(row) = row else { + return Ok(None); + }; + let count = |name: &str| -> Result { + usize::try_from(row.try_get::("", name).map_err(internal)?).map_err(internal) + }; + let wide = |name: &str| -> Result { + u64::try_from(row.try_get::("", name).map_err(internal)?).map_err(internal) + }; + Ok(Some(CertifiedReusableDirectory { + page_id: digest_column(&row, "root_page")?, + proof: RootedReuseRoot { + generation: row.try_get("", "root_generation").map_err(internal)?, + attestation_id: uuid::Uuid::parse_str( + &row.try_get::("", "attestation_id") + .map_err(internal)?, + ) + .map_err(internal)?, + attestation_digest: digest_column(&row, "attestation_digest")?, + certificate_digest: digest_column(&row, "certificate_digest")?, + }, + relative_path_bytes: count("relative_path_bytes")?, + relative_components: count("relative_components")?, + closure_nodes_upper: count("closure_nodes_upper")?, + closure_edges_upper: count("closure_edges_upper")?, + closure_bytes_upper: wide("closure_bytes_upper")?, + closure_entries_upper: usize::try_from(wide("closure_entries_upper")?) + .map_err(internal)?, + })) + } +} + +fn sql(text: impl Into, values: impl IntoIterator) -> Statement { + Statement::from_sql_and_values(DbBackend::Postgres, text, values) +} +fn internal(error: impl std::fmt::Display) -> SnapshotError { + SnapshotError::new( + crate::ceres::snapshot::error::SnapshotErrorCode::Internal, + error.to_string(), + ) +} +pub(super) fn is_lock_unavailable(error: &sea_orm::DbErr) -> bool { + let runtime = match error { + sea_orm::DbErr::Exec(runtime) | sea_orm::DbErr::Query(runtime) => runtime, + _ => return false, + }; + let sea_orm::RuntimeErr::SqlxError(sqlx_error) = runtime else { + return false; + }; + let sea_orm::sqlx::Error::Database(database_error) = sqlx_error.as_ref() else { + return false; + }; + database_error.code().as_deref() == Some("55P03") +} +fn database_error(error: sea_orm::DbErr) -> SnapshotError { + if is_lock_unavailable(&error) { + return SnapshotError::new( + crate::ceres::snapshot::error::SnapshotErrorCode::TemporaryUnavailable, + "qualified source is being updated; retry the operation", + ); + } + internal(error) +} +fn integrity(message: &str) -> SnapshotError { + SnapshotError::new( + crate::ceres::snapshot::error::SnapshotErrorCode::IntegrityError, + message, + ) +} +fn unavailable(message: &str) -> SnapshotError { + SnapshotError::new( + crate::ceres::snapshot::error::SnapshotErrorCode::ObjectUnavailable, + message, + ) +} +fn digest_column(row: &QueryResult, name: &str) -> Result<[u8; 32], SnapshotError> { + let bytes: Vec = row.try_get("", name).map_err(internal)?; + bytes.as_slice().try_into().map_err(internal) +} +fn encode_bindings(bindings: &BTreeMap<[u8; 32], (i64, u64)>) -> Vec { + let mut bytes = Vec::with_capacity(12 + 48 * bindings.len()); + bytes.extend_from_slice(b"MST2GEN1"); + bytes.extend_from_slice(&(bindings.len() as u32).to_be_bytes()); + for (page, (generation, size)) in bindings { + bytes.extend_from_slice(page); + bytes.extend_from_slice(&generation.to_be_bytes()); + bytes.extend_from_slice(&size.to_be_bytes()); + } + bytes +} +fn decode_bindings( + bytes: &[u8], + plan: &RootedMetadataInstallPlan, +) -> Result, SnapshotError> { + if bytes.len() != 12 + 48 * plan.delta.len() || bytes.get(..8) != Some(b"MST2GEN1") { + return Err(integrity( + "rooted delta binding encoding differs from its plan", + )); + } + let count = u32::from_be_bytes(bytes[8..12].try_into().map_err(internal)?) as usize; + if count != plan.delta.len() { + return Err(integrity( + "rooted delta binding count differs from its plan", + )); + } + let mut bindings = BTreeMap::new(); + for ((expected, size), record) in plan.delta.iter().zip(bytes[12..].as_chunks::<48>().0) { + let page: [u8; 32] = record[..32].try_into().map_err(internal)?; + let generation = i64::from_be_bytes(record[32..40].try_into().map_err(internal)?); + let recorded_size = u64::from_be_bytes(record[40..48].try_into().map_err(internal)?); + if page != *expected || recorded_size != *size || generation <= 0 { + return Err(integrity("rooted exact delta lifetime binding is invalid")); + } + bindings.insert(page, (generation, recorded_size)); + } + Ok(bindings) +} + +async fn committed( + txn: DatabaseTransaction, + result: Result, + operation: &str, + digest: [u8; 32], + phase: super::super::native_metadata_install::MetadataCommitPhase, +) -> Result { + match result { + Err(error) => { + let _ = txn.rollback().await; + Err(error.into()) + } + Ok(value) => match txn.commit().await { + Ok(()) => Ok(value), + Err(_) => Err(MetadataInstallError::CommitUncertain { + operation_id: operation.into(), + manifest_digest: digest, + phase, + }), + }, + } +} + +impl RootedQualifiedMetadataRepository { + pub(crate) async fn open( + core: &DatabaseConnection, + config: &DbConfig, + ) -> Result { + let captured = captured_core(core).await?; + let namespace = registered(core, &captured) + .await? + .ok_or_else(|| rejected("rooted qualified family is not provisioned"))?; + let mut q_config = config.clone(); + q_config.db_url = pool_url(&config.db_url, &namespace)?; + q_config.max_connection = q_config.max_connection.clamp(1, 4); + q_config.min_connection = 1; + let connection = postgres_connection(&q_config).await?; + let txn = connection + .begin_with_config(Some(IsolationLevel::ReadCommitted), None) + .await?; + namespace + .enter(&txn) + .await + .map_err(|e| rejected(&e.to_string()))?; + let scope:serde_json::Value=txn.query_one_raw(sql( + "SELECT jsonb_build_array(s.storage_uuid,current_database(),d.oid::bigint,current_schema(),n.oid::bigint, + inet_server_addr()::text,inet_server_port()) AS scope FROM mst2_metadata_storage_scope s + JOIN pg_catalog.pg_database d ON d.datname=current_database() + JOIN pg_catalog.pg_namespace n ON n.nspname=current_schema() WHERE s.singleton=1 AND NOT pg_is_in_recovery()",[], + )).await?.ok_or_else(||rejected("rooted captured primary scope is missing"))?.try_get("","scope")?; + let primary_scope = serde_json::to_vec(&scope).map_err(|e| rejected(&e.to_string()))?; + txn.commit().await?; + Ok(Self { + connection, + namespace, + primary_scope, + maintenance_state: tokio::sync::Mutex::default(), + }) + } + + async fn transaction(&self) -> Result { + let txn = self + .connection + .begin_with_config(Some(IsolationLevel::ReadCommitted), None) + .await + .map_err(database_error)?; + self.namespace.enter(&txn).await?; + let valid: bool = txn + .query_one_raw(sql( + "SELECT mst2_metadata_scope_matches($1) AS valid", + [self.primary_scope.clone().into()], + )) + .await + .map_err(database_error)? + .ok_or_else(|| integrity("rooted primary scope is missing"))? + .try_get("", "valid") + .map_err(internal)?; + if !valid { + return Err(integrity("rooted writer left its captured primary scope")); + } + Ok(txn) + } + + async fn load_intent( + &self, + db: &C, + operation: &str, + plan: &RootedMetadataInstallPlan, + ) -> Result, SnapshotError> { + let Some(row)=db.query_one_raw(sql("SELECT prepare_id,plan_kind,state,manifest_digest,canonical_plan,canonical_bindings, + bindings_digest,primary_scope,storage_seal FROM mst2_metadata_prepare WHERE operation_id=$1",[operation.into()])) + .await.map_err(internal)? else {return Ok(None);}; + let digest = plan.digest()?; + if row.try_get::("", "plan_kind").map_err(internal)? != "ROOTED" + || digest_column(&row, "manifest_digest")? != digest + { + return Err(SnapshotError::new( + crate::ceres::snapshot::error::SnapshotErrorCode::Conflict, + "operation is bound to a different rooted manifest", + )); + } + let stored: Vec = row.try_get("", "canonical_plan").map_err(internal)?; + if RootedMetadataInstallPlan::decode(&stored, &digest)? != *plan { + return Err(integrity( + "rooted stored plan differs from its exact source identity", + )); + } + let encoded_bindings: Vec = row.try_get("", "canonical_bindings").map_err(internal)?; + let bindings_digest: [u8; 32] = Sha256::digest(&encoded_bindings).into(); + if bindings_digest != digest_column(&row, "bindings_digest")? { + return Err(integrity( + "rooted stored lifetime binding digest is invalid", + )); + } + let primary_scope: Vec = row.try_get("", "primary_scope").map_err(internal)?; + if primary_scope != self.primary_scope { + return Err(integrity( + "rooted operation belongs to another physical primary", + )); + } + let bindings = decode_bindings(&encoded_bindings, plan)?; + let root_generation = bindings + .get(&plan.root) + .map(|b| b.0) + .or_else(|| plan.reused.get(&plan.root).map(|b| b.generation)) + .ok_or_else(|| integrity("rooted root lifetime is missing"))?; + Ok(Some(( + RootedPrepareIntent { + prepare_id: row.try_get("", "prepare_id").map_err(internal)?, + operation_id: operation.into(), + plan: Arc::new(plan.clone()), + bindings, + manifest_digest: digest, + bindings_digest, + primary_scope, + storage_seal: digest_column(&row, "storage_seal")?, + root_generation, + }, + row.try_get("", "state").map_err(internal)?, + ))) + } + + pub(crate) async fn begin_intent( + &self, + operation: &str, + plan: &RootedMetadataInstallPlan, + ) -> Result { + plan.validate()?; + if operation.is_empty() || operation.len() > 255 || operation.contains('\0') { + return Err(integrity("invalid rooted operation ID").into()); + } + let manifest_digest = plan.digest()?; + let txn = self.transaction().await?; + let result=async { + if let Some((intent,state))=self.load_intent(&txn,operation,plan).await? { + if state=="ABORTED" {return Err(unavailable("rooted preparation was definitively aborted"));} + return Ok(intent); + } + let pages:Vec<_>=plan.delta.iter().map(|(page,size)|json!({"page":hex::encode(page),"size":size})).collect(); + let encoded=serde_json::to_string(&pages).map_err(internal)?; + txn.execute_raw(sql("INSERT INTO mst2_metadata_lifetime(page_id,node_id,generation,state,metadata_codec,expected_size,graph_domain) + SELECT decode(p.page,'hex'),'page:sha256:'||p.page,coalesce(cur.generation+1,1),'RESERVED',1,p.size,'qualified-v1' + FROM jsonb_to_recordset($1::jsonb) p(page text,size integer) + LEFT JOIN mst2_metadata_current cur ON cur.page_id=decode(p.page,'hex') + LEFT JOIN mst2_metadata_lifetime previous ON previous.page_id=cur.page_id AND previous.generation=cur.generation + WHERE (cur.page_id IS NULL AND NOT EXISTS(SELECT 1 FROM mst2_metadata_lifetime life WHERE life.page_id=decode(p.page,'hex'))) + OR (previous.state='REMOVED' AND cur.generation<9223372036854775807 + AND EXISTS(SELECT 1 FROM mst2_metadata_gc_op proof WHERE proof.page_id=cur.page_id + AND proof.generation=cur.generation AND proof.state='APPLIED'))", + [encoded.clone().into()])).await.map_err(internal)?; + txn.execute_raw(sql("INSERT INTO mst2_metadata_current(page_id,generation) + SELECT life.page_id,life.generation FROM jsonb_to_recordset($1::jsonb) p(page text,size integer) + JOIN mst2_metadata_lifetime life ON life.page_id=decode(p.page,'hex') AND life.generation=1 AND life.state='RESERVED' + WHERE NOT EXISTS(SELECT 1 FROM mst2_metadata_current cur WHERE cur.page_id=life.page_id)", + [encoded.clone().into()])).await.map_err(internal)?; + txn.execute_raw(sql("UPDATE mst2_metadata_current cur SET generation=fresh.generation + FROM jsonb_to_recordset($1::jsonb) p(page text,size integer),mst2_metadata_lifetime previous, + mst2_metadata_lifetime fresh + WHERE cur.page_id=decode(p.page,'hex') AND previous.page_id=cur.page_id AND previous.generation=cur.generation + AND previous.state='REMOVED' AND cur.generation<9223372036854775807 + AND fresh.page_id=cur.page_id AND fresh.generation=cur.generation+1 AND fresh.state='RESERVED' + AND fresh.expected_size=p.size AND fresh.metadata_codec=1 AND fresh.graph_domain='qualified-v1' + AND EXISTS(SELECT 1 FROM mst2_metadata_gc_op proof WHERE proof.page_id=cur.page_id + AND proof.generation=cur.generation AND proof.state='APPLIED')",[encoded.clone().into()])) + .await.map_err(internal)?; + let rows=txn.query_all_raw(sql("SELECT cur.page_id,cur.generation,life.expected_size,life.metadata_codec,life.graph_domain,life.state, + n.state AS graph_state,n.certificate_digest,body.byte_size FROM jsonb_to_recordset($1::jsonb) p(page text,size integer) + JOIN mst2_metadata_current cur ON cur.page_id=decode(p.page,'hex') JOIN mst2_metadata_lifetime life USING(page_id,generation) + LEFT JOIN mst2_metadata_graph_node n USING(page_id,generation) LEFT JOIN mst2_metadata_payload body USING(page_id,generation) + WHERE NOT EXISTS(SELECT 1 FROM mst2_metadata_gc_op gc WHERE gc.page_id=cur.page_id AND gc.generation=cur.generation) + ORDER BY cur.page_id",[encoded.into()])).await.map_err(internal)?; + let mut bindings=BTreeMap::new(); + for row in rows { + let page=digest_column(&row,"page_id")?; let size=row.try_get::("","expected_size").map_err(internal)? as u64; + let state:String=row.try_get("","state").map_err(internal)?; + if plan.delta.get(&page)!=Some(&size) || row.try_get::("","metadata_codec").map_err(internal)?!=1 + || row.try_get::("","graph_domain").map_err(internal)?!="qualified-v1" || !matches!(state.as_str(),"RESERVED"|"LIVE") + || state=="LIVE" && (row.try_get::>("","graph_state").map_err(internal)?.as_deref()!=Some("LIVE") + || row.try_get::>("","byte_size").map_err(internal)?!=Some(size as i32)) { + return Err(unavailable("rooted requested delta lifetime is not exactly installable")); + } + bindings.insert(page,(row.try_get("","generation").map_err(internal)?,size)); + } + if bindings.len()!=plan.delta.len() {return Err(unavailable("rooted delta lifetime coverage is incomplete"));} + let canonical_bindings=encode_bindings(&bindings); let bindings_digest:[u8;32]=Sha256::digest(&canonical_bindings).into(); + let prepare_id=uuid::Uuid::new_v4().to_string(); + let root_generation=bindings.get(&plan.root).map(|b|b.0).or_else(||plan.reused.get(&plan.root).map(|b|b.generation)) + .ok_or_else(||integrity("rooted root lifetime is missing"))?; + let mut seal=Sha256::new(); seal.update(b"mega.mst2.rooted-storage-seal.v1\0"); + seal.update(prepare_id.as_bytes()); seal.update(manifest_digest); seal.update(bindings_digest); + seal.update((self.primary_scope.len() as u64).to_be_bytes()); seal.update(&self.primary_scope); + seal.update(plan.root); seal.update(root_generation.to_be_bytes()); + let storage_seal:[u8;32]=seal.finalize().into(); let identity=&plan.identity; + txn.execute_raw(sql("INSERT INTO mst2_metadata_prepare(prepare_id,operation_id,manifest_digest,canonical_plan,plan_kind, + source_domain,tagged_root_tree_oid,scope,schema_version,metadata_codec,materialization_policy,fs_semantics,access_projection, + verification_revision,projection_revision,metadata_root,node_count,edge_count,total_bytes,state,canonical_bindings,bindings_digest, + primary_scope,storage_seal,graph_domain) VALUES($1,$2,$3,$4,'ROOTED',$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,'PREPARING',$19,$20,$21,$22,'qualified-v1')", + [prepare_id.clone().into(),operation.into(),manifest_digest.to_vec().into(),plan.encode()?.into(),identity.source_domain.clone().into(), + identity.tagged_root_tree_oid.clone().into(),identity.scope.clone().into(),(identity.schema_version as i16).into(), + (identity.metadata_codec as i16).into(),(identity.materialization_policy as i16).into(),(identity.fs_semantics as i16).into(), + (identity.access_projection as i16).into(),identity.verification_revision.into(),(identity.projection_revision as i16).into(), + plan.root.to_vec().into(),(plan.delta.len() as i32).into(),(plan.edges.len() as i32).into(),(plan.delta_bytes()? as i64).into(), + canonical_bindings.into(),bindings_digest.to_vec().into(),self.primary_scope.clone().into(),storage_seal.to_vec().into()])).await.map_err(internal)?; + let delta:Vec<_>=bindings.iter().map(|(page,(generation,size))|json!({"page":hex::encode(page),"generation":generation,"size":size})).collect(); + txn.execute_raw(sql("INSERT INTO mst2_metadata_prepare_page(prepare_id,page_id,generation,expected_size) + SELECT $1,decode(p.page,'hex'),p.generation,p.size FROM jsonb_to_recordset($2::jsonb) p(page text,generation bigint,size integer)", + [prepare_id.clone().into(),serde_json::to_string(&delta).map_err(internal)?.into()])).await.map_err(internal)?; + let reused:Vec<_>=plan.reused.iter().map(|(page,proof)|json!({"page":hex::encode(page),"generation":proof.generation, + "attestation_id":proof.attestation_id.to_string(),"attestation_digest":hex::encode(proof.attestation_digest)})).collect(); + txn.execute_raw(sql("INSERT INTO mst2_metadata_prepare_reuse_root(prepare_id,root_page,root_generation,attestation_id,attestation_digest) + SELECT $1,decode(r.page,'hex'),r.generation,r.attestation_id::uuid,decode(r.attestation_digest,'hex') + FROM jsonb_to_recordset($2::jsonb) r(page text,generation bigint,attestation_id text,attestation_digest text)", + [prepare_id.clone().into(),serde_json::to_string(&reused).map_err(internal)?.into()])).await.map_err(internal)?; + txn.execute_raw(sql("INSERT INTO mst2_metadata_root_anchor(anchor_id,anchor_kind,owner_key,prepare_id,root_page,root_generation,root_certificate_digest) + SELECT gen_random_uuid(),'REUSE',$1,$1,r.root_page,r.root_generation,a.root_certificate_digest + FROM mst2_metadata_prepare_reuse_root r JOIN mst2_metadata_source_root_attestation a USING(attestation_id) WHERE r.prepare_id=$1", + [prepare_id.clone().into()])).await.map_err(internal)?; + Ok(RootedPrepareIntent {prepare_id,operation_id:operation.into(),plan:Arc::new(plan.clone()),bindings,manifest_digest, + bindings_digest,primary_scope:self.primary_scope.clone(),storage_seal,root_generation}) + }.await; + committed( + txn, + result, + operation, + manifest_digest, + super::super::native_metadata_install::MetadataCommitPhase::Intent, + ) + .await + } + + async fn require_intent( + &self, + db: &C, + expected: &RootedPrepareIntent, + ) -> Result { + let (stored, state) = self + .load_intent(db, &expected.operation_id, &expected.plan) + .await? + .ok_or_else(|| unavailable("rooted preparation is missing"))?; + if stored != *expected { + return Err(integrity( + "rooted preparation crossed its immutable physical binding", + )); + } + if state == "ABORTED" { + return Err(unavailable("rooted preparation was definitively aborted")); + } + Ok(state) + } + + pub(crate) async fn install_pages( + &self, + intent: &RootedPrepareIntent, + payloads: &[MetadataPagePayload], + ) -> Result<(), MetadataInstallError> { + if payloads.is_empty() || payloads.len() > 64 { + return Err(integrity("rooted payload batch requires 1..=64 pages").into()); + } + let mut pages = BTreeMap::new(); + for payload in payloads { + let &(generation, size) = intent + .bindings + .get(&payload.id) + .ok_or_else(|| integrity("rooted payload is outside its fixed delta"))?; + if size != payload.size + || payload.bytes.len() as u64 != size + || page_id(&payload.bytes) != payload.id + { + return Err(integrity( + "rooted delta payload differs from its exact page digest or size", + ) + .into()); + } + Page::decode(&payload.bytes).map_err(internal)?; + if pages + .insert( + payload.id, + json!({"page":hex::encode(payload.id),"generation":generation,"size":size, + "payload":hex::encode(&payload.bytes)}), + ) + .is_some() + { + return Err(integrity("rooted payload batch has duplicate pages").into()); + } + } + let encoded = + serde_json::to_string(&pages.into_values().collect::>()).map_err(internal)?; + let txn = self.transaction().await?; + let result=async { + let state=self.require_intent(&txn,intent).await?; + if state=="PREPARING" { + txn.execute_raw(sql("INSERT INTO mst2_metadata_payload(page_id,generation,metadata_codec,byte_size,payload) + SELECT decode(p.page,'hex'),p.generation,1,p.size,decode(p.payload,'hex') + FROM jsonb_to_recordset($1::jsonb) p(page text,generation bigint,size integer,payload text) + WHERE NOT EXISTS(SELECT 1 FROM mst2_metadata_payload body WHERE body.page_id=decode(p.page,'hex')) + ON CONFLICT(page_id) DO NOTHING",[encoded.clone().into()])).await.map_err(internal)?; + } + if txn.query_one_raw(sql("SELECT p.page FROM jsonb_to_recordset($1::jsonb) p(page text,generation bigint,size integer,payload text) + LEFT JOIN mst2_metadata_payload body ON body.page_id=decode(p.page,'hex') + LEFT JOIN mst2_metadata_current cur ON cur.page_id=body.page_id AND cur.generation=body.generation + WHERE body.page_id IS NULL OR cur.page_id IS NULL OR body.generation<>p.generation OR body.metadata_codec<>1 + OR body.byte_size<>p.size OR body.payload<>decode(p.payload,'hex') LIMIT 1",[encoded.into()])) + .await.map_err(internal)?.is_some() {return Err(integrity("rooted durable delta payload conflicts with its exact incarnation"));} + Ok(()) + }.await; + committed( + txn, + result, + &intent.operation_id, + intent.manifest_digest, + super::super::native_metadata_install::MetadataCommitPhase::Payload, + ) + .await + } + + pub(crate) async fn finalize( + &self, + intent: &RootedPrepareIntent, + ) -> Result { + // Rehash/decode the actual delta outside the core route lock. A cold + // preparation additionally keeps the full Rust DAG validator as oracle. + let payloads = self.read_delta(intent).await?; + if intent.plan.reused.is_empty() { + use crate::ceres::snapshot::retention_dag::{ + MetadataDagCandidate, MetadataDagLimits, ValidatedMetadataDag, + }; + ValidatedMetadataDag::validate( + MetadataDagCandidate { + metadata_codec: intent.plan.identity.metadata_codec, + root: intent.plan.root, + pages: payloads, + edges: intent.plan.edges.iter().copied().collect(), + }, + MetadataDagLimits::default(), + )?; + } + let txn = self.transaction().await?; + let result=async { + txn.query_one_raw(sql("SELECT prepare_id FROM mst2_metadata_prepare WHERE prepare_id=$1 FOR UPDATE", + [intent.prepare_id.clone().into()])).await.map_err(internal)?; + let state=self.require_intent(&txn,intent).await?; + if state=="COMMITTED" {return self.receipt(&txn,intent).await;} + let ordered:Vec<_>=intent.plan.child_first_delta()?.iter().map(|page|json!({"page":hex::encode(page), + "generation":intent.bindings[page].0})).collect(); + if !ordered.is_empty() { + let count:i32=txn.query_one_raw(sql("SELECT mst2_metadata_certify_batch($1,$2::jsonb) AS certified", + [intent.prepare_id.clone().into(),serde_json::to_string(&ordered).map_err(internal)?.into()])) + .await.map_err(internal)?.ok_or_else(||integrity("rooted certification result is missing"))?.try_get("","certified").map_err(internal)?; + if count as usize!=ordered.len() {return Err(integrity("rooted certification did not cover its exact delta"));} + } + txn.execute_raw(sql("INSERT INTO mst2_metadata_root_anchor(anchor_id,anchor_kind,owner_key,prepare_id,root_page,root_generation,root_certificate_digest) + SELECT gen_random_uuid(),'PREPARE',$1,$1,c.page_id,c.generation,c.certificate_digest + FROM mst2_metadata_page_certificate c WHERE c.page_id=$2 AND c.generation=$3 + ON CONFLICT(anchor_kind,owner_key,root_page,root_generation) DO NOTHING", + [intent.prepare_id.clone().into(),intent.plan.root.to_vec().into(),intent.root_generation.into()])).await.map_err(internal)?; + let sources:Vec<_>=intent.plan.source_roots.iter().map(|(tree,page)|json!({"tree":tree,"page":hex::encode(page)})).collect(); + let sources=txn.query_all_raw(sql("SELECT source.tree,cur.page_id FROM jsonb_to_recordset($1::jsonb) source(tree text,page text) + JOIN mst2_metadata_current cur ON cur.page_id=decode(source.page,'hex') + JOIN mst2_metadata_page_certificate proof USING(page_id,generation) ORDER BY proof.rank,source.tree", + [serde_json::to_string(&sources).map_err(internal)?.into()])).await.map_err(internal)?; + if sources.len()!=intent.plan.source_roots.len() {return Err(unavailable("rooted source certificate order is incomplete"));} + for source in sources { + let tree:String=source.try_get("","tree").map_err(internal)?; + let page=digest_column(&source,"page_id")?; + let generation=intent.bindings.get(&page).map(|b|b.0).or_else(||intent.plan.reused.get(&page).map(|b|b.generation)) + .ok_or_else(||integrity("rooted directory source has no exact lifetime"))?; + let reused=txn.query_one_raw(sql("SELECT a.attestation_id FROM mst2_metadata_prepare_reuse_root r + JOIN mst2_metadata_source_root_attestation boundary ON boundary.attestation_id=r.attestation_id + JOIN mst2_metadata_source_root_attestation a ON a.root_page=r.root_page AND a.root_generation=r.root_generation + AND a.root_certificate_digest=boundary.root_certificate_digest + JOIN mst2_metadata_prepare origin ON origin.prepare_id=a.origin_prepare_id + JOIN mst2_metadata_current cur ON cur.page_id=a.root_page AND cur.generation=a.root_generation + JOIN mst2_metadata_lifetime life USING(page_id,generation) + JOIN $CORE$.mega_tree source ON source.tree_id=split_part(a.tagged_tree_oid,':',2) + WHERE r.prepare_id=$1 AND r.root_page=$2 AND r.root_generation=$3 AND a.tagged_tree_oid=$4 + AND origin.state='COMMITTED' AND life.state='LIVE' AND a.source_profile=mst2_metadata_native_profile($1) + AND $CORE$.mst2_route_source_tree_matches(split_part(a.tagged_tree_oid,':',2),a.source_revision,a.source_body_digest) + ORDER BY a.attestation_id LIMIT 1".replace("$CORE$",&identifier(&self.namespace.core_schema)), + [intent.prepare_id.clone().into(),page.to_vec().into(),generation.into(),tree.clone().into()])).await.map_err(internal)?; + if reused.is_some() {continue;} + txn.execute_raw(sql("INSERT INTO mst2_metadata_source_root_attestation(attestation_id,namespace_uuid,origin_prepare_id,tagged_tree_oid, + source_profile,profile_digest,source_body_digest,root_page,root_generation,root_certificate_digest,source_proof,attestation_digest) + SELECT gen_random_uuid(),(proof->>'namespace')::uuid,$1,$2,proof->'source_profile',decode(proof->>'profile_digest','hex'), + decode(proof->>'source_body_digest','hex'),$3,$4,decode(proof->>'root_certificate','hex'),proof,decode(proof->>'attestation','hex') + FROM (SELECT mst2_metadata_compute_source_proof($1,$2,$3,$4) AS proof) input", + [intent.prepare_id.clone().into(),tree.clone().into(),page.to_vec().into(),generation.into()])).await.map_err(internal)?; + } + let changed=txn.execute_raw(sql("UPDATE mst2_metadata_prepare SET state='COMMITTED',committed_at=clock_timestamp() + WHERE prepare_id=$1 AND state='PREPARING' AND storage_seal=$2", + [intent.prepare_id.clone().into(),intent.storage_seal.to_vec().into()])).await.map_err(internal)?; + if changed.rows_affected()!=1 {return Err(integrity("rooted finalize lost its exact prepare transition"));} + txn.execute_raw(sql("UPDATE mst2_metadata_lifetime life SET state='LIVE' FROM mst2_metadata_prepare_page member + WHERE member.prepare_id=$1 AND life.page_id=member.page_id AND life.generation=member.generation AND life.state='RESERVED'", + [intent.prepare_id.clone().into()])).await.map_err(internal)?; + txn.execute_raw(sql("INSERT INTO mst2_metadata_reuse_index(profile_digest,tagged_tree_oid,attestation_id,root_page,root_generation,attestation_digest) + SELECT a.profile_digest,a.tagged_tree_oid,a.attestation_id,a.root_page,a.root_generation,a.attestation_digest + FROM mst2_metadata_source_root_attestation a WHERE a.origin_prepare_id=$1 + ON CONFLICT(profile_digest,tagged_tree_oid) DO NOTHING",[intent.prepare_id.clone().into()])).await.map_err(internal)?; + self.receipt(&txn,intent).await + }.await; + committed( + txn, + result, + &intent.operation_id, + intent.manifest_digest, + super::super::native_metadata_install::MetadataCommitPhase::Finalize, + ) + .await + } + + async fn read_delta( + &self, + intent: &RootedPrepareIntent, + ) -> Result, SnapshotError> { + let rows=self.connection.query_all_raw(sql("SELECT member.page_id,member.generation,member.expected_size,body.metadata_codec, + body.byte_size,body.payload FROM mst2_metadata_prepare_page member JOIN mst2_metadata_payload body USING(page_id,generation) + JOIN mst2_metadata_current cur USING(page_id,generation) WHERE member.prepare_id=$1 ORDER BY member.page_id LIMIT 4097", + [intent.prepare_id.clone().into()])).await.map_err(internal)?; + if rows.len() != intent.bindings.len() { + return Err(unavailable("rooted durable delta is incomplete")); + } + let mut payloads = Vec::with_capacity(rows.len()); + for row in rows { + let page = digest_column(&row, "page_id")?; + let &(generation, size) = intent + .bindings + .get(&page) + .ok_or_else(|| integrity("rooted durable delta has an extra member"))?; + let bytes: Vec = row.try_get("", "payload").map_err(internal)?; + if row.try_get::("", "generation").map_err(internal)? != generation + || row.try_get::("", "expected_size").map_err(internal)? as u64 != size + || row.try_get::("", "byte_size").map_err(internal)? as u64 != size + || row.try_get::("", "metadata_codec").map_err(internal)? != 1 + || bytes.len() as u64 != size + || page_id(&bytes) != page + { + return Err(integrity( + "rooted durable delta crossed its fixed byte and lifetime binding", + )); + } + Page::decode(&bytes).map_err(internal)?; + payloads.push(MetadataPagePayload { + id: page, + size, + bytes, + }); + } + Ok(payloads) + } + + async fn receipt( + &self, + db: &C, + intent: &RootedPrepareIntent, + ) -> Result { + let row=db.query_one_raw(sql("SELECT c.certificate_digest,a.attestation_id::text,a.attestation_digest + FROM mst2_metadata_prepare q JOIN mst2_metadata_current cur ON cur.page_id=q.metadata_root AND cur.generation=$2 + JOIN mst2_metadata_lifetime life USING(page_id,generation) JOIN mst2_metadata_graph_node n USING(page_id,generation) + JOIN mst2_metadata_page_certificate c USING(page_id,generation) + JOIN mst2_metadata_source_root_attestation a ON a.root_page=c.page_id AND a.root_generation=c.generation + JOIN mst2_metadata_prepare origin ON origin.prepare_id=a.origin_prepare_id + JOIN $CORE$.mega_tree source ON source.tree_id=split_part(a.tagged_tree_oid,':',2) + WHERE q.prepare_id=$1 AND q.plan_kind='ROOTED' AND q.state='COMMITTED' AND life.state='LIVE' AND n.state='LIVE' + AND n.certificate_digest=c.certificate_digest AND a.root_certificate_digest=c.certificate_digest + AND origin.state='COMMITTED' AND a.source_profile=mst2_metadata_native_profile($1) + AND a.tagged_tree_oid=mst2_metadata_rooted_scope_tree($1) + AND $CORE$.mst2_route_source_tree_matches(split_part(a.tagged_tree_oid,':',2),a.source_revision,a.source_body_digest) + AND NOT EXISTS(SELECT 1 FROM mst2_metadata_gc_op gc WHERE gc.page_id=c.page_id AND gc.generation=c.generation) + AND (q.coverage_retired_at IS NULL AND EXISTS(SELECT 1 FROM mst2_metadata_root_anchor anchor + WHERE anchor.anchor_kind='PREPARE' AND anchor.prepare_id=q.prepare_id AND anchor.owner_key=q.prepare_id + AND anchor.root_page=c.page_id AND anchor.root_generation=c.generation + AND anchor.root_certificate_digest=c.certificate_digest) OR mst2_metadata_session_covers_prepare(q.prepare_id)) + ORDER BY (a.origin_prepare_id=q.prepare_id) DESC,a.attestation_id LIMIT 1".replace("$CORE$",&identifier(&self.namespace.core_schema)), + [intent.prepare_id.clone().into(),intent.root_generation.into()])).await.map_err(internal)? + .ok_or_else(||unavailable("rooted definitive receipt has no exact active canonical source root"))?; + Ok(RootedMetadataReceipt { + intent: intent.clone(), + certificate_digest: digest_column(&row, "certificate_digest")?, + attestation_id: uuid::Uuid::parse_str( + &row.try_get::("", "attestation_id") + .map_err(internal)?, + ) + .map_err(internal)?, + attestation_digest: digest_column(&row, "attestation_digest")?, + }) + } + + pub(crate) async fn recover( + &self, + operation: &str, + plan: &RootedMetadataInstallPlan, + ) -> Result, SnapshotError> { + let txn = self.transaction().await?; + let Some((intent, state)) = self.load_intent(&txn, operation, plan).await? else { + txn.commit().await.map_err(internal)?; + return Ok(None); + }; + let receipt = if state == "COMMITTED" { + Some(self.receipt(&txn, &intent).await?) + } else { + None + }; + txn.commit().await.map_err(internal)?; + Ok(receipt) + } +} diff --git a/src/jupiter/storage/qualified_metadata_rooted.sql b/src/jupiter/storage/qualified_metadata_rooted.sql new file mode 100644 index 00000000..8116bc61 --- /dev/null +++ b/src/jupiter/storage/qualified_metadata_rooted.sql @@ -0,0 +1,358 @@ +CREATE FUNCTION mst2_metadata_read_be(b bytea,p integer,w integer) RETURNS numeric +LANGUAGE plpgsql IMMUTABLE STRICT SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE v numeric:=0; i integer; +BEGIN + IF w NOT IN (2,4,8) OR p<0 OR p>octet_length(b)-w THEN RAISE EXCEPTION 'rooted integer is out of bounds'; END IF; + FOR i IN 0..w-1 LOOP v:=v*256+get_byte(b,p+i); END LOOP; + RETURN v; +END $$; + +CREATE FUNCTION mst2_metadata_rooted_string(b bytea,p integer,maximum integer) RETURNS jsonb +LANGUAGE plpgsql IMMUTABLE STRICT SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE n numeric; value text; +BEGIN + n:=mst2_metadata_read_be(b,p,4); p:=p+4; + IF n>maximum OR n>octet_length(b)-p THEN RAISE EXCEPTION 'rooted string exceeds its exact byte boundary'; END IF; + value:=convert_from(substring(b FROM p+1 FOR n::integer),'UTF8'); + RETURN jsonb_build_object('end',p+n::integer,'text',value); +END $$; + +CREATE FUNCTION mst2_metadata_decode_rooted_plan(b bytea) RETURNS jsonb +LANGUAGE plpgsql IMMUTABLE STRICT SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE domain bytea:=convert_to('mega.mst2.rooted-install.v1','UTF8')||decode('00','hex'); + cursor_pos integer; part jsonb; source_domain text; tree_oid text; scope text; hash_kind text; identity jsonb; + root bytea; count numeric; i integer; page bytea; parent bytea; child bytea; previous bytea; previous_tree text; + size numeric; generation numeric; attestation bytea; attestation_digest bytea; certificate_digest bytea; + delta_rows jsonb[]:=ARRAY[]::jsonb[]; edge_rows jsonb[]:=ARRAY[]::jsonb[]; + reuse_rows jsonb[]:=ARRAY[]::jsonb[]; source_rows jsonb[]:=ARRAY[]::jsonb[]; total_bytes bigint:=0; + delta_index jsonb; node_index jsonb; adjacency jsonb; +BEGIN + IF octet_length(b)>2097152 OR substring(b FROM 1 FOR octet_length(domain))<>domain THEN + RAISE EXCEPTION 'rooted preparation domain or byte budget is invalid'; + END IF; + cursor_pos:=octet_length(domain); + IF mst2_metadata_read_be(b,cursor_pos,2)<>1 THEN RAISE EXCEPTION 'rooted preparation version is unsupported'; END IF; + cursor_pos:=cursor_pos+2; + part:=mst2_metadata_rooted_string(b,cursor_pos,64); cursor_pos:=(part->>'end')::integer; source_domain:=part->>'text'; + part:=mst2_metadata_rooted_string(b,cursor_pos,128); cursor_pos:=(part->>'end')::integer; tree_oid:=part->>'text'; + part:=mst2_metadata_rooted_string(b,cursor_pos,4096); cursor_pos:=(part->>'end')::integer; scope:=part->>'text'; + IF source_domain<>'native-git' OR tree_oid !~ '^(sha1:[0-9a-f]{40}|sha256:[0-9a-f]{64}|blake3:[0-9a-f]{64})$' + OR scope NOT LIKE '/%' OR scope<>'/' AND (scope LIKE '%/' OR scope LIKE '%//%' OR EXISTS( + SELECT 1 FROM unnest(string_to_array(substring(scope FROM 2),'/')) name WHERE name IN ('','.','..') + OR octet_length(name)>255) OR cardinality(string_to_array(substring(scope FROM 2),'/'))>256) THEN + RAISE EXCEPTION 'rooted source identity or scope is invalid'; END IF; + hash_kind:=split_part(tree_oid,':',1); + identity:=jsonb_build_object('source_domain',source_domain,'tagged_root_tree_oid',tree_oid,'scope',scope, + 'schema_version',mst2_metadata_read_be(b,cursor_pos,2), + 'metadata_codec',mst2_metadata_read_be(b,cursor_pos+2,2), + 'materialization_policy',mst2_metadata_read_be(b,cursor_pos+4,2), + 'fs_semantics',mst2_metadata_read_be(b,cursor_pos+6,2), + 'access_projection',mst2_metadata_read_be(b,cursor_pos+8,2), + 'verification_revision',mst2_metadata_read_be(b,cursor_pos+10,4), + 'projection_revision',mst2_metadata_read_be(b,cursor_pos+14,2)); + cursor_pos:=cursor_pos+16; + IF (identity->>'schema_version')::integer<>2 OR (identity->>'metadata_codec')::integer<>1 + OR (identity->>'materialization_policy')::integer<>1 OR (identity->>'fs_semantics')::integer<>1 + OR (identity->>'access_projection')::integer<>0 OR (identity->>'verification_revision')::integer<>2 + OR (identity->>'projection_revision')::integer<>1 THEN RAISE EXCEPTION 'rooted source profile is not current native'; END IF; + IF cursor_pos>octet_length(b)-32 THEN RAISE EXCEPTION 'rooted metadata root is truncated'; END IF; + root:=substring(b FROM cursor_pos+1 FOR 32); cursor_pos:=cursor_pos+32; + count:=mst2_metadata_read_be(b,cursor_pos,4); cursor_pos:=cursor_pos+4; + IF count>4096 THEN RAISE EXCEPTION 'rooted delta exceeds its node budget'; END IF; + IF count>0 THEN FOR i IN 1..count::integer LOOP + IF cursor_pos>octet_length(b)-40 THEN RAISE EXCEPTION 'rooted delta member is truncated'; END IF; + page:=substring(b FROM cursor_pos+1 FOR 32); size:=mst2_metadata_read_be(b,cursor_pos+32,8); cursor_pos:=cursor_pos+40; + IF previous IS NOT NULL AND previous>=page OR size NOT BETWEEN 20 AND 16384 THEN + RAISE EXCEPTION 'rooted delta is not exactly ordered or has invalid size'; END IF; + previous:=page; total_bytes:=total_bytes+size::bigint; + IF total_bytes>67108864 THEN RAISE EXCEPTION 'rooted delta exceeds its metadata byte budget'; END IF; + delta_rows:=array_append(delta_rows,jsonb_build_object('page',encode(page,'hex'),'size',size)); + END LOOP; END IF; + previous:=NULL; count:=mst2_metadata_read_be(b,cursor_pos,4); cursor_pos:=cursor_pos+4; + IF count>16384 THEN RAISE EXCEPTION 'rooted delta exceeds its edge budget'; END IF; + IF count>0 THEN FOR i IN 1..count::integer LOOP + IF cursor_pos>octet_length(b)-64 THEN RAISE EXCEPTION 'rooted edge is truncated'; END IF; + parent:=substring(b FROM cursor_pos+1 FOR 32); child:=substring(b FROM cursor_pos+33 FOR 32); cursor_pos:=cursor_pos+64; + IF previous IS NOT NULL AND previous>=parent||child OR parent=child THEN RAISE EXCEPTION 'rooted edges are not unique and ordered'; END IF; + previous:=parent||child; + edge_rows:=array_append(edge_rows,jsonb_build_object('parent',encode(parent,'hex'),'child',encode(child,'hex'))); + END LOOP; END IF; + previous:=NULL; count:=mst2_metadata_read_be(b,cursor_pos,4); cursor_pos:=cursor_pos+4; + IF count>4096 OR coalesce(array_length(delta_rows,1),0)+count>4096 THEN RAISE EXCEPTION 'rooted delta and boundaries exceed node budget'; END IF; + IF count>0 THEN FOR i IN 1..count::integer LOOP + IF cursor_pos>octet_length(b)-120 THEN RAISE EXCEPTION 'rooted reuse boundary is truncated'; END IF; + page:=substring(b FROM cursor_pos+1 FOR 32); generation:=mst2_metadata_read_be(b,cursor_pos+32,8); + attestation:=substring(b FROM cursor_pos+41 FOR 16); attestation_digest:=substring(b FROM cursor_pos+57 FOR 32); + certificate_digest:=substring(b FROM cursor_pos+89 FOR 32); cursor_pos:=cursor_pos+120; + IF previous IS NOT NULL AND previous>=page OR generation NOT BETWEEN 1 AND 9223372036854775807 THEN + RAISE EXCEPTION 'rooted reuse boundaries are not exact positive ordered lifetimes'; END IF; + previous:=page; + reuse_rows:=array_append(reuse_rows,jsonb_build_object('page',encode(page,'hex'),'generation',generation, + 'attestation_id',encode(attestation,'hex')::uuid,'attestation_digest',encode(attestation_digest,'hex'), + 'certificate_digest',encode(certificate_digest,'hex'))); + END LOOP; END IF; + count:=mst2_metadata_read_be(b,cursor_pos,4); cursor_pos:=cursor_pos+4; + IF count NOT BETWEEN 1 AND 4096 THEN RAISE EXCEPTION 'rooted source-root budget is invalid'; END IF; + FOR i IN 1..count::integer LOOP + part:=mst2_metadata_rooted_string(b,cursor_pos,128); cursor_pos:=(part->>'end')::integer; tree_oid:=part->>'text'; + IF cursor_pos>octet_length(b)-32 THEN RAISE EXCEPTION 'rooted source root is truncated'; END IF; + page:=substring(b FROM cursor_pos+1 FOR 32); cursor_pos:=cursor_pos+32; + IF tree_oid !~ '^(sha1:[0-9a-f]{40}|sha256:[0-9a-f]{64}|blake3:[0-9a-f]{64})$' + OR split_part(tree_oid,':',1)<>hash_kind OR previous_tree IS NOT NULL AND convert_to(previous_tree,'UTF8')>=convert_to(tree_oid,'UTF8') THEN + RAISE EXCEPTION 'rooted source roots are not unique ordered same-profile identities'; END IF; + previous_tree:=tree_oid; source_rows:=array_append(source_rows,jsonb_build_object('tree_oid',tree_oid,'page',encode(page,'hex'))); + END LOOP; + IF cursor_pos<>octet_length(b) THEN RAISE EXCEPTION 'rooted preparation has trailing bytes'; END IF; + SELECT coalesce(jsonb_object_agg(value->>'page',true),'{}'::jsonb) INTO delta_index FROM unnest(delta_rows) d(value); + SELECT coalesce(jsonb_object_agg(value->>'page',true),'{}'::jsonb) INTO node_index FROM ( + SELECT value FROM unnest(delta_rows) UNION ALL SELECT value FROM unnest(reuse_rows)) nodes; + IF coalesce(array_length(delta_rows,1),0)+coalesce(array_length(reuse_rows,1),0)=0 + OR EXISTS(SELECT 1 FROM unnest(reuse_rows) r(value) WHERE delta_index ? (r.value->>'page')) + OR NOT node_index ? encode(root,'hex') + OR EXISTS(SELECT 1 FROM unnest(edge_rows) e(value) WHERE NOT delta_index ? (e.value->>'parent') + OR NOT node_index ? (e.value->>'child')) + OR EXISTS(SELECT 1 FROM unnest(source_rows) s(value) WHERE NOT node_index ? (s.value->>'page')) + OR NOT EXISTS(SELECT 1 FROM unnest(source_rows) s(value) WHERE value->>'page'=encode(root,'hex')) THEN + RAISE EXCEPTION 'rooted plan has overlap or an unbound graph/source endpoint'; + END IF; + SELECT coalesce(jsonb_object_agg(parent,children),'{}'::jsonb) INTO adjacency FROM ( + SELECT value->>'parent' AS parent,jsonb_agg(value->>'child') AS children FROM unnest(edge_rows) e(value) + GROUP BY value->>'parent') grouped; + IF EXISTS(WITH RECURSIVE reached(page) AS (SELECT encode(root,'hex') UNION + SELECT child FROM reached r CROSS JOIN LATERAL jsonb_array_elements_text(adjacency->r.page) children(child)) + SELECT 1 FROM (SELECT value FROM unnest(delta_rows) UNION ALL SELECT value FROM unnest(reuse_rows)) nodes + WHERE NOT EXISTS(SELECT 1 FROM reached r WHERE r.page=nodes.value->>'page')) THEN + RAISE EXCEPTION 'rooted plan includes members outside its bounded delta and boundary closure'; + END IF; + RETURN identity||jsonb_build_object('root',encode(root,'hex'),'delta',to_jsonb(delta_rows),'edges',to_jsonb(edge_rows), + 'reused',to_jsonb(reuse_rows),'source_roots',to_jsonb(source_rows),'total_delta_bytes',total_bytes); +END $$; + +CREATE FUNCTION mst2_metadata_decode_delta_bindings(b bytea,plan jsonb) RETURNS jsonb +LANGUAGE plpgsql IMMUTABLE STRICT SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE count numeric; cursor_pos integer:=12; i integer; page bytea; generation numeric; size numeric; + previous bytea; binding_rows jsonb[]:=ARRAY[]::jsonb[]; +BEGIN + IF octet_length(b) NOT BETWEEN 12 AND 196620 OR substring(b FROM 1 FOR 8)<>convert_to('MST2GEN1','UTF8') THEN + RAISE EXCEPTION 'rooted generation binding encoding is invalid'; END IF; + count:=mst2_metadata_read_be(b,8,4); + IF count<>jsonb_array_length(plan->'delta') OR octet_length(b)<>12+48*count THEN + RAISE EXCEPTION 'rooted delta binding count differs from its immutable plan'; END IF; + IF count>0 THEN FOR i IN 1..count::integer LOOP + page:=substring(b FROM cursor_pos+1 FOR 32); generation:=mst2_metadata_read_be(b,cursor_pos+32,8); + size:=mst2_metadata_read_be(b,cursor_pos+40,8); cursor_pos:=cursor_pos+48; + IF previous IS NOT NULL AND previous>=page OR generation NOT BETWEEN 1 AND 9223372036854775807 + OR encode(page,'hex') IS DISTINCT FROM plan->'delta'->(i-1)->>'page' + OR size IS DISTINCT FROM (plan->'delta'->(i-1)->>'size')::numeric THEN + RAISE EXCEPTION 'rooted generation binding is not its exact positive ordered delta member'; END IF; + previous:=page; binding_rows:=array_append(binding_rows,jsonb_build_object('page',encode(page,'hex'),'generation',generation,'size',size)); + END LOOP; END IF; + RETURN to_jsonb(binding_rows); +END $$; + +CREATE FUNCTION mst2_metadata_rooted_manifest(q mst2_metadata_prepare) RETURNS jsonb +LANGUAGE plpgsql IMMUTABLE STRICT SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE plan jsonb:=mst2_metadata_decode_rooted_plan(q.canonical_plan); bindings jsonb; +BEGIN + bindings:=mst2_metadata_decode_delta_bindings(q.canonical_bindings,plan); + IF q.plan_kind<>'ROOTED' OR q.source_domain IS DISTINCT FROM plan->>'source_domain' + OR q.tagged_root_tree_oid IS DISTINCT FROM plan->>'tagged_root_tree_oid' OR q.scope IS DISTINCT FROM plan->>'scope' + OR q.schema_version IS DISTINCT FROM (plan->>'schema_version')::smallint + OR q.metadata_codec IS DISTINCT FROM (plan->>'metadata_codec')::smallint + OR q.materialization_policy IS DISTINCT FROM (plan->>'materialization_policy')::smallint + OR q.fs_semantics IS DISTINCT FROM (plan->>'fs_semantics')::smallint + OR q.access_projection IS DISTINCT FROM (plan->>'access_projection')::smallint + OR q.verification_revision IS DISTINCT FROM (plan->>'verification_revision')::integer + OR q.projection_revision IS DISTINCT FROM (plan->>'projection_revision')::smallint + OR q.metadata_root IS DISTINCT FROM decode(plan->>'root','hex') + OR q.node_count<>jsonb_array_length(plan->'delta') OR q.edge_count<>jsonb_array_length(plan->'edges') + OR q.total_bytes<>(plan->>'total_delta_bytes')::bigint THEN + RAISE EXCEPTION 'rooted preparation fields differ from its independently decoded manifest'; END IF; + RETURN plan||jsonb_build_object('bindings',bindings); +END $$; + +CREATE FUNCTION mst2_metadata_check_rooted_members(pid text) RETURNS jsonb +LANGUAGE plpgsql VOLATILE STRICT SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE q mst2_metadata_prepare%ROWTYPE; plan jsonb; +BEGIN + SELECT * INTO STRICT q FROM mst2_metadata_prepare WHERE prepare_id=pid AND plan_kind='ROOTED'; + plan:=mst2_metadata_rooted_manifest(q); + IF EXISTS((SELECT decode(value->>'page','hex'),(value->>'generation')::bigint,(value->>'size')::integer + FROM jsonb_array_elements(plan->'bindings')) EXCEPT + (SELECT page_id,generation,expected_size FROM mst2_metadata_prepare_page WHERE prepare_id=pid)) + OR EXISTS((SELECT page_id,generation,expected_size FROM mst2_metadata_prepare_page WHERE prepare_id=pid) EXCEPT + (SELECT decode(value->>'page','hex'),(value->>'generation')::bigint,(value->>'size')::integer + FROM jsonb_array_elements(plan->'bindings'))) + OR EXISTS((SELECT decode(value->>'page','hex'),(value->>'generation')::bigint,(value->>'attestation_id')::uuid, + decode(value->>'attestation_digest','hex'),decode(value->>'certificate_digest','hex') + FROM jsonb_array_elements(plan->'reused')) EXCEPT + (SELECT r.root_page,r.root_generation,r.attestation_id,r.attestation_digest,a.root_certificate_digest + FROM mst2_metadata_prepare_reuse_root r JOIN mst2_metadata_source_root_attestation a USING(attestation_id) + WHERE r.prepare_id=pid)) + OR EXISTS((SELECT r.root_page,r.root_generation,r.attestation_id,r.attestation_digest,a.root_certificate_digest + FROM mst2_metadata_prepare_reuse_root r JOIN mst2_metadata_source_root_attestation a USING(attestation_id) + WHERE r.prepare_id=pid) EXCEPT + (SELECT decode(value->>'page','hex'),(value->>'generation')::bigint,(value->>'attestation_id')::uuid, + decode(value->>'attestation_digest','hex'),decode(value->>'certificate_digest','hex') + FROM jsonb_array_elements(plan->'reused'))) THEN + RAISE EXCEPTION 'rooted preparation has missing or extra exact delta/reuse lifetime bindings'; END IF; + RETURN plan; +END $$; + +CREATE FUNCTION mst2_metadata_rooted_members_complete() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +BEGIN + IF EXISTS(SELECT 1 FROM mst2_metadata_prepare WHERE prepare_id=NEW.prepare_id AND plan_kind='ROOTED') THEN + PERFORM mst2_metadata_check_rooted_members(NEW.prepare_id); + END IF; + RETURN NULL; +END $$; +CREATE CONSTRAINT TRIGGER mst2_metadata_rooted_prepare_complete AFTER INSERT OR UPDATE OF bindings_revision ON mst2_metadata_prepare + DEFERRABLE INITIALLY DEFERRED FOR EACH ROW EXECUTE FUNCTION mst2_metadata_rooted_members_complete(); + +CREATE FUNCTION mst2_metadata_rooted_members_added() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +BEGIN + -- Every actual membership statement queues one authoritative commit check + -- per affected prepare, including a late raw-SQL addition. No caller state + -- can disable the decoded exact-coverage proof. + UPDATE mst2_metadata_prepare q SET bindings_revision=q.bindings_revision+1 + WHERE q.plan_kind='ROOTED' AND q.prepare_id IN (SELECT DISTINCT prepare_id FROM added_members); + RETURN NULL; +END $$; +CREATE TRIGGER mst2_metadata_rooted_delta_added AFTER INSERT ON mst2_metadata_prepare_page + REFERENCING NEW TABLE AS added_members FOR EACH STATEMENT EXECUTE FUNCTION mst2_metadata_rooted_members_added(); +CREATE TRIGGER mst2_metadata_rooted_reuse_added AFTER INSERT ON mst2_metadata_prepare_reuse_root + REFERENCING NEW TABLE AS added_members FOR EACH STATEMENT EXECUTE FUNCTION mst2_metadata_rooted_members_added(); + +CREATE TABLE mst2_metadata_scope_source_reference ( + prepare_id text PRIMARY KEY REFERENCES mst2_metadata_prepare(prepare_id), + scope_tree_oid text NOT NULL CHECK(scope_tree_oid ~ '^(sha1:[0-9a-f]{40}|sha256:[0-9a-f]{64}|blake3:[0-9a-f]{64})$'), + ancestor_revisions jsonb NOT NULL CHECK((jsonb_typeof(ancestor_revisions)='array' AND jsonb_array_length(ancestor_revisions)<=256) IS TRUE) +); +CREATE FUNCTION mst2_metadata_derive_scope_source(pid text) RETURNS jsonb LANGUAGE plpgsql VOLATILE STRICT +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE q mst2_metadata_prepare%ROWTYPE; tree_oid text; component text; body bytea; item jsonb; scanned_bytes bigint:=0; + digest bytea; revision uuid; ancestors jsonb[]:=ARRAY[]::jsonb[]; +BEGIN + SELECT * INTO STRICT q FROM mst2_metadata_prepare WHERE prepare_id=pid AND plan_kind='ROOTED' AND state='PREPARING'; + tree_oid:=q.tagged_root_tree_oid; + IF q.scope='/' THEN RETURN jsonb_build_object('scope_tree_oid',tree_oid,'ancestor_revisions','[]'::jsonb); END IF; + FOREACH component IN ARRAY string_to_array(substring(q.scope FROM 2),'/') LOOP + SELECT sub_trees INTO body FROM $CORE_SCHEMA$.mega_tree WHERE tree_id=split_part(tree_oid,':',2); + IF NOT FOUND THEN RAISE EXCEPTION 'rooted source scope has a missing fixed ancestor'; END IF; + scanned_bytes:=scanned_bytes+octet_length(body); + IF scanned_bytes>67108864 THEN RAISE EXCEPTION 'rooted source-scope walk exceeds its fixed byte-work budget'; END IF; + digest:=sha256(body); + revision:=$CORE_SCHEMA$.mst2_route_capture_source_tree(split_part(tree_oid,':',2),digest); + ancestors:=array_append(ancestors,jsonb_build_object('tree_oid',tree_oid,'revision',revision::text,'body_digest',encode(digest,'hex'))); + SELECT value INTO item FROM jsonb_array_elements(mst2_metadata_decode_git_tree(body,split_part(tree_oid,':',1))) + WHERE decode(value->>'name','hex')=convert_to(component,'UTF8'); + IF NOT FOUND OR (item->>'kind')::integer<>4 THEN RAISE EXCEPTION 'rooted source scope is not its exact fixed directory'; END IF; + tree_oid:=item->>'oid'; + END LOOP; + RETURN jsonb_build_object('scope_tree_oid',tree_oid,'ancestor_revisions',to_jsonb(ancestors)); +END $$; +CREATE FUNCTION mst2_metadata_scope_source_guard() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE proof jsonb; +BEGIN + IF TG_OP<>'INSERT' THEN RAISE EXCEPTION 'rooted scope source proof is immutable'; END IF; + proof:=mst2_metadata_derive_scope_source(NEW.prepare_id); + IF NEW.scope_tree_oid IS DISTINCT FROM proof->>'scope_tree_oid' + OR NEW.ancestor_revisions IS DISTINCT FROM proof->'ancestor_revisions' THEN + RAISE EXCEPTION 'rooted scope source proof was not independently derived from actual core ancestors'; END IF; + RETURN NEW; +END $$; +CREATE TRIGGER mst2_metadata_scope_source_guard BEFORE INSERT OR UPDATE OR DELETE ON mst2_metadata_scope_source_reference + FOR EACH ROW EXECUTE FUNCTION mst2_metadata_scope_source_guard(); + +CREATE FUNCTION mst2_metadata_rooted_scope_tree(pid text) RETURNS text LANGUAGE plpgsql VOLATILE STRICT +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE bound mst2_metadata_scope_source_reference%ROWTYPE; proof jsonb; ancestor jsonb; +BEGIN + SELECT * INTO bound FROM mst2_metadata_scope_source_reference WHERE prepare_id=pid; + IF NOT FOUND THEN + proof:=mst2_metadata_derive_scope_source(pid); + INSERT INTO mst2_metadata_scope_source_reference(prepare_id,scope_tree_oid,ancestor_revisions) + VALUES(pid,proof->>'scope_tree_oid',proof->'ancestor_revisions') RETURNING * INTO bound; + END IF; + FOR ancestor IN SELECT value FROM jsonb_array_elements(bound.ancestor_revisions) LOOP + IF NOT $CORE_SCHEMA$.mst2_route_source_tree_matches(split_part(ancestor->>'tree_oid',':',2), + (ancestor->>'revision')::uuid,decode(ancestor->>'body_digest','hex')) THEN + RAISE EXCEPTION 'rooted fixed scope ancestor lost its exact current source revision'; END IF; + END LOOP; + RETURN bound.scope_tree_oid; +END $$; + +CREATE FUNCTION mst2_metadata_rooted_finalize_proof(pid text) RETURNS jsonb LANGUAGE plpgsql VOLATILE STRICT +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE q mst2_metadata_prepare%ROWTYPE; plan jsonb; selected_root_generation bigint; root_certificate bytea; + profile jsonb; source jsonb; source_generation bigint; scoped_tree text; + generation_index jsonb; +BEGIN + SELECT * INTO STRICT q FROM mst2_metadata_prepare WHERE prepare_id=pid AND plan_kind='ROOTED' AND state='PREPARING'; + plan:=mst2_metadata_check_rooted_members(pid); profile:=mst2_metadata_native_profile(pid); + SELECT jsonb_object_agg(value->>'page',value->'generation') INTO generation_index FROM ( + SELECT value FROM jsonb_array_elements(plan->'bindings') UNION ALL + SELECT value FROM jsonb_array_elements(plan->'reused')) members; + selected_root_generation:=(generation_index->>(plan->>'root'))::bigint; + SELECT c.certificate_digest INTO root_certificate FROM mst2_metadata_page_certificate c + JOIN mst2_metadata_graph_node n USING(page_id,generation) JOIN mst2_metadata_current cur USING(page_id,generation) + JOIN mst2_metadata_lifetime life USING(page_id,generation) + WHERE c.page_id=q.metadata_root AND c.generation=selected_root_generation AND n.state='LIVE' + AND n.certificate_digest=c.certificate_digest AND life.state IN ('RESERVED','LIVE') AND life.graph_domain='qualified-v1' + AND c.relative_path_bytes+CASE WHEN q.scope='/' THEN 0 ELSE octet_length(q.scope) END<=4096 + AND c.relative_components+CASE WHEN q.scope='/' THEN 0 ELSE cardinality(string_to_array(substring(q.scope FROM 2),'/')) END<=256 + AND NOT EXISTS(SELECT 1 FROM mst2_metadata_gc_op gc WHERE gc.page_id=c.page_id AND gc.generation=c.generation); + IF NOT FOUND OR NOT EXISTS(SELECT 1 FROM mst2_metadata_root_anchor a WHERE a.anchor_kind='PREPARE' + AND a.owner_key=pid AND a.prepare_id=pid AND a.root_page=q.metadata_root AND a.root_generation=selected_root_generation + AND a.root_certificate_digest=root_certificate) THEN RAISE EXCEPTION 'rooted finalize lacks its independently certified owned root'; END IF; + IF EXISTS(SELECT 1 FROM mst2_metadata_prepare_page m + LEFT JOIN mst2_metadata_current cur USING(page_id,generation) LEFT JOIN mst2_metadata_lifetime life USING(page_id,generation) + LEFT JOIN mst2_metadata_payload body USING(page_id,generation) LEFT JOIN mst2_metadata_graph_node n USING(page_id,generation) + LEFT JOIN mst2_metadata_page_certificate c USING(page_id,generation) + WHERE m.prepare_id=pid AND (cur.page_id IS NULL OR life.page_id IS NULL OR body.page_id IS NULL OR n.page_id IS NULL OR c.page_id IS NULL + OR life.state NOT IN ('RESERVED','LIVE') OR life.graph_domain<>'qualified-v1' OR life.metadata_codec<>q.metadata_codec + OR body.metadata_codec<>q.metadata_codec OR n.metadata_codec<>q.metadata_codec OR c.metadata_codec<>q.metadata_codec + OR life.expected_size<>m.expected_size OR body.byte_size<>m.expected_size OR n.bytes<>m.expected_size OR c.byte_size<>m.expected_size + OR n.state<>'LIVE' OR n.certificate_digest<>c.certificate_digest + OR life.state='RESERVED' AND c.origin_prepare_id<>pid + OR n.incoming_refs<>(SELECT count(*) FROM mst2_metadata_graph_edge e WHERE e.child_page=m.page_id AND e.child_generation=m.generation) + OR EXISTS(SELECT 1 FROM mst2_metadata_gc_op gc WHERE gc.page_id=m.page_id AND gc.generation=m.generation))) THEN + RAISE EXCEPTION 'rooted finalize lost its exact durable canonical delta graph'; END IF; + IF EXISTS((SELECT decode(value->>'parent','hex'),decode(value->>'child','hex') FROM jsonb_array_elements(plan->'edges')) EXCEPT + (SELECT e.parent_page,e.child_page FROM mst2_metadata_prepare_page m JOIN mst2_metadata_graph_edge e + ON e.parent_page=m.page_id AND e.parent_generation=m.generation WHERE m.prepare_id=pid)) + OR EXISTS((SELECT e.parent_page,e.child_page FROM mst2_metadata_prepare_page m JOIN mst2_metadata_graph_edge e + ON e.parent_page=m.page_id AND e.parent_generation=m.generation WHERE m.prepare_id=pid) EXCEPT + (SELECT decode(value->>'parent','hex'),decode(value->>'child','hex') FROM jsonb_array_elements(plan->'edges'))) + OR EXISTS(SELECT 1 FROM mst2_metadata_prepare_reuse_root r WHERE r.prepare_id=pid AND NOT EXISTS( + SELECT 1 FROM mst2_metadata_root_anchor a WHERE a.prepare_id=pid AND a.owner_key=pid AND a.anchor_kind='REUSE' + AND a.root_page=r.root_page AND a.root_generation=r.root_generation)) THEN + RAISE EXCEPTION 'rooted finalize differs from its exact delta edges or owned reuse boundaries'; END IF; + scoped_tree:=mst2_metadata_rooted_scope_tree(pid); + IF NOT EXISTS(SELECT 1 FROM jsonb_array_elements(plan->'source_roots') binding(value) + WHERE value->>'tree_oid'=scoped_tree AND value->>'page'=plan->>'root') THEN + RAISE EXCEPTION 'rooted metadata root is not bound to its independently selected fixed source scope'; END IF; + FOR source IN SELECT value FROM jsonb_array_elements(plan->'source_roots') LOOP + source_generation:=(generation_index->>(source->>'page'))::bigint; + IF NOT EXISTS(SELECT 1 FROM mst2_metadata_source_root_attestation a + JOIN mst2_metadata_current cur ON cur.page_id=a.root_page AND cur.generation=a.root_generation + JOIN mst2_metadata_lifetime life USING(page_id,generation) JOIN mst2_metadata_graph_node n USING(page_id,generation) + JOIN mst2_metadata_prepare origin ON origin.prepare_id=a.origin_prepare_id + JOIN $CORE_SCHEMA$.mega_tree t ON t.tree_id=split_part(a.tagged_tree_oid,':',2) + WHERE a.tagged_tree_oid=source->>'tree_oid' AND a.root_page=decode(source->>'page','hex') AND a.root_generation=source_generation + AND a.source_profile=profile + AND $CORE_SCHEMA$.mst2_route_source_tree_matches(split_part(a.tagged_tree_oid,':',2),a.source_revision,a.source_body_digest) AND n.state='LIVE' + AND n.certificate_digest=a.root_certificate_digest + AND (a.origin_prepare_id=pid AND origin.state='PREPARING' AND life.state IN ('RESERVED','LIVE') + OR origin.state='COMMITTED' AND life.state='LIVE' AND EXISTS(SELECT 1 FROM mst2_metadata_prepare_reuse_root r + WHERE r.prepare_id=pid AND r.root_page=a.root_page AND r.root_generation=a.root_generation + AND EXISTS(SELECT 1 FROM mst2_metadata_source_root_attestation boundary + WHERE boundary.attestation_id=r.attestation_id AND boundary.root_certificate_digest=a.root_certificate_digest)))) THEN + RAISE EXCEPTION 'rooted source binding lacks its independently attested exact current directory'; END IF; + END LOOP; + RETURN jsonb_build_object('root',plan->>'root','generation',selected_root_generation,'certificate',encode(root_certificate,'hex'), + 'scoped_tree_oid',scoped_tree,'delta_nodes',q.node_count,'delta_edges',q.edge_count,'delta_bytes',q.total_bytes); +END $$; diff --git a/src/jupiter/storage/qualified_metadata_serving.sql b/src/jupiter/storage/qualified_metadata_serving.sql new file mode 100644 index 00000000..fa3d729e --- /dev/null +++ b/src/jupiter/storage/qualified_metadata_serving.sql @@ -0,0 +1,581 @@ +-- Root ownership is derived from durable identities, never an application flag. +CREATE FUNCTION mst2_metadata_publication_valid(instance text,commit_id text,tree_id text, + sequence_id bigint,epoch_id bigint,receipt_id bigint,current_head boolean DEFAULT false) +RETURNS boolean LANGUAGE sql VOLATILE SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ + SELECT EXISTS(SELECT 1 FROM $CORE_SCHEMA$.mst2_native_publication c + JOIN $CORE_SCHEMA$.mst2_publication p ON p.id=c.receipt_id + JOIN $CORE_SCHEMA$.mst2_publication_outbox o ON o.operation_id=p.operation_id + JOIN $CORE_SCHEMA$.mega_commit source ON source.commit_id=c.root_commit AND source.tree=c.root_tree + JOIN $CORE_SCHEMA$.mega_commit previous ON previous.commit_id=c.old_root_commit AND previous.tree=c.old_root_tree + JOIN $CORE_SCHEMA$.mega_commit path_source ON path_source.commit_id=c.path_commit AND path_source.tree=c.path_tree + WHERE c.receipt_id=$6 AND c.namespace='/' AND c.instance_id=$1 + AND c.root_commit=$2 AND c.root_tree=$3 AND c.sequence=$4 AND c.writer_epoch=$5 + AND $4>0 AND $5>0 AND p.writer_epoch=$5 AND p.writer_kind='trunk_push' + AND p.native_certificate_version=1 AND p.request_digest_version=1 + AND p.request_digest ~ '^sha256:[0-9a-f]{64}$' AND c.origin_ref='refs/heads/main' + AND c.origin_path=p.namespace AND c.path_commit=p.new_oid AND c.old_root_commit=p.old_oid + AND c.old_path_commit IS DISTINCT FROM c.path_commit AND o.namespace=p.namespace AND o.sequence=p.sequence + AND (NOT $7 OR (EXISTS(SELECT 1 FROM $CORE_SCHEMA$.mst2_native_head h + WHERE h.namespace='/' AND h.state='READY' AND h.instance_id=$1 AND h.root_commit=$2 + AND h.root_tree=$3 AND h.sequence=$4 AND h.writer_epoch=$5 + AND h.certificate_receipt_id=$6) + AND (SELECT count(*) FROM $CORE_SCHEMA$.mega_refs r + WHERE r.path='/' AND r.ref_name='refs/heads/main' AND NOT r.is_cl)=1 + AND EXISTS(SELECT 1 FROM $CORE_SCHEMA$.mega_refs r WHERE r.path='/' + AND r.ref_name='refs/heads/main' AND NOT r.is_cl AND r.ref_commit_hash=$2 AND r.ref_tree_hash=$3)))) +$$; + +CREATE FUNCTION mst2_metadata_descriptor(pid text,instance text,commit_id text,tree_id text) +RETURNS bytea LANGUAGE plpgsql VOLATILE STRICT SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE p record; scope_bytes bytea; view_digest bytea; instance_bytes bytea; +BEGIN + SELECT tagged_root_tree_oid,scope,metadata_root INTO p FROM mst2_metadata_prepare WHERE prepare_id=pid AND plan_kind='ROOTED' + AND state='COMMITTED' AND graph_domain='qualified-v1' AND mst2_metadata_scope_matches(primary_scope); + IF NOT FOUND OR instance IS DISTINCT FROM (instance::uuid)::text + OR split_part(p.tagged_root_tree_oid,':',2) IS DISTINCT FROM tree_id + OR NOT EXISTS(SELECT 1 FROM $CORE_SCHEMA$.mega_commit c WHERE c.commit_id=$3 AND c.tree=$4) + OR octet_length(commit_id)<>octet_length(tree_id) + OR commit_id !~ '^([0-9a-f]{40}|[0-9a-f]{64})$' THEN + RAISE EXCEPTION 'qualified descriptor has no exact canonical instance and fixed source'; + END IF; + PERFORM mst2_metadata_native_profile(pid); + scope_bytes:=convert_to(p.scope,'UTF8'); + IF p.scope !~ '^/' OR octet_length(scope_bytes)>4096 OR p.scope<>'/' AND + (p.scope ~ '/$|//|/(\.|\.\.)(/|$)' OR cardinality(string_to_array(substring(p.scope FROM 2),'/'))>256) + OR EXISTS(SELECT 1 FROM unnest(string_to_array(substring(p.scope FROM 2),'/')) component + WHERE octet_length(component)>255) THEN + RAISE EXCEPTION 'qualified descriptor scope is not canonical'; + END IF; + instance_bytes:=decode(replace(instance,'-',''),'hex'); + view_digest:=sha256(convert_to('mega.mst2.namespaceview','UTF8')||decode('00','hex')||convert_to(commit_id,'UTF8')); + RETURN convert_to('MSD2','UTF8')||decode('00020001','hex')||instance_bytes||view_digest + ||decode(lpad(to_hex(octet_length(scope_bytes)),4,'0'),'hex')||scope_bytes + ||decode('0001000100000000','hex')||p.metadata_root; +END $$; + +CREATE FUNCTION mst2_metadata_root_live(p bytea,g bigint,c bytea) RETURNS boolean +LANGUAGE sql VOLATILE STRICT SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ + SELECT EXISTS(SELECT 1 FROM mst2_metadata_current cur JOIN mst2_metadata_lifetime life USING(page_id,generation) + JOIN mst2_metadata_graph_node node USING(page_id,generation) + JOIN mst2_metadata_page_certificate proof USING(page_id,generation) + JOIN mst2_metadata_payload body USING(page_id,generation) + WHERE cur.page_id=p AND cur.generation=g AND life.state='LIVE' AND life.graph_domain='qualified-v1' + AND life.metadata_codec=1 AND node.state='LIVE' AND node.metadata_codec=1 AND body.metadata_codec=1 + AND node.certificate_digest=c AND proof.certificate_digest=c AND proof.proof_revision=1 + AND proof.namespace_uuid='$NAMESPACE_UUID$'::uuid AND life.expected_size=proof.byte_size + AND node.bytes=proof.byte_size AND body.byte_size=proof.byte_size AND octet_length(body.payload)=proof.byte_size + AND p=sha256(convert_to('mega.mst2.metapage','UTF8')||decode('00','hex')||body.payload) + AND NOT EXISTS(SELECT 1 FROM mst2_metadata_gc_op gc WHERE gc.page_id=p AND gc.generation=g)) +$$; + +CREATE FUNCTION mst2_metadata_serving_source(pid text,a_id uuid,p bytea,g bigint,c bytea) +RETURNS boolean LANGUAGE plpgsql VOLATILE STRICT SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE a record; scope text; certificate record; +BEGIN + SELECT proof.tagged_tree_oid,proof.source_revision,proof.source_body_digest,proof.profile_digest,proof.source_profile + INTO a FROM mst2_metadata_source_root_attestation proof + JOIN mst2_metadata_prepare origin ON origin.prepare_id=proof.origin_prepare_id + WHERE proof.attestation_id=a_id AND proof.namespace_uuid='$NAMESPACE_UUID$'::uuid + AND proof.root_page=p AND proof.root_generation=g AND proof.root_certificate_digest=c + AND origin.state='COMMITTED' AND proof.source_profile=mst2_metadata_native_profile(pid) + AND proof.tagged_tree_oid=mst2_metadata_rooted_scope_tree(pid); + IF NOT FOUND THEN RETURN false; END IF; + SELECT relative_path_bytes,relative_components INTO certificate FROM mst2_metadata_page_certificate WHERE page_id=p AND generation=g AND certificate_digest=c; + IF NOT FOUND THEN RETURN false; END IF; + SELECT q.scope INTO STRICT scope FROM mst2_metadata_prepare q WHERE q.prepare_id=pid; + IF (CASE WHEN scope='/' THEN 0 ELSE octet_length(scope) END)::bigint+certificate.relative_path_bytes>4096 + OR (CASE WHEN scope='/' THEN 0 ELSE cardinality(string_to_array(substring(scope FROM 2),'/')) END)::bigint + +certificate.relative_components>256 THEN RETURN false; END IF; + RETURN $CORE_SCHEMA$.mst2_route_source_tree_matches(split_part(a.tagged_tree_oid,':',2),a.source_revision,a.source_body_digest) + AND a.profile_digest=sha256(convert_to('mega.mst2.native-profile.v1','UTF8')||decode('00','hex') + ||convert_to(a.source_profile::text,'UTF8')); +END $$; + +CREATE FUNCTION mst2_metadata_snapshot_candidate(sid text,descriptor bytea,instance text,commit_id text, + tree_id text,root bytea,profile jsonb) RETURNS boolean LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE p record; a record; +BEGIN + IF sid IS DISTINCT FROM 'sha256:'||encode(sha256(convert_to('mega.mst2.descriptor','UTF8') + ||decode('00','hex')||descriptor),'hex') THEN RETURN false; END IF; + FOR p IN SELECT q.prepare_id FROM mst2_metadata_prepare q WHERE q.metadata_root=root AND q.state='COMMITTED' + AND q.plan_kind='ROOTED' AND q.graph_domain='qualified-v1' AND q.coverage_retired_at IS NULL + AND q.tagged_root_tree_oid=profile->>'tagged_root_tree_oid' AND q.scope=profile->>'scope' + AND $CORE_SCHEMA$.mst2_route_profile(q.source_domain,q.tagged_root_tree_oid,q.scope,q.schema_version, + q.metadata_codec,q.materialization_policy,q.fs_semantics,q.access_projection, + q.verification_revision,q.projection_revision)=profile ORDER BY q.prepare_id LIMIT 1 LOOP + IF descriptor IS DISTINCT FROM mst2_metadata_descriptor(p.prepare_id,instance,commit_id,tree_id) THEN CONTINUE; END IF; + FOR a IN SELECT proof.attestation_id,proof.root_generation,proof.root_certificate_digest FROM mst2_metadata_source_root_attestation proof + JOIN mst2_metadata_current current_root ON current_root.page_id=proof.root_page AND current_root.generation=proof.root_generation + WHERE proof.root_page=root AND proof.tagged_tree_oid=mst2_metadata_rooted_scope_tree(p.prepare_id) + AND proof.source_profile=mst2_metadata_native_profile(p.prepare_id) + AND $CORE_SCHEMA$.mst2_route_source_tree_matches(split_part(proof.tagged_tree_oid,':',2),proof.source_revision,proof.source_body_digest) + ORDER BY proof.attestation_id LIMIT 1 LOOP + IF mst2_metadata_serving_source(p.prepare_id,a.attestation_id,root,a.root_generation,a.root_certificate_digest) + AND mst2_metadata_root_live(root,a.root_generation,a.root_certificate_digest) THEN RETURN true; END IF; + END LOOP; + END LOOP; + RETURN false; +END $$; + +CREATE FUNCTION mst2_metadata_incarnation_proof(sid text,inc uuid,require_live boolean DEFAULT false) +RETURNS boolean LANGUAGE plpgsql VOLATILE SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE s mst2_qualified_session_incarnation%ROWTYPE; p record; + a record; profile jsonb; +BEGIN + SELECT * INTO s FROM mst2_qualified_session_incarnation WHERE snapshot_id=sid AND session_incarnation=inc; + IF NOT FOUND OR s.namespace_uuid<>'$NAMESPACE_UUID$'::uuid OR s.authorization_epoch<>1 THEN RETURN false; END IF; + SELECT prepare_id,source_domain,tagged_root_tree_oid,scope,schema_version,metadata_codec, + materialization_policy,fs_semantics,access_projection,verification_revision,projection_revision + INTO p FROM mst2_metadata_prepare WHERE prepare_id=s.prepare_id AND state='COMMITTED' + AND plan_kind='ROOTED' AND storage_seal=s.storage_seal AND metadata_root=s.metadata_root; + IF NOT FOUND THEN RETURN false; END IF; + profile:=$CORE_SCHEMA$.mst2_route_profile(p.source_domain,p.tagged_root_tree_oid,p.scope,p.schema_version, + p.metadata_codec,p.materialization_policy,p.fs_semantics,p.access_projection,p.verification_revision,p.projection_revision); + IF s.source_profile IS DISTINCT FROM convert_to(profile::text,'UTF8') + OR s.canonical_descriptor IS DISTINCT FROM mst2_metadata_descriptor(p.prepare_id,s.instance_id,s.commit_oid,s.root_tree_oid) + OR s.snapshot_id IS DISTINCT FROM 'sha256:'||encode(sha256(convert_to('mega.mst2.descriptor','UTF8') + ||decode('00','hex')||s.canonical_descriptor),'hex') + OR NOT EXISTS(SELECT 1 FROM $CORE_SCHEMA$.mst2_snapshot_storage_route r + WHERE r.snapshot_id=s.snapshot_id AND r.namespace_uuid=s.namespace_uuid AND r.canonical_descriptor=s.canonical_descriptor + AND r.instance_id=s.instance_id AND r.commit_oid=s.commit_oid AND r.root_tree_oid=s.root_tree_oid + AND r.metadata_root=s.metadata_root AND r.source_profile=profile) + OR NOT mst2_metadata_publication_valid(s.instance_id,s.commit_oid,s.root_tree_oid, + s.publication_sequence,s.writer_epoch,s.certificate_receipt_id,false) THEN RETURN false; END IF; + SELECT attestation_id,root_certificate_digest INTO a FROM mst2_metadata_source_root_attestation WHERE attestation_id=s.attestation_id + AND attestation_digest=s.attestation_digest AND root_page=s.metadata_root AND root_generation=s.root_generation; + IF NOT FOUND OR NOT mst2_metadata_serving_source(s.prepare_id,a.attestation_id,s.metadata_root, + s.root_generation,a.root_certificate_digest) THEN RETURN false; END IF; + IF require_live AND (s.state<>'READY' OR NOT mst2_metadata_root_live(s.metadata_root,s.root_generation,a.root_certificate_digest) + OR NOT EXISTS(SELECT 1 FROM mst2_metadata_root_anchor anchor WHERE anchor.anchor_kind='SESSION' + AND anchor.owner_key=s.snapshot_id||':'||s.session_incarnation::text AND anchor.snapshot_id=s.snapshot_id + AND anchor.session_incarnation=s.session_incarnation AND anchor.root_page=s.metadata_root + AND anchor.root_generation=s.root_generation AND anchor.root_certificate_digest=a.root_certificate_digest)) THEN RETURN false; END IF; + RETURN true; +END $$; + +CREATE FUNCTION mst2_metadata_snapshot_route_proof(sid text,descriptor bytea,instance text,commit_id text, + tree_id text,root bytea,profile jsonb) RETURNS boolean LANGUAGE sql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ + SELECT EXISTS(SELECT 1 FROM mst2_qualified_session_incarnation s WHERE s.snapshot_id=sid + AND s.canonical_descriptor=descriptor AND s.instance_id=instance AND s.commit_oid=commit_id AND s.root_tree_oid=tree_id + AND s.metadata_root=root AND s.source_profile=convert_to(profile::text,'UTF8') + AND mst2_metadata_incarnation_proof(s.snapshot_id,s.session_incarnation,false)) +$$; + +CREATE FUNCTION mst2_metadata_lease_route_proof(lid text,sid text,namespace uuid,inc uuid,pid text,root bytea, + auth bigint,publication bigint,epoch bigint,receipt bigint) RETURNS boolean LANGUAGE sql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ + SELECT EXISTS(SELECT 1 FROM mst2_qualified_lease_binding l JOIN mst2_qualified_session_incarnation s + ON s.snapshot_id=l.snapshot_id AND s.session_incarnation=l.session_incarnation + WHERE l.lease_id=lid AND l.snapshot_id=sid AND l.namespace_uuid=namespace AND namespace='$NAMESPACE_UUID$'::uuid + AND l.session_incarnation=inc AND l.prepare_id=pid AND l.metadata_root=root + AND l.authorization_epoch=auth AND l.publication_sequence=publication AND l.writer_epoch=epoch + AND l.certificate_receipt_id=receipt AND l.storage_seal=s.storage_seal AND l.root_generation=s.root_generation + AND ROW(l.authorization_epoch,l.publication_sequence,l.writer_epoch,l.certificate_receipt_id) + IS NOT DISTINCT FROM ROW(s.authorization_epoch,s.publication_sequence,s.writer_epoch,s.certificate_receipt_id) + AND mst2_metadata_incarnation_proof(s.snapshot_id,s.session_incarnation,false)) +$$; + +CREATE UNIQUE INDEX mst2_qualified_snapshot_ready ON mst2_qualified_session_incarnation(snapshot_id) WHERE state='READY'; +CREATE INDEX mst2_metadata_committed_source ON mst2_metadata_prepare(metadata_root,tagged_root_tree_oid,scope,prepare_id) + WHERE state='COMMITTED' AND plan_kind='ROOTED'; +CREATE INDEX mst2_qualified_lease_expiry ON mst2_qualified_lease_binding(expires_at_unix,lease_id) WHERE state='ACTIVE'; +CREATE FUNCTION mst2_metadata_session_guard() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE p record; a record; profile jsonb; +BEGIN + IF TG_OP='DELETE' THEN RAISE EXCEPTION 'qualified session incarnation history is immutable'; END IF; + IF TG_OP='UPDATE' THEN + IF (to_jsonb(NEW)-'state') IS DISTINCT FROM (to_jsonb(OLD)-'state') OR OLD.state='RETIRED' AND NEW.state<>'RETIRED' + OR NEW.state NOT IN ('READY','RETIRED') THEN RAISE EXCEPTION 'qualified fixed incarnation cannot change or revive'; END IF; + IF NEW.state='RETIRED' AND EXISTS(SELECT 1 FROM mst2_qualified_lease_binding l + WHERE l.snapshot_id=OLD.snapshot_id AND l.session_incarnation=OLD.session_incarnation AND l.state='ACTIVE') THEN + RAISE EXCEPTION 'qualified ready session still has active lease identities'; END IF; + RETURN NEW; + END IF; + SELECT prepare_id,source_domain,tagged_root_tree_oid,scope,schema_version,metadata_codec, + materialization_policy,fs_semantics,access_projection,verification_revision,projection_revision + INTO p FROM mst2_metadata_prepare WHERE prepare_id=NEW.prepare_id AND state='COMMITTED' + AND plan_kind='ROOTED' AND storage_seal=NEW.storage_seal AND metadata_root=NEW.metadata_root AND coverage_retired_at IS NULL; + IF NOT FOUND OR NEW.namespace_uuid<>'$NAMESPACE_UUID$'::uuid OR NEW.state<>'READY' OR NEW.authorization_epoch<>1 + OR substr(NEW.session_incarnation::text,15,1)<>'4' OR substr(NEW.session_incarnation::text,20,1) NOT IN ('8','9','a','b') + OR EXISTS(SELECT 1 FROM mst2_qualified_session_incarnation WHERE snapshot_id=NEW.snapshot_id AND state='READY') THEN + RAISE EXCEPTION 'qualified incarnation requires a fresh server identity and definitive rooted preparation'; END IF; + profile:=$CORE_SCHEMA$.mst2_route_profile(p.source_domain,p.tagged_root_tree_oid,p.scope,p.schema_version, + p.metadata_codec,p.materialization_policy,p.fs_semantics,p.access_projection,p.verification_revision,p.projection_revision); + SELECT attestation_id,root_certificate_digest INTO a FROM mst2_metadata_source_root_attestation WHERE attestation_id=NEW.attestation_id + AND attestation_digest=NEW.attestation_digest AND root_page=NEW.metadata_root AND root_generation=NEW.root_generation; + IF NOT FOUND OR NEW.source_profile IS DISTINCT FROM convert_to(profile::text,'UTF8') + OR NEW.canonical_descriptor IS DISTINCT FROM mst2_metadata_descriptor(p.prepare_id,NEW.instance_id,NEW.commit_oid,NEW.root_tree_oid) + OR NEW.snapshot_id IS DISTINCT FROM 'sha256:'||encode(sha256(convert_to('mega.mst2.descriptor','UTF8') + ||decode('00','hex')||NEW.canonical_descriptor),'hex') + OR NOT mst2_metadata_serving_source(p.prepare_id,a.attestation_id,NEW.metadata_root,NEW.root_generation,a.root_certificate_digest) + OR NOT mst2_metadata_root_live(NEW.metadata_root,NEW.root_generation,a.root_certificate_digest) + OR NOT EXISTS(SELECT 1 FROM mst2_metadata_root_anchor anchor WHERE anchor.anchor_kind='PREPARE' + AND anchor.prepare_id=NEW.prepare_id AND anchor.owner_key=NEW.prepare_id AND anchor.root_page=NEW.metadata_root + AND anchor.root_generation=NEW.root_generation AND anchor.root_certificate_digest=a.root_certificate_digest) + OR NOT mst2_metadata_publication_valid(NEW.instance_id,NEW.commit_oid,NEW.root_tree_oid, + NEW.publication_sequence,NEW.writer_epoch,NEW.certificate_receipt_id,true) + OR NOT EXISTS(SELECT 1 FROM $CORE_SCHEMA$.mst2_snapshot_storage_route r WHERE r.snapshot_id=NEW.snapshot_id + AND r.namespace_uuid=NEW.namespace_uuid AND r.canonical_descriptor=NEW.canonical_descriptor + AND r.instance_id=NEW.instance_id AND r.commit_oid=NEW.commit_oid AND r.root_tree_oid=NEW.root_tree_oid + AND r.metadata_root=NEW.metadata_root AND r.source_profile=profile) THEN + RAISE EXCEPTION 'qualified incarnation differs from its independently derived source and permanent route'; END IF; + RETURN NEW; +END $$; +CREATE TRIGGER mst2_metadata_session_guard BEFORE INSERT OR UPDATE OR DELETE ON mst2_qualified_session_incarnation + FOR EACH ROW EXECUTE FUNCTION mst2_metadata_session_guard(); + +CREATE FUNCTION mst2_metadata_lease_guard() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE s mst2_qualified_session_incarnation%ROWTYPE; now_unix bigint:=floor(extract(epoch FROM clock_timestamp()))::bigint; +BEGIN + IF TG_OP='DELETE' THEN RAISE EXCEPTION 'qualified lease binding history is immutable'; END IF; + IF TG_OP='UPDATE' THEN + IF (to_jsonb(NEW)-ARRAY['state','expires_at_unix','lease_epoch']) IS DISTINCT FROM + (to_jsonb(OLD)-ARRAY['state','expires_at_unix','lease_epoch']) OR OLD.state<>'ACTIVE' AND NEW IS DISTINCT FROM OLD THEN + RAISE EXCEPTION 'qualified fixed lease cannot change or revive'; END IF; + IF NEW.state='ACTIVE' THEN + IF OLD.expires_at_unix<=now_unix OR NEW.expires_at_unix<=now_unix OR NEW.expires_at_unix>now_unix+3600 + OR NEW.lease_epoch<>OLD.lease_epoch THEN RAISE EXCEPTION 'qualified renewal requires its still-active bounded lease'; END IF; + ELSIF NEW.state IN ('RELEASED','EXPIRED') THEN + IF NEW.expires_at_unix<>OLD.expires_at_unix OR NEW.lease_epoch<>OLD.lease_epoch+1 + OR NEW.state='EXPIRED' AND OLD.expires_at_unix>now_unix THEN + RAISE EXCEPTION 'qualified terminal lease differs from its exact deadline and epoch'; END IF; + ELSE RAISE EXCEPTION 'qualified lease state is unsupported'; END IF; + RETURN NEW; + END IF; + SELECT * INTO s FROM mst2_qualified_session_incarnation + WHERE snapshot_id=NEW.snapshot_id AND session_incarnation=NEW.session_incarnation AND state='READY'; + IF NOT FOUND OR NEW.namespace_uuid<>'$NAMESPACE_UUID$'::uuid OR NEW.namespace_uuid<>s.namespace_uuid + OR NEW.state<>'ACTIVE' OR NEW.lease_epoch<>1 OR NEW.expires_at_unix<=now_unix OR NEW.expires_at_unix>now_unix+3600 + OR NEW.lease_id IS DISTINCT FROM (NEW.lease_id::uuid)::text OR substr(NEW.lease_id,15,1)<>'4' + OR substr(NEW.lease_id,20,1) NOT IN ('8','9','a','b') + OR ROW(NEW.prepare_id,NEW.storage_seal,NEW.metadata_root,NEW.root_generation,NEW.authorization_epoch, + NEW.publication_sequence,NEW.writer_epoch,NEW.certificate_receipt_id) IS DISTINCT FROM + ROW(s.prepare_id,s.storage_seal,s.metadata_root,s.root_generation,s.authorization_epoch, + s.publication_sequence,s.writer_epoch,s.certificate_receipt_id) + OR NOT mst2_metadata_incarnation_proof(s.snapshot_id,s.session_incarnation,true) THEN + RAISE EXCEPTION 'qualified lease differs from its exact active incarnation and source'; END IF; + RETURN NEW; +END $$; +CREATE TRIGGER mst2_metadata_lease_guard BEFORE INSERT OR UPDATE OR DELETE ON mst2_qualified_lease_binding + FOR EACH ROW EXECUTE FUNCTION mst2_metadata_lease_guard(); + +CREATE FUNCTION mst2_metadata_reader_guard() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE l mst2_qualified_lease_binding%ROWTYPE; now_unix bigint:=floor(extract(epoch FROM clock_timestamp()))::bigint; +BEGIN + IF TG_OP='DELETE' THEN RAISE EXCEPTION 'qualified reader operation history is immutable'; END IF; + IF TG_OP='UPDATE' THEN + IF (to_jsonb(NEW)-'state') IS DISTINCT FROM (to_jsonb(OLD)-'state') + OR OLD.state<>'ACTIVE' AND NEW.state<>OLD.state OR NEW.state NOT IN ('ACTIVE','FINISHED','EXPIRED') + OR NEW.state='EXPIRED' AND OLD.hard_deadline_unix>now_unix THEN + RAISE EXCEPTION 'qualified reader identity cannot change or be prematurely expired'; END IF; + RETURN NEW; + END IF; + SELECT * INTO l FROM mst2_qualified_lease_binding WHERE lease_id=NEW.lease_id AND state='ACTIVE' AND expires_at_unix>now_unix; + IF NOT FOUND OR NEW.state<>'ACTIVE' OR substr(NEW.operation_id::text,15,1)<>'4' + OR substr(NEW.operation_id::text,20,1) NOT IN ('8','9','a','b') + OR ROW(NEW.snapshot_id,NEW.session_incarnation,NEW.root_page,NEW.root_generation,NEW.lease_epoch) IS DISTINCT FROM + ROW(l.snapshot_id,l.session_incarnation,l.metadata_root,l.root_generation,l.lease_epoch) + OR NEW.hard_deadline_unix<=now_unix OR NEW.hard_deadline_unix>least(l.expires_at_unix,now_unix+60) + OR NOT mst2_metadata_incarnation_proof(l.snapshot_id,l.session_incarnation,true) THEN + RAISE EXCEPTION 'qualified reader lacks its exact active lease and bounded deadline'; END IF; + RETURN NEW; +END $$; +CREATE TRIGGER mst2_metadata_reader_guard BEFORE INSERT OR UPDATE OR DELETE ON mst2_metadata_reader_operation + FOR EACH ROW EXECUTE FUNCTION mst2_metadata_reader_guard(); + +CREATE FUNCTION mst2_metadata_serving_complete() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE s mst2_qualified_session_incarnation%ROWTYPE; l mst2_qualified_lease_binding%ROWTYPE; + r mst2_metadata_reader_operation%ROWTYPE; +BEGIN + IF TG_TABLE_NAME='mst2_qualified_session_incarnation' THEN + SELECT * INTO STRICT s FROM mst2_qualified_session_incarnation + WHERE snapshot_id=NEW.snapshot_id AND session_incarnation=NEW.session_incarnation; + IF NOT mst2_metadata_incarnation_proof(s.snapshot_id,s.session_incarnation,s.state='READY') + OR s.state='READY' AND NOT EXISTS(SELECT 1 FROM mst2_qualified_lease_binding lease + WHERE lease.snapshot_id=s.snapshot_id AND lease.session_incarnation=s.session_incarnation AND lease.state='ACTIVE') + OR s.state='RETIRED' AND EXISTS(SELECT 1 FROM mst2_metadata_root_anchor a + WHERE a.anchor_kind='SESSION' AND a.snapshot_id=s.snapshot_id AND a.session_incarnation=s.session_incarnation) THEN + RAISE EXCEPTION 'qualified session cannot commit without its exact final serving roots'; END IF; + ELSIF TG_TABLE_NAME='mst2_qualified_lease_binding' THEN + SELECT * INTO STRICT l FROM mst2_qualified_lease_binding WHERE lease_id=NEW.lease_id; + IF NOT EXISTS(SELECT 1 FROM $CORE_SCHEMA$.mst2_lease_storage_route route + WHERE route.lease_id=l.lease_id AND route.namespace_uuid=l.namespace_uuid AND route.snapshot_id=l.snapshot_id + AND route.session_incarnation=l.session_incarnation AND route.prepare_id=l.prepare_id AND route.metadata_root=l.metadata_root + AND route.authorization_epoch=l.authorization_epoch AND route.publication_sequence=l.publication_sequence + AND route.writer_epoch=l.writer_epoch AND route.certificate_receipt_id=l.certificate_receipt_id) + OR l.state='ACTIVE' AND (l.expires_at_unix<=floor(extract(epoch FROM clock_timestamp()))::bigint + OR NOT mst2_metadata_incarnation_proof(l.snapshot_id,l.session_incarnation,true) + OR NOT EXISTS(SELECT 1 FROM mst2_metadata_root_anchor a WHERE a.anchor_kind='LEASE' AND a.owner_key=l.lease_id + AND a.lease_id=l.lease_id AND a.root_page=l.metadata_root AND a.root_generation=l.root_generation)) + OR l.state<>'ACTIVE' AND NOT EXISTS(SELECT 1 FROM mst2_metadata_reader_operation operation + WHERE operation.lease_id=l.lease_id AND operation.state='ACTIVE') + AND EXISTS(SELECT 1 FROM mst2_metadata_root_anchor a WHERE a.anchor_kind='LEASE' AND a.lease_id=l.lease_id) THEN + RAISE EXCEPTION 'qualified lease cannot commit without its exact final route and owned protection'; END IF; + ELSE + SELECT * INTO STRICT r FROM mst2_metadata_reader_operation WHERE operation_id=NEW.operation_id; + IF r.state='ACTIVE' AND (r.hard_deadline_unix<=floor(extract(epoch FROM clock_timestamp()))::bigint + OR (SELECT count(*) FROM mst2_metadata_root_anchor a WHERE a.reader_operation_id=r.operation_id + AND a.anchor_kind IN ('REQUEST','READER') AND a.owner_key=r.operation_id::text + AND a.lease_id=r.lease_id AND a.snapshot_id=r.snapshot_id AND a.session_incarnation=r.session_incarnation + AND a.root_page=r.root_page AND a.root_generation=r.root_generation)<>2) + OR r.state<>'ACTIVE' AND EXISTS(SELECT 1 FROM mst2_metadata_root_anchor a WHERE a.reader_operation_id=r.operation_id) THEN + RAISE EXCEPTION 'qualified reader cannot commit without both exact owned roots or definitive cleanup'; END IF; + END IF; + RETURN NULL; +END $$; +CREATE CONSTRAINT TRIGGER mst2_metadata_session_complete AFTER INSERT OR UPDATE ON mst2_qualified_session_incarnation + DEFERRABLE INITIALLY DEFERRED FOR EACH ROW EXECUTE FUNCTION mst2_metadata_serving_complete(); +CREATE CONSTRAINT TRIGGER mst2_metadata_lease_complete AFTER INSERT OR UPDATE ON mst2_qualified_lease_binding + DEFERRABLE INITIALLY DEFERRED FOR EACH ROW EXECUTE FUNCTION mst2_metadata_serving_complete(); +CREATE CONSTRAINT TRIGGER mst2_metadata_reader_complete AFTER INSERT OR UPDATE ON mst2_metadata_reader_operation + DEFERRABLE INITIALLY DEFERRED FOR EACH ROW EXECUTE FUNCTION mst2_metadata_serving_complete(); + +CREATE FUNCTION mst2_metadata_session_row(sid text,lid text,instance text) +RETURNS TABLE(canonical_descriptor bytea,commit_oid text,root_tree_oid text,expires_at_unix bigint, + authorization_epoch bigint,session_incarnation uuid,root_generation bigint,certificate_digest bytea,attestation_id uuid) +LANGUAGE plpgsql VOLATILE SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE l mst2_qualified_lease_binding%ROWTYPE; s mst2_qualified_session_incarnation%ROWTYPE; + c bytea; +BEGIN + SELECT * INTO l FROM mst2_qualified_lease_binding lease WHERE lease.lease_id=lid AND lease.snapshot_id=sid + AND lease.state='ACTIVE' AND lease.expires_at_unix>floor(extract(epoch FROM clock_timestamp()))::bigint; + IF NOT FOUND THEN RETURN; END IF; + SELECT * INTO s FROM mst2_qualified_session_incarnation session WHERE session.snapshot_id=sid + AND session.session_incarnation=l.session_incarnation AND session.state='READY' AND session.instance_id=instance; + IF NOT FOUND THEN RETURN; END IF; + SELECT proof.root_certificate_digest INTO c FROM mst2_metadata_source_root_attestation proof + WHERE proof.attestation_id=s.attestation_id AND proof.attestation_digest=s.attestation_digest; + IF NOT FOUND OR NOT mst2_metadata_incarnation_proof(sid,s.session_incarnation,true) + OR NOT $CORE_SCHEMA$.mst2_route_qualified_namespace_valid('$NAMESPACE_UUID$'::uuid) + OR NOT EXISTS(SELECT 1 FROM $CORE_SCHEMA$.mst2_lease_storage_route route + WHERE route.lease_id=lid AND route.snapshot_id=sid AND route.namespace_uuid=l.namespace_uuid + AND route.session_incarnation=l.session_incarnation AND route.prepare_id=l.prepare_id AND route.metadata_root=l.metadata_root + AND route.authorization_epoch=l.authorization_epoch AND route.publication_sequence=l.publication_sequence + AND route.writer_epoch=l.writer_epoch AND route.certificate_receipt_id=l.certificate_receipt_id) + OR l.namespace_uuid IS DISTINCT FROM s.namespace_uuid + OR ROW(l.prepare_id,l.storage_seal,l.metadata_root,l.root_generation,l.authorization_epoch, + l.publication_sequence,l.writer_epoch,l.certificate_receipt_id) IS DISTINCT FROM + ROW(s.prepare_id,s.storage_seal,s.metadata_root,s.root_generation,s.authorization_epoch, + s.publication_sequence,s.writer_epoch,s.certificate_receipt_id) + OR NOT EXISTS(SELECT 1 FROM mst2_metadata_root_anchor anchor WHERE anchor.anchor_kind='LEASE' AND anchor.owner_key=lid + AND anchor.lease_id=lid AND anchor.snapshot_id=sid AND anchor.session_incarnation=l.session_incarnation + AND anchor.root_page=l.metadata_root AND anchor.root_generation=l.root_generation AND anchor.root_certificate_digest=c) THEN + RAISE EXCEPTION 'qualified durable session route, source, current lifetime or root ownership changed'; END IF; + RETURN QUERY SELECT s.canonical_descriptor,s.commit_oid,s.root_tree_oid,l.expires_at_unix, + l.authorization_epoch,s.session_incarnation,s.root_generation,c,s.attestation_id; +END $$; + +CREATE FUNCTION mst2_metadata_handoff(request jsonb) RETURNS TABLE(lease_id text,expires_at_unix bigint) +LANGUAGE plpgsql VOLATILE SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE sid text:=request->>'snapshot_id'; descriptor bytea:=decode(request->>'canonical_descriptor','hex'); + instance text:=request->>'instance_id'; commit_id text:=request->>'commit_oid'; tree_id text:=request->>'root_tree_oid'; + lid text:=request->>'lease_id'; inc uuid:=(request->>'session_incarnation')::uuid; pid text:=request->>'prepare_id'; + p record; s mst2_qualified_session_incarnation%ROWTYPE; + a record; profile jsonb; now_unix bigint; deadline bigint; +BEGIN + PERFORM $CORE_SCHEMA$.mst2_route_enter($CORE_LITERAL$); + PERFORM mst2_metadata_cleanup_expired(64); + now_unix:=floor(extract(epoch FROM clock_timestamp()))::bigint; + deadline:=now_unix+least(3600,greatest(1,(request->>'lease_seconds')::bigint)); + IF request->>'authorization_epoch' IS DISTINCT FROM '1' OR lid IS NULL OR inc IS NULL OR descriptor IS NULL + OR NOT mst2_metadata_publication_valid(instance,commit_id,tree_id,(request->>'publication_sequence')::bigint, + (request->>'writer_epoch')::bigint,(request->>'certificate_receipt_id')::bigint,true) THEN + RAISE EXCEPTION 'qualified handoff differs from the independently observed current publication'; END IF; + SELECT * INTO s FROM mst2_qualified_session_incarnation WHERE snapshot_id=sid AND state='READY'; + IF FOUND THEN + IF s.state<>'READY' OR s.canonical_descriptor IS DISTINCT FROM descriptor OR s.instance_id IS DISTINCT FROM instance + OR s.commit_oid IS DISTINCT FROM commit_id OR s.root_tree_oid IS DISTINCT FROM tree_id + OR ROW(s.publication_sequence,s.writer_epoch,s.certificate_receipt_id) IS DISTINCT FROM + ROW((request->>'publication_sequence')::bigint,(request->>'writer_epoch')::bigint,(request->>'certificate_receipt_id')::bigint) + OR NOT mst2_metadata_incarnation_proof(sid,s.session_incarnation,true) THEN + RAISE EXCEPTION 'qualified handoff cannot revive or replace a historical incarnation'; END IF; + ELSE + SELECT metadata_root,storage_seal,source_domain,tagged_root_tree_oid,scope,schema_version,metadata_codec, + materialization_policy,fs_semantics,access_projection,verification_revision,projection_revision + INTO p FROM mst2_metadata_prepare WHERE prepare_id=pid AND state='COMMITTED' AND plan_kind='ROOTED' + AND storage_seal=decode(request->>'storage_seal','hex') AND coverage_retired_at IS NULL; + IF NOT FOUND THEN RAISE EXCEPTION 'qualified handoff has no exact definitive rooted receipt'; END IF; + SELECT attestation_id,attestation_digest,root_generation,root_certificate_digest + INTO a FROM mst2_metadata_source_root_attestation WHERE attestation_id=(request->>'attestation_id')::uuid + AND attestation_digest=decode(request->>'attestation_digest','hex') AND root_page=p.metadata_root + AND root_generation=(request->>'root_generation')::bigint AND root_certificate_digest=decode(request->>'certificate_digest','hex'); + IF NOT FOUND OR descriptor IS DISTINCT FROM mst2_metadata_descriptor(pid,instance,commit_id,tree_id) + OR NOT mst2_metadata_serving_source(pid,a.attestation_id,p.metadata_root,a.root_generation,a.root_certificate_digest) + OR NOT mst2_metadata_root_live(p.metadata_root,a.root_generation,a.root_certificate_digest) + OR NOT EXISTS(SELECT 1 FROM mst2_metadata_root_anchor anchor WHERE anchor.anchor_kind='PREPARE' + AND anchor.prepare_id=pid AND anchor.owner_key=pid AND anchor.root_page=p.metadata_root + AND anchor.root_generation=a.root_generation AND anchor.root_certificate_digest=a.root_certificate_digest) THEN + RAISE EXCEPTION 'qualified handoff lacks its continuously owned exact canonical source root'; END IF; + profile:=$CORE_SCHEMA$.mst2_route_profile(p.source_domain,p.tagged_root_tree_oid,p.scope,p.schema_version, + p.metadata_codec,p.materialization_policy,p.fs_semantics,p.access_projection,p.verification_revision,p.projection_revision); + INSERT INTO $CORE_SCHEMA$.mst2_snapshot_storage_route(snapshot_id,namespace_uuid,canonical_descriptor,instance_id, + commit_oid,root_tree_oid,metadata_root,source_profile) + VALUES(sid,'$NAMESPACE_UUID$'::uuid,descriptor,instance,commit_id,tree_id,p.metadata_root,profile) + ON CONFLICT(snapshot_id) DO NOTHING; + INSERT INTO mst2_qualified_session_incarnation(snapshot_id,session_incarnation,namespace_uuid,prepare_id,storage_seal, + metadata_root,root_generation,source_profile,instance_id,commit_oid,root_tree_oid,authorization_epoch, + publication_sequence,writer_epoch,certificate_receipt_id,canonical_descriptor,attestation_id,attestation_digest,state) + VALUES(sid,inc,'$NAMESPACE_UUID$'::uuid,pid,p.storage_seal,p.metadata_root,a.root_generation,convert_to(profile::text,'UTF8'), + instance,commit_id,tree_id,1,(request->>'publication_sequence')::bigint,(request->>'writer_epoch')::bigint, + (request->>'certificate_receipt_id')::bigint,descriptor,a.attestation_id,a.attestation_digest,'READY') RETURNING * INTO s; + INSERT INTO mst2_metadata_root_anchor(anchor_id,anchor_kind,owner_key,root_page,root_generation,root_certificate_digest, + snapshot_id,session_incarnation) VALUES(gen_random_uuid(),'SESSION',sid||':'||inc::text, + s.metadata_root,s.root_generation,a.root_certificate_digest,sid,inc); + END IF; + SELECT root_certificate_digest INTO STRICT a FROM mst2_metadata_source_root_attestation WHERE attestation_id=s.attestation_id; + INSERT INTO mst2_qualified_lease_binding(lease_id,snapshot_id,session_incarnation,prepare_id,storage_seal,metadata_root, + root_generation,namespace_uuid,authorization_epoch,publication_sequence,writer_epoch,certificate_receipt_id, + expires_at_unix,state,lease_epoch) VALUES(lid,sid,s.session_incarnation,s.prepare_id,s.storage_seal,s.metadata_root, + s.root_generation,s.namespace_uuid,s.authorization_epoch,s.publication_sequence,s.writer_epoch,s.certificate_receipt_id, + deadline,'ACTIVE',1); + INSERT INTO $CORE_SCHEMA$.mst2_lease_storage_route(lease_id,snapshot_id,namespace_uuid,session_incarnation,prepare_id, + metadata_root,authorization_epoch,publication_sequence,writer_epoch,certificate_receipt_id) + VALUES(lid,sid,s.namespace_uuid,s.session_incarnation,s.prepare_id,s.metadata_root,s.authorization_epoch, + s.publication_sequence,s.writer_epoch,s.certificate_receipt_id); + INSERT INTO mst2_metadata_root_anchor(anchor_id,anchor_kind,owner_key,root_page,root_generation,root_certificate_digest, + snapshot_id,session_incarnation,lease_id) VALUES(gen_random_uuid(),'LEASE',lid,s.metadata_root,s.root_generation, + a.root_certificate_digest,sid,s.session_incarnation,lid); + IF pid IS NOT NULL THEN + SELECT coverage_retired_at INTO p FROM mst2_metadata_prepare WHERE prepare_id=pid AND state='COMMITTED' AND plan_kind='ROOTED' + AND storage_seal=decode(request->>'storage_seal','hex') AND metadata_root=s.metadata_root; + IF NOT FOUND OR s.root_generation IS DISTINCT FROM (request->>'root_generation')::bigint + OR a.root_certificate_digest IS DISTINCT FROM decode(request->>'certificate_digest','hex') + OR descriptor IS DISTINCT FROM mst2_metadata_descriptor(pid,instance,commit_id,tree_id) + OR NOT mst2_metadata_session_covers_prepare(pid) + OR NOT EXISTS(SELECT 1 FROM mst2_metadata_source_root_attestation receipt + WHERE receipt.attestation_id=(request->>'attestation_id')::uuid + AND receipt.attestation_digest=decode(request->>'attestation_digest','hex') + AND receipt.root_page=s.metadata_root AND receipt.root_generation=s.root_generation + AND receipt.root_certificate_digest=a.root_certificate_digest + AND mst2_metadata_serving_source(pid,receipt.attestation_id,s.metadata_root,s.root_generation,a.root_certificate_digest)) THEN + RAISE EXCEPTION 'qualified losing preparation cannot retire a different serving identity'; END IF; + IF p.coverage_retired_at IS NULL THEN + UPDATE mst2_metadata_prepare SET coverage_retired_at=clock_timestamp() WHERE prepare_id=pid; + DELETE FROM mst2_metadata_root_anchor WHERE prepare_id=pid AND anchor_kind IN ('PREPARE','REUSE'); + END IF; + END IF; + RETURN QUERY SELECT lid,deadline; +END $$; + +CREATE FUNCTION mst2_metadata_cleanup_lease(lid text) RETURNS void LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE l mst2_qualified_lease_binding%ROWTYPE; +BEGIN + SELECT * INTO l FROM mst2_qualified_lease_binding WHERE lease_id=lid; + IF NOT FOUND THEN RETURN; END IF; + IF l.state<>'ACTIVE' AND NOT EXISTS(SELECT 1 FROM mst2_metadata_reader_operation WHERE lease_id=lid AND state='ACTIVE') THEN + DELETE FROM mst2_metadata_root_anchor WHERE lease_id=lid AND anchor_kind='LEASE'; + END IF; + IF NOT EXISTS(SELECT 1 FROM mst2_qualified_lease_binding lease WHERE lease.snapshot_id=l.snapshot_id + AND lease.session_incarnation=l.session_incarnation AND lease.state='ACTIVE') THEN + UPDATE mst2_qualified_session_incarnation SET state='RETIRED' + WHERE snapshot_id=l.snapshot_id AND session_incarnation=l.session_incarnation AND state='READY'; + DELETE FROM mst2_metadata_root_anchor WHERE snapshot_id=l.snapshot_id + AND session_incarnation=l.session_incarnation AND anchor_kind='SESSION'; + END IF; +END $$; + +CREATE FUNCTION mst2_metadata_cleanup_expired(maximum integer DEFAULT 64) RETURNS void LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE item record; now_unix bigint; bound integer:=least(64,greatest(0,maximum)); +BEGIN + PERFORM $CORE_SCHEMA$.mst2_route_enter($CORE_LITERAL$); + now_unix:=floor(extract(epoch FROM clock_timestamp()))::bigint; + FOR item IN SELECT * FROM ( + (SELECT 'READER'::text AS kind,operation_id::text AS owner,lease_id,hard_deadline_unix AS deadline + FROM mst2_metadata_reader_operation WHERE state='ACTIVE' AND hard_deadline_unix<=now_unix + ORDER BY hard_deadline_unix,operation_id LIMIT bound) + UNION ALL + (SELECT 'LEASE'::text,lease_id,lease_id,expires_at_unix + FROM mst2_qualified_lease_binding WHERE state='ACTIVE' AND expires_at_unix<=now_unix + ORDER BY expires_at_unix,lease_id LIMIT bound) + ) expired ORDER BY deadline,kind,owner LIMIT bound LOOP + IF item.kind='READER' THEN + UPDATE mst2_metadata_reader_operation SET state='EXPIRED' WHERE operation_id=item.owner::uuid AND state='ACTIVE'; + DELETE FROM mst2_metadata_root_anchor WHERE reader_operation_id=item.owner::uuid AND anchor_kind IN ('REQUEST','READER'); + ELSE + UPDATE mst2_qualified_lease_binding SET state='EXPIRED',lease_epoch=lease_epoch+1 WHERE lease_id=item.owner AND state='ACTIVE'; + END IF; + PERFORM mst2_metadata_cleanup_lease(item.lease_id); + END LOOP; +END $$; + +CREATE FUNCTION mst2_metadata_renew_lease(lid text,seconds bigint,instance text) +RETURNS TABLE(snapshot_id text,expires_at_unix bigint) LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE l mst2_qualified_lease_binding%ROWTYPE; now_unix bigint; deadline bigint; +BEGIN + PERFORM $CORE_SCHEMA$.mst2_route_enter($CORE_LITERAL$); + PERFORM mst2_metadata_cleanup_expired(64); + now_unix:=floor(extract(epoch FROM clock_timestamp()))::bigint; + SELECT * INTO l FROM mst2_qualified_lease_binding WHERE lease_id=lid AND state='ACTIVE'; + IF NOT FOUND THEN RETURN; END IF; + IF l.expires_at_unix<=now_unix THEN + UPDATE mst2_qualified_lease_binding SET state='EXPIRED',lease_epoch=lease_epoch+1 WHERE lease_id=lid; + PERFORM mst2_metadata_cleanup_lease(lid); RETURN; + END IF; + IF NOT EXISTS(SELECT 1 FROM mst2_metadata_session_row(l.snapshot_id,lid,instance)) THEN RETURN; END IF; + deadline:=greatest(l.expires_at_unix,now_unix+least(3600,greatest(1,seconds))); + UPDATE mst2_qualified_lease_binding lease SET expires_at_unix=deadline WHERE lease.lease_id=lid; + RETURN QUERY SELECT l.snapshot_id,deadline; +END $$; + +CREATE FUNCTION mst2_metadata_release_lease(lid text) RETURNS boolean LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE l mst2_qualified_lease_binding%ROWTYPE; changed boolean; +BEGIN + PERFORM $CORE_SCHEMA$.mst2_route_enter($CORE_LITERAL$); + PERFORM mst2_metadata_cleanup_expired(64); + SELECT * INTO l FROM mst2_qualified_lease_binding WHERE lease_id=lid; + IF NOT FOUND THEN RETURN false; END IF; + IF NOT mst2_metadata_lease_route_proof(lid,l.snapshot_id,l.namespace_uuid,l.session_incarnation,l.prepare_id,l.metadata_root, + l.authorization_epoch,l.publication_sequence,l.writer_epoch,l.certificate_receipt_id) THEN + RAISE EXCEPTION 'qualified release lost its fixed historical source identity'; END IF; + changed:=l.state='ACTIVE'; + IF changed THEN UPDATE mst2_qualified_lease_binding SET state='RELEASED',lease_epoch=lease_epoch+1 WHERE lease_id=lid; END IF; + PERFORM mst2_metadata_cleanup_lease(lid); + RETURN changed; +END $$; + +CREATE FUNCTION mst2_metadata_begin_reader(sid text,lid text,instance text) +RETURNS TABLE(operation_id uuid,root_generation bigint,certificate_digest bytea) LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE l mst2_qualified_lease_binding%ROWTYPE; s record; op uuid:=gen_random_uuid(); deadline bigint; kind text; +BEGIN + PERFORM $CORE_SCHEMA$.mst2_route_enter($CORE_LITERAL$); + PERFORM mst2_metadata_cleanup_expired(64); + SELECT * INTO s FROM mst2_metadata_session_row(sid,lid,instance); + IF NOT FOUND THEN RETURN; END IF; + SELECT * INTO STRICT l FROM mst2_qualified_lease_binding WHERE lease_id=lid; + deadline:=least(l.expires_at_unix,floor(extract(epoch FROM clock_timestamp()))::bigint+60); + INSERT INTO mst2_metadata_reader_operation(operation_id,lease_id,snapshot_id,session_incarnation,root_page,root_generation, + lease_epoch,hard_deadline_unix,state) VALUES(op,lid,sid,l.session_incarnation,l.metadata_root,l.root_generation,l.lease_epoch,deadline,'ACTIVE'); + FOREACH kind IN ARRAY ARRAY['REQUEST','READER'] LOOP + INSERT INTO mst2_metadata_root_anchor(anchor_id,anchor_kind,owner_key,root_page,root_generation,root_certificate_digest, + snapshot_id,session_incarnation,lease_id,reader_operation_id) VALUES(gen_random_uuid(),kind,op::text, + l.metadata_root,l.root_generation,s.certificate_digest,sid,l.session_incarnation,lid,op); + END LOOP; + RETURN QUERY SELECT op,l.root_generation,s.certificate_digest::bytea; +END $$; + +CREATE FUNCTION mst2_metadata_finish_reader(op uuid) RETURNS void LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE r mst2_metadata_reader_operation%ROWTYPE; +BEGIN + PERFORM $CORE_SCHEMA$.mst2_route_enter($CORE_LITERAL$); + SELECT * INTO r FROM mst2_metadata_reader_operation WHERE operation_id=op; + IF NOT FOUND THEN RETURN; END IF; + IF r.state='ACTIVE' THEN UPDATE mst2_metadata_reader_operation SET state='FINISHED' WHERE operation_id=op; END IF; + DELETE FROM mst2_metadata_root_anchor WHERE reader_operation_id=op AND anchor_kind IN ('REQUEST','READER'); + PERFORM mst2_metadata_cleanup_lease(r.lease_id); +END $$; + +DROP TRIGGER mst2_01_family_closed ON mst2_qualified_session_incarnation; +DROP TRIGGER mst2_01_family_closed ON mst2_qualified_lease_binding; +DROP TRIGGER mst2_01_family_closed ON mst2_metadata_reader_operation; diff --git a/src/jupiter/storage/qualified_metadata_session.rs b/src/jupiter/storage/qualified_metadata_session.rs new file mode 100644 index 00000000..8e29b301 --- /dev/null +++ b/src/jupiter/storage/qualified_metadata_session.rs @@ -0,0 +1,291 @@ +//! Rooted session handoff and lease operations over the captured physical Q. + +use mst2_codec::descriptor::ServingDescriptor; + +use super::*; +use crate::{ + ceres::snapshot::{ + descriptor::BuiltDescriptor, + rooted_metadata_projection::PreparedRootedNativeMetadata, + runtime::{LeaseRenewed, SnapshotContext}, + view::{SnapshotView, hex}, + }, + jupiter::storage::native_publication_storage::NativePublicationHead, +}; + +fn gone() -> SnapshotError { + SnapshotError::new( + crate::ceres::snapshot::error::SnapshotErrorCode::LeaseExpired, + "lease is not active for this snapshot; re-resolve", + ) +} + +pub(super) async fn finish( + txn: DatabaseTransaction, + result: Result, +) -> Result { + match result { + Ok(value) => { + txn.commit().await.map_err(|_| { + SnapshotError::new( + crate::ceres::snapshot::error::SnapshotErrorCode::TemporaryUnavailable, + "qualified transaction outcome unknown; retry the original operation", + ) + })?; + Ok(value) + } + Err(error) => { + let _ = txn.rollback().await; + Err(error) + } + } +} + +pub(super) fn context( + row: &QueryResult, + sid: &str, + lease: &str, + instance: &str, +) -> Result { + let bytes: Vec = row.try_get("", "canonical_descriptor").map_err(internal)?; + let descriptor = ServingDescriptor::decode(&bytes).map_err(internal)?; + let commit: String = row.try_get("", "commit_oid").map_err(internal)?; + let tree: String = row.try_get("", "root_tree_oid").map_err(internal)?; + let view = SnapshotView::from_commit(&commit, &tree); + if format!( + "sha256:{}", + hex(&descriptor.snapshot_id().map_err(internal)?) + ) != sid + || uuid::Uuid::from_bytes(descriptor.instance_uuid).to_string() != instance + || format!("sha256:{}", hex(&descriptor.namespace_view_id)) != view.view_id + { + return Err(integrity( + "qualified durable descriptor differs from its fixed source", + )); + } + let deadline = u64::try_from( + row.try_get::("", "expires_at_unix") + .map_err(internal)?, + ) + .map_err(internal)?; + let epoch = u64::try_from( + row.try_get::("", "authorization_epoch") + .map_err(internal)?, + ) + .map_err(internal)?; + let metadata_root = format!("sha256:{}", hex(&descriptor.metadata_root)); + Ok(SnapshotContext { + built: BuiltDescriptor { + descriptor, + instance_id: instance.into(), + snapshot_id: sid.into(), + metadata_root, + }, + commit_oid: commit, + root_tree_oid: tree, + lease_id: lease.into(), + lease_expires_at_unix: deadline, + authorization_epoch: epoch, + }) +} + +impl RootedQualifiedMetadataRepository { + pub(crate) async fn install( + &self, + built: &BuiltDescriptor, + prepared: &PreparedRootedNativeMetadata, + ) -> Result { + if prepared.plan.root != built.descriptor.metadata_root + || prepared.plan.identity.scope != built.descriptor.scope + { + return Err(integrity("rooted projection differs from the serving descriptor").into()); + } + // Each attempt has its own durable identity. A previous retired + // incarnation may have the same cold plan but a different generation; + // its historical receipt cannot install or resurrect this attempt. + let operation = format!("rooted-http:{}:{}", built.snapshot_id, uuid::Uuid::new_v4()); + let intent = self.begin_intent(&operation, &prepared.plan).await?; + for pages in prepared.payloads.chunks(64) { + self.install_pages(&intent, pages).await?; + } + self.finalize(&intent).await + } + + pub(crate) async fn open_session( + &self, + expected: &NativePublicationHead, + built: &BuiltDescriptor, + receipt: Option<&RootedMetadataReceipt>, + seconds: u64, + ) -> Result, SnapshotError> { + let txn = self.transaction().await?; + let result = async { + if receipt.is_none() { + txn.execute_raw(sql("SELECT mst2_metadata_cleanup_expired(64)",[])).await.map_err(database_error)?; + } + let present = txn.query_one_raw(sql("SELECT session_incarnation FROM mst2_qualified_session_incarnation + WHERE snapshot_id=$1 AND state='READY'", [built.snapshot_id.clone().into()])).await.map_err(database_error)?; + if present.is_none() && receipt.is_none() { return Ok(None); } + if let Some(receipt) = receipt { + self.require_intent(&txn, &receipt.intent).await?; + if self.receipt(&txn, &receipt.intent).await? != *receipt + || receipt.metadata_root() != built.descriptor.metadata_root + { return Err(integrity("qualified handoff receipt differs from its definitive canonical root")); } + } + let request = json!({ + "snapshot_id":built.snapshot_id,"canonical_descriptor":hex::encode(built.descriptor.encode().map_err(internal)?), + "instance_id":expected.instance_id,"commit_oid":expected.root.commit,"root_tree_oid":expected.root.tree, + "publication_sequence":expected.token.sequence,"writer_epoch":expected.token.epoch, + "certificate_receipt_id":expected.token.certificate,"authorization_epoch":1, + "lease_id":uuid::Uuid::new_v4().to_string(),"lease_seconds":seconds.clamp(1,3600), + "session_incarnation":uuid::Uuid::new_v4().to_string(), + "prepare_id":receipt.map(|r|r.intent.prepare_id.as_str()), + "storage_seal":receipt.map(|r|hex::encode(r.intent.storage_seal)), + "root_generation":receipt.map(|r|r.intent.root_generation), + "attestation_id":receipt.map(|r|r.attestation_id.to_string()), + "attestation_digest":receipt.map(|r|hex::encode(r.attestation_digest)), + "certificate_digest":receipt.map(|r|hex::encode(r.certificate_digest)), + }); + let row = txn.query_one_raw(sql("SELECT * FROM mst2_metadata_handoff($1::jsonb)", [request.into()])) + .await.map_err(database_error)?.ok_or_else(|| integrity("qualified handoff returned no lease"))?; + let lease: String = row.try_get("", "lease_id").map_err(internal)?; + let row = self.session_row(&txn, &built.snapshot_id, &lease, &expected.instance_id).await?; + let context = context(&row, &built.snapshot_id, &lease, &expected.instance_id)?; + if context.built.descriptor != built.descriptor || context.commit_oid != expected.root.commit + || context.root_tree_oid != expected.root.tree { return Err(integrity("qualified handoff fixed source changed")); } + Ok(Some(context)) + }.await; + finish(txn, result).await + } + + pub(super) async fn session_row( + &self, + db: &C, + sid: &str, + lease: &str, + instance: &str, + ) -> Result { + db.query_one_raw(sql( + "SELECT * FROM mst2_metadata_session_row($1,$2,$3)", + [sid.into(), lease.into(), instance.into()], + )) + .await + .map_err(database_error)? + .ok_or_else(gone) + } + + pub(crate) async fn context( + &self, + sid: &str, + lease: &str, + instance: &str, + ) -> Result { + // Read-only revalidation grants no mutable authority. The helper checks + // the exact route, incarnation, publication, LIVE root and owned roots. + let txn = self.read_transaction().await?; + let result = async { + let row = self.session_row(&txn, sid, lease, instance).await?; + context(&row, sid, lease, instance) + } + .await; + finish(txn, result).await + } + + pub(crate) async fn renew( + &self, + lease: &str, + seconds: u64, + instance: &str, + ) -> Result { + let txn = self.transaction().await?; + let result = async { + let row = txn + .query_one_raw(sql( + "SELECT * FROM mst2_metadata_renew_lease($1,$2,$3)", + [ + lease.into(), + (seconds.clamp(1, 3600) as i64).into(), + instance.into(), + ], + )) + .await + .map_err(database_error)? + .ok_or_else(gone)?; + Ok(LeaseRenewed { + lease_id: lease.into(), + snapshot_id: row.try_get("", "snapshot_id").map_err(internal)?, + expires_at_unix: u64::try_from( + row.try_get::("", "expires_at_unix") + .map_err(internal)?, + ) + .map_err(internal)?, + }) + } + .await; + finish(txn, result).await + } + + pub(crate) async fn release(&self, lease: &str) -> Result { + let txn = self.transaction().await?; + let result = async { + txn.query_one_raw(sql( + "SELECT mst2_metadata_release_lease($1) AS released", + [lease.into()], + )) + .await + .map_err(database_error)? + .ok_or_else(|| integrity("qualified release result missing"))? + .try_get("", "released") + .map_err(internal) + } + .await; + finish(txn, result).await + } + + pub(super) async fn read_transaction(&self) -> Result { + let txn = self + .connection + .begin_with_config(Some(IsolationLevel::ReadCommitted), None) + .await + .map_err(database_error)?; + if registered( + &txn, + &(self.namespace.core_schema.clone(), self.namespace.core_oid), + ) + .await + .map_err(|error| match error { + crate::common::errors::MegaError::Db(error) => database_error(error), + error => internal(error), + })? + .as_ref() + != Some(&self.namespace) + { + return Err(integrity( + "qualified read physical namespace or authority catalog changed", + )); + } + let valid: bool = txn + .query_one_raw(sql( + "SELECT mst2_metadata_scope_matches($1) AS valid", + [self.primary_scope.clone().into()], + )) + .await + .map_err(database_error)? + .ok_or_else(|| integrity("qualified read scope missing"))? + .try_get("", "valid") + .map_err(internal)?; + if !valid { + return Err(integrity("qualified read left its captured primary scope")); + } + #[cfg(test)] + if super::reader::READER_TEMP_SOURCE_SHADOW + .try_with(|shadow| *shadow) + .unwrap_or(false) + { + txn.execute_unprepared("CREATE TEMP TABLE mega_tree(id bigint,tree_id text,sub_trees bytea) ON COMMIT DROP; + CREATE TEMP TABLE mst2_rooted_source_tree_revision(tree_id text,tree_row_id bigint,revision uuid,body_digest bytea,valid boolean) ON COMMIT DROP") + .await.map_err(database_error)?; + } + Ok(txn) + } +} diff --git a/src/jupiter/storage/qualified_metadata_source_read.sql b/src/jupiter/storage/qualified_metadata_source_read.sql new file mode 100644 index 00000000..232a0e6d --- /dev/null +++ b/src/jupiter/storage/qualified_metadata_source_read.sql @@ -0,0 +1,132 @@ +-- Indexed fixed-source names are independently derived by the attestation +-- proof. Selected windows revalidate current file facts without source scans. +CREATE TABLE mst2_metadata_source_entry_reference ( + attestation_id uuid NOT NULL REFERENCES mst2_metadata_source_root_attestation(attestation_id), + name bytea NOT NULL CHECK(octet_length(name) BETWEEN 1 AND 255), + git_oid text NOT NULL CHECK(git_oid ~ '^(sha1:[0-9a-f]{40}|sha256:[0-9a-f]{64}|blake3:[0-9a-f]{64})$'), + kind smallint NOT NULL CHECK(kind BETWEEN 1 AND 4), + byte_size bigint,content_digest bytea,child_root bytea,child_generation bigint,child_certificate_digest bytea, + PRIMARY KEY(attestation_id,name), + CHECK(((kind=4 AND byte_size IS NULL AND content_digest IS NULL AND octet_length(child_root)=32 + AND child_generation>0 AND octet_length(child_certificate_digest)=32) + OR (kind<>4 AND byte_size BETWEEN 0 AND 8796093022208 AND octet_length(content_digest)=32 + AND child_root IS NULL AND child_generation IS NULL AND child_certificate_digest IS NULL)) IS TRUE), + CHECK((kind<>3 OR byte_size BETWEEN 1 AND 4095) IS TRUE) +); + +CREATE FUNCTION mst2_metadata_source_entry_guard() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +BEGIN + RAISE EXCEPTION 'qualified fixed source entry history is immutable'; +END $$; +CREATE TRIGGER mst2_metadata_source_entry_guard BEFORE UPDATE OR DELETE ON mst2_metadata_source_entry_reference + FOR EACH ROW EXECUTE FUNCTION mst2_metadata_source_entry_guard(); + +CREATE FUNCTION mst2_metadata_source_entries_proof() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE source_id uuid; expected jsonb; supplied jsonb; +BEGIN + FOR source_id IN SELECT DISTINCT attestation_id FROM added_source_entries LOOP + SELECT source_proof->'source_entries' INTO expected FROM mst2_metadata_source_root_attestation + WHERE attestation_id=source_id; + SELECT jsonb_object_agg(encode(reference.name,'hex'), + jsonb_build_object('kind',reference.kind,'git_oid',reference.git_oid)||CASE WHEN reference.kind=4 + THEN jsonb_build_object('child_root',encode(reference.child_root,'hex'), + 'child_generation',reference.child_generation,'child_certificate',encode(reference.child_certificate_digest,'hex')) + ELSE jsonb_build_object('size',reference.byte_size,'content_digest',encode(reference.content_digest,'hex')) END) + INTO supplied FROM added_source_entries reference WHERE reference.attestation_id=source_id; + IF expected IS NULL OR expected IS DISTINCT FROM supplied THEN + RAISE EXCEPTION 'fixed source entry batch differs from independently derived complete body and fact proof'; END IF; + END LOOP; + RETURN NULL; +END $$; +CREATE TRIGGER mst2_metadata_source_entries_proof AFTER INSERT ON mst2_metadata_source_entry_reference + REFERENCING NEW TABLE AS added_source_entries FOR EACH STATEMENT EXECUTE FUNCTION mst2_metadata_source_entries_proof(); + +CREATE FUNCTION mst2_metadata_source_entries_install() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +BEGIN + INSERT INTO mst2_metadata_source_entry_reference(attestation_id,name,git_oid,kind,byte_size,content_digest, + child_root,child_generation,child_certificate_digest) + SELECT NEW.attestation_id,decode(key,'hex'),value->>'git_oid',(value->>'kind')::smallint, + (value->>'size')::bigint,decode(value->>'content_digest','hex'),decode(value->>'child_root','hex'), + (value->>'child_generation')::bigint,decode(value->>'child_certificate','hex') + FROM jsonb_each(NEW.source_proof->'source_entries'); + RETURN NULL; +END $$; +CREATE TRIGGER mst2_metadata_source_entries_install AFTER INSERT ON mst2_metadata_source_root_attestation + FOR EACH ROW EXECUTE FUNCTION mst2_metadata_source_entries_install(); + +CREATE FUNCTION mst2_metadata_source_entries_complete() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +BEGIN + IF (SELECT count(*) FROM mst2_metadata_source_entry_reference WHERE attestation_id=NEW.attestation_id) + IS DISTINCT FROM (NEW.source_proof->>'source_entry_count')::bigint THEN + RAISE EXCEPTION 'source attestation cannot commit with incomplete exact name references'; END IF; + RETURN NULL; +END $$; +CREATE CONSTRAINT TRIGGER mst2_metadata_source_entries_complete AFTER INSERT ON mst2_metadata_source_root_attestation + DEFERRABLE INITIALLY DEFERRED FOR EACH ROW EXECUTE FUNCTION mst2_metadata_source_entries_complete(); + +CREATE FUNCTION mst2_metadata_read_source_entries(op uuid,source_id uuid,p bytea,g bigint,c bytea,names jsonb) +RETURNS TABLE(name bytea,git_oid text,kind smallint,byte_size bigint,content_digest bytea,child_root bytea, + child_generation bigint,child_certificate_digest bytea,child_attestation_id uuid,fact_state text) +LANGUAGE plpgsql VOLATILE SET search_path=$Q_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE a record; profile jsonb; +BEGIN + IF jsonb_typeof(names)<>'array' OR jsonb_array_length(names)>256 + OR EXISTS(SELECT 1 FROM jsonb_array_elements_text(names) wanted + WHERE wanted !~ '^([0-9a-f]{2}){1,255}$') THEN + RAISE EXCEPTION 'qualified source-name read exceeds its bounded exact request'; END IF; + SELECT source.namespace_uuid,source.source_profile,source.tagged_tree_oid,source.source_body_digest,source.source_revision, + source.root_page,source.root_generation,source.root_certificate_digest + INTO a FROM mst2_metadata_source_root_attestation source + JOIN mst2_metadata_prepare origin ON origin.prepare_id=source.origin_prepare_id + WHERE source.attestation_id=source_id AND origin.state='COMMITTED' + AND source.root_page=p AND source.root_generation=g AND source.root_certificate_digest=c + AND source.namespace_uuid='$NAMESPACE_UUID$'::uuid; + IF NOT FOUND THEN RAISE EXCEPTION 'qualified selected directory has no definitive source attestation'; END IF; + SELECT mst2_metadata_native_profile(session.prepare_id) INTO profile + FROM mst2_metadata_reader_operation reader JOIN mst2_qualified_session_incarnation session + ON session.snapshot_id=reader.snapshot_id AND session.session_incarnation=reader.session_incarnation + WHERE reader.operation_id=op AND reader.state='ACTIVE' + AND reader.hard_deadline_unix>floor(extract(epoch FROM clock_timestamp()))::bigint + AND EXISTS(SELECT 1 FROM mst2_metadata_root_anchor anchor WHERE anchor.reader_operation_id=reader.operation_id + AND anchor.anchor_kind='READER' AND anchor.root_page=reader.root_page AND anchor.root_generation=reader.root_generation); + IF NOT FOUND OR profile IS DISTINCT FROM a.source_profile + OR NOT mst2_metadata_root_live(a.root_page,a.root_generation,a.root_certificate_digest) THEN + RAISE EXCEPTION 'qualified source read lost its exact reader profile and current directory'; END IF; + IF NOT $CORE_SCHEMA$.mst2_route_source_tree_matches(split_part(a.tagged_tree_oid,':',2),a.source_revision,a.source_body_digest) THEN + RAISE EXCEPTION 'qualified selected directory source body changed'; END IF; + RETURN QUERY SELECT reference.name,reference.git_oid,reference.kind,reference.byte_size,reference.content_digest, + reference.child_root,reference.child_generation,reference.child_certificate_digest,child.attestation_id, + CASE WHEN reference.kind=4 THEN CASE WHEN child.attestation_id IS NULL THEN 'SOURCE_UNAVAILABLE' ELSE 'READY' END + WHEN fact.git_oid IS NULL THEN 'MISSING' + WHEN fact.state<>'VERIFIED' OR fact.verification_version NOT IN (1,2) + OR fact.size NOT BETWEEN 0 AND 8796093022208 OR octet_length(fact.raw_sha256)<>32 THEN 'INVALID' + WHEN fact.verification_version=1 THEN 'MISSING' + WHEN fact.size IS DISTINCT FROM reference.byte_size OR reference.kind=3 AND fact.size NOT BETWEEN 1 AND 4095 + OR CASE WHEN octet_length(fact.raw_sha256)=32 THEN fact.raw_sha256 ELSE NULL END + IS DISTINCT FROM reference.content_digest THEN 'INVALID' + ELSE 'READY' END + FROM (SELECT DISTINCT decode(value,'hex') AS name FROM jsonb_array_elements_text(names)) wanted + JOIN mst2_metadata_source_entry_reference reference ON reference.attestation_id=source_id AND reference.name=wanted.name + LEFT JOIN LATERAL ( + SELECT verified.git_oid,verified.state,verified.verification_version,verified.size,verified.raw_sha256 + FROM $CORE_SCHEMA$.mst2_verified_object verified WHERE reference.kind<>4 AND verified.storage_domain='git' + AND verified.object_kind='blob' AND verified.git_oid=split_part(reference.git_oid,':',2) + FOR SHARE OF verified NOWAIT + ) fact ON true + LEFT JOIN LATERAL ( + SELECT candidate.attestation_id FROM mst2_metadata_source_root_attestation candidate + JOIN mst2_metadata_prepare origin ON origin.prepare_id=candidate.origin_prepare_id + JOIN $CORE_SCHEMA$.mega_tree source_tree ON source_tree.tree_id=split_part(candidate.tagged_tree_oid,':',2) + WHERE reference.kind=4 AND candidate.namespace_uuid=a.namespace_uuid AND origin.state='COMMITTED' + AND candidate.tagged_tree_oid=reference.git_oid AND candidate.source_profile=a.source_profile + AND candidate.root_page=reference.child_root AND candidate.root_generation=reference.child_generation + AND candidate.root_certificate_digest=reference.child_certificate_digest + AND $CORE_SCHEMA$.mst2_route_source_tree_matches(split_part(candidate.tagged_tree_oid,':',2),candidate.source_revision,candidate.source_body_digest) + AND mst2_metadata_root_live(candidate.root_page,candidate.root_generation,candidate.root_certificate_digest) + ORDER BY candidate.attestation_id LIMIT 1 + ) child ON true ORDER BY reference.name; +END $$; diff --git a/src/jupiter/storage/qualified_metadata_source_revision_tests.rs b/src/jupiter/storage/qualified_metadata_source_revision_tests.rs new file mode 100644 index 00000000..9bbb6c54 --- /dev/null +++ b/src/jupiter/storage/qualified_metadata_source_revision_tests.rs @@ -0,0 +1,329 @@ +use super::{canonical_tests::seeded_rooted_plan, *}; +use crate::ceres::snapshot::rooted_metadata_projection::RootedReuseLookup; + +async fn revision(core: &DatabaseConnection, oid: &str) -> String { + core.query_one_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + "SELECT revision::text FROM mst2_rooted_source_tree_revision WHERE tree_id=$1", + [oid.into()], + )) + .await + .unwrap() + .unwrap() + .try_get_by_index(0) + .unwrap() +} + +#[tokio::test] +async fn source_revision_rejects_forgery_preserves_unchanged_writes_and_requires_fresh_attestation() +{ + let (config, core, namespace, q, _guard) = fixture().await; + let (plan, payload) = seeded_rooted_plan(&core, 'a', "file", 1).await; + let writer = RootedQualifiedMetadataRepository::open(&core, &config) + .await + .unwrap(); + let intent = writer + .begin_intent("source-revision-first", &plan) + .await + .unwrap(); + writer + .install_pages(&intent, std::slice::from_ref(&payload)) + .await + .unwrap(); + writer.finalize(&intent).await.unwrap(); + let oid = "a".repeat(40); + let original = revision(&core, &oid).await; + for statement in [ + "UPDATE mst2_rooted_source_tree_revision SET revision=gen_random_uuid()", + "UPDATE mst2_rooted_source_tree_revision SET body_digest=decode(repeat('11',32),'hex')", + "UPDATE mst2_rooted_source_tree_revision SET valid=false", + "DELETE FROM mst2_rooted_source_tree_revision", + "TRUNCATE mst2_rooted_source_tree_revision", + ] { + assert!( + core.execute_unprepared(statement).await.is_err(), + "{statement}" + ); + } + assert_eq!(revision(&core, &oid).await, original); + core.execute_unprepared( + "CREATE TABLE source_revision_spoof(id integer); + CREATE FUNCTION source_revision_spoof_trigger() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN UPDATE mst2_rooted_source_tree_revision SET valid=false; RETURN NEW; END $$; + CREATE TRIGGER source_revision_spoof AFTER INSERT ON source_revision_spoof + FOR EACH ROW EXECUTE FUNCTION source_revision_spoof_trigger()", + ) + .await + .unwrap(); + assert!( + core.execute_unprepared("INSERT INTO source_revision_spoof VALUES(1)") + .await + .is_err() + ); + assert_eq!(revision(&core, &oid).await, original); + core.execute_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + "UPDATE mega_tree SET sub_trees=sub_trees,pack_offset=pack_offset WHERE tree_id=$1", + [oid.clone().into()], + )) + .await + .unwrap(); + assert_eq!(revision(&core, &oid).await, original); + assert!( + writer + .lookup_reuse(&plan.identity.tagged_root_tree_oid, &plan.identity) + .await + .unwrap() + .is_some() + ); + core.execute_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + "UPDATE mega_tree SET sub_trees=set_byte(sub_trees,7,120) WHERE tree_id=$1", + [oid.clone().into()], + )) + .await + .unwrap(); + assert_ne!(revision(&core, &oid).await, original); + assert_eq!( + count( + &core, + "SELECT count(*) FROM mst2_rooted_source_tree_revision WHERE valid" + ) + .await, + 0 + ); + assert!( + writer + .lookup_reuse(&plan.identity.tagged_root_tree_oid, &plan.identity) + .await + .unwrap() + .is_none() + ); + let rejected=core.execute_raw(Statement::from_sql_and_values(DbBackend::Postgres, + "SELECT mst2_route_capture_source_tree($1,(SELECT source_body_digest FROM {q}.mst2_metadata_source_root_attestation LIMIT 1))" + .replace("{q}",&identifier(&namespace.schema)),[oid.clone().into()])).await; + assert!(rejected.is_err()); + // Restoring the same old bytes does not silently resurrect the old revision. + core.execute_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + "UPDATE mega_tree SET sub_trees=set_byte(sub_trees,7,102) WHERE tree_id=$1", + [oid.clone().into()], + )) + .await + .unwrap(); + assert_eq!( + count( + &core, + "SELECT count(*) FROM mst2_rooted_source_tree_revision WHERE valid" + ) + .await, + 0 + ); + let fresh = writer + .begin_intent("source-revision-reattest", &plan) + .await + .unwrap(); + writer.install_pages(&fresh, &[payload]).await.unwrap(); + writer.finalize(&fresh).await.unwrap(); + assert_eq!( + count( + &core, + "SELECT count(*) FROM mst2_rooted_source_tree_revision WHERE valid" + ) + .await, + 1 + ); + assert_ne!(revision(&core, &oid).await, original); + assert_eq!( + count( + &q, + "SELECT count(*) FROM mst2_metadata_source_root_attestation" + ) + .await, + 2 + ); + assert_eq!(count(&q,&format!("SELECT count(*) FROM mst2_metadata_source_root_attestation a WHERE NOT {}.mst2_route_source_tree_matches( + split_part(a.tagged_tree_oid,':',2),a.source_revision,a.source_body_digest)",identifier(&namespace.core_schema))).await,1); +} + +#[tokio::test] +async fn deleted_and_reinserted_source_oid_requires_new_independent_revision() { + let (config, core, namespace, q, _guard) = fixture().await; + let (plan, payload) = seeded_rooted_plan(&core, 'a', "file", 1).await; + let writer = RootedQualifiedMetadataRepository::open(&core, &config) + .await + .unwrap(); + let intent = writer + .begin_intent("source-delete-original", &plan) + .await + .unwrap(); + writer + .install_pages(&intent, std::slice::from_ref(&payload)) + .await + .unwrap(); + writer.finalize(&intent).await.unwrap(); + let oid = "a".repeat(40); + let original = revision(&core, &oid).await; + let saved: String = core + .query_one_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + "SELECT row_to_json(t)::text FROM mega_tree t WHERE tree_id=$1", + [oid.clone().into()], + )) + .await + .unwrap() + .unwrap() + .try_get_by_index(0) + .unwrap(); + core.execute_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + "DELETE FROM mega_tree WHERE tree_id=$1", + [oid.clone().into()], + )) + .await + .unwrap(); + let deleted = revision(&core, &oid).await; + assert_ne!(deleted, original); + assert_eq!( + count( + &core, + "SELECT count(*) FROM mst2_rooted_source_tree_revision WHERE valid" + ) + .await, + 0 + ); + assert!( + writer + .lookup_reuse(&plan.identity.tagged_root_tree_oid, &plan.identity) + .await + .unwrap() + .is_none() + ); + // Even an identical OID, row ID, and byte body cannot revive the old stamp. + core.execute_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + "INSERT INTO mega_tree SELECT (json_populate_record(NULL::mega_tree,$1::json)).*", + [saved.into()], + )) + .await + .unwrap(); + assert_eq!(revision(&core, &oid).await, deleted); + assert_eq!( + count( + &core, + "SELECT count(*) FROM mst2_rooted_source_tree_revision WHERE valid" + ) + .await, + 0 + ); + assert!( + writer + .lookup_reuse(&plan.identity.tagged_root_tree_oid, &plan.identity) + .await + .unwrap() + .is_none() + ); + let fresh = writer + .begin_intent("source-delete-reattest", &plan) + .await + .unwrap(); + writer.install_pages(&fresh, &[payload]).await.unwrap(); + writer.finalize(&fresh).await.unwrap(); + assert_ne!(revision(&core, &oid).await, deleted); + assert_eq!( + count( + &core, + "SELECT count(*) FROM mst2_rooted_source_tree_revision WHERE valid" + ) + .await, + 1 + ); + assert_eq!(count(&q,&format!("SELECT count(*) FROM mst2_metadata_source_root_attestation a WHERE NOT {}.mst2_route_source_tree_matches( + split_part(a.tagged_tree_oid,':',2),a.source_revision,a.source_body_digest)",identifier(&namespace.core_schema))).await,1); +} + +#[tokio::test] +async fn current_source_revision_uses_captured_core_relations_under_temp_shadow() { + let (config, core, namespace, q, _guard) = fixture().await; + let (plan, payload) = seeded_rooted_plan(&core, 'a', "file", 1).await; + let writer = RootedQualifiedMetadataRepository::open(&core, &config) + .await + .unwrap(); + let intent = writer + .begin_intent("source-temp-shadow", &plan) + .await + .unwrap(); + writer.install_pages(&intent, &[payload]).await.unwrap(); + writer.finalize(&intent).await.unwrap(); + q.execute_unprepared("CREATE TEMP TABLE mega_tree(id bigint,tree_id text,sub_trees bytea); + CREATE TEMP TABLE mst2_rooted_source_tree_revision(tree_id text,tree_row_id bigint,revision uuid,body_digest bytea,valid boolean)") + .await.unwrap(); + assert_eq!(count(&q,&format!("SELECT {}.mst2_route_source_tree_matches(split_part(tagged_tree_oid,':',2),source_revision,source_body_digest)::bigint + FROM mst2_metadata_source_root_attestation",identifier(&namespace.core_schema))).await,1); +} + +#[tokio::test] +async fn current_source_revision_share_fence_orders_real_source_update_after_read() { + let (config, core, namespace, q, _guard) = fixture().await; + let (plan, payload) = seeded_rooted_plan(&core, 'a', "file", 1).await; + let writer = RootedQualifiedMetadataRepository::open(&core, &config) + .await + .unwrap(); + let intent = writer + .begin_intent("source-share-fence", &plan) + .await + .unwrap(); + writer.install_pages(&intent, &[payload]).await.unwrap(); + writer.finalize(&intent).await.unwrap(); + let read = q.begin().await.unwrap(); + assert_eq!(count(&read,&format!("SELECT {}.mst2_route_source_tree_matches(split_part(tagged_tree_oid,':',2),source_revision,source_body_digest)::bigint + FROM mst2_metadata_source_root_attestation",identifier(&namespace.core_schema))).await,1); + let core_writer = core.clone(); + let (ready, received) = tokio::sync::oneshot::channel(); + let pending = tokio::spawn(async move { + let txn = core_writer.begin().await.unwrap(); + let pid: i32 = txn + .query_one_raw(Statement::from_string( + DbBackend::Postgres, + "SELECT pg_backend_pid()", + )) + .await + .unwrap() + .unwrap() + .try_get_by_index(0) + .unwrap(); + ready.send(pid).unwrap(); + txn.execute_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + "UPDATE mega_tree SET sub_trees=set_byte(sub_trees,7,120) WHERE tree_id=$1", + ["a".repeat(40).into()], + )) + .await + .unwrap(); + txn.commit().await + }); + let pid = received.await.unwrap(); + tokio::time::timeout(Duration::from_secs(4),async { + loop { + let waiting:bool=core.query_one_raw(Statement::from_sql_and_values(DbBackend::Postgres, + "SELECT coalesce(wait_event_type='Lock',false) FROM pg_stat_activity WHERE pid=$1",[pid.into()])) + .await.unwrap().unwrap().try_get_by_index(0).unwrap(); + if waiting {break;} tokio::task::yield_now().await; + } + }).await.unwrap(); + assert!(!pending.is_finished()); + read.commit().await.unwrap(); + tokio::time::timeout(Duration::from_secs(4), pending) + .await + .unwrap() + .unwrap() + .unwrap(); + assert_eq!( + count( + &core, + "SELECT count(*) FROM mst2_rooted_source_tree_revision WHERE valid" + ) + .await, + 0 + ); +} diff --git a/src/jupiter/storage/qualified_source_revision.sql b/src/jupiter/storage/qualified_source_revision.sql new file mode 100644 index 00000000..855112ce --- /dev/null +++ b/src/jupiter/storage/qualified_source_revision.sql @@ -0,0 +1,124 @@ +-- This records only trees actually attested by Q, not the global Git inventory. +CREATE TABLE mst2_rooted_source_tree_revision ( + tree_id text PRIMARY KEY CHECK(tree_id ~ '^([0-9a-f]{40}|[0-9a-f]{64})$'), + tree_row_id bigint NOT NULL,revision uuid NOT NULL,body_digest bytea NOT NULL CHECK(octet_length(body_digest)=32), + valid boolean NOT NULL, + CHECK(substr(revision::text,15,1)='4' AND substr(revision::text,20,1) IN ('8','9','a','b')) +); + +CREATE FUNCTION mst2_route_source_tree_revision_guard() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$CORE_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE source_id bigint; body bytea; +BEGIN + IF TG_OP='DELETE' OR TG_OP='TRUNCATE' THEN RAISE EXCEPTION 'rooted source revision watermarks are immutable'; END IF; + IF pg_is_in_recovery() OR current_setting('transaction_isolation')<>'read committed' + OR TG_RELID<>'mst2_rooted_source_tree_revision'::regclass THEN + RAISE EXCEPTION 'rooted source revision requires its captured primary relation'; END IF; + IF TG_OP='UPDATE' THEN + IF NEW.tree_id IS DISTINCT FROM OLD.tree_id THEN RAISE EXCEPTION 'rooted source revision cannot retarget its OID'; END IF; + IF NEW.valid IS FALSE THEN + -- A nested caller cannot invent invalidation: independently observe the + -- actual core mutation after it happened in this still-uncommitted writer. + IF pg_trigger_depth()<2 OR NOT OLD.valid OR NEW.tree_row_id IS DISTINCT FROM OLD.tree_row_id + OR NEW.body_digest IS DISTINCT FROM OLD.body_digest OR NEW.revision IS DISTINCT FROM OLD.revision THEN + RAISE EXCEPTION 'rooted source invalidation is outside its actual source writer'; END IF; + PERFORM mst2_route_enter($CORE_LITERAL$); + SELECT id,sub_trees INTO source_id,body FROM mega_tree WHERE tree_id=OLD.tree_id; + IF FOUND AND source_id=OLD.tree_row_id AND octet_length(body)<=67108864 + AND sha256(body)=OLD.body_digest THEN + RAISE EXCEPTION 'rooted source invalidation has no actual changed or deleted core bytes'; END IF; + NEW.revision:=gen_random_uuid(); RETURN NEW; + END IF; + IF OLD.valid OR NEW.revision IS DISTINCT FROM OLD.revision OR NEW.valid IS DISTINCT FROM true THEN + RAISE EXCEPTION 'ordinary DML cannot rewrite a valid rooted source revision'; END IF; + END IF; + PERFORM mst2_route_enter($CORE_LITERAL$); + SELECT id,sub_trees INTO source_id,body FROM mega_tree WHERE tree_id=NEW.tree_id FOR SHARE; + IF NOT FOUND OR octet_length(body)>67108864 THEN RAISE EXCEPTION 'rooted source capture has no bounded actual core tree'; END IF; + IF NEW.body_digest IS DISTINCT FROM sha256(body) THEN + RAISE EXCEPTION 'rooted source revision was not independently derived from its actual core bytes'; END IF; + NEW.tree_row_id:=source_id; NEW.revision:=gen_random_uuid(); NEW.valid:=true; + RETURN NEW; +END $$; +CREATE TRIGGER mst2_route_source_tree_revision_guard BEFORE INSERT OR UPDATE OR DELETE ON mst2_rooted_source_tree_revision + FOR EACH ROW EXECUTE FUNCTION mst2_route_source_tree_revision_guard(); +CREATE TRIGGER mst2_route_source_tree_revision_truncate BEFORE TRUNCATE ON mst2_rooted_source_tree_revision + FOR EACH STATEMENT EXECUTE FUNCTION mst2_route_source_tree_revision_guard(); + +CREATE FUNCTION mst2_route_source_tree_inventory_guard() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$CORE_SCHEMA$,pg_catalog,pg_temp AS $$ +BEGIN + IF (SELECT count(*) FROM (SELECT 1 FROM mst2_rooted_source_tree_revision LIMIT 65537) bounded)>65536 THEN + RAISE EXCEPTION 'rooted attested source inventory capacity is exceeded'; END IF; + RETURN NULL; +END $$; +CREATE TRIGGER mst2_route_source_tree_inventory_guard AFTER INSERT ON mst2_rooted_source_tree_revision + FOR EACH STATEMENT EXECUTE FUNCTION mst2_route_source_tree_inventory_guard(); + +CREATE FUNCTION mst2_route_source_tree_writer_enter() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$CORE_SCHEMA$,pg_catalog,pg_temp AS $$ +BEGIN + IF TG_RELID<>'mega_tree'::regclass THEN RAISE EXCEPTION 'rooted source writer left its captured core relation'; END IF; + IF TG_OP='TRUNCATE' THEN RAISE EXCEPTION 'core tree truncation cannot bypass rooted source revision invalidation'; END IF; + PERFORM mst2_route_enter($CORE_LITERAL$); + RETURN NULL; +END $$; +CREATE TRIGGER mst2_route_source_tree_writer_enter BEFORE INSERT OR UPDATE OR DELETE OR TRUNCATE ON mega_tree + FOR EACH STATEMENT EXECUTE FUNCTION mst2_route_source_tree_writer_enter(); + +CREATE FUNCTION mst2_route_source_tree_invalidate() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$CORE_SCHEMA$,pg_catalog,pg_temp AS $$ +BEGIN + IF TG_RELID<>'mega_tree'::regclass THEN RAISE EXCEPTION 'rooted invalidation left its captured core relation'; END IF; + IF TG_OP='UPDATE' AND NEW.tree_id IS NOT DISTINCT FROM OLD.tree_id AND NEW.id IS NOT DISTINCT FROM OLD.id + AND NEW.sub_trees IS NOT DISTINCT FROM OLD.sub_trees THEN RETURN NEW; END IF; + UPDATE mst2_rooted_source_tree_revision SET valid=false WHERE tree_id=OLD.tree_id AND valid; + IF TG_OP='DELETE' THEN RETURN OLD; END IF; + RETURN NEW; +END $$; +CREATE TRIGGER mst2_route_source_tree_invalidate AFTER UPDATE OR DELETE ON mega_tree + FOR EACH ROW EXECUTE FUNCTION mst2_route_source_tree_invalidate(); +CREATE FUNCTION mst2_route_source_tree_inserted() RETURNS trigger LANGUAGE plpgsql VOLATILE +SET search_path=$CORE_SCHEMA$,pg_catalog,pg_temp AS $$ +BEGIN + IF TG_RELID<>'mega_tree'::regclass THEN RAISE EXCEPTION 'rooted insertion left its captured core relation'; END IF; + UPDATE mst2_rooted_source_tree_revision SET valid=false WHERE tree_id=NEW.tree_id AND valid; + RETURN NULL; +END $$; +CREATE TRIGGER mst2_route_source_tree_inserted AFTER INSERT ON mega_tree + FOR EACH ROW EXECUTE FUNCTION mst2_route_source_tree_inserted(); + +CREATE FUNCTION mst2_route_capture_source_tree(oid_text text,expected_digest bytea) RETURNS uuid LANGUAGE plpgsql VOLATILE +SET search_path=$CORE_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE captured mst2_rooted_source_tree_revision%ROWTYPE; actual_id bigint; +BEGIN + PERFORM mst2_route_enter($CORE_LITERAL$); + SELECT id INTO actual_id FROM mega_tree WHERE tree_id=oid_text FOR SHARE; + IF NOT FOUND OR expected_digest IS NULL OR octet_length(expected_digest)<>32 THEN + RAISE EXCEPTION 'rooted source capture has no exact actual row and body digest'; END IF; + SELECT * INTO captured FROM mst2_rooted_source_tree_revision WHERE tree_id=oid_text FOR UPDATE; + IF FOUND THEN + IF captured.valid THEN + IF captured.tree_row_id<>actual_id OR captured.body_digest IS DISTINCT FROM expected_digest THEN + RAISE EXCEPTION 'rooted source bytes changed behind their exact immutable revision'; END IF; + RETURN captured.revision; + END IF; + UPDATE mst2_rooted_source_tree_revision SET valid=true,body_digest=expected_digest WHERE tree_id=oid_text + RETURNING revision INTO captured.revision; + ELSE + INSERT INTO mst2_rooted_source_tree_revision(tree_id,tree_row_id,revision,body_digest,valid) + VALUES(oid_text,actual_id,gen_random_uuid(),expected_digest,true) RETURNING revision INTO captured.revision; + END IF; + RETURN captured.revision; +END $$; + +CREATE FUNCTION mst2_route_source_tree_matches(oid_text text,exact_revision uuid,exact_digest bytea) +RETURNS boolean LANGUAGE plpgsql VOLATILE STRICT SET search_path=$CORE_SCHEMA$,pg_catalog,pg_temp AS $$ +DECLARE captured mst2_rooted_source_tree_revision%ROWTYPE; +BEGIN + IF pg_is_in_recovery() OR current_setting('transaction_isolation')<>'read committed' THEN RETURN false; END IF; + SELECT revision.* INTO captured FROM mst2_rooted_source_tree_revision revision + JOIN mega_tree source ON source.id=revision.tree_row_id AND source.tree_id=revision.tree_id + WHERE revision.tree_id=oid_text FOR SHARE OF revision NOWAIT; + RETURN FOUND AND captured.valid AND captured.revision=exact_revision AND captured.body_digest=exact_digest; +END $$; diff --git a/src/jupiter/tests.rs b/src/jupiter/tests.rs index 4f39f1ad..aa3dd08d 100644 --- a/src/jupiter/tests.rs +++ b/src/jupiter/tests.rs @@ -223,6 +223,45 @@ fn drop_test_schema(admin_url: &str, schema: &str) { [Value::from(schema)], )) .await?; + // A bootstrap-created physical Q family belongs to this exact test + // core schema. Drop the pair even on panic, before its core FK targets. + let registry = admin + .query_one_raw(Statement::from_sql_and_values( + DatabaseBackend::Postgres, + "SELECT pg_catalog.to_regclass($1) IS NOT NULL AS present", + [format!("{schema}.mst2_metadata_namespace").into()], + )) + .await? + .unwrap() + .try_get::("", "present")?; + if registry { + let families = admin + .query_all_raw(Statement::from_string( + DatabaseBackend::Postgres, + format!( + "SELECT n.nspname FROM {schema}.mst2_metadata_namespace q + JOIN pg_catalog.pg_namespace n ON n.oid=q.metadata_schema_oid + JOIN pg_catalog.pg_namespace c ON c.oid=q.core_schema_oid + WHERE q.graph_domain='qualified-v1' AND c.nspname='{schema}' + AND q.core_schema='{schema}' AND q.family_identity='v3-rooted-qualified-1'" + ), + )) + .await?; + for family in families { + let q_schema: String = family.try_get("", "nspname")?; + if !q_schema.starts_with("mst2q_") { + return Err(sea_orm::DbErr::Custom( + "test Q schema escaped its generated prefix".into(), + )); + } + let quoted = format!("\"{}\"", q_schema.replace('"', "\"\"")); + admin + .execute_unprepared(&format!( + "SET lock_timeout='120s'; DROP SCHEMA {quoted} CASCADE" + )) + .await?; + } + } // The timeout turns a lock held by anything else into a leaked schema // and a message, not a hung test run. admin @@ -380,6 +419,8 @@ pub async fn test_storage_with_config(temp_dir: impl AsRef, config: Config native_projection_cache: Arc::default(), native_snapshot_sessions: Arc::default(), native_chunk_maps: Arc::default(), + shadow_qualified_metadata: Arc::default(), + rooted_qualified_metadata: Arc::default(), projection_observation_sink: None, cl_service: CLService::mock(), push_queue_service: PushQueueService::new(