From 7d21362bb9a5cdb6acc83355d92103443753473e Mon Sep 17 00:00:00 2001 From: Elrond <28d6302a099e5225b02c4155ac4236e4912603df2ab08dbfc2f4fef08ce598c8@buzz.block.builderlab.xyz> Date: Wed, 14 Oct 2026 13:49:57 -0400 Subject: [PATCH 01/10] buzz-db: check events created_at floor in the app writers (phase 1) Every channel-bearing INSERT INTO events now checks created_at against database clock_timestamp() minus CREATED_AT_FLOOR_SECS on the admitted transaction before inserting, and returns the typed, non-retryable DbError::CreatedAtFloorRejected. The check samples the clock after BEGIN, so the replica-fence wall min(oldest_xact_start, S) - floor - margin still covers it. Migration 0021's deferred trigger and the writer-pool GUC stay in place as the mixed-version backstop until phase 2. A source policy requires the check on every channel-bearing events INSERT and forbids production UPDATEs of events.created_at/channel_id. The admin seed script now refuses non-local PGHOST. Refs BUZZ-316. Signed-off-by: Elrond <28d6302a099e5225b02c4155ac4236e4912603df2ab08dbfc2f4fef08ce598c8@buzz.block.builderlab.xyz> --- crates/buzz-db/src/error.rs | 7 + crates/buzz-db/src/runtime/replica_fence.rs | 49 ++- crates/buzz-db/src/runtime/tests.rs | 74 +++-- .../tests/thread_window_postgres_tests.rs | 4 +- crates/buzz-db/src/store/channel_members.rs | 6 + crates/buzz-db/src/store/event.rs | 12 +- .../src/store/event_floor_postgres_tests.rs | 313 ++++++++++++++++++ crates/buzz-db/src/store/mod.rs | 3 + .../participation_postgres_tests.rs | 6 +- .../src/store/personal_read/postgres_tests.rs | 9 +- .../personal_read/threads_postgres_tests.rs | 4 +- crates/buzz-db/src/store/replaceable.rs | 5 + crates/buzz-db/src/test_support.rs | 48 +++ crates/buzz-db/tests/observability_source.rs | 201 ++++++++++- scripts/seed-admin-dashboard.sh | 9 + 15 files changed, 685 insertions(+), 65 deletions(-) create mode 100644 crates/buzz-db/src/store/event_floor_postgres_tests.rs diff --git a/crates/buzz-db/src/error.rs b/crates/buzz-db/src/error.rs index 8c051a7a793..a7eae563108 100644 --- a/crates/buzz-db/src/error.rs +++ b/crates/buzz-db/src/error.rs @@ -81,6 +81,13 @@ pub enum DbError { #[error("invalid timestamp: {0}")] InvalidTimestamp(i64), + /// A channel event is too old for replica-fence completeness. Retrying the + /// same signed event cannot make its timestamp newer. Like the mixed-version + /// trigger's SQLx error, this remains a database error at the relay boundary + /// (WS OK false / HTTP 500), rather than a different ingest rejection class. + #[error("database error: events.created_at is below the replica-fence floor")] + CreatedAtFloorRejected, + /// A roster mutation would remove the last effective relay Operator, /// leaving no one able to administer the deployment through the API. /// The transaction is rolled back and the caller must add a replacement diff --git a/crates/buzz-db/src/runtime/replica_fence.rs b/crates/buzz-db/src/runtime/replica_fence.rs index 520ff9e8698..1f85f20c5f0 100644 --- a/crates/buzz-db/src/runtime/replica_fence.rs +++ b/crates/buzz-db/src/runtime/replica_fence.rs @@ -3,12 +3,11 @@ //! A read replica may serve a cursor page only when every row the page could //! contain is provably present on the replica. The proof rests on two parts: //! -//! 1. **Commit-time floor** (migration 0021): a deferred constraint trigger -//! aborts, at COMMIT, any transaction inserting a channel-bearing `events` -//! row with `created_at` more than `floor` seconds before commit time -//! (`clock_timestamp()`, evaluated inside commit processing). Enforcement -//! is armed per session via the `buzz.created_at_floor` GUC, which the -//! relay's writer pool sets on every connection. +//! 1. **Insert-time floor**: supported channel-event writers check `created_at` +//! against database `clock_timestamp()` on their admitted transaction before +//! INSERT. This is the primary guard. Migration 0021's deferred trigger and +//! writer-pool `buzz.created_at_floor` GUC remain the mixed-version backstop +//! until phase 2; they also reject rows that age below the floor before COMMIT. //! 2. **Ordered heartbeat handshake** (this module): on one pinned writer //! connection, one transaction first takes the exclusive replica-floor //! advisory lock (draining compliant shared-lock channel writers), then @@ -25,7 +24,7 @@ //! commit, so the replica session has replayed it; //! (b) open at the activity scan — represented by `xact_start`, so it is //! bounded by the `oldest_xact_start` term; -//! (c) started after the activity scan — its deferred floor guard runs +//! (c) started after the activity scan — its insert-time app guard runs //! after `S`, so it cannot commit a row with //! `created_at < S - floor`. //! There is no fourth bucket. Each committed token `M` therefore proves a @@ -53,9 +52,9 @@ //! mismatch, or an observed token below every retained entry all route the //! request back to the writer — degraded capacity, never holes. //! -//! Operational bypasses (sessions without the GUC, `session_replication_role -//! = replica` restores) are outside the proof by design and require holding -//! the fence closed for their duration; see `migrations/0021`. +//! Raw SQL without the armed GUC and `session_replication_role = replica` +//! restores bypass the supported app writers. These operational bypasses are +//! outside the proof and require holding the fence closed; see `migrations/0021`. use std::collections::VecDeque; use std::sync::{Arc, Mutex}; @@ -67,7 +66,7 @@ use uuid::Uuid; use buzz_datastore_tracing::datastore_span; -/// Seconds of `created_at` history the commit-time floor guard tolerates. +/// Seconds of `created_at` history the app and mixed-version trigger tolerate. /// /// Must exceed the relay's ingest envelope (±900 s) by enough slack that a /// legitimately accepted event still commits within the floor even under @@ -75,6 +74,34 @@ use buzz_datastore_tracing::datastore_span; /// and the fence subtracts it; the two uses must never diverge. pub const CREATED_AT_FLOOR_SECS: i64 = 960; +/// Enforce the channel-event floor on the admitted INSERT connection. +/// +/// One preceding SELECT keeps the five existing INSERT shapes intact. Its +/// database clock is sampled after BEGIN: transactions open at the probe scan +/// are bounded by xact_start; transactions starting later sample after S. No +/// commit hook, candidate tracking or ordering against TTL refresh is needed. +/// NULL-channel rows never enter covered channel windows and need no query. +pub(crate) async fn guard_event_created_at_floor( + tx: &mut crate::AdmittedTx, + created_at: DateTime, + channel_id: Option, +) -> crate::Result<()> { + if channel_id.is_none() { + return Ok(()); + } + let below_floor: bool = sqlx::query_scalar( + "SELECT $1::timestamptz < clock_timestamp() - make_interval(secs => $2::double precision)", + ) + .bind(created_at) + .bind(CREATED_AT_FLOOR_SECS as f64) + .fetch_one(tx.conn()) + .await?; + if below_floor { + return Err(crate::DbError::CreatedAtFloorRejected); + } + Ok(()) +} + /// Deployment-global advisory-lock key ordering compliant channel-event write /// transactions against the writer probe handshake. /// diff --git a/crates/buzz-db/src/runtime/tests.rs b/crates/buzz-db/src/runtime/tests.rs index 2e0e260bd26..d3f429baee0 100644 --- a/crates/buzz-db/src/runtime/tests.rs +++ b/crates/buzz-db/src/runtime/tests.rs @@ -908,27 +908,27 @@ fn signed_event_at(keys: &nostr::Keys, content: &str, secs: u64) -> nostr::Event .expect("sign event") } +// Routing tests intentionally model stored history and divergent replicas. +// Raw fixture SQL keeps them independent of serving-time floor admission. async fn insert_top_level(pool: &PgPool, community: Uuid, channel: Uuid, ev: &nostr::Event) { - let ts = chrono::DateTime::from_timestamp(ev.created_at.as_secs() as i64, 0).expect("valid ts"); - event::insert_event_with_thread_metadata( + crate::test_support::insert_historical_event( pool, CommunityId::from_uuid(community), ev, - Some(channel), - Some(event::ThreadMetadataParams { - event_id: ev.id.as_bytes(), - event_created_at: ts, - channel_id: channel, - parent_event_id: None, - parent_event_created_at: None, - root_event_id: None, - root_event_created_at: None, - depth: 0, - broadcast: true, - }), + channel, + ) + .await; + sqlx::query( + "INSERT INTO thread_metadata (community_id,event_id,event_created_at,channel_id,depth,broadcast) \ + VALUES ($1,$2,to_timestamp($3),$4,0,true)", ) + .bind(community) + .bind(ev.id.as_bytes().as_slice()) + .bind(ev.created_at.as_secs() as f64) + .bind(channel) + .execute(pool) .await - .expect("insert top-level event"); + .expect("seed top-level metadata"); } async fn insert_thread_reply( @@ -938,29 +938,37 @@ async fn insert_thread_reply( root: &nostr::Event, reply: &nostr::Event, ) { - let reply_ts = - chrono::DateTime::from_timestamp(reply.created_at.as_secs() as i64, 0).expect("valid ts"); - let root_ts = - chrono::DateTime::from_timestamp(root.created_at.as_secs() as i64, 0).expect("valid ts"); - event::insert_event_with_thread_metadata( + crate::test_support::insert_historical_event( pool, CommunityId::from_uuid(community), reply, - Some(channel), - Some(event::ThreadMetadataParams { - event_id: reply.id.as_bytes(), - event_created_at: reply_ts, - channel_id: channel, - parent_event_id: Some(root.id.as_bytes()), - parent_event_created_at: Some(root_ts), - root_event_id: Some(root.id.as_bytes()), - root_event_created_at: Some(root_ts), - depth: 1, - broadcast: false, - }), + channel, + ) + .await; + sqlx::query( + "INSERT INTO thread_metadata \ + (community_id,event_id,event_created_at,channel_id,parent_event_id,parent_event_created_at, \ + root_event_id,root_event_created_at,depth,broadcast) \ + VALUES ($1,$2,to_timestamp($3),$4,$5,to_timestamp($6),$5,to_timestamp($6),1,false)", + ) + .bind(community) + .bind(reply.id.as_bytes().as_slice()) + .bind(reply.created_at.as_secs() as f64) + .bind(channel) + .bind(root.id.as_bytes().as_slice()) + .bind(root.created_at.as_secs() as f64) + .execute(pool) + .await + .expect("seed reply metadata"); + sqlx::query( + "UPDATE thread_metadata SET reply_count=reply_count+1,descendant_count=descendant_count+1, \ + last_reply_at=now() WHERE community_id=$1 AND event_id=$2", ) + .bind(community) + .bind(root.id.as_bytes().as_slice()) + .execute(pool) .await - .expect("insert reply"); + .expect("seed root counters"); } /// Composite thread cursor: 8-byte BE seconds + raw event id. diff --git a/crates/buzz-db/src/runtime/tests/thread_window_postgres_tests.rs b/crates/buzz-db/src/runtime/tests/thread_window_postgres_tests.rs index beda4b199fc..1f0d1927e18 100644 --- a/crates/buzz-db/src/runtime/tests/thread_window_postgres_tests.rs +++ b/crates/buzz-db/src/runtime/tests/thread_window_postgres_tests.rs @@ -87,9 +87,7 @@ async fn thread_window_upper_fence_terminal_snapshot_and_fallback() { .sign_with_keys(&keys) .unwrap(); for pool in [&writer, &replica] { - event::insert_event(pool, community, &edit, Some(channel)) - .await - .unwrap(); + crate::test_support::insert_historical_event(pool, community, &edit, channel).await; event::insert_event(pool, community, &deletion, None) .await .unwrap(); diff --git a/crates/buzz-db/src/store/channel_members.rs b/crates/buzz-db/src/store/channel_members.rs index 1994de14ed4..82bf9d52822 100644 --- a/crates/buzz-db/src/store/channel_members.rs +++ b/crates/buzz-db/src/store/channel_members.rs @@ -396,6 +396,12 @@ impl LockedMemberSnapshot { )); } } + crate::replica_fence::guard_event_created_at_floor( + &mut self.tx, + created_at, + Some(channel_id), + ) + .await?; sqlx::query("UPDATE events SET deleted_at = NOW() WHERE community_id = $1 AND kind = $2 AND pubkey = $3 AND channel_id = $4 AND deleted_at IS NULL") .bind(community_id.as_uuid()).bind(kind).bind(pubkey.as_slice()).bind(channel_id) .execute(self.tx.conn()).await?; diff --git a/crates/buzz-db/src/store/event.rs b/crates/buzz-db/src/store/event.rs index 183585e2785..fc4cd4571bc 100644 --- a/crates/buzz-db/src/store/event.rs +++ b/crates/buzz-db/src/store/event.rs @@ -480,6 +480,7 @@ async fn insert_event_on( let received_at = Utc::now(); let d_tag = extract_d_tag(event); let not_before = extract_not_before(event); + crate::replica_fence::guard_event_created_at_floor(tx, created_at, channel_id).await?; let result = sqlx::query( r#" INSERT INTO events (community_id, id, pubkey, created_at, kind, tags, content, sig, received_at, channel_id, d_tag, not_before) @@ -1731,6 +1732,7 @@ pub(crate) async fn insert_event_with_thread_metadata_tx( let d_tag = extract_d_tag(event); let not_before = extract_not_before(event); + crate::replica_fence::guard_event_created_at_floor(tx, created_at, channel_id).await?; let result = sqlx::query( r#" INSERT INTO events (community_id, id, pubkey, created_at, kind, tags, content, sig, received_at, channel_id, d_tag, not_before) @@ -3872,7 +3874,15 @@ mod postgres_tests { // ─── canvas CAS tests ───────────────────────────────────────────────────── fn make_canvas_event_at(content: &str, created_at: u64) -> nostr::Event { - make_event_at(buzz_core::kind::KIND_CANVAS as u16, content, created_at) + // Keep the CAS fixtures' logical seconds and ties, anchored at a fresh + // timestamp so they exercise the serving writer's floor admission. + static BASE: std::sync::OnceLock = std::sync::OnceLock::new(); + let base = BASE.get_or_init(|| nostr::Timestamp::now().as_secs() - 1000); + make_event_at( + buzz_core::kind::KIND_CANVAS as u16, + content, + base + created_at, + ) } /// Build two canvas events sharing `created_at`, returned `(lower, higher)` diff --git a/crates/buzz-db/src/store/event_floor_postgres_tests.rs b/crates/buzz-db/src/store/event_floor_postgres_tests.rs new file mode 100644 index 00000000000..857d91ba686 --- /dev/null +++ b/crates/buzz-db/src/store/event_floor_postgres_tests.rs @@ -0,0 +1,313 @@ +//! App-floor regressions with every floor-trigger clone disabled in the +//! nextest-provided per-test database, so the DB backstop cannot mask a bypass. + +use buzz_core::CommunityId; +use chrono::{DateTime, Utc}; +use nostr::{Event, EventBuilder, Keys, Kind, Tag, Timestamp}; +use sqlx::postgres::PgPoolOptions; +use uuid::Uuid; + +use crate::replica_fence::{ + probe_once, ReplicaFence, CREATED_AT_FLOOR_SECS, FENCE_CLOCK_MARGIN_SECS, +}; +use crate::{event, AdmittedTx, Db, DbError}; + +async fn fixture() -> (Db, CommunityId, Uuid, Keys) { + let pool = PgPoolOptions::new() + .max_connections(4) + .connect(&crate::test_support::database_url()) + .await + .expect("isolated test database"); + sqlx::query("ALTER TABLE events DISABLE TRIGGER events_created_at_floor") + .execute(&pool) + .await + .expect("disable floor trigger recursively on the partitioned table"); + let (total, enabled): (i64, i64) = sqlx::query_as( + "SELECT count(*), count(*) FILTER (WHERE tgenabled <> 'D') FROM pg_trigger \ + WHERE tgname = 'events_created_at_floor' \ + AND tgrelid IN (SELECT relid FROM pg_partition_tree('events'))", + ) + .fetch_one(&pool) + .await + .expect("check every trigger clone"); + assert!(total > 0, "the test must disable an existing backstop"); + assert_eq!(enabled, 0, "no partition may mask a missing app check"); + let community = CommunityId::from_uuid(Uuid::new_v4()); + sqlx::query("INSERT INTO communities (id, host) VALUES ($1, $2)") + .bind(community.as_uuid()) + .bind(format!("floor-{}.invalid", community.as_uuid())) + .execute(&pool) + .await + .expect("community"); + let channel = Uuid::new_v4(); + let keys = Keys::generate(); + sqlx::query( + "INSERT INTO channels (id, community_id, name, created_by) VALUES ($1,$2,'floor',$3)", + ) + .bind(channel) + .bind(community.as_uuid()) + .bind(keys.public_key().to_bytes().as_slice()) + .execute(&pool) + .await + .expect("channel"); + (Db::from_pool(pool), community, channel, keys) +} + +async fn database_clock(db: &Db) -> DateTime { + sqlx::query_scalar("SELECT clock_timestamp()") + .fetch_one(db.pool()) + .await + .expect("database clock") +} + +fn signed(keys: &Keys, kind: u16, channel: Uuid, time: DateTime) -> Event { + EventBuilder::new(Kind::Custom(kind), "floor-test") + .tags([Tag::parse(["d", &channel.to_string()]).expect("coordinate")]) + .custom_created_at(Timestamp::from(time.timestamp() as u64)) + .sign_with_keys(keys) + .expect("signed event") +} + +async fn count(db: &Db, community: CommunityId, event: &Event) -> i64 { + sqlx::query_scalar("SELECT count(*) FROM events WHERE community_id=$1 AND id=$2") + .bind(community.as_uuid()) + .bind(event.id.as_bytes().as_slice()) + .fetch_one(db.pool()) + .await + .expect("event count") +} + +async fn begin(db: &Db, community: CommunityId) -> AdmittedTx { + db.begin_event_write_transaction(community) + .await + .expect("admission") +} + +fn rejected(result: crate::Result) { + assert!( + matches!(result, Err(DbError::CreatedAtFloorRejected)), + "expected the typed app-floor rejection, got {result:?}" + ); +} + +#[tokio::test] +#[ignore = "requires Postgres"] +async fn insert_event_floor_rejects_old_and_accepts_fresh() { + let (db, community, channel, keys) = fixture().await; + let now = database_clock(&db).await; + let old = signed( + &keys, + 9, + channel, + now - chrono::Duration::seconds(CREATED_AT_FLOOR_SECS + 60), + ); + rejected(event::insert_event(db.pool(), community, &old, Some(channel)).await); + assert_eq!(count(&db, community, &old).await, 0); + let fresh = signed(&keys, 9, channel, now); + assert!( + event::insert_event(db.pool(), community, &fresh, Some(channel)) + .await + .expect("fresh") + .1 + ); + assert_eq!(count(&db, community, &fresh).await, 1); + // Channel-NULL history is structurally outside covered windows. + assert!( + event::insert_event(db.pool(), community, &old, None) + .await + .expect("NULL-channel history") + .1 + ); +} + +#[tokio::test] +#[ignore = "requires Postgres"] +async fn thread_metadata_event_floor_rejects_old_and_accepts_fresh() { + let (db, community, channel, keys) = fixture().await; + let now = database_clock(&db).await; + for (time, old) in [ + ( + now - chrono::Duration::seconds(CREATED_AT_FLOOR_SECS + 60), + true, + ), + (now, false), + ] { + let event = signed(&keys, 9, channel, time); + let meta = event::ThreadMetadataParams { + event_id: event.id.as_bytes().as_slice(), + event_created_at: time, + channel_id: channel, + parent_event_id: None, + parent_event_created_at: None, + root_event_id: None, + root_event_created_at: None, + depth: 0, + broadcast: false, + }; + let mut tx = begin(&db, community).await; + let result = + event::insert_event_with_thread_metadata_tx(&mut tx, &event, Some(channel), Some(meta)) + .await; + if old { + rejected(result); + tx.rollback().await.expect("rollback refusal"); + assert_eq!(count(&db, community, &event).await, 0); + } else { + assert!(result.expect("fresh").1); + tx.commit().await.expect("commit"); + assert_eq!(count(&db, community, &event).await, 1); + } + } +} + +#[tokio::test] +#[ignore = "requires Postgres"] +async fn parameterized_event_floor_rejects_old_and_accepts_fresh() { + let (db, community, channel, keys) = fixture().await; + let now = database_clock(&db).await; + let old = signed( + &keys, + 30078, + channel, + now - chrono::Duration::seconds(CREATED_AT_FLOOR_SECS + 60), + ); + rejected( + db.replace_parameterized_event(community, &old, &channel.to_string(), Some(channel)) + .await, + ); + assert_eq!(count(&db, community, &old).await, 0); + let fresh = signed(&keys, 30078, channel, now); + assert!( + db.replace_parameterized_event(community, &fresh, &channel.to_string(), Some(channel)) + .await + .expect("fresh") + .1 + ); + assert_eq!(count(&db, community, &fresh).await, 1); +} + +#[tokio::test] +#[ignore = "requires Postgres"] +async fn addressable_event_floor_rejects_old_and_accepts_fresh() { + let (db, community, channel, keys) = fixture().await; + let now = database_clock(&db).await; + let old = signed( + &keys, + 39000, + channel, + now - chrono::Duration::seconds(CREATED_AT_FLOOR_SECS + 60), + ); + rejected( + db.replace_addressable_event(community, &old, Some(channel)) + .await, + ); + assert_eq!(count(&db, community, &old).await, 0); + let fresh = signed(&keys, 39000, channel, now); + assert!( + db.replace_addressable_event(community, &fresh, Some(channel)) + .await + .expect("fresh") + .1 + ); + assert_eq!(count(&db, community, &fresh).await, 1); +} + +#[tokio::test] +#[ignore = "requires Postgres"] +async fn member_snapshot_event_floor_rejects_old_and_accepts_fresh() { + let (db, community, channel, keys) = fixture().await; + let now = database_clock(&db).await; + let old = signed( + &keys, + 39002, + channel, + now - chrono::Duration::seconds(CREATED_AT_FLOOR_SECS + 60), + ); + let mut snapshot = db + .lock_member_snapshot(community, channel, &keys.public_key().to_bytes()) + .await + .expect("snapshot"); + rejected(snapshot.replace_member_event(&old).await); + // A refusal must leave the held snapshot usable and write no event. + assert_eq!(count(&db, community, &old).await, 0); + let fresh = signed(&keys, 39002, channel, now); + assert!( + snapshot + .replace_member_event(&fresh) + .await + .expect("fresh") + .1 + ); + snapshot.release().await.expect("commit snapshot"); + assert_eq!(count(&db, community, &fresh).await, 1); +} + +#[tokio::test] +#[ignore = "requires Postgres"] +async fn cluster_global_event_floor_held_transaction_cannot_write_below_probe_wall() { + let (db, community, channel, keys) = fixture().await; + let mut tx = begin(&db, community).await; + let start: DateTime = sqlx::query_scalar("SELECT now()") + .fetch_one(tx.conn()) + .await + .expect("transaction start"); + let entry = probe_once(db.pool(), &ReplicaFence::new()) + .await + .expect("heartbeat while writer is open"); + assert!( + entry.fence_wall + <= start - chrono::Duration::seconds(CREATED_AT_FLOOR_SECS + FENCE_CLOCK_MARGIN_SECS) + ); + let below_wall = signed( + &keys, + 9, + channel, + entry.fence_wall - chrono::Duration::seconds(1), + ); + rejected(event::insert_event_in_transaction(&mut tx, &below_wall, Some(channel)).await); + // Sample just after the probe; a near-floor accepted insert is above its + // wall even when this transaction commits after the heartbeat. + let fresh = signed( + &keys, + 9, + channel, + database_clock(&db).await - chrono::Duration::seconds(CREATED_AT_FLOOR_SECS - 5), + ); + let created_at = + DateTime::from_timestamp(fresh.created_at.as_secs() as i64, 0).expect("timestamp"); + assert!(created_at > entry.fence_wall); + assert!( + event::insert_event_in_transaction(&mut tx, &fresh, Some(channel)) + .await + .expect("near floor") + .1 + ); + tx.commit().await.expect("commit after probe"); + assert_eq!(count(&db, community, &below_wall).await, 0); + assert_eq!(count(&db, community, &fresh).await, 1); +} + +#[tokio::test] +#[ignore = "requires Postgres"] +async fn event_floor_samples_statement_clock_after_transaction_start() { + let (db, community, channel, keys) = fixture().await; + let mut tx = begin(&db, community).await; + // The smallest integral event timestamp inside the floor at BEGIN. + let secs: i64 = + sqlx::query_scalar("SELECT ceil(extract(epoch FROM now()) - $1::double precision)::bigint") + .bind(CREATED_AT_FLOOR_SECS as f64) + .fetch_one(tx.conn()) + .await + .expect("BEGIN boundary"); + let time = DateTime::from_timestamp(secs, 0).expect("timestamp"); + // Wait on the DB clock until this exact boundary expires (at most 1.1 s), + // not an arbitrary delay and not a Rust clock. now() remains frozen inside. + tokio::time::timeout(std::time::Duration::from_secs(3), + sqlx::query("SELECT pg_sleep(GREATEST(0, extract(epoch FROM ($1::timestamptz + make_interval(secs => $2::double precision) - clock_timestamp()))) + 0.1)") + .bind(time).bind(CREATED_AT_FLOOR_SECS as f64).execute(tx.conn())) + .await.expect("bounded boundary wait").expect("database wait"); + let event = signed(&keys, 9, channel, time); + rejected(event::insert_event_in_transaction(&mut tx, &event, Some(channel)).await); + tx.rollback().await.expect("rollback"); + assert_eq!(count(&db, community, &event).await, 0); +} diff --git a/crates/buzz-db/src/store/mod.rs b/crates/buzz-db/src/store/mod.rs index efa46f17485..756a276c207 100644 --- a/crates/buzz-db/src/store/mod.rs +++ b/crates/buzz-db/src/store/mod.rs @@ -26,6 +26,9 @@ pub(crate) mod event_follow_up; #[cfg(test)] mod event_follow_up_postgres_tests; +#[cfg(test)] +mod event_floor_postgres_tests; + /// Home feed queries. pub mod feed; /// Git repository name registry (NIP-34 kind:30617). diff --git a/crates/buzz-db/src/store/personal_read/participation_postgres_tests.rs b/crates/buzz-db/src/store/personal_read/participation_postgres_tests.rs index 5a3d3a0acad..2800b9b1804 100644 --- a/crates/buzz-db/src/store/personal_read/participation_postgres_tests.rs +++ b/crates/buzz-db/src/store/personal_read/participation_postgres_tests.rs @@ -70,10 +70,8 @@ impl World { .custom_created_at(Timestamp::from(at)) .sign_with_keys(author) .unwrap(); - self.db - .insert_event(self.community, &event, Some(channel)) - .await - .unwrap(); + crate::test_support::insert_historical_event(&self.pool, self.community, &event, channel) + .await; event } diff --git a/crates/buzz-db/src/store/personal_read/postgres_tests.rs b/crates/buzz-db/src/store/personal_read/postgres_tests.rs index cba0d992a47..7678f4c13cb 100644 --- a/crates/buzz-db/src/store/personal_read/postgres_tests.rs +++ b/crates/buzz-db/src/store/personal_read/postgres_tests.rs @@ -164,8 +164,9 @@ async fn personal_read_sidebar_author_window_excludes_expired_and_late_old_messa "an empty author-time window has nothing unread" ); assert_eq!(row.latest_id.as_deref(), Some(latest.id.to_hex().as_str())); - // Acceptance time is irrelevant: a message accepted now with an author - // time beyond the horizon stays out; one authored now is counted. + // Stored historical data with an author time beyond the horizon stays out; + // one authored now is counted. The serving writer now rejects the old + // timestamp, so seed the historical row through unarmed fixture SQL. let now = nostr::Timestamp::now().as_secs(); for (age, unread, mentions) in [(40 * 86400, false, 0), (0, true, 1)] { let event = EventBuilder::new(Kind::Custom(9), "accepted now") @@ -173,9 +174,7 @@ async fn personal_read_sidebar_author_window_excludes_expired_and_late_old_messa .custom_created_at(nostr::Timestamp::from(now - age)) .sign_with_keys(&Keys::generate()) .unwrap(); - db.insert_event(community, &event, Some(channel)) - .await - .unwrap(); + crate::test_support::insert_historical_event(&pool, community, &event, channel).await; let row = &sidebar(&db, community, &actor).await.channels[0]; assert_eq!((row.unread, row.mentions), (unread, mentions), "age={age}"); } diff --git a/crates/buzz-db/src/store/personal_read/threads_postgres_tests.rs b/crates/buzz-db/src/store/personal_read/threads_postgres_tests.rs index 2bb86aab8da..09786f55856 100644 --- a/crates/buzz-db/src/store/personal_read/threads_postgres_tests.rs +++ b/crates/buzz-db/src/store/personal_read/threads_postgres_tests.rs @@ -32,9 +32,7 @@ async fn post_as( .custom_created_at(nostr::Timestamp::from(at)) .sign_with_keys(author) .unwrap(); - db.insert_event(community, &event, Some(channel)) - .await - .unwrap(); + crate::test_support::insert_historical_event(&db.pool, community, &event, channel).await; event } diff --git a/crates/buzz-db/src/store/replaceable.rs b/crates/buzz-db/src/store/replaceable.rs index 1bce5766a46..8dccadcb5c2 100644 --- a/crates/buzz-db/src/store/replaceable.rs +++ b/crates/buzz-db/src/store/replaceable.rs @@ -251,6 +251,8 @@ async fn replace_parameterized_event_in_transaction_impl( )); } + crate::replica_fence::guard_event_created_at_floor(tx, created_at, channel_id).await?; + // Everything from the supersede through the mention index runs under one // savepoint on the admitted transaction. A duplicate insert or any failure // (including mention indexing) rolls the whole replacement back and leaves @@ -485,6 +487,9 @@ impl Db { } } + crate::replica_fence::guard_event_created_at_floor(&mut tx, created_at, channel_id) + .await?; + // Soft-delete the old event (if any). IS NOT DISTINCT FROM for NULL safety. sqlx::query( "UPDATE events SET deleted_at = NOW() \ diff --git a/crates/buzz-db/src/test_support.rs b/crates/buzz-db/src/test_support.rs index e94b8b14491..ec1e816e391 100644 --- a/crates/buzz-db/src/test_support.rs +++ b/crates/buzz-db/src/test_support.rs @@ -8,6 +8,54 @@ pub(crate) fn database_url() -> String { .unwrap_or_else(|_| DEFAULT_DATABASE_URL.to_owned()) } +/// Seed historical read fixtures through raw SQL on an unarmed test pool. +/// Supported serving APIs reject these old timestamps; this represents data +/// already in storage, while retaining admission, mentions and event follow-ups. +#[cfg(test)] // Keep raw fixture SQL visibly test-only to the source-policy scan. +pub(crate) async fn insert_historical_event( + pool: &sqlx::PgPool, + community: buzz_core::CommunityId, + event: &nostr::Event, + channel: uuid::Uuid, +) { + let mut tx = crate::begin_community_write_transaction( + pool, + community, + crate::observability::WriterOperation::EventWrite, + ) + .await + .expect("admit historical fixture"); + sqlx::query( + "INSERT INTO events (community_id,id,pubkey,created_at,kind,tags,content,sig,channel_id,d_tag) \ + VALUES ($1,$2,$3,to_timestamp($4),$5,$6,$7,$8,$9,$10)", + ) + .bind(community.as_uuid()) + .bind(event.id.as_bytes().as_slice()) + .bind(event.pubkey.to_bytes().as_slice()) + .bind(event.created_at.as_secs() as f64) + .bind(i32::from(event.kind.as_u16())) + .bind(serde_json::to_value(&event.tags).expect("fixture tags")) + .bind(&event.content) + .bind(event.sig.serialize().as_slice()) + .bind(channel) + .bind(crate::event::extract_d_tag(event)) + .execute(tx.conn()) + .await + .expect("insert historical fixture"); + crate::store::event_follow_up::after_admitted_insert( + &mut tx, + event.id.as_bytes(), + i32::from(event.kind.as_u16()), + Some(channel), + ) + .await + .expect("fixture follow-ups"); + crate::insert_mentions_in_transaction(&mut tx, event, Some(channel)) + .await + .expect("fixture mentions"); + tx.commit().await.expect("commit historical fixture"); +} + /// Move a community fixture into a deletion lifecycle state the way the /// executor does: the tombstone trigger only accepts lifecycle changes from a /// transaction carrying the executor GUCs for that community, including its diff --git a/crates/buzz-db/tests/observability_source.rs b/crates/buzz-db/tests/observability_source.rs index 4d4edcf8e98..40179a269ca 100644 --- a/crates/buzz-db/tests/observability_source.rs +++ b/crates/buzz-db/tests/observability_source.rs @@ -1626,11 +1626,9 @@ fn masker_blanks_every_literal_form_and_keeps_code() { } } -/// Whether `source` inserts into `events` itself (not `event_mentions` or any -/// other `events_*` table), in any letter case, across line breaks and string -/// continuations, `\n`/`\t`/`\r` escapes, and with or without a `public.` -/// schema prefix. -fn inserts_events_row(source: &str) -> bool { +/// Normalize SQL case, whitespace and Rust string continuations/escapes for +/// the event INSERT and UPDATE policies. +fn normalized_sql_source(source: &str) -> String { let mut normalized = String::with_capacity(source.len()); let mut chars = source.chars().peekable(); while let Some(ch) = chars.next() { @@ -1645,6 +1643,13 @@ fn inserts_events_row(source: &str) -> bool { normalized.push(ch.to_ascii_lowercase()); } } + normalized +} + +/// Whether `source` inserts into `events` itself (not `event_mentions` or any +/// other `events_*` table), with or without a `public.` schema prefix. +fn inserts_events_row(source: &str) -> bool { + let normalized = normalized_sql_source(source); ["insert into events", "insert into public.events"] .iter() .any(|marker| { @@ -1935,3 +1940,189 @@ fn every_production_event_insert_runs_the_follow_up_hook() { "expected at least seven hooked event writers, found {hooked_writers}" ); } + +// These startup probes always roll back; they deliberately exercise the DB +// backstops rather than the app check. The two serving writers are exempt only +// while their INSERT remains structurally channel-NULL (checked below). +const EVENT_FLOOR_PROBE_EXCEPTIONS: [&str; 2] = [ + "pub async fn verify_floor_guard_behavior(", + "pub async fn verify_channel_roster_fence_behavior(", +]; + +fn is_channel_null_event_writer(function: &str) -> bool { + let header = function_header(function); + if header.starts_with("pub async fn publish_nip43_membership_locked(") { + // Kind 13534: the channel bind is always NULL. + return function.contains(".bind::>(None)"); + } + if header.starts_with("pub async fn accept_lease_event(") { + // Kind 30350: channel_id is the literal NULL in VALUES. + return normalized_sql_source(function).contains("now(),null,$8)"); + } + false +} + +fn updates_event_floor_columns(function: &str) -> bool { + let normalized = normalized_sql_source(function); + ["update events set ", "update public.events set "] + .iter() + .any(|marker| { + normalized.match_indices(marker).any(|(index, marker)| { + let assignments = normalized[index + marker.len()..] + .split(" where ") + .next() + .expect("split has first part"); + assignments.split(',').any(|assignment| { + assignment.split_once('=').is_some_and(|(column, _)| { + matches!(column.trim(), "created_at" | "channel_id") + }) + }) + }) + }) +} + +fn event_floor_policy_violations(production: &str) -> Vec { + function_slices(production) + .into_iter() + .filter(|function| { + let header = function_header(function); + if EVENT_FLOOR_PROBE_EXCEPTIONS + .iter() + .any(|exception| header.starts_with(exception)) + { + return false; + } + updates_event_floor_columns(function) + || (inserts_events_row(function) + && !is_channel_null_event_writer(function) + && !mask_comments_and_literals(function) + .contains("replica_fence::guard_event_created_at_floor(")) + }) + .map(|function| function_header(function).to_owned()) + .collect() +} + +#[test] +fn event_floor_policy_rejects_unguarded_inserts_and_floor_column_updates() { + let source = r#" +async fn missing_guard(tx: &mut AdmittedTx) { + sqlx::query("INSERT INTO events (id) VALUES ($1)"); + // replica_fence::guard_event_created_at_floor(tx, created_at, channel).await?; +} +async fn guarded(tx: &mut AdmittedTx) { + crate::replica_fence::guard_event_created_at_floor(tx, created_at, channel).await?; + sqlx::query("INSERT INTO public.events (id) VALUES ($1)"); +} +async fn rewrite_time(tx: &mut AdmittedTx) { + sqlx::query("UPDATE events SET tags = $1, created_at = $2 WHERE id = $3"); +} +async fn attach_channel(tx: &mut AdmittedTx) { + sqlx::query("uPdAtE public.events\nSET channel_id=$1 WHERE id=$2"); +} +async fn delete_only(tx: &mut AdmittedTx) { + sqlx::query("UPDATE events SET deleted_at=now() WHERE channel_id=$1 AND created_at<$2"); +} +"#; + assert_eq!( + event_floor_policy_violations(source), + [ + "async fn missing_guard(tx: &mut AdmittedTx) {", + "async fn rewrite_time(tx: &mut AdmittedTx) {", + "async fn attach_channel(tx: &mut AdmittedTx) {", + ] + ); +} + +#[test] +fn every_production_channel_event_insert_checks_the_floor_and_never_rewrites_it() { + fn collect(dir: &std::path::Path, files: &mut Vec) { + for entry in std::fs::read_dir(dir).expect("read source directory") { + let path = entry.expect("source entry").path(); + if path.is_dir() { + collect(&path, files); + } else if should_scan_guarded_write_source_file(&path) { + files.push(path); + } + } + } + let mut files = Vec::new(); + collect( + &std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("src"), + &mut files, + ); + let mut guarded_writers = 0; + let mut violations = Vec::new(); + for path in files { + let production = strip_cfg_test_items(&std::fs::read_to_string(&path).expect("source")); + guarded_writers += function_slices(&production) + .into_iter() + .filter(|function| { + inserts_events_row(function) + && mask_comments_and_literals(function) + .contains("replica_fence::guard_event_created_at_floor(") + }) + .count(); + violations.extend( + event_floor_policy_violations(&production) + .into_iter() + .map(|header| format!("{}: {header}", path.display())), + ); + } + assert!( + violations.is_empty(), + "event floor policy violations: {violations:?}" + ); + assert!( + guarded_writers >= 5, + "expected five channel-bearing event writers, found {guarded_writers}" + ); +} + +#[test] +fn event_floor_guard_uses_the_database_statement_clock_and_shared_floor() { + let source = strip_cfg_test_items(include_str!("../src/runtime/replica_fence.rs")); + let guard = function_slices(&source) + .into_iter() + .find(|function| function_header(function).contains("fn guard_event_created_at_floor(")) + .expect("app floor guard"); + assert!( + guard.contains("SELECT $1::timestamptz < clock_timestamp()"), + "the guard must sample the database statement clock, after BEGIN" + ); + assert!( + guard.contains(".bind(CREATED_AT_FLOOR_SECS as f64)"), + "app check, GUC and fence wall must share the same floor constant" + ); +} + +#[cfg(unix)] +#[test] +fn admin_seed_refuses_remote_database_before_invoking_psql() { + use std::os::unix::fs::PermissionsExt; + let root = std::env::temp_dir().join(format!("buzz-seed-policy-{}", uuid::Uuid::new_v4())); + let scripts = root.join("scripts"); + std::fs::create_dir_all(&scripts).expect("scratch scripts directory"); + let script = scripts.join("seed-admin-dashboard.sh"); + std::fs::write( + &script, + include_str!("../../../scripts/seed-admin-dashboard.sh"), + ) + .expect("copy script without developer .env"); + let psql = root.join("psql"); + std::fs::write(&psql, "#!/bin/sh\ntouch \"$SEED_PSQL_WITNESS\"\nexit 1\n").expect("stub psql"); + std::fs::set_permissions(&psql, std::fs::Permissions::from_mode(0o755)) + .expect("executable stub"); + let witness = root.join("psql-was-invoked"); + let output = std::process::Command::new("/bin/bash") + .arg(script) + .env("PGHOST", "shared-db.example.invalid") + .env("PATH", format!("{}:/usr/bin:/bin", root.display())) + .env("SEED_PSQL_WITNESS", &witness) + .output() + .expect("run local seed guard"); + let invoked = witness.exists(); + std::fs::remove_dir_all(root).expect("remove scratch fixture"); + assert!(!invoked, "a non-local target reached psql"); + assert!(!output.status.success()); + assert!(String::from_utf8_lossy(&output.stderr).contains("requires a local PostgreSQL target")); +} diff --git a/scripts/seed-admin-dashboard.sh b/scripts/seed-admin-dashboard.sh index 32320718fad..5372e1d014d 100755 --- a/scripts/seed-admin-dashboard.sh +++ b/scripts/seed-admin-dashboard.sh @@ -19,6 +19,15 @@ export PGUSER="${PGUSER:-buzz}" export PGPASSWORD="${PGPASSWORD:-buzz_dev}" export PGDATABASE="${PGDATABASE:-buzz}" +# This raw fixture writer is unarmed and must never target a live replica fence. +case "${PGHOST}" in + localhost|127.0.0.1|::1|/*) ;; + *) + echo "error: admin seed requires a local PostgreSQL target (PGHOST=${PGHOST})" >&2 + exit 1 + ;; +esac + if command -v psql >/dev/null 2>&1; then run_psql() { PGPASSWORD="${PGPASSWORD}" psql -h "${PGHOST}" -p "${PGPORT}" \ From 91b507a66cac3a7a8f5a4660f529ff1ca804f845 Mon Sep 17 00:00:00 2001 From: Elrond <28d6302a099e5225b02c4155ac4236e4912603df2ab08dbfc2f4fef08ce598c8@buzz.block.builderlab.xyz> Date: Wed, 14 Oct 2026 14:25:17 -0400 Subject: [PATCH 02/10] buzz-db: check the floor only on newly inserted event rows Review rework for the insert-time created_at floor (BUZZ-316 phase 1). - Ordinary and thread event writers now run the floor check after their ON CONFLICT DO NOTHING INSERT, only when a row was inserted and before follow-ups. Replaying an already-stored id stays Duplicate even when its created_at has aged below the floor, matching the trigger. - Source policy: UPDATE rule catches ONLY, aliases and row-constructor column lists; quoted identifiers are normalized; the floor policy scans every crate's src via the shared workspace walk. - NIP-43 membership insert writes a literal NULL channel_id, pinned positionally; the bind-based exemption is gone. - Canvas CAS fixtures take a per-test base instead of a process-wide OnceLock. - seed-admin-dashboard.sh also refuses PGHOSTADDR, PGSERVICE and multi-host PGHOST. Signed-off-by: Elrond <28d6302a099e5225b02c4155ac4236e4912603df2ab08dbfc2f4fef08ce598c8@buzz.block.builderlab.xyz> --- crates/buzz-db/src/runtime/replica_fence.rs | 14 +- crates/buzz-db/src/store/event.rs | 95 +++++--- .../src/store/event_floor_postgres_tests.rs | 70 ++++++ crates/buzz-db/src/store/relay_members.rs | 4 +- crates/buzz-db/tests/observability_source.rs | 209 ++++++++++++++---- scripts/seed-admin-dashboard.sh | 4 + 6 files changed, 309 insertions(+), 87 deletions(-) diff --git a/crates/buzz-db/src/runtime/replica_fence.rs b/crates/buzz-db/src/runtime/replica_fence.rs index 1f85f20c5f0..06401a5a102 100644 --- a/crates/buzz-db/src/runtime/replica_fence.rs +++ b/crates/buzz-db/src/runtime/replica_fence.rs @@ -4,10 +4,11 @@ //! contain is provably present on the replica. The proof rests on two parts: //! //! 1. **Insert-time floor**: supported channel-event writers check `created_at` -//! against database `clock_timestamp()` on their admitted transaction before -//! INSERT. This is the primary guard. Migration 0021's deferred trigger and -//! writer-pool `buzz.created_at_floor` GUC remain the mixed-version backstop -//! until phase 2; they also reject rows that age below the floor before COMMIT. +//! against database `clock_timestamp()` on their admitted transaction for +//! each new row. Duplicate INSERTs need no floor check. This is the primary +//! guard. Migration 0021's deferred trigger and the writer-pool +//! `buzz.created_at_floor` GUC remain the mixed-version backstop until phase 2; +//! they also reject rows that age below the floor before COMMIT. //! 2. **Ordered heartbeat handshake** (this module): on one pinned writer //! connection, one transaction first takes the exclusive replica-floor //! advisory lock (draining compliant shared-lock channel writers), then @@ -76,7 +77,10 @@ pub const CREATED_AT_FLOOR_SECS: i64 = 960; /// Enforce the channel-event floor on the admitted INSERT connection. /// -/// One preceding SELECT keeps the five existing INSERT shapes intact. Its +/// One SELECT keeps the five existing INSERT shapes intact. Ordinary/thread +/// writers check after INSERT only when it inserted a row, before follow-ups; +/// replacements check after replay exits, before superseding the old head. +/// The caller must propagate failure and roll back the write transaction. Its /// database clock is sampled after BEGIN: transactions open at the probe scan /// are bounded by xact_start; transactions starting later sample after S. No /// commit hook, candidate tracking or ordering against TTL refresh is needed. diff --git a/crates/buzz-db/src/store/event.rs b/crates/buzz-db/src/store/event.rs index fc4cd4571bc..1fe5a9b5df3 100644 --- a/crates/buzz-db/src/store/event.rs +++ b/crates/buzz-db/src/store/event.rs @@ -439,7 +439,8 @@ pub async fn insert_event( /// /// This is the transaction-composition seam for callers that must keep the /// event insert open while performing related work. The caller owns commit or -/// rollback. +/// rollback. On error, roll back or drop the transaction: the event row may +/// already have been inserted before the app floor check rejected it. pub async fn insert_event_in_transaction( tx: &mut AdmittedTx, event: &Event, @@ -480,7 +481,6 @@ async fn insert_event_on( let received_at = Utc::now(); let d_tag = extract_d_tag(event); let not_before = extract_not_before(event); - crate::replica_fence::guard_event_created_at_floor(tx, created_at, channel_id).await?; let result = sqlx::query( r#" INSERT INTO events (community_id, id, pubkey, created_at, kind, tags, content, sig, received_at, channel_id, d_tag, not_before) @@ -505,6 +505,7 @@ async fn insert_event_on( let was_inserted = result.rows_affected() > 0; if was_inserted { + crate::replica_fence::guard_event_created_at_floor(tx, created_at, channel_id).await?; crate::store::event_follow_up::after_admitted_insert( tx, id_bytes.as_slice(), @@ -1732,7 +1733,6 @@ pub(crate) async fn insert_event_with_thread_metadata_tx( let d_tag = extract_d_tag(event); let not_before = extract_not_before(event); - crate::replica_fence::guard_event_created_at_floor(tx, created_at, channel_id).await?; let result = sqlx::query( r#" INSERT INTO events (community_id, id, pubkey, created_at, kind, tags, content, sig, received_at, channel_id, d_tag, not_before) @@ -1758,6 +1758,7 @@ pub(crate) async fn insert_event_with_thread_metadata_tx( let was_inserted = result.rows_affected() > 0; if was_inserted { + crate::replica_fence::guard_event_created_at_floor(tx, created_at, channel_id).await?; crate::store::event_follow_up::after_admitted_insert( tx, id_bytes.as_slice(), @@ -3873,11 +3874,8 @@ mod postgres_tests { // ─── canvas CAS tests ───────────────────────────────────────────────────── - fn make_canvas_event_at(content: &str, created_at: u64) -> nostr::Event { - // Keep the CAS fixtures' logical seconds and ties, anchored at a fresh - // timestamp so they exercise the serving writer's floor admission. - static BASE: std::sync::OnceLock = std::sync::OnceLock::new(); - let base = BASE.get_or_init(|| nostr::Timestamp::now().as_secs() - 1000); + fn make_canvas_event_at(base: u64, content: &str, created_at: u64) -> nostr::Event { + // Each test owns its fresh base; logical seconds and ties stay explicit. make_event_at( buzz_core::kind::KIND_CANVAS as u16, content, @@ -3888,9 +3886,9 @@ mod postgres_tests { /// Build two canvas events sharing `created_at`, returned `(lower, higher)` /// by event id. Regenerates until the ids differ (always, since keys differ) /// so tests can assert deterministic head ordering on the id tiebreak. - fn same_second_ordered_pair(created_at: u64) -> (nostr::Event, nostr::Event) { - let a = make_canvas_event_at("# A", created_at); - let b = make_canvas_event_at("# B", created_at); + fn same_second_ordered_pair(base: u64, created_at: u64) -> (nostr::Event, nostr::Event) { + let a = make_canvas_event_at(base, "# A", created_at); + let b = make_canvas_event_at(base, "# B", created_at); if a.id.as_bytes() <= b.id.as_bytes() { (a, b) } else { @@ -3898,6 +3896,20 @@ mod postgres_tests { } } + #[test] + fn canvas_fixture_base_is_owned_by_each_test() { + let first_base = 1_700_000_000; + let second_base = first_base + crate::replica_fence::CREATED_AT_FLOOR_SECS as u64 * 2; + let first = make_canvas_event_at(first_base, "first test", 1000); + let second = make_canvas_event_at(second_base, "later test", 1000); + assert_eq!(first.created_at.as_secs(), first_base + 1000); + assert_eq!(second.created_at.as_secs(), second_base + 1000); + let (lower, higher) = same_second_ordered_pair(second_base, 1000); + assert_eq!(lower.created_at, higher.created_at); + assert_eq!(lower.created_at.as_secs(), second_base + 1000); + assert!(lower.id.as_bytes() < higher.id.as_bytes()); + } + #[tokio::test] async fn canvas_head_checked_rejects_non_canvas_kinds_before_opening_a_transaction() { // A lazy pool that is never connected: the kind check must reject @@ -3925,10 +3937,11 @@ mod postgres_tests { #[tokio::test] #[ignore = "requires Postgres"] async fn channel_head_checked_expect_no_head_creates_first_canvas() { + let canvas_base = nostr::Timestamp::now().as_secs() - 1000; let pool = setup_pool().await; let community = CommunityId::from_uuid(make_test_community(&pool).await); let channel = make_test_channel(&pool, community.as_uuid().to_owned(), None).await; - let event = make_canvas_event_at("# First", 1000); + let event = make_canvas_event_at(canvas_base, "# First", 1000); let (_, status) = insert_canvas_head_checked( &pool, @@ -3945,10 +3958,11 @@ mod postgres_tests { #[tokio::test] #[ignore = "requires Postgres"] async fn channel_head_checked_expect_no_head_rejects_when_head_exists() { + let canvas_base = nostr::Timestamp::now().as_secs() - 1000; let pool = setup_pool().await; let community = CommunityId::from_uuid(make_test_community(&pool).await); let channel = make_test_channel(&pool, community.as_uuid().to_owned(), None).await; - let first = make_canvas_event_at("# First", 1000); + let first = make_canvas_event_at(canvas_base, "# First", 1000); insert_canvas_head_checked( &pool, community, @@ -3959,7 +3973,7 @@ mod postgres_tests { .await .expect("first canvas"); - let second = make_canvas_event_at("# Racing create", 1001); + let second = make_canvas_event_at(canvas_base, "# Racing create", 1001); let (_, status) = insert_canvas_head_checked( &pool, community, @@ -3985,10 +3999,11 @@ mod postgres_tests { #[tokio::test] #[ignore = "requires Postgres"] async fn channel_head_checked_expected_head_matches_and_advances() { + let canvas_base = nostr::Timestamp::now().as_secs() - 1000; let pool = setup_pool().await; let community = CommunityId::from_uuid(make_test_community(&pool).await); let channel = make_test_channel(&pool, community.as_uuid().to_owned(), None).await; - let first = make_canvas_event_at("# First", 1000); + let first = make_canvas_event_at(canvas_base, "# First", 1000); insert_canvas_head_checked( &pool, community, @@ -3999,7 +4014,7 @@ mod postgres_tests { .await .expect("first canvas"); - let second = make_canvas_event_at("# Second", 1001); + let second = make_canvas_event_at(canvas_base, "# Second", 1001); let head = first.id.as_bytes(); let (_, status) = insert_canvas_head_checked( &pool, @@ -4016,10 +4031,11 @@ mod postgres_tests { #[tokio::test] #[ignore = "requires Postgres"] async fn channel_head_checked_expected_head_mismatch_rejects() { + let canvas_base = nostr::Timestamp::now().as_secs() - 1000; let pool = setup_pool().await; let community = CommunityId::from_uuid(make_test_community(&pool).await); let channel = make_test_channel(&pool, community.as_uuid().to_owned(), None).await; - let first = make_canvas_event_at("# First", 1000); + let first = make_canvas_event_at(canvas_base, "# First", 1000); insert_canvas_head_checked( &pool, community, @@ -4030,7 +4046,7 @@ mod postgres_tests { .await .expect("first canvas"); - let stale = make_canvas_event_at("# Stale edit", 1002); + let stale = make_canvas_event_at(canvas_base, "# Stale edit", 1002); let wrong_head = [0u8; 32]; let (_, status) = insert_canvas_head_checked( &pool, @@ -4047,10 +4063,11 @@ mod postgres_tests { #[tokio::test] #[ignore = "requires Postgres"] async fn channel_head_checked_expected_head_missing_rejects() { + let canvas_base = nostr::Timestamp::now().as_secs() - 1000; let pool = setup_pool().await; let community = CommunityId::from_uuid(make_test_community(&pool).await); let channel = make_test_channel(&pool, community.as_uuid().to_owned(), None).await; - let event = make_canvas_event_at("# Edit with no head", 1000); + let event = make_canvas_event_at(canvas_base, "# Edit with no head", 1000); let some_head = [1u8; 32]; let (_, status) = insert_canvas_head_checked( @@ -4068,10 +4085,11 @@ mod postgres_tests { #[tokio::test] #[ignore = "requires Postgres"] async fn channel_head_checked_replay_of_head_is_duplicate() { + let canvas_base = nostr::Timestamp::now().as_secs() - 1000; let pool = setup_pool().await; let community = CommunityId::from_uuid(make_test_community(&pool).await); let channel = make_test_channel(&pool, community.as_uuid().to_owned(), None).await; - let event = make_canvas_event_at("# First", 1000); + let event = make_canvas_event_at(canvas_base, "# First", 1000); insert_canvas_head_checked( &pool, community, @@ -4103,10 +4121,11 @@ mod postgres_tests { #[tokio::test] #[ignore = "requires Postgres"] async fn channel_head_checked_replay_skips_stale_precondition() { + let canvas_base = nostr::Timestamp::now().as_secs() - 1000; let pool = setup_pool().await; let community = CommunityId::from_uuid(make_test_community(&pool).await); let channel = make_test_channel(&pool, community.as_uuid().to_owned(), None).await; - let event = make_canvas_event_at("# First", 1000); + let event = make_canvas_event_at(canvas_base, "# First", 1000); insert_canvas_head_checked( &pool, community, @@ -4138,11 +4157,12 @@ mod postgres_tests { #[tokio::test] #[ignore = "requires Postgres"] async fn channel_head_checked_rejects_same_second_higher_id() { + let canvas_base = nostr::Timestamp::now().as_secs() - 1000; let pool = setup_pool().await; let community = CommunityId::from_uuid(make_test_community(&pool).await); let channel = make_test_channel(&pool, community.as_uuid().to_owned(), None).await; - let (lower, higher) = same_second_ordered_pair(1000); + let (lower, higher) = same_second_ordered_pair(canvas_base, 1000); insert_canvas_head_checked( &pool, community, @@ -4181,11 +4201,12 @@ mod postgres_tests { #[tokio::test] #[ignore = "requires Postgres"] async fn channel_head_checked_accepts_same_second_lower_id() { + let canvas_base = nostr::Timestamp::now().as_secs() - 1000; let pool = setup_pool().await; let community = CommunityId::from_uuid(make_test_community(&pool).await); let channel = make_test_channel(&pool, community.as_uuid().to_owned(), None).await; - let (lower, higher) = same_second_ordered_pair(1000); + let (lower, higher) = same_second_ordered_pair(canvas_base, 1000); // Seed the higher-id event as the head first. insert_canvas_head_checked( &pool, @@ -4215,10 +4236,11 @@ mod postgres_tests { #[tokio::test] #[ignore = "requires Postgres"] async fn channel_head_checked_rejects_behind_clock_writer() { + let canvas_base = nostr::Timestamp::now().as_secs() - 1000; let pool = setup_pool().await; let community = CommunityId::from_uuid(make_test_community(&pool).await); let channel = make_test_channel(&pool, community.as_uuid().to_owned(), None).await; - let head = make_canvas_event_at("# Head", 2000); + let head = make_canvas_event_at(canvas_base, "# Head", 2000); insert_canvas_head_checked( &pool, community, @@ -4230,7 +4252,7 @@ mod postgres_tests { .expect("head at t=2000"); // Writer's clock is behind — created_at earlier than head. - let behind = make_canvas_event_at("# Behind clock", 1100); + let behind = make_canvas_event_at(canvas_base, "# Behind clock", 1100); let (_, status) = insert_canvas_head_checked( &pool, community, @@ -4314,12 +4336,13 @@ mod postgres_tests { #[tokio::test] #[ignore = "requires Postgres"] async fn channel_head_checked_deleted_replay_returns_revision_mismatch() { + let canvas_base = nostr::Timestamp::now().as_secs() - 1000; let pool = setup_pool().await; let community = CommunityId::from_uuid(make_test_community(&pool).await); let channel = make_test_channel(&pool, community.as_uuid().to_owned(), None).await; // H: seed the initial head. - let head = make_canvas_event_at("# Head", 1000); + let head = make_canvas_event_at(canvas_base, "# Head", 1000); insert_canvas_head_checked( &pool, community, @@ -4332,7 +4355,7 @@ mod postgres_tests { let head_id = head.id.to_bytes().to_vec(); // A: insert a second revision. - let a = make_canvas_event_at("# A", 1001); + let a = make_canvas_event_at(canvas_base, "# A", 1001); insert_canvas_head_checked( &pool, community, @@ -4437,11 +4460,12 @@ mod postgres_tests { #[tokio::test] #[ignore = "requires Postgres"] async fn channel_head_checked_concurrent_authors_only_one_advances() { + let canvas_base = nostr::Timestamp::now().as_secs() - 1000; let pool = setup_pool().await; let community = CommunityId::from_uuid(make_test_community(&pool).await); let channel = make_test_channel(&pool, community.as_uuid().to_owned(), None).await; - let base = make_canvas_event_at("# Base", 1000); + let base = make_canvas_event_at(canvas_base, "# Base", 1000); insert_canvas_head_checked( &pool, community, @@ -4466,8 +4490,8 @@ mod postgres_tests { // Both edits assert `base` as their head and are timestamped strictly // ahead of it (writer discipline), so neither trips SupersedeFailed — // only the advisory-lock serialization decides the winner. - let a = make_canvas_event_at("# Author A", 1001); - let b = make_canvas_event_at("# Author B", 1002); + let a = make_canvas_event_at(canvas_base, "# Author A", 1001); + let b = make_canvas_event_at(canvas_base, "# Author B", 1002); let (pool_a, pool_b) = (pool.clone(), pool.clone()); let (id_a, id_b) = (base_id.clone(), base_id.clone()); @@ -4578,6 +4602,7 @@ mod postgres_tests { #[tokio::test] #[ignore = "requires Postgres"] async fn channel_head_checked_concurrent_first_create_only_one_wins() { + let canvas_base = nostr::Timestamp::now().as_secs() - 1000; let pool = setup_pool().await; let community = CommunityId::from_uuid(make_test_community(&pool).await); let channel = make_test_channel(&pool, community.as_uuid().to_owned(), None).await; @@ -4591,8 +4616,8 @@ mod postgres_tests { .await .expect("blocker acquires key"); - let a = make_canvas_event_at("# First A", 1000); - let b = make_canvas_event_at("# First B", 1001); + let a = make_canvas_event_at(canvas_base, "# First A", 1000); + let b = make_canvas_event_at(canvas_base, "# First B", 1001); let (pool_a, pool_b) = (pool.clone(), pool.clone()); let ta = tokio::spawn(async move { @@ -4699,6 +4724,7 @@ mod postgres_tests { #[tokio::test] #[ignore = "requires Postgres"] async fn db_untagged_canvas_append_serializes_on_advisory_key() { + let canvas_base = nostr::Timestamp::now().as_secs() - 1000; let pool = setup_pool().await; let db = crate::Db::from_pool(pool.clone()); let community = CommunityId::from_uuid(make_test_community(&pool).await); @@ -4712,7 +4738,7 @@ mod postgres_tests { .await .expect("holder acquires key"); - let event = make_canvas_event_at("# Untagged", 1000); + let event = make_canvas_event_at(canvas_base, "# Untagged", 1000); let write_task = tokio::spawn(async move { db.insert_event_with_thread_metadata(community, &event, Some(channel), None) .await @@ -4754,13 +4780,14 @@ mod postgres_tests { #[tokio::test] #[ignore = "requires Postgres"] async fn channel_head_canvas_deletion_serializes_on_advisory_key() { + let canvas_base = nostr::Timestamp::now().as_secs() - 1000; let pool = setup_pool().await; let community = CommunityId::from_uuid(make_test_community(&pool).await); let channel = make_test_channel(&pool, community.as_uuid().to_owned(), None).await; let lock_key = canvas_lock_key(community, channel); // Insert a canvas event to delete. - let event = make_canvas_event_at("# To delete", 1000); + let event = make_canvas_event_at(canvas_base, "# To delete", 1000); insert_canvas_head_checked( &pool, community, diff --git a/crates/buzz-db/src/store/event_floor_postgres_tests.rs b/crates/buzz-db/src/store/event_floor_postgres_tests.rs index 857d91ba686..741e3b7d52a 100644 --- a/crates/buzz-db/src/store/event_floor_postgres_tests.rs +++ b/crates/buzz-db/src/store/event_floor_postgres_tests.rs @@ -120,6 +120,62 @@ async fn insert_event_floor_rejects_old_and_accepts_fresh() { ); } +#[tokio::test] +#[ignore = "requires Postgres"] +async fn insert_event_floor_accepts_expired_duplicate() { + let (db, community, channel, keys) = fixture().await; + let old = signed( + &keys, + 9, + channel, + database_clock(&db).await - chrono::Duration::seconds(CREATED_AT_FLOOR_SECS + 60), + ); + crate::test_support::insert_historical_event(db.pool(), community, &old, channel).await; + let (_, inserted) = event::insert_event(db.pool(), community, &old, Some(channel)) + .await + .expect("expired duplicate is idempotent"); + assert!(!inserted); + assert_eq!(count(&db, community, &old).await, 1); +} + +#[tokio::test] +#[ignore = "requires Postgres"] +async fn thread_metadata_event_floor_accepts_expired_duplicate() { + let (db, community, channel, keys) = fixture().await; + let time = database_clock(&db).await - chrono::Duration::seconds(CREATED_AT_FLOOR_SECS + 60); + let old = signed(&keys, 9, channel, time); + crate::test_support::insert_historical_event(db.pool(), community, &old, channel).await; + let mut tx = begin(&db, community).await; + let meta = event::ThreadMetadataParams { + event_id: old.id.as_bytes(), + event_created_at: DateTime::from_timestamp(old.created_at.as_secs() as i64, 0) + .expect("timestamp"), + channel_id: channel, + parent_event_id: None, + parent_event_created_at: None, + root_event_id: None, + root_event_created_at: None, + depth: 0, + broadcast: true, + }; + let (_, inserted) = + event::insert_event_with_thread_metadata_tx(&mut tx, &old, Some(channel), Some(meta)) + .await + .expect("expired duplicate is idempotent"); + assert!(!inserted); + tx.commit().await.expect("commit duplicate result"); + assert_eq!(count(&db, community, &old).await, 1); + let metadata: i64 = sqlx::query_scalar( + "SELECT count(*) FROM thread_metadata WHERE community_id=$1 AND event_id=$2", + ) + .bind(community.as_uuid()) + .bind(old.id.as_bytes().as_slice()) + .fetch_one(db.pool()) + .await + .expect("metadata count"); + assert_eq!(metadata, 0, "a duplicate must not run thread follow-ups"); +} + #[tokio::test] #[ignore = "requires Postgres"] async fn thread_metadata_event_floor_rejects_old_and_accepts_fresh() { @@ -264,7 +320,21 @@ async fn cluster_global_event_floor_held_transaction_cannot_write_below_probe_wa channel, entry.fence_wall - chrono::Duration::seconds(1), ); + sqlx::query("SAVEPOINT rejected_floor_insert") + .execute(tx.conn()) + .await + .expect("isolate rejected insert"); rejected(event::insert_event_in_transaction(&mut tx, &below_wall, Some(channel)).await); + // The check runs after INSERT. Roll back that write before continuing in + // the same held transaction, just as a caller must on any insert failure. + sqlx::query("ROLLBACK TO SAVEPOINT rejected_floor_insert") + .execute(tx.conn()) + .await + .expect("roll back rejected row"); + sqlx::query("RELEASE SAVEPOINT rejected_floor_insert") + .execute(tx.conn()) + .await + .expect("release rejection savepoint"); // Sample just after the probe; a near-floor accepted insert is above its // wall even when this transaction commits after the heartbeat. let fresh = signed( diff --git a/crates/buzz-db/src/store/relay_members.rs b/crates/buzz-db/src/store/relay_members.rs index 6b21c7ce01a..61d7104da14 100644 --- a/crates/buzz-db/src/store/relay_members.rs +++ b/crates/buzz-db/src/store/relay_members.rs @@ -10,7 +10,6 @@ use buzz_core::StoredEvent; use buzz_datastore_tracing::datastore_span; use chrono::{DateTime, Utc}; use sqlx::{PgPool, Row as _}; -use uuid::Uuid; use crate::error::{DbError, Result}; use crate::{observability, replaceable, CommunityId, Db, RouteDecision, RoutePredicate}; @@ -1372,7 +1371,7 @@ impl Db { let insert_result = sqlx::query( "INSERT INTO events (community_id, id, pubkey, created_at, kind, tags, content, sig, received_at, channel_id, d_tag) \ - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) \ + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, NULL, $10) \ ON CONFLICT DO NOTHING", ) .bind(community_id.as_uuid()) @@ -1384,7 +1383,6 @@ impl Db { .bind(&event.content) .bind(sig_bytes.as_slice()) .bind(received_at) - .bind::>(None) .bind(d_tag.as_deref()) .execute(tx.conn()) .await?; diff --git a/crates/buzz-db/tests/observability_source.rs b/crates/buzz-db/tests/observability_source.rs index 40179a269ca..e7504e54222 100644 --- a/crates/buzz-db/tests/observability_source.rs +++ b/crates/buzz-db/tests/observability_source.rs @@ -1626,12 +1626,19 @@ fn masker_blanks_every_literal_form_and_keeps_code() { } } -/// Normalize SQL case, whitespace and Rust string continuations/escapes for +/// Normalize SQL case, quoted identifiers, whitespace and Rust string escapes for /// the event INSERT and UPDATE policies. fn normalized_sql_source(source: &str) -> String { let mut normalized = String::with_capacity(source.len()); let mut chars = source.chars().peekable(); while let Some(ch) = chars.next() { + if ch == '"' { + continue; + } + if ch == '\\' && chars.peek() == Some(&'"') { + chars.next(); + continue; + } if ch.is_whitespace() || ch == '\\' { if ch == '\\' && matches!(chars.peek(), Some('n' | 't' | 'r')) { chars.next(); @@ -1698,6 +1705,9 @@ pub(crate) async fn unhooked_writer(tx: &mut AdmittedTx) { .await .expect("write"); } +pub(crate) async fn quoted_writer(tx: &mut AdmittedTx) { + sqlx::query("INSERT INTO \"public\".\"events\" (id) VALUES ($1)"); +} pub(crate) async fn push_only_writer(tx: &mut AdmittedTx) { sqlx::query("INSERT INTO events \ (community_id, id) VALUES ($1, $2)") @@ -1862,6 +1872,7 @@ mod tests { event_insert_follow_up_violations(&strip_cfg_test_items(source)), [ "pub(crate) async fn unhooked_writer(tx: &mut AdmittedTx) {", + "pub(crate) async fn quoted_writer(tx: &mut AdmittedTx) {", "pub(crate) async fn push_only_writer(tx: &mut AdmittedTx) {", "pub(crate) async fn lowercase_writer(tx: &mut AdmittedTx) {", "pub(crate) async fn continued_writer(tx: &mut AdmittedTx) {", @@ -1882,8 +1893,7 @@ mod tests { ); } -#[test] -fn every_production_event_insert_runs_the_follow_up_hook() { +fn collect_production_rs_files() -> Vec { use std::path::{Path, PathBuf}; fn collect_rs_files(dir: &Path, out: &mut Vec) { @@ -1908,6 +1918,12 @@ fn every_production_event_insert_runs_the_follow_up_hook() { } } + files +} + +#[test] +fn every_production_event_insert_runs_the_follow_up_hook() { + let files = collect_production_rs_files(); let mut hooked_writers = 0usize; let mut violations = Vec::new(); for path in files { @@ -1951,34 +1967,60 @@ const EVENT_FLOOR_PROBE_EXCEPTIONS: [&str; 2] = [ fn is_channel_null_event_writer(function: &str) -> bool { let header = function_header(function); + let sql = normalized_sql_source(function).replace(' ', ""); if header.starts_with("pub async fn publish_nip43_membership_locked(") { - // Kind 13534: the channel bind is always NULL. - return function.contains(".bind::>(None)"); + // Kind 13534: channel_id is the tenth value, a literal NULL. + return sql.contains("values($1,$2,$3,$4,$5,$6,$7,$8,$9,null,$10)"); } if header.starts_with("pub async fn accept_lease_event(") { - // Kind 30350: channel_id is the literal NULL in VALUES. - return normalized_sql_source(function).contains("now(),null,$8)"); + // Kind 30350: channel_id is the literal NULL after received_at. + return sql.contains("now(),null,$8)"); } false } fn updates_event_floor_columns(function: &str) -> bool { let normalized = normalized_sql_source(function); - ["update events set ", "update public.events set "] - .iter() - .any(|marker| { - normalized.match_indices(marker).any(|(index, marker)| { - let assignments = normalized[index + marker.len()..] - .split(" where ") - .next() - .expect("split has first part"); - assignments.split(',').any(|assignment| { - assignment.split_once('=').is_some_and(|(column, _)| { - matches!(column.trim(), "created_at" | "channel_id") - }) + normalized.match_indices("update ").any(|(index, marker)| { + let Some((target, assignments)) = normalized[index + marker.len()..].split_once(" set ") + else { + return false; + }; + let mut target = target.split_whitespace(); + let mut table = target.next(); + if table == Some("only") { + table = target.next(); + } + if !matches!(table, Some("events" | "public.events")) { + return false; + } + // The target may have an alias, with or without AS. + let alias = target.collect::>(); + if !matches!(alias.as_slice(), [] | [_] | ["as", _]) { + return false; + } + let assignments = assignments.split(" where ").next().expect("first part"); + let mut depth = 0usize; + assignments + .split(|ch| { + match ch { + '(' => depth += 1, + ')' => depth = depth.saturating_sub(1), + _ => {} + } + ch == ',' && depth == 0 + }) + .any(|assignment| { + assignment.split_once('=').is_some_and(|(columns, _)| { + columns + .trim() + .trim_start_matches('(') + .trim_end_matches(')') + .split(',') + .any(|column| matches!(column.trim(), "created_at" | "channel_id")) }) }) - }) + }) } fn event_floor_policy_violations(production: &str) -> Vec { @@ -2019,6 +2061,36 @@ async fn rewrite_time(tx: &mut AdmittedTx) { async fn attach_channel(tx: &mut AdmittedTx) { sqlx::query("uPdAtE public.events\nSET channel_id=$1 WHERE id=$2"); } +async fn alias_time(tx: &mut AdmittedTx) { + sqlx::query("UPDATE events e SET created_at = $1 WHERE e.id = $2"); +} +async fn as_alias_time(tx: &mut AdmittedTx) { + sqlx::query("UPDATE events AS e SET created_at = $1 WHERE e.id = $2"); +} +async fn only_channel(tx: &mut AdmittedTx) { + sqlx::query("UPDATE ONLY events SET channel_id = $1 WHERE id = $2"); +} +async fn only_public_alias(tx: &mut AdmittedTx) { + sqlx::query("UPDATE ONLY public.events AS e SET channel_id = $1 WHERE e.id = $2"); +} +async fn row_time(tx: &mut AdmittedTx) { + sqlx::query("UPDATE events SET (created_at, kind) = ($1, 1) WHERE id = $2"); +} +async fn row_channel(tx: &mut AdmittedTx) { + sqlx::query("UPDATE events SET (kind, channel_id) = (1, $1) WHERE id = $2"); +} +async fn quoted_insert(tx: &mut AdmittedTx) { + sqlx::query("INSERT INTO \"events\" (id) VALUES ($1)"); +} +async fn quoted_schema_insert(tx: &mut AdmittedTx) { + sqlx::query("INSERT INTO \"public\".\"events\" (id) VALUES ($1)"); +} +async fn quoted_update(tx: &mut AdmittedTx) { + sqlx::query("UPDATE \"events\" SET created_at = $1 WHERE id = $2"); +} +async fn quoted_schema_update(tx: &mut AdmittedTx) { + sqlx::query("UPDATE \"public\".\"events\" SET channel_id = $1 WHERE id = $2"); +} async fn delete_only(tx: &mut AdmittedTx) { sqlx::query("UPDATE events SET deleted_at=now() WHERE channel_id=$1 AND created_at<$2"); } @@ -2029,26 +2101,48 @@ async fn delete_only(tx: &mut AdmittedTx) { "async fn missing_guard(tx: &mut AdmittedTx) {", "async fn rewrite_time(tx: &mut AdmittedTx) {", "async fn attach_channel(tx: &mut AdmittedTx) {", + "async fn alias_time(tx: &mut AdmittedTx) {", + "async fn as_alias_time(tx: &mut AdmittedTx) {", + "async fn only_channel(tx: &mut AdmittedTx) {", + "async fn only_public_alias(tx: &mut AdmittedTx) {", + "async fn row_time(tx: &mut AdmittedTx) {", + "async fn row_channel(tx: &mut AdmittedTx) {", + "async fn quoted_insert(tx: &mut AdmittedTx) {", + "async fn quoted_schema_insert(tx: &mut AdmittedTx) {", + "async fn quoted_update(tx: &mut AdmittedTx) {", + "async fn quoted_schema_update(tx: &mut AdmittedTx) {", ] ); } +#[test] +fn event_floor_policy_requires_nip43_channel_null_at_its_insert_position() { + let literal_null = r#" +pub async fn publish_nip43_membership_locked(tx: &mut AdmittedTx) { + sqlx::query("INSERT INTO events (community_id,id,pubkey,kind,content,tags,sig,created_at,received_at,channel_id,d_tag) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,NULL,$10)"); +} +"#; + assert!(event_floor_policy_violations(literal_null).is_empty()); + let unrelated_null_bind = literal_null.replace("$9,NULL,$10", "$9,$10,$11").replace( + "sqlx::query(", + "let unrelated = query.bind::>(None); sqlx::query(", + ); + assert_eq!( + event_floor_policy_violations(&unrelated_null_bind).len(), + 1, + "an unrelated NULL bind must not exempt a channel-bearing INSERT" + ); +} + #[test] fn every_production_channel_event_insert_checks_the_floor_and_never_rewrites_it() { - fn collect(dir: &std::path::Path, files: &mut Vec) { - for entry in std::fs::read_dir(dir).expect("read source directory") { - let path = entry.expect("source entry").path(); - if path.is_dir() { - collect(&path, files); - } else if should_scan_guarded_write_source_file(&path) { - files.push(path); - } - } - } - let mut files = Vec::new(); - collect( - &std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("src"), - &mut files, + let files = collect_production_rs_files(); + let crates_root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .expect("crates root"); + assert!( + files.contains(&crates_root.join("buzz-relay/src/state.rs")), + "floor policy must scan the workspace, including relay writers" ); let mut guarded_writers = 0; let mut violations = Vec::new(); @@ -2113,16 +2207,41 @@ fn admin_seed_refuses_remote_database_before_invoking_psql() { std::fs::set_permissions(&psql, std::fs::Permissions::from_mode(0o755)) .expect("executable stub"); let witness = root.join("psql-was-invoked"); - let output = std::process::Command::new("/bin/bash") - .arg(script) - .env("PGHOST", "shared-db.example.invalid") - .env("PATH", format!("{}:/usr/bin:/bin", root.display())) - .env("SEED_PSQL_WITNESS", &witness) - .output() - .expect("run local seed guard"); - let invoked = witness.exists(); + let cases = [ + ("remote host", "shared-db.example.invalid", "", ""), + ("address override", "localhost", "192.0.2.1", ""), + ("service override", "localhost", "", "production"), + ( + "socket-prefixed host list", + "/tmp,shared-db.example.invalid", + "", + "", + ), + ("host list", "localhost,shared-db.example.invalid", "", ""), + ]; + let results = cases.map(|(name, host, address, service)| { + let output = std::process::Command::new("/bin/bash") + .arg(&script) + .env("PGHOST", host) + .env("PGHOSTADDR", address) + .env("PGSERVICE", service) + .env("PATH", format!("{}:/usr/bin:/bin", root.display())) + .env("SEED_PSQL_WITNESS", &witness) + .output() + .expect("run local seed guard"); + let invoked = witness.exists(); + if invoked { + std::fs::remove_file(&witness).expect("clear witness"); + } + (name, output, invoked) + }); std::fs::remove_dir_all(root).expect("remove scratch fixture"); - assert!(!invoked, "a non-local target reached psql"); - assert!(!output.status.success()); - assert!(String::from_utf8_lossy(&output.stderr).contains("requires a local PostgreSQL target")); + for (name, output, invoked) in results { + assert!(!invoked, "{name} reached psql"); + assert!(!output.status.success(), "{name} accepted"); + assert!( + String::from_utf8_lossy(&output.stderr).contains("requires a local PostgreSQL target"), + "{name}: missing refusal" + ); + } } diff --git a/scripts/seed-admin-dashboard.sh b/scripts/seed-admin-dashboard.sh index 5372e1d014d..e0155604810 100755 --- a/scripts/seed-admin-dashboard.sh +++ b/scripts/seed-admin-dashboard.sh @@ -20,6 +20,10 @@ export PGPASSWORD="${PGPASSWORD:-buzz_dev}" export PGDATABASE="${PGDATABASE:-buzz}" # This raw fixture writer is unarmed and must never target a live replica fence. +if [[ -n "${PGHOSTADDR:-}" || -n "${PGSERVICE:-}" || "${PGHOST}" == *,* ]]; then + echo "error: admin seed requires a local PostgreSQL target without PGHOSTADDR, PGSERVICE or multiple hosts" >&2 + exit 1 +fi case "${PGHOST}" in localhost|127.0.0.1|::1|/*) ;; *) From 93ccf41255bb0f65082ad483bce99b02bf2c58ea Mon Sep 17 00:00:00 2001 From: Elrond <28d6302a099e5225b02c4155ac4236e4912603df2ab08dbfc2f4fef08ce598c8@buzz.block.builderlab.xyz> Date: Fri, 16 Oct 2026 09:53:30 -0400 Subject: [PATCH 03/10] fix(seed): refuse connection-string PGDATABASE in local-only admin seed guard psql -d treats URI and key=value values as full connection strings that override -h, so PGDATABASE could route the unarmed fixture writer to a shared database. Accept only a plain database name before either psql path, and extend the refusal test with URI and key/value cases (host=, hostaddr=, service=) plus a plain-name control that must reach psql. Also note in replica_fence that prepared-transaction coverage comes from the retained deferred trigger, not the insert-time app guard. Signed-off-by: Elrond <28d6302a099e5225b02c4155ac4236e4912603df2ab08dbfc2f4fef08ce598c8@buzz.block.builderlab.xyz> --- crates/buzz-db/src/runtime/replica_fence.rs | 5 +- crates/buzz-db/tests/observability_source.rs | 77 ++++++++++++++++---- scripts/seed-admin-dashboard.sh | 6 ++ 3 files changed, 74 insertions(+), 14 deletions(-) diff --git a/crates/buzz-db/src/runtime/replica_fence.rs b/crates/buzz-db/src/runtime/replica_fence.rs index 06401a5a102..b07a2339903 100644 --- a/crates/buzz-db/src/runtime/replica_fence.rs +++ b/crates/buzz-db/src/runtime/replica_fence.rs @@ -639,7 +639,10 @@ async fn sample_writer(writer: &PgPool) -> Result { // prepared they have left `pg_stat_activity` but can still commit // after the token. Their deferred floor guard already ran at PREPARE, // so `pg_prepared_xacts.prepared` bounds their rows exactly like - // `xact_start`; fold it into the same minimum. + // `xact_start`; fold it into the same minimum. Only the retained + // deferred trigger provides that PREPARE-time check: the insert-time + // app guard runs earlier, and no app writer issues PREPARE TRANSACTION. + // Dropping the trigger must keep this case covered. let row = sqlx::query( r#" SELECT diff --git a/crates/buzz-db/tests/observability_source.rs b/crates/buzz-db/tests/observability_source.rs index e7504e54222..913ae9cad62 100644 --- a/crates/buzz-db/tests/observability_source.rs +++ b/crates/buzz-db/tests/observability_source.rs @@ -2207,24 +2207,13 @@ fn admin_seed_refuses_remote_database_before_invoking_psql() { std::fs::set_permissions(&psql, std::fs::Permissions::from_mode(0o755)) .expect("executable stub"); let witness = root.join("psql-was-invoked"); - let cases = [ - ("remote host", "shared-db.example.invalid", "", ""), - ("address override", "localhost", "192.0.2.1", ""), - ("service override", "localhost", "", "production"), - ( - "socket-prefixed host list", - "/tmp,shared-db.example.invalid", - "", - "", - ), - ("host list", "localhost,shared-db.example.invalid", "", ""), - ]; - let results = cases.map(|(name, host, address, service)| { + let run = |host: &str, address: &str, service: &str, database: &str| { let output = std::process::Command::new("/bin/bash") .arg(&script) .env("PGHOST", host) .env("PGHOSTADDR", address) .env("PGSERVICE", service) + .env("PGDATABASE", database) .env("PATH", format!("{}:/usr/bin:/bin", root.display())) .env("SEED_PSQL_WITNESS", &witness) .output() @@ -2233,9 +2222,71 @@ fn admin_seed_refuses_remote_database_before_invoking_psql() { if invoked { std::fs::remove_file(&witness).expect("clear witness"); } + (output, invoked) + }; + // Control: a local host with a plain database name reaches psql, so the + // refusals below come from the guard rather than an unrunnable fixture. + let (_, control_invoked) = run("localhost", "", "", "buzz"); + let cases = [ + ("remote host", "shared-db.example.invalid", "", "", ""), + ("address override", "localhost", "192.0.2.1", "", ""), + ("service override", "localhost", "", "production", ""), + ( + "socket-prefixed host list", + "/tmp,shared-db.example.invalid", + "", + "", + "", + ), + ( + "host list", + "localhost,shared-db.example.invalid", + "", + "", + "", + ), + ( + "database URI", + "localhost", + "", + "", + "postgresql://buzz@shared-db.example.invalid/buzz", + ), + ( + "short database URI", + "localhost", + "", + "", + "postgres://shared-db.example.invalid/buzz", + ), + ( + "database key/value host", + "localhost", + "", + "", + "host=shared-db.example.invalid dbname=buzz", + ), + ( + "database key/value hostaddr", + "localhost", + "", + "", + "hostaddr=192.0.2.1 dbname=buzz", + ), + ( + "database key/value service", + "localhost", + "", + "", + "service=production", + ), + ]; + let results = cases.map(|(name, host, address, service, database)| { + let (output, invoked) = run(host, address, service, database); (name, output, invoked) }); std::fs::remove_dir_all(root).expect("remove scratch fixture"); + assert!(control_invoked, "plain local target did not reach psql"); for (name, output, invoked) in results { assert!(!invoked, "{name} reached psql"); assert!(!output.status.success(), "{name} accepted"); diff --git a/scripts/seed-admin-dashboard.sh b/scripts/seed-admin-dashboard.sh index e0155604810..38425b4ff67 100755 --- a/scripts/seed-admin-dashboard.sh +++ b/scripts/seed-admin-dashboard.sh @@ -31,6 +31,12 @@ case "${PGHOST}" in exit 1 ;; esac +# psql -d treats URI and key=value values as full connection strings that +# override -h, so accept only a plain database name. +if [[ ! "${PGDATABASE}" =~ ^[A-Za-z0-9_-]+$ ]]; then + echo "error: admin seed requires a local PostgreSQL target with a plain PGDATABASE name" >&2 + exit 1 +fi if command -v psql >/dev/null 2>&1; then run_psql() { From 06a84e119dd287da9505fb4b1f2c89709247dfcf Mon Sep 17 00:00:00 2001 From: Elrond <28d6302a099e5225b02c4155ac4236e4912603df2ab08dbfc2f4fef08ce598c8@buzz.block.builderlab.xyz> Date: Fri, 16 Oct 2026 10:02:38 -0400 Subject: [PATCH 04/10] fix(seed): run psql with -X so psqlrc cannot retarget after the local guard psql -c no longer implies -X, so ~/.psqlrc or PSQLRC (exportable from .env) ran after the guard and could \connect to another host. Pass -X on both the local and Docker psql paths and assert the control invocation carries it. Signed-off-by: Elrond <28d6302a099e5225b02c4155ac4236e4912603df2ab08dbfc2f4fef08ce598c8@buzz.block.builderlab.xyz> --- crates/buzz-db/tests/observability_source.rs | 25 +++++++++++++------- scripts/seed-admin-dashboard.sh | 5 ++-- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/crates/buzz-db/tests/observability_source.rs b/crates/buzz-db/tests/observability_source.rs index 913ae9cad62..60632e75dfb 100644 --- a/crates/buzz-db/tests/observability_source.rs +++ b/crates/buzz-db/tests/observability_source.rs @@ -2203,7 +2203,11 @@ fn admin_seed_refuses_remote_database_before_invoking_psql() { ) .expect("copy script without developer .env"); let psql = root.join("psql"); - std::fs::write(&psql, "#!/bin/sh\ntouch \"$SEED_PSQL_WITNESS\"\nexit 1\n").expect("stub psql"); + std::fs::write( + &psql, + "#!/bin/sh\nprintf '%s\\n' \"$@\" > \"$SEED_PSQL_WITNESS\"\nexit 1\n", + ) + .expect("stub psql"); std::fs::set_permissions(&psql, std::fs::Permissions::from_mode(0o755)) .expect("executable stub"); let witness = root.join("psql-was-invoked"); @@ -2218,15 +2222,16 @@ fn admin_seed_refuses_remote_database_before_invoking_psql() { .env("SEED_PSQL_WITNESS", &witness) .output() .expect("run local seed guard"); - let invoked = witness.exists(); - if invoked { + let args = std::fs::read_to_string(&witness).ok(); + if args.is_some() { std::fs::remove_file(&witness).expect("clear witness"); } - (output, invoked) + (output, args) }; // Control: a local host with a plain database name reaches psql, so the // refusals below come from the guard rather than an unrunnable fixture. - let (_, control_invoked) = run("localhost", "", "", "buzz"); + // It must also pass -X so a psqlrc cannot \connect after the guard. + let (_, control_args) = run("localhost", "", "", "buzz"); let cases = [ ("remote host", "shared-db.example.invalid", "", "", ""), ("address override", "localhost", "192.0.2.1", "", ""), @@ -2282,11 +2287,15 @@ fn admin_seed_refuses_remote_database_before_invoking_psql() { ), ]; let results = cases.map(|(name, host, address, service, database)| { - let (output, invoked) = run(host, address, service, database); - (name, output, invoked) + let (output, args) = run(host, address, service, database); + (name, output, args.is_some()) }); std::fs::remove_dir_all(root).expect("remove scratch fixture"); - assert!(control_invoked, "plain local target did not reach psql"); + let control_args = control_args.expect("plain local target did not reach psql"); + assert!( + control_args.lines().any(|arg| arg == "-X"), + "psql invoked without -X: {control_args:?}" + ); for (name, output, invoked) in results { assert!(!invoked, "{name} reached psql"); assert!(!output.status.success(), "{name} accepted"); diff --git a/scripts/seed-admin-dashboard.sh b/scripts/seed-admin-dashboard.sh index 38425b4ff67..96fd2fb0915 100755 --- a/scripts/seed-admin-dashboard.sh +++ b/scripts/seed-admin-dashboard.sh @@ -38,15 +38,16 @@ if [[ ! "${PGDATABASE}" =~ ^[A-Za-z0-9_-]+$ ]]; then exit 1 fi +# -X skips psqlrc, which runs after this guard and could \connect elsewhere. if command -v psql >/dev/null 2>&1; then run_psql() { - PGPASSWORD="${PGPASSWORD}" psql -h "${PGHOST}" -p "${PGPORT}" \ + PGPASSWORD="${PGPASSWORD}" psql -X -h "${PGHOST}" -p "${PGPORT}" \ -U "${PGUSER}" -d "${PGDATABASE}" "$@" } elif docker exec buzz-postgres psql --version >/dev/null 2>&1; then run_psql() { docker exec -i -e PGPASSWORD="${PGPASSWORD}" buzz-postgres \ - psql -U "${PGUSER}" -d "${PGDATABASE}" "$@" + psql -X -U "${PGUSER}" -d "${PGDATABASE}" "$@" } else echo "error: neither psql nor buzz-postgres docker psql is available" >&2 From b5fd799b9a2deb11ace1b152ab6679bb984f4a60 Mon Sep 17 00:00:00 2001 From: Elrond <28d6302a099e5225b02c4155ac4236e4912603df2ab08dbfc2f4fef08ce598c8@buzz.block.builderlab.xyz> Date: Wed, 14 Oct 2026 14:10:13 -0400 Subject: [PATCH 05/10] buzz-db: drop the events_created_at_floor trigger (phase 2) Phase 2 of BUZZ-316. The insert-time application floor check (phase 1) is now the only created_at floor guard, so drop the deferred constraint trigger, its function, the writer-pool buzz.created_at_floor GUC, and the startup catalog/behavior probes that required them. Migration 0062 drops the trigger on events and every partition plus the function, all IF EXISTS. Desired-state schema, pgschema reconciliation and migration asserts drop the same objects. spawn_fence_probe becomes synchronous; the ordered heartbeat probe still fails closed. DESTRUCTIVE: deploy only after phase 1 runs on every writer, with no rollback expected, out-of-band writers stopped, and the replica breaker held closed through the drop until replay catches up. Signed-off-by: Elrond <28d6302a099e5225b02c4155ac4236e4912603df2ab08dbfc2f4fef08ce598c8@buzz.block.builderlab.xyz> --- ARCHITECTURE.md | 31 +- crates/buzz-db/src/lib.rs | 6 +- .../src/runtime/connection_observability.rs | 39 +- crates/buzz-db/src/runtime/migration.rs | 17 +- crates/buzz-db/src/runtime/mod.rs | 84 +--- crates/buzz-db/src/runtime/observability.rs | 11 +- crates/buzz-db/src/runtime/replica_fence.rs | 236 +--------- crates/buzz-db/src/runtime/tests.rs | 415 +----------------- .../src/runtime/tests/floor_retirement.rs | 161 +++++++ crates/buzz-db/src/store/event.rs | 2 +- .../src/store/event_floor_postgres_tests.rs | 13 +- crates/buzz-db/tests/observability_source.rs | 32 +- crates/buzz-relay/src/api/admin/mod.rs | 71 +-- .../bridge/thread_window/postgres_tests.rs | 2 +- crates/buzz-relay/src/main.rs | 25 +- crates/buzz-relay/src/startup_steps.rs | 2 +- deploy/charts/buzz/README.md | 6 +- migrations/0062_retire_created_at_floor.sql | 26 ++ .../events_created_at_floor_guard.sql | 29 -- schema/schema.sql | 1 - schema/triggers/events.sql | 11 - scripts/reconcile-schema-after-pgschema.sql | 8 - 22 files changed, 313 insertions(+), 915 deletions(-) create mode 100644 crates/buzz-db/src/runtime/tests/floor_retirement.rs create mode 100644 migrations/0062_retire_created_at_floor.sql delete mode 100644 schema/functions/events_created_at_floor_guard.sql diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index e0875618dd0..101542f0eaf 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -503,24 +503,29 @@ shape is an explicit transition, not a flag day: transitions take the matching exclusive lock. A supported serving transaction never spans communities; unmigrated paths still rely on trigger/function backstops. -- **Replica floor (transition foundation):** runtime now provides a shared - replica-floor lock helper for compliant channel-event writers, and the writer - probe handshake takes the exclusive counterpart before sampling `S`, scanning - activity, and committing the heartbeat token last. `probe_once` remains the - sole token/fence-wall publication path; there is no separately persisted active - floor cutoff. Commit-time trigger+GUC enforcement remains authoritative for all - paths. -- **Dual enforcement (current):** application-owned lock+precheck paths run in - front of the existing trigger/function enforcement; commit-time trigger checks - remain authoritative during this phase. Removing those backstops is a separate - gated migration, not part of the lock foundation. +- **Replica floor:** supported channel-event insertions check `created_at` + against `clock_timestamp()` on their admitted transaction using the shared + 960-second floor. The ordered heartbeat handshake and activity scan retain + the same fence-wall calculation. Migration 0062 retires only the deferred + floor trigger/function; writer connections no longer arm its GUC and probe + startup no longer requires those objects. +- **Floor retirement deploy gate:** phase 1 must be deployed to every writer + pod/job, with no rollback expected, before applying the destructive contract. + Drain old connections/in-flight transactions, stop out-of-band writers, and + hold the replica breaker closed through the drop until replay catches up. + Already-running old pods do not reverify the trigger on each heartbeat. +- **Dual enforcement (community fence transition):** application-owned + lock+precheck paths run in front of the existing community trigger/function + enforcement. Those backstops remain in place until their own separately gated + retirement. - **Role separation:** relay/runtime code owns admission and lock protocols; operator maintenance/backfill workflows must either use those protocols or run under explicit reviewed procedures that keep routing fences closed. - **Unsupported shape:** direct owner SQL that bypasses supported transaction and lock protocols is not a supported write path. -- **Fleet gates and reconciliation:** startup/fence probes verify guard catalog + - behavior, pgschema bootstraps run reconciliation, and lock/pool metrics remain +- **Fleet gates and reconciliation:** retained guards keep their catalog and + behavior checks; heartbeat/epoch and activity checks still fail closed. + pgschema bootstraps run reconciliation, and lock/pool metrics remain the operational evidence path for rollout safety. **Does NOT:** cache queries, implement connection pooling logic (delegated to sqlx), or make network calls outside Postgres. diff --git a/crates/buzz-db/src/lib.rs b/crates/buzz-db/src/lib.rs index 2f306c35d08..3d091bc4e80 100644 --- a/crates/buzz-db/src/lib.rs +++ b/crates/buzz-db/src/lib.rs @@ -45,15 +45,15 @@ pub const DB_POOL_ACQUIRE_RAW_SERIES_PER_POD: usize = runtime::observability::POOL_ACQUIRE_RAW_SERIES_PER_POD; /// Valid database connection role/step pairs with a start counter. -pub const DB_CONNECTION_STARTED_STEPS: [(DbPoolRole, DbConnectionStep); 4] = +pub const DB_CONNECTION_STARTED_STEPS: [(DbPoolRole, DbConnectionStep); 3] = runtime::CONNECTION_STARTED_STEPS; /// Valid database connection role/step pairs with a duration histogram. -pub const DB_CONNECTION_DURATION_STEPS: [(DbPoolRole, DbConnectionStep); 4] = +pub const DB_CONNECTION_DURATION_STEPS: [(DbPoolRole, DbConnectionStep); 3] = runtime::CONNECTION_DURATION_STEPS; /// Valid database connection role/step/outcome terminal combinations. -pub const DB_CONNECTION_TERMINALS: [(DbPoolRole, DbConnectionStep, DbConnectionOutcome); 15] = +pub const DB_CONNECTION_TERMINALS: [(DbPoolRole, DbConnectionStep, DbConnectionOutcome); 12] = runtime::CONNECTION_TERMINALS; /// Raw Prometheus series ceiling per pod for connection-step telemetry. diff --git a/crates/buzz-db/src/runtime/connection_observability.rs b/crates/buzz-db/src/runtime/connection_observability.rs index a1ab04022fd..710d3e04a50 100644 --- a/crates/buzz-db/src/runtime/connection_observability.rs +++ b/crates/buzz-db/src/runtime/connection_observability.rs @@ -17,8 +17,6 @@ pub enum DbConnectionStep { WriterPool, /// A physical connection has completed DNS/network/TLS/authentication. PhysicalConnect, - /// Install the created-at replica-fence floor. - CreatedAtFloor, /// Install lock, idle-transaction, and statement timeouts. SessionTimeouts, /// Verify READ COMMITTED transaction isolation. @@ -29,10 +27,9 @@ pub enum DbConnectionStep { impl DbConnectionStep { /// Complete metric-label vocabulary. - pub const ALL: [Self; 6] = [ + pub const ALL: [Self; 5] = [ Self::WriterPool, Self::PhysicalConnect, - Self::CreatedAtFloor, Self::SessionTimeouts, Self::Isolation, Self::Ready, @@ -43,7 +40,6 @@ impl DbConnectionStep { match self { Self::WriterPool => "writer_pool", Self::PhysicalConnect => "physical_connect", - Self::CreatedAtFloor => "created_at_floor", Self::SessionTimeouts => "session_timeouts", Self::Isolation => "isolation", Self::Ready => "ready", @@ -77,18 +73,17 @@ impl DbConnectionOutcome { } /// Valid role/step pairs with an explicit start counter. -pub const CONNECTION_STARTED_STEPS: [(DbPoolRole, DbConnectionStep); 4] = [ +pub const CONNECTION_STARTED_STEPS: [(DbPoolRole, DbConnectionStep); 3] = [ (DbPoolRole::Writer, DbConnectionStep::WriterPool), - (DbPoolRole::Writer, DbConnectionStep::CreatedAtFloor), (DbPoolRole::Writer, DbConnectionStep::SessionTimeouts), (DbPoolRole::Writer, DbConnectionStep::Isolation), ]; /// Valid role/step pairs with a duration histogram. -pub const CONNECTION_DURATION_STEPS: [(DbPoolRole, DbConnectionStep); 4] = CONNECTION_STARTED_STEPS; +pub const CONNECTION_DURATION_STEPS: [(DbPoolRole, DbConnectionStep); 3] = CONNECTION_STARTED_STEPS; /// Valid role/step/outcome terminal combinations. -pub const CONNECTION_TERMINALS: [(DbPoolRole, DbConnectionStep, DbConnectionOutcome); 15] = [ +pub const CONNECTION_TERMINALS: [(DbPoolRole, DbConnectionStep, DbConnectionOutcome); 12] = [ ( DbPoolRole::Writer, DbConnectionStep::WriterPool, @@ -114,21 +109,6 @@ pub const CONNECTION_TERMINALS: [(DbPoolRole, DbConnectionStep, DbConnectionOutc DbConnectionStep::PhysicalConnect, DbConnectionOutcome::Succeeded, ), - ( - DbPoolRole::Writer, - DbConnectionStep::CreatedAtFloor, - DbConnectionOutcome::Succeeded, - ), - ( - DbPoolRole::Writer, - DbConnectionStep::CreatedAtFloor, - DbConnectionOutcome::Failed, - ), - ( - DbPoolRole::Writer, - DbConnectionStep::CreatedAtFloor, - DbConnectionOutcome::Cancelled, - ), ( DbPoolRole::Writer, DbConnectionStep::SessionTimeouts, @@ -166,8 +146,8 @@ pub const CONNECTION_TERMINALS: [(DbPoolRole, DbConnectionStep, DbConnectionOutc ), ]; -/// Four start counters + four 13-series histograms + fifteen terminal counters. -pub const CONNECTION_RAW_SERIES_PER_POD: usize = 4 + (4 * 13) + 15; +/// Three start counters + three 13-series histograms + twelve terminal counters. +pub const CONNECTION_RAW_SERIES_PER_POD: usize = 3 + (3 * 13) + 12; pub(crate) struct DbConnectionStepAttempt { pool_role: DbPoolRole, @@ -282,13 +262,12 @@ mod tests { [ "writer_pool", "physical_connect", - "created_at_floor", "session_timeouts", "isolation", "ready", ] ); - assert_eq!(CONNECTION_RAW_SERIES_PER_POD, 71); + assert_eq!(CONNECTION_RAW_SERIES_PER_POD, 54); } #[test] @@ -297,7 +276,7 @@ mod tests { let snapshotter = recorder.snapshotter(); let _guard = metrics::set_default_local_recorder(&recorder); let attempt = - DbConnectionStepAttempt::start(DbPoolRole::Writer, DbConnectionStep::CreatedAtFloor); + DbConnectionStepAttempt::start(DbPoolRole::Writer, DbConnectionStep::SessionTimeouts); drop(attempt); let metrics = snapshotter.snapshot().into_vec(); @@ -309,7 +288,7 @@ mod tests { .map(|label| (label.key(), label.value())) .collect::>(); assert_eq!(labels.get("pool_role"), Some(&"writer")); - assert_eq!(labels.get("step"), Some(&"created_at_floor")); + assert_eq!(labels.get("step"), Some(&"session_timeouts")); assert!(!labels.contains_key("reason")); assert!(!labels.contains_key("connection_ordinal")); match value { diff --git a/crates/buzz-db/src/runtime/migration.rs b/crates/buzz-db/src/runtime/migration.rs index b6ead54e06f..a0759d99c7d 100644 --- a/crates/buzz-db/src/runtime/migration.rs +++ b/crates/buzz-db/src/runtime/migration.rs @@ -50,14 +50,6 @@ pub(crate) async fn run_migrations_through(pool: &PgPool, target: i64) -> Result async fn run_migrations_locked(conn: &mut PgConnection) -> Result<()> { reject_legacy_nip_rs_cardinality_ambiguity(conn).await?; MIGRATOR.run(&mut *conn).await?; - // The replica-fence proof (see `replica_fence`) requires the commit-time - // `created_at` floor trigger from migration 0021 — correctly shaped — on - // the `events` parent and every partition. `CREATE TABLE .. PARTITION OF` - // clones parent triggers, but a partition attached with `ATTACH - // PARTITION` or created by an older code path would silently escape the - // guard, so migration fails closed if any is missing. (The fence probe - // re-runs this same check at startup on non-migrating relays.) - crate::replica_fence::verify_floor_guard_catalog(&mut *conn).await?; crate::channel::verify_channel_roster_fence_catalog(&mut *conn).await?; Ok(()) } @@ -705,10 +697,11 @@ mod postgres_tests { let mut migrations: Vec<_> = MIGRATOR.iter().collect(); migrations.sort_by_key(|migration| migration.version); - assert_eq!(migrations.len(), 60); + assert_eq!(migrations.len(), 61); assert_eq!(migrations[58].version, 59); assert!(migrations[58].sql.as_str().contains("buzz.push_enabled")); assert_eq!(migrations[59].version, 60); + assert_eq!(migrations[60].version, 62); assert!(migrations[59] .sql .as_str() @@ -1044,12 +1037,6 @@ mod postgres_tests { .as_str() .contains("CREATE TABLE join_policy_acceptances")); - // Replica-fence commit-time floor guard on channel-bearing events. - assert_eq!(migrations[20].version, 21); - assert!(migrations[20] - .sql - .as_str() - .contains("events_created_at_floor_guard")); assert!(!migrations[0] .sql .as_str() diff --git a/crates/buzz-db/src/runtime/mod.rs b/crates/buzz-db/src/runtime/mod.rs index 266003f5a09..0e7178ae874 100644 --- a/crates/buzz-db/src/runtime/mod.rs +++ b/crates/buzz-db/src/runtime/mod.rs @@ -356,8 +356,8 @@ pub(crate) mod route_proof { use uuid::Uuid; /// Proof that a query/page can only return rows with - /// `channel_id IS NOT NULL` — the domain of the commit-time floor guard - /// (migration 0021). `channel_ids` (retains channel-NULL rows) and + /// `channel_id IS NOT NULL` — the domain of the application floor check + /// (see [`replica_fence`]). `channel_ids` (retains channel-NULL rows) and /// `global_only = false` are explicitly NOT proofs. /// /// Each constructor keys off *how* its path proves channel-bearing-ness: @@ -415,18 +415,18 @@ pub(crate) enum RoutePredicate { /// read budget `B` (default off). Bounds TIME — the page misses at most /// the freshest `B` of writes. Sound for ANY query shape, including /// global (channel-NULL) rows: it relies only on heartbeat commit order, - /// not the floor guard. + /// not the floor check. Bounded, /// Completeness: the proved wall must cover the page's upper bound. /// Bounds CONTENT — every row at/below `upper` is present, meaningful /// even when the cursor is hours old, where `B`-freshness says nothing. - /// Sound ONLY on the floor guard's domain (channel-bearing rows), hence + /// Sound ONLY on the application floor's domain (channel-bearing rows), hence /// the proof token. `upper` is non-optional: the no-upper-bound /// post-verifying case is [`RoutePredicate::CoveredPostVerified`]. /// /// Bounds INSERT-completeness only — "no missing rows", not "no extra /// rows". Soft deletes are `UPDATE .. SET deleted_at` commits outside - /// the floor guard and never touch `created_at`, so a covered page can + /// the floor check and never touch `created_at`, so a covered page can /// briefly serve a row the writer already excludes; deletion visibility /// is bounded by replication lag under `FENCE_STALENESS` (30s), not by /// `upper` or `B`. Do not extend the covered arm to a surface that @@ -708,10 +708,6 @@ impl Db { /// When `config.read_database_url` is set, a second pool with the same /// sizing is connected to it for lag-tolerant reads (see [`Db::read`]). /// - /// The writer pool arms the commit-time `created_at` floor guard - /// (migration 0021) on every connection by setting the - /// `buzz.created_at_floor` GUC — this is what makes the replica fence - /// proof hold for every insert path that goes through this pool. pub async fn new(config: &DbConfig) -> Result { let pool = Self::connect_writer_pool(config).await?; let read_max_connections = config @@ -742,11 +738,10 @@ impl Db { /// Connect the writer pool with all session-level safety premises. /// - /// SQLx stores one `after_connect` hook, so the floor guard and transaction - /// isolation assertion must remain in this single closure. Registering a - /// second hook replaces the first and silently disarms the floor trigger. + /// SQLx stores one `after_connect` hook, so session timeouts and the + /// transaction isolation assertion must remain in this single closure. /// Additional writer pools, including the relay audit pool, must use this - /// constructor so they inherit the timeout, floor-guard, and isolation + /// constructor so they inherit the timeout and isolation /// policy installed by [`Db::new`]. pub async fn connect_writer_pool(config: &DbConfig) -> Result { use connection_observability::{ @@ -771,22 +766,6 @@ impl Db { // point at which DNS/network/TLS/authentication succeeded. record_milestone(DbPoolRole::Writer, DbConnectionStep::PhysicalConnect); - // `SET` cannot take bind parameters; `set_config` can. - let floor = DbConnectionStepAttempt::start( - DbPoolRole::Writer, - DbConnectionStep::CreatedAtFloor, - ); - if let Err(error) = - sqlx::query("SELECT set_config('buzz.created_at_floor', $1, false)") - .bind(replica_fence::CREATED_AT_FLOOR_SECS.to_string()) - .execute(&mut *conn) - .await - { - floor.fail(); - return Err(error); - } - floor.succeed(); - // `lock_timeout` fails the waiting statement; it does not // cancel the holder. `idle_in_transaction_session_timeout` // reaps only holders idling inside an open transaction, @@ -893,9 +872,8 @@ impl Db { /// are dialed only on first acquire; the ~10-minute reaper never tops /// the pool back up, which is fine — routed reads re-fill it on demand. /// - /// No floor guard or writer-isolation assertion: replica sessions are - /// read-only, so the commit-time trigger from migration 0021 never fires - /// here and the write fence that depends on READ COMMITTED is never reached. + /// No writer-isolation assertion: replica sessions are read-only, so the + /// write fence that depends on READ COMMITTED is never reached here. fn connect_read_pool(config: &DbConfig, url: &str, max_connections: u32) -> Result { Ok(PgPoolOptions::new() .max_connections(max_connections) @@ -1015,43 +993,19 @@ impl Db { &self.fence } - /// Verify the floor guard end-to-end, then spawn the background fence - /// probe. Returns `Ok(false)` when no replica is configured. - /// - /// Ordering matters (Perci, PR #2084 review): this must run **after** - /// the migration decision. On a relay with `BUZZ_AUTO_MIGRATE` off, the - /// writer pool arms the GUC regardless, but if migration 0021 has not - /// been applied there is no trigger enforcing it — and a heartbeat probe - /// would open the fence over an unenforced floor. So the probe is gated - /// on an unconditional two-part verification against the live schema: - /// catalog shape ([`replica_fence::verify_floor_guard_catalog`]) and - /// observed semantics through this exact pool - /// ([`replica_fence::verify_floor_guard_behavior`]). - /// - /// On any verification failure the probe is never spawned and the fence - /// stays closed: every cursor page routes to the writer. The relay keeps - /// serving — degraded capacity, never holes. - pub async fn spawn_fence_probe(&self) -> Result { + /// Spawn the background fence probe after the migration decision. + /// Returns `false` when no replica is configured. Probe failures keep + /// the fence closed and cursor reads on the writer. Application floor + /// coverage is enforced by the supported-writer contract and fleet gate. + pub fn spawn_fence_probe(&self) -> bool { if self.read_pool.is_none() { - return Ok(false); + return false; } - self.verify_replica_fence_at_boot().await?; tokio::spawn(replica_fence::run_probe( self.pool.clone(), std::sync::Arc::clone(&self.fence), )); - Ok(true) - } - - /// Verify replica-fence catalog shape and behavior through attributed - /// writer/bootstrap acquisitions without starting the recurring probe. - pub(crate) async fn verify_replica_fence_at_boot(&self) -> Result<()> { - let mut connection = - observability::acquire_writer(&self.pool, observability::WriterOperation::Bootstrap) - .await?; - replica_fence::verify_floor_guard_catalog(&mut *connection).await?; - drop(connection); - replica_fence::verify_floor_guard_behavior(&self.pool).await + true } /// The pool for lag-tolerant reads: the read replica when configured, @@ -1613,3 +1567,7 @@ mod pool_role_tests { #[cfg(test)] #[path = "tests.rs"] mod postgres_tests; + +#[cfg(test)] +#[path = "tests/floor_retirement.rs"] +mod floor_retirement_postgres_tests; diff --git a/crates/buzz-db/src/runtime/observability.rs b/crates/buzz-db/src/runtime/observability.rs index d487b9a3621..d6fdd3a4087 100644 --- a/crates/buzz-db/src/runtime/observability.rs +++ b/crates/buzz-db/src/runtime/observability.rs @@ -1478,11 +1478,14 @@ mod tests { let test_scope = CommunityId::from_uuid(Uuid::new_v4()); let query = crate::EventQuery::for_community(test_scope); - routed_db.read_pool_boot_ping_for_tests().await; - routed_db - .verify_replica_fence_at_boot() + writer_db + .ensure_configured_community_for_bootstrap(&format!( + "pool-operation-bootstrap-{}.invalid", + Uuid::new_v4().simple() + )) .await - .expect("real startup fence verification succeeds"); + .expect("real community bootstrap succeeds"); + routed_db.read_pool_boot_ping_for_tests().await; let _ = crate::replica_fence::probe_once(&writer_pool, routed_db.fence()).await; routed_db.fence().force_open_for_tests(Utc::now()); assert_eq!( diff --git a/crates/buzz-db/src/runtime/replica_fence.rs b/crates/buzz-db/src/runtime/replica_fence.rs index b07a2339903..f1fb75bf755 100644 --- a/crates/buzz-db/src/runtime/replica_fence.rs +++ b/crates/buzz-db/src/runtime/replica_fence.rs @@ -5,10 +5,9 @@ //! //! 1. **Insert-time floor**: supported channel-event writers check `created_at` //! against database `clock_timestamp()` on their admitted transaction for -//! each new row. Duplicate INSERTs need no floor check. This is the primary -//! guard. Migration 0021's deferred trigger and the writer-pool -//! `buzz.created_at_floor` GUC remain the mixed-version backstop until phase 2; -//! they also reject rows that age below the floor before COMMIT. +//! each new row. Duplicate INSERTs need no floor check. This is the only +//! guard: migration 0062 retired 0021's deferred trigger and the +//! writer-pool `buzz.created_at_floor` GUC. //! 2. **Ordered heartbeat handshake** (this module): on one pinned writer //! connection, one transaction first takes the exclusive replica-floor //! advisory lock (draining compliant shared-lock channel writers), then @@ -67,12 +66,12 @@ use uuid::Uuid; use buzz_datastore_tracing::datastore_span; -/// Seconds of `created_at` history the app and mixed-version trigger tolerate. +/// Seconds of `created_at` history the application floor check tolerates. /// /// Must exceed the relay's ingest envelope (±900 s) by enough slack that a -/// legitimately accepted event still commits within the floor even under -/// slow validation/lock waits. The writer pool arms the guard with this value -/// and the fence subtracts it; the two uses must never diverge. +/// legitimately accepted event still passes the check after slow +/// validation/lock waits. Application checks and the fence wall use this +/// same value; the two uses must never diverge. pub const CREATED_AT_FLOOR_SECS: i64 = 960; /// Enforce the channel-event floor on the admitted INSERT connection. @@ -110,13 +109,13 @@ pub(crate) async fn guard_event_created_at_floor( /// transactions against the writer probe handshake. /// /// Shared holders are channel-event write transactions validating `created_at` -/// under the armed floor contract; the probe takes the exclusive lock before -/// sampling and publishing its heartbeat token. +/// under the application floor contract; the probe takes the exclusive lock +/// before sampling and publishing its heartbeat token. pub const REPLICA_FLOOR_LOCK_KEY: i64 = 0x62757a7a666c6f72; /// Safety margin subtracted from the fence on top of the floor. /// -/// All proof timestamps (`clock_timestamp()`, `xact_start`, the guard's +/// All proof timestamps (`clock_timestamp()`, `xact_start`, the floor check's /// clock) come from the writer host, so this only needs to absorb /// `created_at` second-truncation and scheduling noise, not clock skew /// between machines. @@ -350,208 +349,6 @@ impl ReplicaFence { } } -/// Catalog-level verification that the commit-time floor guard (migration -/// 0021) is present and correctly shaped on the `events` parent AND every -/// partition: right function, `DEFERRABLE INITIALLY DEFERRED`, row-level, -/// AFTER, firing on both INSERT and UPDATE (an UPDATE can move an exempt -/// channel-NULL row into the guarded set, or rewrite `created_at` downward). -/// -/// This is a name-and-shape check only; it cannot detect a sabotaged -/// function body. [`verify_floor_guard_behavior`] proves the semantics. -/// -/// Generic over the executor so the migration path can run it on the -/// lock-holding connection while the startup probe keeps using the pool. -#[datastore_span(name = "replica_fence_verify_catalog", system = "postgresql")] -pub async fn verify_floor_guard_catalog<'e>( - executor: impl sqlx::PgExecutor<'e>, -) -> crate::Result<()> { - // tgtype bits: 1 = ROW, 2 = BEFORE, 4 = INSERT, 16 = UPDATE, 64 = INSTEAD. - // Required: ROW + INSERT + UPDATE set, BEFORE + INSTEAD clear. - let missing: Vec = sqlx::query_scalar( - r#" - SELECT c.relname::text - FROM ( - SELECT 'events'::regclass AS oid - UNION ALL - SELECT inhrelid FROM pg_inherits WHERE inhparent = 'events'::regclass - ) rels - JOIN pg_class c ON c.oid = rels.oid - WHERE NOT EXISTS ( - SELECT 1 FROM pg_trigger t - WHERE t.tgrelid = rels.oid - AND t.tgname = 'events_created_at_floor' - AND t.tgfoid = 'events_created_at_floor_guard'::regproc - AND t.tgdeferrable - AND t.tginitdeferred - AND t.tgtype & 1 = 1 -- row-level - AND t.tgtype & 2 = 0 -- AFTER, not BEFORE - AND t.tgtype & 64 = 0 -- not INSTEAD OF - AND t.tgtype & 4 = 4 -- fires on INSERT - AND t.tgtype & 16 = 16 -- fires on UPDATE - ) - "#, - ) - .fetch_all(executor) - .await?; - if !missing.is_empty() { - return Err(crate::error::DbError::InvalidData(format!( - "created_at floor guard trigger missing or mis-shaped on: {} \ - (replica fence must stay closed)", - missing.join(", ") - ))); - } - Ok(()) -} - -/// Behavioral verification of the floor guard, end-to-end through the armed -/// pool. A catalog check cannot detect a no-op function body or an unarmed -/// pool; this proves the semantics the fence proof cites, inside one -/// rolled-back transaction: -/// -/// 1. the pool's session GUC equals [`CREATED_AT_FLOOR_SECS`] (arming); -/// 2. an old channel-bearing INSERT raises `check_violation` (23514); -/// 3. a fresh channel-bearing INSERT commits; -/// 4. rewriting a fresh row's `created_at` below the floor raises; -/// 5. an old channel-NULL INSERT is exempt, but flipping its `channel_id` -/// on raises (the `UPDATE OF` arm). -/// -/// `SET CONSTRAINTS ALL IMMEDIATE` makes the deferred trigger fire per -/// statement so each adversary is observable under a savepoint; deferral to -/// COMMIT is separately pinned by the held-transaction fixture. -#[datastore_span(name = "replica_fence_verify_behavior", system = "postgresql")] -pub async fn verify_floor_guard_behavior(pool: &PgPool) -> crate::Result<()> { - use crate::error::DbError; - - let expect_violation = |res: Result, - what: &str| - -> crate::Result<()> { - match res { - Err(sqlx::Error::Database(e)) if e.code().as_deref() == Some("23514") => Ok(()), - Ok(_) => Err(DbError::InvalidData(format!( - "floor guard is inert: {what} was accepted (replica fence must stay closed)" - ))), - Err(e) => Err(DbError::InvalidData(format!( - "floor guard verification failed unexpectedly on {what}: {e}" - ))), - } - }; - - let connection = crate::observability::acquire_writer( - pool, - crate::observability::WriterOperation::Bootstrap, - ) - .await?; - let mut tx = sqlx::Transaction::begin(connection, None).await?; - - // 1. Pool arming (Perci: assert the effective value, not the intent). - let armed: String = sqlx::query_scalar("SHOW buzz.created_at_floor") - .fetch_one(&mut *tx) - .await - .map_err(|e| { - DbError::InvalidData(format!( - "buzz.created_at_floor GUC not set on this pool: {e}" - )) - })?; - if armed != CREATED_AT_FLOOR_SECS.to_string() { - return Err(DbError::InvalidData(format!( - "buzz.created_at_floor is '{armed}', expected '{CREATED_AT_FLOOR_SECS}': \ - pool is not armed" - ))); - } - - sqlx::query("SET CONSTRAINTS ALL IMMEDIATE") - .execute(&mut *tx) - .await?; - - // Scratch community satisfying the FK; the whole transaction rolls back. - let community = uuid::Uuid::new_v4(); - sqlx::query("INSERT INTO communities (id, host) VALUES ($1, $2)") - .bind(community) - .bind(format!("fence-verify-{}.invalid", community.simple())) - .execute(&mut *tx) - .await?; - let channel = uuid::Uuid::new_v4(); - - let insert = |tx_id: [u8; 32], age_secs: i64, ch: Option| { - sqlx::query( - "INSERT INTO events (community_id, id, pubkey, created_at, kind, tags, \ - content, sig, received_at, channel_id) \ - VALUES ($1, $2, $3, clock_timestamp() - make_interval(secs => $4::double precision), \ - 9, '[]', 'fence-verify', $5, NOW(), $6)", - ) - .bind(community) - .bind(tx_id.to_vec()) - .bind(vec![0u8; 32]) - .bind(age_secs as f64) - .bind(vec![0u8; 64]) - .bind(ch) - }; - let old_age = CREATED_AT_FLOOR_SECS + 60; - - // 2. Old channel-bearing insert must raise. - sqlx::query("SAVEPOINT floor_probe") - .execute(&mut *tx) - .await?; - let res = insert(rand_id(), old_age, Some(channel)) - .execute(&mut *tx) - .await; - expect_violation(res, "an old channel-bearing INSERT")?; - sqlx::query("ROLLBACK TO SAVEPOINT floor_probe") - .execute(&mut *tx) - .await?; - - // 3. Fresh channel-bearing insert must pass. - let fresh_id = rand_id(); - insert(fresh_id, 0, Some(channel)).execute(&mut *tx).await?; - - // 4. Rewriting created_at below the floor must raise (in- or - // cross-partition, either arm of the guard catches the NEW row). - sqlx::query("SAVEPOINT floor_probe") - .execute(&mut *tx) - .await?; - let res = sqlx::query( - "UPDATE events SET created_at = clock_timestamp() - make_interval(secs => $1::double precision) \ - WHERE community_id = $2 AND id = $3", - ) - .bind(old_age as f64) - .bind(community) - .bind(fresh_id.to_vec()) - .execute(&mut *tx) - .await; - expect_violation(res, "rewriting created_at below the floor")?; - sqlx::query("ROLLBACK TO SAVEPOINT floor_probe") - .execute(&mut *tx) - .await?; - - // 5. Old channel-NULL insert is exempt; flipping channel_id on must raise. - let null_id = rand_id(); - insert(null_id, old_age, None).execute(&mut *tx).await?; - sqlx::query("SAVEPOINT floor_probe") - .execute(&mut *tx) - .await?; - let res = sqlx::query("UPDATE events SET channel_id = $1 WHERE community_id = $2 AND id = $3") - .bind(channel) - .bind(community) - .bind(null_id.to_vec()) - .execute(&mut *tx) - .await; - expect_violation(res, "moving an old channel-NULL row into a channel")?; - sqlx::query("ROLLBACK TO SAVEPOINT floor_probe") - .execute(&mut *tx) - .await?; - - tx.rollback().await?; - Ok(()) -} - -fn rand_id() -> [u8; 32] { - let mut id = [0u8; 32]; - for chunk in id.chunks_mut(16) { - chunk.copy_from_slice(&uuid::Uuid::new_v4().into_bytes()[..chunk.len()]); - } - id -} - /// One writer-side sample of the ordered handshake. #[derive(Debug)] struct WriterSample { @@ -632,17 +429,14 @@ async fn sample_writer(writer: &PgPool) -> Result { // conservative direction (a long autovacuum merely holds the fence // back), and it keeps the classification simple. // Scope is every other backend regardless of role or application - // name: an admin psql transaction writes under the same trigger and - // must be representable in the oldest-xact term. + // name: every open transaction must be representable in the + // oldest-xact term. Out-of-band writes require a closed breaker. // // Prepared transactions (2PC) are a bucket of their own: while // prepared they have left `pg_stat_activity` but can still commit - // after the token. Their deferred floor guard already ran at PREPARE, - // so `pg_prepared_xacts.prepared` bounds their rows exactly like - // `xact_start`; fold it into the same minimum. Only the retained - // deferred trigger provides that PREPARE-time check: the insert-time - // app guard runs earlier, and no app writer issues PREPARE TRANSACTION. - // Dropping the trigger must keep this case covered. + // after the token. Their floor check must precede PREPARE, so + // `pg_prepared_xacts.prepared` bounds their rows exactly like + // `xact_start`; fold it into the same minimum. let row = sqlx::query( r#" SELECT diff --git a/crates/buzz-db/src/runtime/tests.rs b/crates/buzz-db/src/runtime/tests.rs index d3f429baee0..9c66f7fe44f 100644 --- a/crates/buzz-db/src/runtime/tests.rs +++ b/crates/buzz-db/src/runtime/tests.rs @@ -1933,7 +1933,7 @@ async fn routed_fallback_spends_one_acquire_budget_when_aurora_cache_is_cold() { format!("{}/{}", &base[..idx], wname) }; - // `Db::new` so the writer arms the floor guard and the reader is the + // `Db::new` so the writer uses production session policy and the reader is the // real lazy `connect_read_pool` pool (min_connections=0, 150ms // acquire timeout). Reader is sized 1 so holding one connection // saturates it. @@ -2241,9 +2241,8 @@ async fn routed_reads_are_confined_to_the_requested_community() { } /// D4: a LAZY reader pool (connect_lazy, min_connections=0, never yet -/// used) must still let [`Db::spawn_fence_probe`] verify the writer's -/// floor guard and spawn — reader-down or reader-idle at boot must not -/// disable fence probing. +/// used) must still let [`Db::spawn_fence_probe`] spawn — reader-down or +/// reader-idle at boot must not disable fence probing. #[tokio::test] #[ignore = "requires Postgres"] async fn lazy_reader_pool_still_spawns_fence_probe() { @@ -2258,11 +2257,7 @@ async fn lazy_reader_pool_still_spawns_fence_probe() { let idx = base.rfind('/').expect("db url has a path segment"); format!("{}/{}", &base[..idx], wname) }; - // `Db::new` (not `from_pools`) so the WRITER pool arms the - // `buzz.created_at_floor` GUC — `spawn_fence_probe` verifies the - // floor guard on a writer connection, and `create_scratch_db`'s - // plain `PgPool::connect` never arms it. The reader is still the - // lazy `connect_read_pool` pool this test is about. + // Use the production writer policy and lazy `connect_read_pool`. let db = Db::new(&DbConfig { database_url: writer_url.clone(), read_database_url: Some(writer_url), @@ -2270,12 +2265,9 @@ async fn lazy_reader_pool_still_spawns_fence_probe() { ..DbConfig::default() }) .await - .expect("connect armed Db with lazy reader"); + .expect("connect Db with lazy reader"); - let spawned = db - .spawn_fence_probe() - .await - .expect("floor-guard verification must pass on the migrated writer"); + let spawned = db.spawn_fence_probe(); assert!(spawned, "a configured (lazy) reader must spawn the probe"); drop_scratch_db(&admin, db.pool.clone(), &wname).await; @@ -2568,100 +2560,6 @@ async fn thread_full_replica_page_above_fence_is_reverified_on_writer() { drop_scratch_db(&admin, writer, &wname).await; } -/// Commit-time floor guard (migration 0021), exact held-transaction -/// adversary: a channel-bearing row whose `created_at` is older than the -/// floor at COMMIT time must abort the transaction — the guard runs -/// inside commit processing with `clock_timestamp()`, so holding the -/// transaction open cannot outrun it. channel_id-NULL rows are -/// structurally exempt, and sessions without the GUC are unaffected. -#[tokio::test] -#[ignore = "requires Postgres"] -async fn created_at_floor_guard_aborts_old_channel_rows_at_commit() { - let admin = PgPool::connect(&admin_url().await) - .await - .expect("connect admin"); - let (pool, name) = create_scratch_db(&admin, "floor_guard").await; - - let author = nostr::Keys::generate(); - let community = Uuid::new_v4(); - let channel = Uuid::new_v4(); - seed_community_channel(&pool, community, channel, &author).await; - - let insert_raw = |ev: nostr::Event, channel_id: Option| { - let pool = pool.clone(); - async move { - let mut tx = pool.begin().await.expect("begin"); - // Arm the guard for this transaction only (the relay's - // writer pool arms it per connection; tests are explicit). - sqlx::query("SELECT set_config('buzz.created_at_floor', $1, true)") - .bind(crate::replica_fence::CREATED_AT_FLOOR_SECS.to_string()) - .execute(&mut *tx) - .await - .expect("arm guard"); - sqlx::query( - "INSERT INTO events (community_id, id, pubkey, created_at, kind, tags, \ - content, sig, received_at, channel_id) \ - VALUES ($1, $2, $3, to_timestamp($4), 9, '[]', $5, $6, NOW(), $7)", - ) - .bind(community) - .bind(ev.id.as_bytes().as_slice()) - .bind(ev.pubkey.to_bytes().as_slice()) - .bind(ev.created_at.as_secs() as f64) - .bind(&ev.content) - .bind(ev.sig.serialize().as_slice()) - .bind(channel_id) - .execute(&mut *tx) - .await - .expect("insert inside tx (guard is deferred to commit)"); - // Hold the transaction "open" past the insert, then commit — - // the deferred guard must still see the stale created_at. - sqlx::query("SELECT pg_sleep(0.05)") - .execute(&mut *tx) - .await - .expect("hold tx"); - tx.commit().await - } - }; - - let now_secs = chrono::Utc::now().timestamp() as u64; - let floor = crate::replica_fence::CREATED_AT_FLOOR_SECS as u64; - - // Old channel-bearing row → COMMIT aborts with check_violation. - let old = signed_event_at(&author, "old-held-tx", now_secs - floor - 60); - let err = insert_raw(old, Some(channel)) - .await - .expect_err("below-floor channel row must abort at COMMIT"); - let code = match &err { - sqlx::Error::Database(db_err) => db_err.code().map(|c| c.to_string()), - other => panic!("expected database error, got {other:?}"), - }; - assert_eq!( - code.as_deref(), - Some("23514"), - "guard raises check_violation" - ); - - // Fresh channel-bearing row → commits. - let fresh = signed_event_at(&author, "fresh", now_secs); - insert_raw(fresh, Some(channel)) - .await - .expect("fresh row commits under the armed guard"); - - // Old row WITHOUT a channel (push lease / profile shapes) → - // structurally exempt, commits. - let old_global = signed_event_at(&author, "old-global", now_secs - floor - 60); - insert_raw(old_global, None) - .await - .expect("channel_id-NULL rows are exempt from the floor"); - - // Unarmed session (no GUC) → guard inert; backfills stay possible - // (and must hold the fence closed, per the migration header). - let old_backfill = signed_event_at(&author, "old-backfill", now_secs - floor - 60); - insert_top_level(&pool, community, channel, &old_backfill).await; - - drop_scratch_db(&admin, pool, &name).await; -} - #[test] fn writer_pool_safety_hook_is_single_and_composed() { let source = include_str!("mod.rs"); @@ -2675,13 +2573,10 @@ fn writer_pool_safety_hook_is_single_and_composed() { 1, "SQLx replaces after_connect hooks; writer safety must use exactly one" ); - assert!(connect_pool.contains("buzz.created_at_floor")); assert!(connect_pool.contains("SHOW transaction_isolation")); assert!(connect_pool.contains("'lock_timeout'")); assert!(connect_pool.contains("'idle_in_transaction_session_timeout'")); assert!(connect_pool.contains("'statement_timeout'")); - assert!(!connect_pool.contains("arm_floor_guard")); - assert!(!connect_pool.contains("_arm_floor_guard")); assert!(!connect_pool.contains("allow(unused_variables)")); let reader_doc = source @@ -2738,7 +2633,6 @@ async fn writer_pool_metrics_record_every_initial_connection_ready() { 2 ); for step in [ - DbConnectionStep::CreatedAtFloor, DbConnectionStep::SessionTimeouts, DbConnectionStep::Isolation, ] { @@ -3090,106 +2984,6 @@ async fn session_timeouts_install_through_db_new_and_bound_lock_waits() { drop_scratch_db(&admin, db.pool.clone(), &name).await; } -/// The armed writer pool (`Db::new`) must enforce the floor end-to-end -/// through the public insert APIs, and the session GUC must be verifiably -/// set on pooled connections. -#[tokio::test] -#[ignore = "requires Postgres"] -async fn armed_pool_rejects_old_channel_inserts_through_public_api() { - let admin = PgPool::connect(&admin_url().await) - .await - .expect("connect admin"); - let (seed_pool, name) = create_scratch_db(&admin, "floor_pool").await; - - let author = nostr::Keys::generate(); - let community = Uuid::new_v4(); - let channel = Uuid::new_v4(); - seed_community_channel(&seed_pool, community, channel, &author).await; - - // Connect a Db the production way: after_connect arms the guard. - let base = admin_url().await; - let idx = base.rfind('/').expect("db url has a path segment"); - let scratch_url = format!("{}/{}", &base[..idx], name); - let db = Db::new(&DbConfig { - database_url: scratch_url, - max_connections: 2, - ..DbConfig::default() - }) - .await - .expect("connect armed Db"); - let cid = CommunityId::from_uuid(community); - - // Perci nit: assert the effective session value, not the intent. - let effective: String = sqlx::query_scalar("SHOW buzz.created_at_floor") - .fetch_one(&db.pool) - .await - .expect("SHOW guard GUC"); - assert_eq!( - effective, - crate::replica_fence::CREATED_AT_FLOOR_SECS.to_string(), - "writer pool must arm the floor guard on every connection" - ); - let isolation: String = sqlx::query_scalar("SHOW transaction_isolation") - .fetch_one(&db.pool) - .await - .expect("SHOW writer isolation"); - assert_eq!( - isolation, "read committed", - "the same writer after_connect hook must enforce the isolation premise" - ); - - let now_secs = chrono::Utc::now().timestamp() as u64; - let floor = crate::replica_fence::CREATED_AT_FLOOR_SECS as u64; - - // insert_event (single INSERT, autocommit): old channel row rejected. - let old = signed_event_at(&author, "old-direct", now_secs - floor - 60); - let err = event::insert_event(&db.pool, cid, &old, Some(channel)) - .await - .expect_err("armed pool must reject below-floor channel inserts"); - assert!( - err.to_string().contains("below the replica-fence floor"), - "unexpected error: {err}" - ); - - // insert_event_with_thread_metadata (multi-statement tx): same. - let old2 = signed_event_at(&author, "old-thread-meta", now_secs - floor - 90); - let ts = - chrono::DateTime::from_timestamp(old2.created_at.as_secs() as i64, 0).expect("valid ts"); - let err = event::insert_event_with_thread_metadata( - &db.pool, - cid, - &old2, - Some(channel), - Some(event::ThreadMetadataParams { - event_id: old2.id.as_bytes(), - event_created_at: ts, - channel_id: channel, - parent_event_id: None, - parent_event_created_at: None, - root_event_id: None, - root_event_created_at: None, - depth: 0, - broadcast: true, - }), - ) - .await - .expect_err("armed pool must reject below-floor thread-metadata inserts"); - assert!( - err.to_string().contains("below the replica-fence floor"), - "unexpected error: {err}" - ); - - // Fresh events pass through both APIs. - let fresh = signed_event_at(&author, "fresh-direct", now_secs); - event::insert_event(&db.pool, cid, &fresh, Some(channel)) - .await - .expect("fresh insert passes the armed guard"); - - drop_scratch_db(&admin, seed_pool, &name).await; - // db pool still holds connections to the dropped DB; close it. - db.pool.close().await; -} - /// A writer transaction holding the shared replica-floor advisory lock, the /// shape a floor-compliant writer takes before the exclusive probe can run. async fn begin_replica_floor_locked_writer(db: &Db) -> sqlx::Transaction<'static, sqlx::Postgres> { @@ -3197,7 +2991,7 @@ async fn begin_replica_floor_locked_writer(db: &Db) -> sqlx::Transaction<'static .pool .begin() .await - .expect("begin floor-guarded writer tx"); + .expect("begin floor-compliant writer tx"); sqlx::query("SELECT pg_advisory_xact_lock_shared($1)") .bind(crate::replica_fence::REPLICA_FLOOR_LOCK_KEY) .execute(&mut *tx) @@ -3325,201 +3119,6 @@ async fn replica_floor_probe_waits_for_shared_writer_and_records_after_release() drop_scratch_db(&admin, seed_pool, &name).await; } -/// `spawn_fence_probe` must verify the floor guard before letting the -/// probe run — catalog shape AND observed behavior — and refuse on -/// sabotage. This is the production gate for a relay running with -/// `BUZZ_AUTO_MIGRATE` off: an armed GUC with no enforcing trigger must -/// never yield an open fence. -#[tokio::test] -#[ignore = "requires Postgres"] -async fn fence_probe_refuses_to_start_without_verified_floor_guard() { - let admin = PgPool::connect(&admin_url().await) - .await - .expect("connect admin"); - let (seed_pool, wname) = create_scratch_db(&admin, "fence_gate_w").await; - let (replica_pool, rname) = create_scratch_db(&admin, "fence_gate_r").await; - seed_pool.close().await; - replica_pool.close().await; - - let base = admin_url().await; - let idx = base.rfind('/').expect("db url has a path segment"); - let writer_url = format!("{}/{}", &base[..idx], wname); - let replica_url = format!("{}/{}", &base[..idx], rname); - - // Healthy schema: verification passes, probe starts. A SEPARATE Db - // instance, because its background probe legitimately opens its own - // fence (the heartbeat probe is writer-side only) — the refusal - // assertions below must run against a fence whose spawns were all - // refused. - let db_healthy = Db::new(&DbConfig { - database_url: writer_url.clone(), - read_database_url: Some(replica_url.clone()), - max_connections: 2, - ..DbConfig::default() - }) - .await - .expect("connect armed Db with replica"); - assert!( - db_healthy - .spawn_fence_probe() - .await - .expect("verification passes"), - "probe must start on a verified schema" - ); - - let db = Db::new(&DbConfig { - database_url: writer_url, - read_database_url: Some(replica_url), - max_connections: 2, - ..DbConfig::default() - }) - .await - .expect("connect armed Db with replica"); - - // Sabotage A: catalog-shaped no-op — same trigger, gutted function - // body. Catalog check alone would pass; behavior check must refuse. - sqlx::query( - "CREATE OR REPLACE FUNCTION events_created_at_floor_guard() RETURNS trigger \ - LANGUAGE plpgsql AS $$ BEGIN RETURN NULL; END $$", - ) - .execute(&db.pool) - .await - .expect("gut the guard function"); - let err = db - .spawn_fence_probe() - .await - .expect_err("inert guard body must refuse the probe"); - assert!( - err.to_string().contains("floor guard is inert"), - "unexpected error: {err}" - ); - - // Sabotage B: trigger dropped entirely (the BUZZ_AUTO_MIGRATE=off / - // 0021-unapplied shape). Catalog check must refuse. - sqlx::query("DROP TRIGGER events_created_at_floor ON events") - .execute(&db.pool) - .await - .expect("drop the guard trigger"); - let err = db - .spawn_fence_probe() - .await - .expect_err("missing trigger must refuse the probe"); - assert!( - err.to_string().contains("missing or mis-shaped"), - "unexpected error: {err}" - ); - - // In both refusal states the fence never opened. - assert!( - db.fence().verified_through().is_none(), - "fence must remain closed when verification refuses the probe" - ); - - db_healthy.pool.close().await; - if let Some(rp) = &db_healthy.read_pool { - rp.close().await; - } - db.pool.close().await; - if let Some(rp) = &db.read_pool { - rp.close().await; - } - let _ = sqlx::query(sqlx::AssertSqlSafe(format!( - "DROP DATABASE IF EXISTS {wname} WITH (FORCE)" - ))) - .execute(&admin) - .await; - let _ = sqlx::query(sqlx::AssertSqlSafe(format!( - "DROP DATABASE IF EXISTS {rname} WITH (FORCE)" - ))) - .execute(&admin) - .await; -} - -/// The `UPDATE OF` arm of the floor guard (Perci's second structural -/// hole): an old row legitimately admitted with `channel_id` NULL must -/// not be movable into keyset windows, and a channel row's `created_at` -/// must not be movable below the fence — through raw SQL, at COMMIT. -#[tokio::test] -#[ignore = "requires Postgres"] -async fn floor_guard_blocks_updates_that_move_rows_below_the_fence() { - let admin = PgPool::connect(&admin_url().await) - .await - .expect("connect admin"); - let (pool, name) = create_scratch_db(&admin, "floor_upd").await; - - let author = nostr::Keys::generate(); - let community = Uuid::new_v4(); - let channel = Uuid::new_v4(); - seed_community_channel(&pool, community, channel, &author).await; - - let now_secs = chrono::Utc::now().timestamp() as u64; - let floor = crate::replica_fence::CREATED_AT_FLOOR_SECS as u64; - - // Seed via unarmed session: one old channel-NULL row, one fresh - // channel row. - let old_null = signed_event_at(&author, "old-null", now_secs - floor - 120); - insert_top_level(&pool, community, channel, &old_null).await; - sqlx::query("UPDATE events SET channel_id = NULL WHERE community_id = $1 AND id = $2") - .bind(community) - .bind(old_null.id.as_bytes().as_slice()) - .execute(&pool) - .await - .expect("detach channel (unarmed seed)"); - let fresh = signed_event_at(&author, "fresh-row", now_secs); - insert_top_level(&pool, community, channel, &fresh).await; - - // Armed transaction, deferred to COMMIT (the production shape). - let run_armed_update = |sql: &'static str, id: Vec, age: Option| { - let pool = pool.clone(); - async move { - let mut tx = pool.begin().await.expect("begin"); - sqlx::query("SELECT set_config('buzz.created_at_floor', $1, true)") - .bind(crate::replica_fence::CREATED_AT_FLOOR_SECS.to_string()) - .execute(&mut *tx) - .await - .expect("arm guard"); - let q = sqlx::query(sql).bind(community).bind(id); - let q = match age { - Some(a) => q.bind(a as f64), - None => q, - }; - q.execute(&mut *tx) - .await - .expect("update inside tx (deferred)"); - tx.commit().await - } - }; - - // channel-NULL → channel-bearing on an old row: COMMIT must abort. - let err = run_armed_update( - "UPDATE events SET channel_id = community_id WHERE community_id = $1 AND id = $2", - old_null.id.as_bytes().to_vec(), - None, - ) - .await - .expect_err("moving an old channel-NULL row into a channel must abort at COMMIT"); - assert!( - matches!(&err, sqlx::Error::Database(e) if e.code().as_deref() == Some("23514")), - "unexpected error: {err}" - ); - - // created_at rewrite below the floor on a channel row: COMMIT must abort. - let err = run_armed_update( - "UPDATE events SET created_at = clock_timestamp() - make_interval(secs => $3::double precision) \ - WHERE community_id = $1 AND id = $2", - fresh.id.as_bytes().to_vec(), - Some(floor + 120), - ) - .await - .expect_err("rewriting created_at below the floor must abort at COMMIT"); - assert!( - matches!(&err, sqlx::Error::Database(e) if e.code().as_deref() == Some("23514")), - "unexpected error: {err}" - ); - - drop_scratch_db(&admin, pool, &name).await; -} - // ---- Statement-deadline cancellation vs. replica fallback ---------------- // // A replica read cancelled by `statement_timeout` (SQLSTATE 57014) must be diff --git a/crates/buzz-db/src/runtime/tests/floor_retirement.rs b/crates/buzz-db/src/runtime/tests/floor_retirement.rs new file mode 100644 index 00000000000..6fe582a23e4 --- /dev/null +++ b/crates/buzz-db/src/runtime/tests/floor_retirement.rs @@ -0,0 +1,161 @@ +//! Contract-migration coverage; application insert coverage belongs to phase 1. + +use super::*; + +async fn floor_objects(pool: &PgPool) -> (i64, i64) { + sqlx::query_as( + "SELECT (SELECT count(*) FROM pg_trigger WHERE tgname = 'events_created_at_floor'), \ + (SELECT count(*) FROM pg_proc p JOIN pg_namespace n ON n.oid = p.pronamespace \ + WHERE n.nspname = 'public' AND p.proname = 'events_created_at_floor_guard')", + ) + .fetch_one(pool) + .await + .expect("read floor catalog") +} + +async fn retained_guards(pool: &PgPool) -> Vec<(String, String)> { + sqlx::query_as( + "SELECT c.relname, pg_get_triggerdef(t.oid) FROM pg_trigger t \ + JOIN pg_class c ON c.oid = t.tgrelid \ + JOIN pg_namespace n ON n.oid = c.relnamespace \ + WHERE n.nspname = 'public' AND t.tgname <> 'events_created_at_floor' \ + UNION ALL \ + SELECT c.relname, pg_get_constraintdef(k.oid) FROM pg_constraint k \ + JOIN pg_class c ON c.oid = k.conrelid \ + JOIN pg_namespace n ON n.oid = c.relnamespace \ + WHERE n.nspname = 'public' AND k.contype = 'f' \ + ORDER BY 1, 2", + ) + .fetch_all(pool) + .await + .expect("read retained triggers and foreign keys") +} + +#[tokio::test] +#[ignore = "requires Postgres"] +async fn migration_schema_floor_retirement_drops_parent_and_partition_clones() { + let pool = PgPool::connect(&crate::test_support::database_url()) + .await + .expect("connect isolated migration database"); + crate::migration::run_migrations_through(&pool, 60) + .await + .expect("apply pre-contract schema"); + let partitions: i64 = + sqlx::query_scalar("SELECT count(*) FROM pg_partition_tree('events'::regclass)") + .fetch_one(&pool) + .await + .expect("count parent and partitions"); + assert_eq!(floor_objects(&pool).await, (partitions, 1)); + let before = retained_guards(&pool).await; + crate::migration::run_migrations(&pool) + .await + .expect("apply contract migration and retained startup checks"); + assert_eq!(floor_objects(&pool).await, (0, 0)); + assert_eq!(retained_guards(&pool).await, before); + pool.close().await; +} + +#[tokio::test] +#[ignore = "requires Postgres"] +async fn migration_schema_floor_retirement_cleans_child_copies_idempotently() { + let pool = PgPool::connect(&crate::test_support::database_url()) + .await + .expect("connect isolated migration database"); + crate::migration::run_migrations_through(&pool, 60) + .await + .expect("apply pre-contract schema"); + // Model independently attached copies after the parent guard was removed. + sqlx::raw_sql( + "DROP TRIGGER events_created_at_floor ON events; \ + CREATE CONSTRAINT TRIGGER events_created_at_floor \ + AFTER INSERT OR UPDATE OF created_at, channel_id ON events_p_past \ + DEFERRABLE INITIALLY DEFERRED FOR EACH ROW \ + EXECUTE FUNCTION events_created_at_floor_guard();", + ) + .execute(&pool) + .await + .expect("install independent child copy"); + assert_eq!(floor_objects(&pool).await, (1, 1)); + let before = retained_guards(&pool).await; + crate::migration::run_migrations(&pool) + .await + .expect("retire independent copy"); + assert_eq!(floor_objects(&pool).await, (0, 0)); + let mut tx = pool.begin().await.expect("begin idempotence check"); + sqlx::raw_sql(include_str!( + "../../../../../migrations/0062_retire_created_at_floor.sql" + )) + .execute(&mut *tx) + .await + .expect("repeat contract SQL after retirement"); + tx.commit().await.expect("commit repeated retirement"); + assert_eq!(floor_objects(&pool).await, (0, 0)); + assert_eq!(retained_guards(&pool).await, before); + pool.close().await; +} + +#[tokio::test] +#[ignore = "requires Postgres"] +async fn floor_retirement_desired_and_migrated_catalogs_match() { + // The nextest wrapper bootstraps this database through real pgschema apply + // and reconciliation. Check the retired floor catalog against the real + // migration boot path; unrelated trigger/FK preservation is checked above. + let pool = PgPool::connect(&crate::test_support::database_url()) + .await + .expect("connect desired-state database"); + assert_eq!(floor_objects(&pool).await, (0, 0)); + sqlx::raw_sql("DROP SCHEMA public CASCADE; CREATE SCHEMA public;") + .execute(&pool) + .await + .expect("reset isolated database for migration boot"); + crate::migration::run_migrations(&pool) + .await + .expect("boot migration schema"); + assert_eq!(floor_objects(&pool).await, (0, 0)); + pool.close().await; +} + +#[tokio::test] +#[ignore = "requires Postgres"] +async fn floor_retirement_writer_and_probe_accept_legacy_and_retired_schema() { + let url = crate::test_support::database_url(); + let db = Db::new(&DbConfig { + database_url: url.clone(), + read_database_url: Some(url), + max_connections: 2, + min_connections: 1, + ..DbConfig::default() + }) + .await + .expect("connect production writer with retired schema"); + let armed: Option = + sqlx::query_scalar("SELECT nullif(current_setting('buzz.created_at_floor', true), '')") + .fetch_one(&db.pool) + .await + .expect("inspect writer session setting"); + assert_eq!(armed, None, "writer must no longer arm the retired guard"); + assert_eq!(floor_objects(&db.pool).await, (0, 0)); + assert!(db.spawn_fence_probe()); + crate::replica_fence::probe_once(&db.pool, db.fence()) + .await + .expect("retired schema still supports the ordered heartbeat"); + + // This binary can also run before migration, without changing legacy SQL. + sqlx::raw_sql(include_str!( + "../../../../../migrations/0021_created_at_fence_floor.sql" + )) + .execute(&db.pool) + .await + .expect("restore actual legacy guard"); + assert!(floor_objects(&db.pool).await.0 > 1); + assert!(db.spawn_fence_probe()); + crate::replica_fence::probe_once(&db.pool, db.fence()) + .await + .expect("legacy schema still supports ordered heartbeat"); + db.pool.close().await; + db.read_pool + .as_ref() + .expect("reader configured") + .close() + .await; +} diff --git a/crates/buzz-db/src/store/event.rs b/crates/buzz-db/src/store/event.rs index 1fe5a9b5df3..341a9a8da8c 100644 --- a/crates/buzz-db/src/store/event.rs +++ b/crates/buzz-db/src/store/event.rs @@ -2322,7 +2322,7 @@ impl Db { /// /// Counts route on the BOUNDED arm only, never covered: the covered /// arm bounds insert-completeness but not deletion visibility (soft - /// deletes are UPDATEs outside the floor guard), and a count has no + /// deletes are UPDATEs outside the floor check), and a count has no /// downstream per-row re-filter to absorb extra rows — a silently /// inflated number for up to `FENCE_STALENESS` is a different product /// statement than a page briefly showing a deleted row. `Bounded` ties diff --git a/crates/buzz-db/src/store/event_floor_postgres_tests.rs b/crates/buzz-db/src/store/event_floor_postgres_tests.rs index 741e3b7d52a..bf7451763fb 100644 --- a/crates/buzz-db/src/store/event_floor_postgres_tests.rs +++ b/crates/buzz-db/src/store/event_floor_postgres_tests.rs @@ -1,4 +1,4 @@ -//! App-floor regressions with every floor-trigger clone disabled in the +//! App-floor regressions with no enabled floor-trigger clone in the //! nextest-provided per-test database, so the DB backstop cannot mask a bypass. use buzz_core::CommunityId; @@ -18,19 +18,14 @@ async fn fixture() -> (Db, CommunityId, Uuid, Keys) { .connect(&crate::test_support::database_url()) .await .expect("isolated test database"); - sqlx::query("ALTER TABLE events DISABLE TRIGGER events_created_at_floor") - .execute(&pool) - .await - .expect("disable floor trigger recursively on the partitioned table"); - let (total, enabled): (i64, i64) = sqlx::query_as( - "SELECT count(*), count(*) FILTER (WHERE tgenabled <> 'D') FROM pg_trigger \ - WHERE tgname = 'events_created_at_floor' \ + let enabled: i64 = sqlx::query_scalar( + "SELECT count(*) FROM pg_trigger \ + WHERE tgname = 'events_created_at_floor' AND tgenabled <> 'D' \ AND tgrelid IN (SELECT relid FROM pg_partition_tree('events'))", ) .fetch_one(&pool) .await .expect("check every trigger clone"); - assert!(total > 0, "the test must disable an existing backstop"); assert_eq!(enabled, 0, "no partition may mask a missing app check"); let community = CommunityId::from_uuid(Uuid::new_v4()); sqlx::query("INSERT INTO communities (id, host) VALUES ($1, $2)") diff --git a/crates/buzz-db/tests/observability_source.rs b/crates/buzz-db/tests/observability_source.rs index 60632e75dfb..1aab895598a 100644 --- a/crates/buzz-db/tests/observability_source.rs +++ b/crates/buzz-db/tests/observability_source.rs @@ -444,22 +444,11 @@ fn p0_pool_acquisitions_use_typed_operation_pairs_without_other() { let runtime = include_str!("../src/runtime/mod.rs"); assert!(runtime.contains("observability::refresh_pool_waiters(self.read_pool.is_some())")); - assert!(runtime.contains("self.verify_replica_fence_at_boot().await?")); - let fence_boot = runtime - .split_once("pub(crate) async fn verify_replica_fence_at_boot") - .expect("runtime must expose attributed boot fence verification") - .1 - .split_once("/// The pool for lag-tolerant reads") - .expect("boot fence verification must precede routed-read plumbing") - .0; - assert!(fence_boot.contains("WriterOperation::Bootstrap")); - let replica_fence = include_str!("../src/runtime/replica_fence.rs"); let replica_fence_production = replica_fence .split("\n#[cfg(test)]") .next() .expect("replica-fence production source"); - assert!(replica_fence_production.contains("WriterOperation::Bootstrap")); assert!(replica_fence_production.contains("WriterOperation::Maintenance")); assert!(!replica_fence_production.contains("pool.begin().await")); assert!(!replica_fence_production.contains("writer.acquire().await")); @@ -915,8 +904,7 @@ fn has_any_tenant_local_chokepoint(source: &str) -> bool { const ADMITTED_TX_SIGNATURE_MARKERS: [&str; 2] = ["&mut AdmittedTx", "&mut crate::AdmittedTx"]; // Narrow reviewed exceptions for non-serving verification probes only. -const GUARDED_WRITE_FUNCTION_EXCEPTIONS: [&str; 3] = [ - "pub async fn verify_floor_guard_behavior(", +const GUARDED_WRITE_FUNCTION_EXCEPTIONS: [&str; 2] = [ "pub async fn verify_channel_roster_fence_behavior(", "pub async fn backfill_d_tags(&self) -> Result {", ]; @@ -1422,10 +1410,8 @@ fn serving_table_writes_expose_syntactic_chokepoint_or_guarded_tx_routes() { // Rolled-back verification probes write `events` rows that never commit, so // they owe no push job or TTL refresh. -const EVENT_INSERT_FOLLOW_UP_EXCEPTIONS: [&str; 2] = [ - "pub async fn verify_floor_guard_behavior(", - "pub async fn verify_channel_roster_fence_behavior(", -]; +const EVENT_INSERT_FOLLOW_UP_EXCEPTIONS: [&str; 1] = + ["pub async fn verify_channel_roster_fence_behavior("]; /// `source` with every `#[cfg(test)]` item removed, wherever it sits in the /// file. Each item runs to the `;` that ends it or to the brace that closes @@ -1743,7 +1729,7 @@ pub(crate) async fn mention_writer(tx: &mut AdmittedTx) { .await .expect("write"); } -pub async fn verify_floor_guard_behavior(pool: &PgPool) { +pub async fn verify_channel_roster_fence_behavior(pool: &PgPool) { sqlx::query("INSERT INTO events (community_id, id) VALUES ($1, $2)") .execute(pool) .await @@ -1957,13 +1943,11 @@ fn every_production_event_insert_runs_the_follow_up_hook() { ); } -// These startup probes always roll back; they deliberately exercise the DB -// backstops rather than the app check. The two serving writers are exempt only +// This startup probe always rolls back; it deliberately exercises the roster +// DB backstop rather than the app check. The two serving writers are exempt only // while their INSERT remains structurally channel-NULL (checked below). -const EVENT_FLOOR_PROBE_EXCEPTIONS: [&str; 2] = [ - "pub async fn verify_floor_guard_behavior(", - "pub async fn verify_channel_roster_fence_behavior(", -]; +const EVENT_FLOOR_PROBE_EXCEPTIONS: [&str; 1] = + ["pub async fn verify_channel_roster_fence_behavior("]; fn is_channel_null_event_writer(function: &str) -> bool { let header = function_header(function); diff --git a/crates/buzz-relay/src/api/admin/mod.rs b/crates/buzz-relay/src/api/admin/mod.rs index b5cfa1ef490..c9e071b6b94 100644 --- a/crates/buzz-relay/src/api/admin/mod.rs +++ b/crates/buzz-relay/src/api/admin/mod.rs @@ -9029,18 +9029,6 @@ mod postgres_tests { // `fail_outbox_row` (not `mark_outbox_delivered`) when the insert itself // fails — so nothing is durably persisted, and the row is NOT marked delivered. // - // The failure is induced AFTER tenant resolution, inside `emit_system_message`'s - // `insert_event` call, by: - // 1. Building a dedicated test pool whose `after_connect` sets the - // `buzz.created_at_floor` GUC session-locally (not database-globally). - // Every connection from that pool inherits the floor; no other pool or - // test is affected, and there is no cleanup race on panic. - // 2. Backdating the outbox row's `created_at` beyond that floor. - // `emit_system_message` derives the Nostr event's `created_at` from - // `row.created_at` (the idempotency timestamp). With the floor active, the - // deferrable trigger fires on INSERT and raises a check_violation, which - // `insert_event` propagates as `Err`. The `?` in `emit_system_message` then - // propagates it up through `deliver_tombstone → deliver_one → fail_outbox_row`. let pool = e2e_pool().await; let (community_id, _host) = e2e_community(&pool, "race-c3").await; let target = vec![24u8; 32]; @@ -9144,43 +9132,22 @@ mod postgres_tests { .await .expect("insert fail-outbox row"); - // Backdate the outbox row's created_at so emit_system_message uses an old - // idempotency_ts. The events_created_at_floor trigger will reject the INSERT - // once we arm the GUC below. - sqlx::query( - "UPDATE relay_admin_outbox SET created_at = now() - interval '10 seconds' WHERE id = $1", + // Inject a durable insert failure only in this per-test database. This + // exercises error propagation and retry state independently of the floor. + sqlx::raw_sql( + "CREATE FUNCTION reject_c3_system_message() RETURNS trigger LANGUAGE plpgsql AS $$ \ + BEGIN IF NEW.kind = 40099 THEN \ + RAISE EXCEPTION 'injected system-message insert failure' USING ERRCODE = '23514'; \ + END IF; RETURN NEW; END $$; \ + CREATE TRIGGER reject_c3_system_message BEFORE INSERT ON events \ + FOR EACH ROW EXECUTE FUNCTION reject_c3_system_message();", ) - .bind(fail_outbox_id) .execute(&pool) .await - .expect("backdate outbox created_at"); - - // Build a dedicated pool whose after_connect sets buzz.created_at_floor = 5 - // session-locally on each connection (set_config 3rd arg false = session scope). - // A floor of 5 s means any event with created_at > 5 s ago is rejected. - // Our outbox row's created_at is ~10 s ago → trigger fires on insert_event. - // This pool is fully isolated: no other pool or test is affected, and there - // is no cleanup dependence (dropping the pool closes all its connections). - let db_url = database_url(); - let floor_pool = sqlx::postgres::PgPoolOptions::new() - .max_connections(4) - .after_connect(|conn, _meta| { - Box::pin(async move { - sqlx::query("SELECT set_config('buzz.created_at_floor', '5', false)") - .execute(conn) - .await?; - Ok(()) - }) - }) - .connect(&db_url) - .await - .expect("connect floor pool"); - - // Build an AppState around the floor pool so deliver_one's insert_event call - // runs on a connection where the deferrable trigger is active. - let fresh_state = state_from_pool(floor_pool.clone()).await; + .expect("install isolated insert failure"); + let fresh_state = state_from_pool(pool.clone()).await; - // Claim the row via the floor pool so deliver_one has a real claim token. + // Claim the row so deliver_one has a real claim token. let lease_until = chrono::Utc::now() + chrono::Duration::seconds(30); let mut batch = fresh_state .db @@ -9193,14 +9160,18 @@ mod postgres_tests { .expect("fail_outbox_id must be in batch"); let row = batch.remove(row_idx); - // deliver_one fails inside emit_system_message at insert_event (deferrable - // floor-guard trigger → check_violation) and must call fail_outbox_row — + // deliver_one fails inside emit_system_message at insert_event and + // must call fail_outbox_row — // NOT mark_outbox_delivered. crate::handlers::admin_outbox_worker::deliver_one(&fresh_state, &row).await; - // Drop the floor pool — all its connections close, GUC vanishes with them. - // No ALTER DATABASE, no global state, no reset required. - drop(floor_pool); + sqlx::raw_sql( + "DROP TRIGGER reject_c3_system_message ON events; \ + DROP FUNCTION reject_c3_system_message();", + ) + .execute(&pool) + .await + .expect("remove isolated insert failure"); // No tombstone event was persisted — the failure was inside insert_event. let post_event_count: i64 = sqlx::query_scalar( diff --git a/crates/buzz-relay/src/api/bridge/thread_window/postgres_tests.rs b/crates/buzz-relay/src/api/bridge/thread_window/postgres_tests.rs index 02ef106cdae..3f39f2e4101 100644 --- a/crates/buzz-relay/src/api/bridge/thread_window/postgres_tests.rs +++ b/crates/buzz-relay/src/api/bridge/thread_window/postgres_tests.rs @@ -30,7 +30,7 @@ impl Fixture { .await .unwrap(); let mut state = (*state).clone(); - // Use production after_connect policy (floor guard, isolation and + // Use production after_connect policy (isolation and // session timeouts), not raw SQLx pools that mask deployed failures. state.db = production_db(buzz_db::DbConfig::default()).await; Arc::make_mut(&mut state.config).require_auth_token = true; diff --git a/crates/buzz-relay/src/main.rs b/crates/buzz-relay/src/main.rs index 4ed127dfffa..031ef8e8de0 100644 --- a/crates/buzz-relay/src/main.rs +++ b/crates/buzz-relay/src/main.rs @@ -551,26 +551,11 @@ async fn run_relay_main(boot: BootTracker) -> anyhow::Result<()> { info!("Community deletion serving fences verified"); step.finish(); - // Freshness fence probe: cursor pages route to the replica only for - // history the probe has verified as fully replayed. Deliberately AFTER - // the migration decision: spawn_fence_probe first verifies the - // commit-time floor guard (catalog shape + observed behavior through the - // armed pool) against the live schema, so a relay running with - // BUZZ_AUTO_MIGRATE off and migration 0021 unapplied can never open the - // fence over an unenforced floor. Verification failure is loud but - // non-fatal: the fence stays closed and every cursor page routes to the - // writer. - let mut step = StepTimer::start(StartupStep::ReplicaFenceProbe); - match db.spawn_fence_probe().await { - Ok(true) => info!("Replica fence probe started (floor guard verified)"), - Ok(false) => {} - Err(e) => { - step.degrade(); - error!( - "Replica fence disabled — floor guard verification failed: {e}. \ - All cursor reads stay on the writer." - ); - } + // The ordered heartbeat probe opens replica routing only after a + // successful sample. Errors keep cursor reads on the writer. + let step = StepTimer::start(StartupStep::ReplicaFenceProbe); + if db.spawn_fence_probe() { + info!("Replica fence probe started"); } step.finish(); diff --git a/crates/buzz-relay/src/startup_steps.rs b/crates/buzz-relay/src/startup_steps.rs index 3980b6b86af..ee248e6395a 100644 --- a/crates/buzz-relay/src/startup_steps.rs +++ b/crates/buzz-relay/src/startup_steps.rs @@ -47,7 +47,7 @@ pub enum StartupStep { PartitionEnsure, /// Verify the community-deletion serving fences. DeletionFenceVerify, - /// Verify the replica floor guard and start the fence probe. + /// Start the replica freshness fence probe. ReplicaFenceProbe, /// Ensure the deployment community, backfill the allowlist, and bootstrap the owner. MembershipBootstrap, diff --git a/deploy/charts/buzz/README.md b/deploy/charts/buzz/README.md index 5704a7622b8..4cc1901ea4a 100644 --- a/deploy/charts/buzz/README.md +++ b/deploy/charts/buzz/README.md @@ -354,7 +354,7 @@ can give it to a caller. | `buzz_db_connection_step_duration_seconds` | histogram | `pool_role`, `step` | | `buzz_db_connection_step_attempts_total` | counter | `pool_role`, `step`, `outcome` | -The fixed steps are `writer_pool`, `physical_connect`, `created_at_floor`, +The fixed steps are `writer_pool`, `physical_connect`, `session_timeouts`, `isolation`, and `ready`. Measurable phases emit start, duration, and terminal evidence. `physical_connect` and `ready` are success milestones: SQLx 0.9 exposes `after_connect` only after DNS, network, TLS, and @@ -368,8 +368,8 @@ start counter to derive the number of attempts currently in progress. The session phases run in order, so a later phase start also proves the earlier phases succeeded for that attempt. -Four start counters, four 13-series histograms, and fifteen terminal counters -create a hard ceiling of 71 raw Prometheus series per pod. Connection ordinals, +Three start counters, three 13-series histograms, and twelve terminal counters +create a hard ceiling of 54 raw Prometheus series per pod. Connection ordinals, database URLs, hosts, usernames, SQL, and raw errors are forbidden as metric labels. diff --git a/migrations/0062_retire_created_at_floor.sql b/migrations/0062_retire_created_at_floor.sql new file mode 100644 index 00000000000..fac3f74364d --- /dev/null +++ b/migrations/0062_retire_created_at_floor.sql @@ -0,0 +1,26 @@ +SET LOCAL lock_timeout = '5s'; + +-- DESTRUCTIVE CONTRACT: deploy phase 1's application floor check to every +-- writer pod/job first, with no rollback expected. Stop out-of-band writers +-- and hold the replica breaker closed through this drop until replay catches +-- up. Old pods that already started do not recheck the guard on each probe. +-- Keep the 960-second application floor and the ordered heartbeat handshake. +-- No other trigger, function, FK, or tenant fence is retired here. +DROP TRIGGER IF EXISTS events_created_at_floor ON events; + +-- Dropping the parent removes its inherited clones (including partitions +-- created since 0021). Also remove any independently attached child copy. +DO $$ +DECLARE + partition_table regclass; +BEGIN + FOR partition_table IN + SELECT relid FROM pg_partition_tree('events'::regclass) + WHERE relid <> 'events'::regclass + LOOP + EXECUTE format('DROP TRIGGER IF EXISTS events_created_at_floor ON %s', partition_table); + END LOOP; +END +$$; + +DROP FUNCTION IF EXISTS events_created_at_floor_guard(); diff --git a/schema/functions/events_created_at_floor_guard.sql b/schema/functions/events_created_at_floor_guard.sql deleted file mode 100644 index 263886f83f9..00000000000 --- a/schema/functions/events_created_at_floor_guard.sql +++ /dev/null @@ -1,29 +0,0 @@ --- Replica-fence floor guard (keep in sync with migrations/0021). A deferred --- constraint trigger re-checks, inside COMMIT processing, that channel-bearing --- event rows are no older than `buzz.created_at_floor` seconds before commit --- time (clock_timestamp(), NOT the transaction-frozen now()). This turns the --- relay's ingest-time created_at envelope into a commit-time storage --- invariant, which is what lets keyset-cursor pages below the replica fence --- be served by a read replica without holes. Enforcement is armed per session --- via the GUC (set by the relay's writer pool on connect); sessions without --- the GUC (pg_restore, manual backfills) bypass it and must hold the replica --- fence closed for their duration. The only structural exemption is --- channel_id IS NULL: those rows never appear in keyset-paged windows. -CREATE FUNCTION events_created_at_floor_guard() RETURNS trigger -LANGUAGE plpgsql AS $$ -DECLARE - floor_secs numeric := nullif(current_setting('buzz.created_at_floor', true), '')::numeric; -BEGIN - IF floor_secs IS NOT NULL - AND floor_secs > 0 - AND NEW.channel_id IS NOT NULL - AND NEW.created_at < clock_timestamp() - make_interval(secs => floor_secs) - THEN - RAISE EXCEPTION - 'events.created_at % is more than % s before commit time %; below the replica-fence floor', - NEW.created_at, floor_secs, clock_timestamp() - USING ERRCODE = 'check_violation'; - END IF; - RETURN NULL; -END -$$; diff --git a/schema/schema.sql b/schema/schema.sql index 5ec8bd5829a..30670abb820 100644 --- a/schema/schema.sql +++ b/schema/schema.sql @@ -67,7 +67,6 @@ CREATE EXTENSION IF NOT EXISTS pgcrypto; \i functions/enqueue_push_match_job.sql \i functions/refresh_channel_ttl_after_event_insert.sql \i functions/guard_channel_roster_snapshot.sql -\i functions/events_created_at_floor_guard.sql \i functions/prevent_community_deletion_request_retargeting.sql \i functions/prevent_community_deletion_approval_removal.sql \i functions/protect_community_deletion_manifest_keys.sql diff --git a/schema/triggers/events.sql b/schema/triggers/events.sql index 5491d1631d6..aea4facc0e9 100644 --- a/schema/triggers/events.sql +++ b/schema/triggers/events.sql @@ -10,15 +10,4 @@ CREATE TRIGGER trg_events_guard_channel_roster_snapshot BEFORE INSERT ON events FOR EACH ROW EXECUTE FUNCTION guard_channel_roster_snapshot(); --- INSERT OR UPDATE OF: an UPDATE can move a previously exempt row into the --- guarded set (channel_id NULL -> NOT NULL) or move a channel row's --- created_at below the fence, so both mutation paths re-run the guard on the --- NEW row. A created_at rewrite that crosses partition bounds runs as --- DELETE + INSERT and hits the cloned AFTER INSERT guard on the destination --- partition; an in-partition rewrite fires the UPDATE OF arm. -CREATE CONSTRAINT TRIGGER events_created_at_floor - AFTER INSERT OR UPDATE OF created_at, channel_id ON events - DEFERRABLE INITIALLY DEFERRED - FOR EACH ROW - EXECUTE FUNCTION events_created_at_floor_guard(); SELECT attach_community_write_fence('events'); diff --git a/scripts/reconcile-schema-after-pgschema.sql b/scripts/reconcile-schema-after-pgschema.sql index 7eef8c4bbe5..131c1085bd3 100644 --- a/scripts/reconcile-schema-after-pgschema.sql +++ b/scripts/reconcile-schema-after-pgschema.sql @@ -18,7 +18,6 @@ BEGIN -- triggers while attaching and rejects same-named child triggers. DROP TRIGGER IF EXISTS events_enqueue_push_match ON events_p_past; DROP TRIGGER IF EXISTS events_refresh_channel_ttl ON events_p_past; - DROP TRIGGER IF EXISTS events_created_at_floor ON events_p_past; DROP TRIGGER IF EXISTS community_write_fence_events ON events_p_past; DROP TRIGGER IF EXISTS trg_events_guard_channel_roster_snapshot ON events_p_past; ALTER TABLE events ATTACH PARTITION events_p_past @@ -32,7 +31,6 @@ BEGIN ) THEN DROP TRIGGER IF EXISTS events_enqueue_push_match ON events_p2026_01; DROP TRIGGER IF EXISTS events_refresh_channel_ttl ON events_p2026_01; - DROP TRIGGER IF EXISTS events_created_at_floor ON events_p2026_01; DROP TRIGGER IF EXISTS community_write_fence_events ON events_p2026_01; DROP TRIGGER IF EXISTS trg_events_guard_channel_roster_snapshot ON events_p2026_01; ALTER TABLE events ATTACH PARTITION events_p2026_01 @@ -46,7 +44,6 @@ BEGIN ) THEN DROP TRIGGER IF EXISTS events_enqueue_push_match ON events_p2026_02; DROP TRIGGER IF EXISTS events_refresh_channel_ttl ON events_p2026_02; - DROP TRIGGER IF EXISTS events_created_at_floor ON events_p2026_02; DROP TRIGGER IF EXISTS community_write_fence_events ON events_p2026_02; DROP TRIGGER IF EXISTS trg_events_guard_channel_roster_snapshot ON events_p2026_02; ALTER TABLE events ATTACH PARTITION events_p2026_02 @@ -60,7 +57,6 @@ BEGIN ) THEN DROP TRIGGER IF EXISTS events_enqueue_push_match ON events_p2026_03; DROP TRIGGER IF EXISTS events_refresh_channel_ttl ON events_p2026_03; - DROP TRIGGER IF EXISTS events_created_at_floor ON events_p2026_03; DROP TRIGGER IF EXISTS community_write_fence_events ON events_p2026_03; DROP TRIGGER IF EXISTS trg_events_guard_channel_roster_snapshot ON events_p2026_03; ALTER TABLE events ATTACH PARTITION events_p2026_03 @@ -74,7 +70,6 @@ BEGIN ) THEN DROP TRIGGER IF EXISTS events_enqueue_push_match ON events_p2026_04; DROP TRIGGER IF EXISTS events_refresh_channel_ttl ON events_p2026_04; - DROP TRIGGER IF EXISTS events_created_at_floor ON events_p2026_04; DROP TRIGGER IF EXISTS community_write_fence_events ON events_p2026_04; DROP TRIGGER IF EXISTS trg_events_guard_channel_roster_snapshot ON events_p2026_04; ALTER TABLE events ATTACH PARTITION events_p2026_04 @@ -88,7 +83,6 @@ BEGIN ) THEN DROP TRIGGER IF EXISTS events_enqueue_push_match ON events_p2026_05; DROP TRIGGER IF EXISTS events_refresh_channel_ttl ON events_p2026_05; - DROP TRIGGER IF EXISTS events_created_at_floor ON events_p2026_05; DROP TRIGGER IF EXISTS community_write_fence_events ON events_p2026_05; DROP TRIGGER IF EXISTS trg_events_guard_channel_roster_snapshot ON events_p2026_05; ALTER TABLE events ATTACH PARTITION events_p2026_05 @@ -102,7 +96,6 @@ BEGIN ) THEN DROP TRIGGER IF EXISTS events_enqueue_push_match ON events_p2026_06; DROP TRIGGER IF EXISTS events_refresh_channel_ttl ON events_p2026_06; - DROP TRIGGER IF EXISTS events_created_at_floor ON events_p2026_06; DROP TRIGGER IF EXISTS community_write_fence_events ON events_p2026_06; DROP TRIGGER IF EXISTS trg_events_guard_channel_roster_snapshot ON events_p2026_06; ALTER TABLE events ATTACH PARTITION events_p2026_06 @@ -116,7 +109,6 @@ BEGIN ) THEN DROP TRIGGER IF EXISTS events_enqueue_push_match ON events_p_future; DROP TRIGGER IF EXISTS events_refresh_channel_ttl ON events_p_future; - DROP TRIGGER IF EXISTS events_created_at_floor ON events_p_future; DROP TRIGGER IF EXISTS community_write_fence_events ON events_p_future; DROP TRIGGER IF EXISTS trg_events_guard_channel_roster_snapshot ON events_p_future; ALTER TABLE events ATTACH PARTITION events_p_future From 09fb7443c7c781c080289b43761bedb3d67731aa Mon Sep 17 00:00:00 2001 From: Elrond <28d6302a099e5225b02c4155ac4236e4912603df2ab08dbfc2f4fef08ce598c8@buzz.block.builderlab.xyz> Date: Wed, 14 Oct 2026 14:23:09 -0400 Subject: [PATCH 06/10] buzz-db: trim retirement tests and fix stale floor prose Address review of the phase-2 floor retirement: - the workspace-wide floor source policy now comes from phase 1's shared crates/*/src walk (which pins buzz-relay coverage); this commit only drops the GUC from its constant-parity message - drop the legacy-0021 half of the writer/probe test, which could not fail, and rename the desired-state test to what it checks - fix stale GUC prose and document the ACCESS EXCLUSIVE lock in 0062 Signed-off-by: Elrond <28d6302a099e5225b02c4155ac4236e4912603df2ab08dbfc2f4fef08ce598c8@buzz.block.builderlab.xyz> --- crates/buzz-db/src/runtime/replica_fence.rs | 6 ++-- .../src/runtime/tests/floor_retirement.rs | 28 +++---------------- crates/buzz-db/tests/observability_source.rs | 2 +- migrations/0062_retire_created_at_floor.sql | 3 ++ 4 files changed, 11 insertions(+), 28 deletions(-) diff --git a/crates/buzz-db/src/runtime/replica_fence.rs b/crates/buzz-db/src/runtime/replica_fence.rs index f1fb75bf755..8117a2690c2 100644 --- a/crates/buzz-db/src/runtime/replica_fence.rs +++ b/crates/buzz-db/src/runtime/replica_fence.rs @@ -52,9 +52,9 @@ //! mismatch, or an observed token below every retained entry all route the //! request back to the writer — degraded capacity, never holes. //! -//! Raw SQL without the armed GUC and `session_replication_role = replica` -//! restores bypass the supported app writers. These operational bypasses are -//! outside the proof and require holding the fence closed; see `migrations/0021`. +//! Raw SQL and `session_replication_role = replica` restores bypass the +//! application floor check. These operational bypasses are outside the proof +//! and require holding the fence closed. use std::collections::VecDeque; use std::sync::{Arc, Mutex}; diff --git a/crates/buzz-db/src/runtime/tests/floor_retirement.rs b/crates/buzz-db/src/runtime/tests/floor_retirement.rs index 6fe582a23e4..fbf3e20c5ea 100644 --- a/crates/buzz-db/src/runtime/tests/floor_retirement.rs +++ b/crates/buzz-db/src/runtime/tests/floor_retirement.rs @@ -96,28 +96,20 @@ async fn migration_schema_floor_retirement_cleans_child_copies_idempotently() { #[tokio::test] #[ignore = "requires Postgres"] -async fn floor_retirement_desired_and_migrated_catalogs_match() { +async fn floor_retirement_desired_state_has_no_floor_objects() { // The nextest wrapper bootstraps this database through real pgschema apply - // and reconciliation. Check the retired floor catalog against the real - // migration boot path; unrelated trigger/FK preservation is checked above. + // and reconciliation; desired-state/migration parity is checked by + // `admin_schema_parity_between_desired_state_and_migrations`. let pool = PgPool::connect(&crate::test_support::database_url()) .await .expect("connect desired-state database"); assert_eq!(floor_objects(&pool).await, (0, 0)); - sqlx::raw_sql("DROP SCHEMA public CASCADE; CREATE SCHEMA public;") - .execute(&pool) - .await - .expect("reset isolated database for migration boot"); - crate::migration::run_migrations(&pool) - .await - .expect("boot migration schema"); - assert_eq!(floor_objects(&pool).await, (0, 0)); pool.close().await; } #[tokio::test] #[ignore = "requires Postgres"] -async fn floor_retirement_writer_and_probe_accept_legacy_and_retired_schema() { +async fn floor_retirement_writer_and_probe_run_on_retired_schema() { let url = crate::test_support::database_url(); let db = Db::new(&DbConfig { database_url: url.clone(), @@ -140,18 +132,6 @@ async fn floor_retirement_writer_and_probe_accept_legacy_and_retired_schema() { .await .expect("retired schema still supports the ordered heartbeat"); - // This binary can also run before migration, without changing legacy SQL. - sqlx::raw_sql(include_str!( - "../../../../../migrations/0021_created_at_fence_floor.sql" - )) - .execute(&db.pool) - .await - .expect("restore actual legacy guard"); - assert!(floor_objects(&db.pool).await.0 > 1); - assert!(db.spawn_fence_probe()); - crate::replica_fence::probe_once(&db.pool, db.fence()) - .await - .expect("legacy schema still supports ordered heartbeat"); db.pool.close().await; db.read_pool .as_ref() diff --git a/crates/buzz-db/tests/observability_source.rs b/crates/buzz-db/tests/observability_source.rs index 1aab895598a..3e176621aa3 100644 --- a/crates/buzz-db/tests/observability_source.rs +++ b/crates/buzz-db/tests/observability_source.rs @@ -2169,7 +2169,7 @@ fn event_floor_guard_uses_the_database_statement_clock_and_shared_floor() { ); assert!( guard.contains(".bind(CREATED_AT_FLOOR_SECS as f64)"), - "app check, GUC and fence wall must share the same floor constant" + "app check and fence wall must share the same floor constant" ); } diff --git a/migrations/0062_retire_created_at_floor.sql b/migrations/0062_retire_created_at_floor.sql index fac3f74364d..057171e9752 100644 --- a/migrations/0062_retire_created_at_floor.sql +++ b/migrations/0062_retire_created_at_floor.sql @@ -5,6 +5,9 @@ SET LOCAL lock_timeout = '5s'; -- and hold the replica breaker closed through this drop until replay catches -- up. Old pods that already started do not recheck the guard on each probe. -- Keep the 960-second application floor and the ordered heartbeat handshake. +-- DROP TRIGGER takes ACCESS EXCLUSIVE on events and every partition, so it +-- can stall events reads/writes for up to lock_timeout. Run off-peak; a lock +-- timeout aborts the whole migration and is safe to retry. -- No other trigger, function, FK, or tenant fence is retired here. DROP TRIGGER IF EXISTS events_created_at_floor ON events; From 6f0d888a8aa638f8f11adc0a1284bff16c170e44 Mon Sep 17 00:00:00 2001 From: Elrond <28d6302a099e5225b02c4155ac4236e4912603df2ab08dbfc2f4fef08ce598c8@buzz.block.builderlab.xyz> Date: Wed, 14 Oct 2026 15:33:42 -0400 Subject: [PATCH 07/10] buzz-db: catch SET( and ON CONFLICT DO UPDATE in the floor policy With the floor trigger retired, the source policy is the only guard against rewriting created_at/channel_id on existing event rows. Close two matcher gaps found in review: - `UPDATE events SET(created_at, ...)` with no space before the parenthesis slipped past the " set " split - `INSERT INTO events ... ON CONFLICT ... DO UPDATE SET channel_id = ...` names no table, so it passed even though it can re-home an existing row below the floor; bind each DO UPDATE to the INSERT it belongs to and treat it as an events update when that INSERT targets events Share the SET-list check between both paths, and pin each gap with a negative fixture plus controls: a non-floor-column events upsert, DO NOTHING, and a guarded events insert next to a watermark upsert that sets created_at (the shape of the replaceable-event writer). Signed-off-by: Elrond <28d6302a099e5225b02c4155ac4236e4912603df2ab08dbfc2f4fef08ce598c8@buzz.block.builderlab.xyz> --- crates/buzz-db/tests/observability_source.rs | 96 +++++++++++++++----- 1 file changed, 73 insertions(+), 23 deletions(-) diff --git a/crates/buzz-db/tests/observability_source.rs b/crates/buzz-db/tests/observability_source.rs index 3e176621aa3..ba628a715a5 100644 --- a/crates/buzz-db/tests/observability_source.rs +++ b/crates/buzz-db/tests/observability_source.rs @@ -1963,9 +1963,36 @@ fn is_channel_null_event_writer(function: &str) -> bool { false } +/// Whether a `SET` list assigns `created_at` or `channel_id`, including the +/// row form `(a, b) = (...)`. Stops at the first `WHERE`. +fn assigns_event_floor_column(assignments: &str) -> bool { + let assignments = assignments.split(" where ").next().expect("first part"); + let mut depth = 0usize; + assignments + .split(|ch| { + match ch { + '(' => depth += 1, + ')' => depth = depth.saturating_sub(1), + _ => {} + } + ch == ',' && depth == 0 + }) + .any(|assignment| { + assignment.split_once('=').is_some_and(|(columns, _)| { + columns + .trim() + .trim_start_matches('(') + .trim_end_matches(')') + .split(',') + .any(|column| matches!(column.trim(), "created_at" | "channel_id")) + }) + }) +} + fn updates_event_floor_columns(function: &str) -> bool { - let normalized = normalized_sql_source(function); - normalized.match_indices("update ").any(|(index, marker)| { + // `SET(` with no space is still a row assignment. + let normalized = normalized_sql_source(function).replace(" set(", " set ("); + let direct_update = normalized.match_indices("update ").any(|(index, marker)| { let Some((target, assignments)) = normalized[index + marker.len()..].split_once(" set ") else { return false; @@ -1983,28 +2010,24 @@ fn updates_event_floor_columns(function: &str) -> bool { if !matches!(alias.as_slice(), [] | [_] | ["as", _]) { return false; } - let assignments = assignments.split(" where ").next().expect("first part"); - let mut depth = 0usize; - assignments - .split(|ch| { - match ch { - '(' => depth += 1, - ')' => depth = depth.saturating_sub(1), - _ => {} - } - ch == ',' && depth == 0 - }) - .any(|assignment| { - assignment.split_once('=').is_some_and(|(columns, _)| { - columns - .trim() - .trim_start_matches('(') - .trim_end_matches(')') - .split(',') - .any(|column| matches!(column.trim(), "created_at" | "channel_id")) + assigns_event_floor_column(assignments) + }); + // `ON CONFLICT ... DO UPDATE SET` names no table: it updates the target of + // the INSERT it belongs to, so an events upsert can re-home an existing row. + let events_upsert = normalized + .match_indices("do update set ") + .any(|(index, marker)| { + normalized[..index] + .rfind("insert into ") + .is_some_and(|insert| { + let table = normalized[insert + "insert into ".len()..] + .split(|ch: char| !(ch.is_ascii_alphanumeric() || ch == '_' || ch == '.')) + .next(); + matches!(table, Some("events" | "public.events")) }) - }) - }) + && assigns_event_floor_column(&normalized[index + marker.len()..]) + }); + direct_update || events_upsert } fn event_floor_policy_violations(production: &str) -> Vec { @@ -2078,6 +2101,30 @@ async fn quoted_schema_update(tx: &mut AdmittedTx) { async fn delete_only(tx: &mut AdmittedTx) { sqlx::query("UPDATE events SET deleted_at=now() WHERE channel_id=$1 AND created_at<$2"); } +async fn row_time_no_space(tx: &mut AdmittedTx) { + sqlx::query("UPDATE events SET(created_at,kind)=($1,1) WHERE id=$2"); +} +async fn guarded_upsert_rehome(tx: &mut AdmittedTx) { + crate::replica_fence::guard_event_created_at_floor(tx, created_at, channel).await?; + sqlx::query("INSERT INTO events (id, channel_id) VALUES ($1, $2) ON CONFLICT (id) DO UPDATE SET channel_id = EXCLUDED.channel_id"); +} +async fn guarded_upsert_row_time(tx: &mut AdmittedTx) { + crate::replica_fence::guard_event_created_at_floor(tx, created_at, channel).await?; + sqlx::query("INSERT INTO events (id) VALUES ($1) ON CONFLICT (id) DO UPDATE SET(kind, created_at) = (1, $2)"); +} +async fn guarded_upsert_other_column(tx: &mut AdmittedTx) { + crate::replica_fence::guard_event_created_at_floor(tx, created_at, channel).await?; + sqlx::query("INSERT INTO events (id) VALUES ($1) ON CONFLICT (id) DO UPDATE SET tags = EXCLUDED.tags"); +} +async fn guarded_upsert_do_nothing(tx: &mut AdmittedTx) { + crate::replica_fence::guard_event_created_at_floor(tx, created_at, channel).await?; + sqlx::query("INSERT INTO events (id) VALUES ($1) ON CONFLICT (id) DO NOTHING"); +} +async fn guarded_insert_then_other_table_upsert(tx: &mut AdmittedTx) { + crate::replica_fence::guard_event_created_at_floor(tx, created_at, channel).await?; + sqlx::query("INSERT INTO events (id) VALUES ($1) ON CONFLICT DO NOTHING"); + sqlx::query("INSERT INTO parameterized_event_watermarks (d_tag, created_at) VALUES ($1, $2) ON CONFLICT (d_tag) DO UPDATE SET created_at = EXCLUDED.created_at"); +} "#; assert_eq!( event_floor_policy_violations(source), @@ -2095,6 +2142,9 @@ async fn delete_only(tx: &mut AdmittedTx) { "async fn quoted_schema_insert(tx: &mut AdmittedTx) {", "async fn quoted_update(tx: &mut AdmittedTx) {", "async fn quoted_schema_update(tx: &mut AdmittedTx) {", + "async fn row_time_no_space(tx: &mut AdmittedTx) {", + "async fn guarded_upsert_rehome(tx: &mut AdmittedTx) {", + "async fn guarded_upsert_row_time(tx: &mut AdmittedTx) {", ] ); } From 8752ce1245be270751f2a806633971cf3ad59763 Mon Sep 17 00:00:00 2001 From: Elrond <28d6302a099e5225b02c4155ac4236e4912603df2ab08dbfc2f4fef08ce598c8@buzz.block.builderlab.xyz> Date: Fri, 16 Oct 2026 10:04:31 -0400 Subject: [PATCH 08/10] buzz-db: fail the floor policy closed on SQL it cannot attribute With the trigger gone the source policy is the only guard. Expose SQL the per-function scan could not see: include_str! is expanded from the file it loads and concat! literals are joined, so both reach the existing insert/update checks. Fail closed on what stays opaque: a non-literal include path, a concat! that spells SQL around a non-literal argument, event-write SQL in a const, static or macro body outside any function, and PREPARE TRANSACTION in any production writer. Cut each function slice to its own item so trailing consts are judged at item level. The replica fence no longer folds pg_prepared_xacts.prepared into the wall: only the retired trigger checked prepared rows at PREPARE, and the insert-time guard's clock is not exposed there. A prepared transaction now fails the sample closed (PreparedTransactions). Signed-off-by: Elrond <28d6302a099e5225b02c4155ac4236e4912603df2ab08dbfc2f4fef08ce598c8@buzz.block.builderlab.xyz> --- crates/buzz-db/src/runtime/replica_fence.rs | 93 ++++- crates/buzz-db/tests/observability_source.rs | 390 ++++++++++++++++++- 2 files changed, 462 insertions(+), 21 deletions(-) diff --git a/crates/buzz-db/src/runtime/replica_fence.rs b/crates/buzz-db/src/runtime/replica_fence.rs index 8117a2690c2..9a9e378db46 100644 --- a/crates/buzz-db/src/runtime/replica_fence.rs +++ b/crates/buzz-db/src/runtime/replica_fence.rs @@ -23,7 +23,8 @@ //! (a) finished before the activity scan — its commit precedes `M`'s //! commit, so the replica session has replayed it; //! (b) open at the activity scan — represented by `xact_start`, so it is -//! bounded by the `oldest_xact_start` term; +//! bounded by the `oldest_xact_start` term (a prepared transaction has +//! no `xact_start` to read, so one existing fails the sample closed); //! (c) started after the activity scan — its insert-time app guard runs //! after `S`, so it cannot commit a row with //! `created_at < S - floor`. @@ -383,6 +384,17 @@ pub enum ProbeError { /// Number of other client backends with masked/unknown state. masked: i64, }, + /// A prepared (2PC) transaction exists. Its rows were floor-checked at + /// insert time, which `pg_prepared_xacts` cannot bound, and no supported + /// writer prepares transactions — treat it as out-of-band. + #[error( + "{prepared} prepared transaction(s) in pg_prepared_xacts — out-of-band writer; \ + replica floor cannot bound their rows" + )] + PreparedTransactions { + /// Number of prepared transactions in the cluster. + prepared: i64, + }, /// The single heartbeat row (migration 0026) is missing on the writer. #[error("replica_heartbeat row missing on the writer — migration 0026 not applied?")] HeartbeatRowMissing, @@ -434,18 +446,19 @@ async fn sample_writer(writer: &PgPool) -> Result { // // Prepared transactions (2PC) are a bucket of their own: while // prepared they have left `pg_stat_activity` but can still commit - // after the token. Their floor check must precede PREPARE, so - // `pg_prepared_xacts.prepared` bounds their rows exactly like - // `xact_start`; fold it into the same minimum. + // after the token. Only the retired deferred trigger checked their + // rows at PREPARE; the insert-time app guard runs earlier, at a clock + // `pg_prepared_xacts` does not expose, so `prepared` no longer bounds + // their rows. No supported writer issues PREPARE TRANSACTION (the + // source policy enforces it) and `max_prepared_transactions` defaults + // to 0, so any prepared transaction is out-of-band: fail closed. let row = sqlx::query( r#" SELECT - least( - (SELECT min(xact_start) - FROM pg_stat_activity - WHERE pid <> pg_backend_pid()), - (SELECT min(prepared) FROM pg_prepared_xacts) - ) AS oldest_xact_start, + (SELECT min(xact_start) + FROM pg_stat_activity + WHERE pid <> pg_backend_pid()) AS oldest_xact_start, + (SELECT count(*) FROM pg_prepared_xacts) AS prepared, (SELECT count(*) FROM pg_stat_activity WHERE pid <> pg_backend_pid() @@ -462,6 +475,10 @@ async fn sample_writer(writer: &PgPool) -> Result { if masked > 0 { return Err(ProbeError::MaskedActivity { masked }); } + let prepared: i64 = row.get("prepared"); + if prepared > 0 { + return Err(ProbeError::PreparedTransactions { prepared }); + } let oldest_xact_start: Option> = row.get("oldest_xact_start"); // 3. Token commit LAST, on the same pinned connection. The single-row @@ -978,6 +995,62 @@ mod postgres_tests { .expect("drop role"); } + /// With the floor trigger retired, nothing bounds a prepared (2PC) + /// transaction's rows by `pg_prepared_xacts.prepared`, so the sample must + /// fail closed while one exists. On a server with the default + /// `max_prepared_transactions = 0` (CI, compose, chart) the test instead + /// pins that 2PC is unavailable, which is the supported-writer exclusion. + #[tokio::test] + #[ignore = "requires Postgres"] + async fn cluster_global_sample_writer_fails_closed_on_prepared_transaction() { + let pool = PgPool::connect(&crate::test_support::database_url()) + .await + .expect("connect"); + let max_prepared: String = sqlx::query_scalar("SHOW max_prepared_transactions") + .fetch_one(&pool) + .await + .expect("show max_prepared_transactions"); + + let gid = format!("fence_probe_{}", uuid::Uuid::new_v4().simple()); + let mut conn = pool.acquire().await.expect("acquire"); + sqlx::query("BEGIN") + .execute(&mut *conn) + .await + .expect("begin"); + sqlx::query("SELECT 1") + .execute(&mut *conn) + .await + .expect("touch tx"); + let prepare = sqlx::query(sqlx::AssertSqlSafe(format!("PREPARE TRANSACTION '{gid}'"))) + .execute(&mut *conn) + .await; + + if max_prepared == "0" { + prepare.expect_err("PREPARE TRANSACTION must fail when 2PC is disabled"); + sqlx::query("ROLLBACK") + .execute(&mut *conn) + .await + .expect("rollback"); + return; + } + + prepare.expect("prepare transaction"); + drop(conn); + let result = sample_writer(&pool).await; + sqlx::query(sqlx::AssertSqlSafe(format!("ROLLBACK PREPARED '{gid}'"))) + .execute(&pool) + .await + .expect("rollback prepared"); + let err = result.expect_err("a prepared transaction must fail the sample closed"); + assert!( + matches!(err, ProbeError::PreparedTransactions { prepared } if prepared >= 1), + "expected PreparedTransactions, got {err:?}" + ); + sample_writer(&pool) + .await + .expect("sample succeeds once no transaction is prepared"); + } + /// The Aurora PostgreSQL identity function name is exact — the /// MySQL-family near-miss (`aurora_server_id`) would make the /// capability probe cache a permanent false on real Aurora and diff --git a/crates/buzz-db/tests/observability_source.rs b/crates/buzz-db/tests/observability_source.rs index ba628a715a5..939384e325f 100644 --- a/crates/buzz-db/tests/observability_source.rs +++ b/crates/buzz-db/tests/observability_source.rs @@ -1483,21 +1483,60 @@ fn raw_literal_prefix_starts_token(bytes: &[u8], index: usize) -> bool { fn mask_comments_and_literals(source: &str) -> String { let bytes = source.as_bytes(); let mut masked = bytes.to_vec(); - let mut blank = |from: usize, to: usize| { - for byte in &mut masked[from..to.min(bytes.len())] { + for span in source_spans(source) { + for byte in &mut masked[span.start..span.end] { if *byte != b'\n' { *byte = b' '; } } - }; + } + String::from_utf8(masked).expect("masking replaces whole ASCII-delimited spans") +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum SpanKind { + Comment, + /// A plain (`"…"`) string literal; `start` is its opening quote. + Str, + /// A raw string literal; `start` is its `r`, followed by `hashes` `#`s. + RawStr { + hashes: usize, + }, + Char, +} + +struct SourceSpan { + start: usize, + end: usize, + kind: SpanKind, +} + +impl SourceSpan { + /// The text between a string literal's delimiters, escapes left as written. + fn string_body<'a>(&self, source: &'a str) -> Option<&'a str> { + match self.kind { + SpanKind::Str => source.get(self.start + 1..self.end.saturating_sub(1)), + SpanKind::RawStr { hashes } => { + source.get(self.start + 2 + hashes..self.end.saturating_sub(1 + hashes)) + } + SpanKind::Comment | SpanKind::Char => None, + } + } +} + +/// Every comment, string, raw string, and char literal in `source`, in order. +fn source_spans(source: &str) -> Vec { + let bytes = source.as_bytes(); + let mut spans = Vec::new(); let mut index = 0usize; while index < bytes.len() { let start = index; - match bytes[index] { + let kind = match bytes[index] { b'/' if bytes.get(index + 1) == Some(&b'/') => { while index < bytes.len() && bytes[index] != b'\n' { index += 1; } + SpanKind::Comment } b'/' if bytes.get(index + 1) == Some(&b'*') => { let mut depth = 0usize; @@ -1515,6 +1554,7 @@ fn mask_comments_and_literals(source: &str) -> String { index += 1; } } + SpanKind::Comment } b'r' if matches!(bytes.get(index + 1), Some(b'"' | b'#')) && raw_literal_prefix_starts_token(bytes, index) @@ -1536,6 +1576,7 @@ fn mask_comments_and_literals(source: &str) -> String { index = source[body..] .find(&close) .map_or(bytes.len(), |offset| body + offset + close.len()); + SpanKind::RawStr { hashes } } b'"' => { index += 1; @@ -1543,18 +1584,24 @@ fn mask_comments_and_literals(source: &str) -> String { index += if bytes[index] == b'\\' { 2 } else { 1 }; } index += 1; + SpanKind::Str } b'\'' if char_literal_end(source, index).is_some() => { index = char_literal_end(source, index).expect("guard matched"); + SpanKind::Char } _ => { index += 1; continue; } - } - blank(start, index); + }; + spans.push(SourceSpan { + start, + end: index.min(bytes.len()), + kind, + }); } - String::from_utf8(masked).expect("masking replaces whole ASCII-delimited spans") + spans } /// The end (exclusive) of the char literal whose opening quote is at `start`, @@ -2030,8 +2077,247 @@ fn updates_event_floor_columns(function: &str) -> bool { direct_update || events_upsert } -fn event_floor_policy_violations(production: &str) -> Vec { +/// The offset of the delimiter that closes the bracket opened at `open` in +/// `masked` (comments and literals already blanked). +fn matching_close(masked: &str, open: usize) -> Option { + let mut depth = 0usize; + for (index, byte) in masked.bytes().enumerate().skip(open) { + match byte { + b'(' | b'[' | b'{' => depth += 1, + b')' | b']' | b'}' => { + depth = depth.checked_sub(1)?; + if depth == 0 { + return Some(index); + } + } + _ => {} + } + } + None +} + +/// `text` as a plain Rust string literal. +fn plain_literal(text: &str) -> String { + format!("\"{}\"", text.replace('\\', "\\\\").replace('"', "\\\"")) +} + +/// The string literal that is the whole of `arg`, if it is one. +fn sole_string_literal(arg: &str) -> Option { + let trimmed = arg.trim(); + let offset = arg.len() - arg.trim_start().len(); + let mut spans = source_spans(trimmed).into_iter(); + let span = spans.next()?; + (spans.next().is_none() + && span.start == 0 + && span.end == trimmed.len() + && matches!(span.kind, SpanKind::Str | SpanKind::RawStr { .. })) + .then_some(SourceSpan { + start: span.start + offset, + end: span.end + offset, + kind: span.kind, + }) +} + +/// Whether literal text spells event-write SQL the floor policy must see. +fn spells_event_write_sql(text: &str) -> bool { + let normalized = normalized_sql_source(text); + normalized.contains("insert into") || normalized.contains("update ") +} + +/// Bring SQL that per-function literal scans cannot see into view, so the +/// floor policy fails closed on it now that no trigger backs it up: each +/// `include_str!("…")` becomes one literal holding the included file +/// (resolved against `path`), and the literal arguments of each `concat!` are +/// joined into one literal. Forms that stay opaque are violations: an +/// `include_str!` whose path is not one literal or cannot be read, and a +/// `concat!` that spells SQL around a non-literal argument. +fn sql_visible_source(path: &std::path::Path, source: &str) -> (String, Vec) { + let mut violations = Vec::new(); + let mut text = source.to_owned(); + for (macro_name, expand_include) in [("include_str!(", true), ("concat!(", false)] { + // Last occurrence first: a rewrite never moves an earlier offset, and + // an inner `concat!` is joined before the one around it. + loop { + let masked = mask_comments_and_literals(&text); + let Some(at) = masked + .match_indices(macro_name) + .map(|(at, _)| at) + .filter(|at| { + !masked[..*at] + .chars() + .next_back() + .is_some_and(|ch| ch.is_ascii_alphanumeric() || ch == '_') + }) + .last() + else { + break; + }; + let open = at + macro_name.len() - 1; + let Some(close) = matching_close(&masked, open) else { + violations.push(format!("{macro_name}…) is unterminated")); + break; + }; + let args = &text[open + 1..close]; + let replacement = if expand_include { + let included = sole_string_literal(args) + .and_then(|span| span.string_body(args).map(str::to_owned)) + .and_then(|relative| { + let resolved = path.parent()?.join(&relative); + std::fs::read_to_string(resolved).ok() + }); + match included { + Some(contents) => plain_literal(&contents), + None => { + violations.push(format!("include_str!({}) is opaque", args.trim())); + plain_literal("") + } + } + } else { + let masked_args = &masked[open + 1..close]; + let mut pieces = Vec::new(); + let mut joined = String::new(); + let mut opaque = false; + let mut depth = 0usize; + let mut arg_start = 0usize; + for (index, byte) in masked_args.bytes().chain(std::iter::once(b',')).enumerate() { + match byte { + b'(' | b'[' | b'{' => depth += 1, + b')' | b']' | b'}' => depth = depth.saturating_sub(1), + b',' if depth == 0 => { + let arg = &args[arg_start..index.min(args.len())]; + arg_start = index + 1; + if arg.trim().is_empty() { + continue; + } + match sole_string_literal(arg) { + Some(span) => { + let body = span.string_body(arg).unwrap_or_default(); + let body = if matches!(span.kind, SpanKind::Str) { + body.to_owned() + } else { + body.replace('\\', "\\\\").replace('"', "\\\"") + }; + joined.push_str(&body); + } + None => { + opaque = true; + if !joined.is_empty() { + pieces.push(format!("\"{joined}\"")); + } + joined.clear(); + pieces.push(arg.trim().to_owned()); + } + } + } + _ => {} + } + } + if !joined.is_empty() { + pieces.push(format!("\"{joined}\"")); + } + if opaque && spells_event_write_sql(args) { + violations.push(format!( + "concat!({}) spells SQL around a non-literal argument", + args.trim() + )); + } + if opaque { + // Keep the opaque form visible to the per-function scan but + // drop the macro name so this loop terminates. + format!("concat_opaque!({})", pieces.join(", ")) + } else { + pieces.pop().unwrap_or_else(|| plain_literal("")) + } + }; + text.replace_range(at..=close, &replacement); + } + } + (text, violations) +} + +/// Length of the function item that starts `masked_function` (comments and +/// literals blanked): through the `}` closing its body, or through the `;` of +/// a bodiless declaration. Unlike [`cfg_test_item_len`], a `;` inside the +/// signature's parentheses or brackets (`[u8; 4]`) does not end it. Without a +/// body the whole input counts, so an unrecognized shape never hides SQL. +fn function_item_len(masked_function: &str) -> usize { + let mut nesting = 0usize; + for (index, byte) in masked_function.bytes().enumerate() { + match byte { + b'(' | b'[' => nesting += 1, + b')' | b']' => nesting = nesting.saturating_sub(1), + b';' if nesting == 0 => return index + 1, + b'{' if nesting == 0 => { + return matching_close(masked_function, index) + .map_or(masked_function.len(), |close| close + 1); + } + _ => {} + } + } + masked_function.len() +} + +/// Each function slice cut to the function's own item, so a const or static +/// after its body is not attributed to it (see [`item_level_literals`]). +fn function_items(production: &str) -> Vec<&str> { function_slices(production) + .into_iter() + .map(|function| &function[..function_item_len(&mask_comments_and_literals(function))]) + .collect() +} + +/// String literal bodies outside every function body: consts, statics and +/// macro bodies. A function's floor guard cannot be attributed to SQL there. +fn item_level_literals(production: &str) -> String { + let base = production.as_ptr() as usize; + let functions: Vec<(usize, usize)> = function_items(production) + .into_iter() + .map(|function| { + let start = function.as_ptr() as usize - base; + (start, start + function.len()) + }) + .collect(); + let mut literals = String::new(); + for span in source_spans(production) { + if functions + .iter() + .any(|(start, end)| *start <= span.start && span.end <= *end) + { + continue; + } + if let Some(body) = span.string_body(production) { + literals.push_str(body); + literals.push('\n'); + } + } + literals +} + +/// Every floor-policy violation in one production file, after +/// [`sql_visible_source`] has exposed included and concatenated SQL. +fn file_event_floor_policy_violations(path: &std::path::Path, source: &str) -> Vec { + let (production, mut violations) = sql_visible_source(path, &strip_cfg_test_items(source)); + let item_level = item_level_literals(&production); + if inserts_events_row(&item_level) || updates_event_floor_columns(&item_level) { + violations.push("event-write SQL outside any function body".to_owned()); + } + // The fence cannot bound a prepared transaction's rows without the + // retired trigger (see `replica_fence::sample_writer`), so no supported + // writer may prepare one. + let literals: String = source_spans(&production) + .iter() + .filter_map(|span| span.string_body(&production)) + .collect::>() + .join("\n"); + if normalized_sql_source(&literals).contains("prepare transaction") { + violations.push("PREPARE TRANSACTION in a production writer".to_owned()); + } + violations.extend(event_floor_policy_violations(&production)); + violations +} + +fn event_floor_policy_violations(production: &str) -> Vec { + function_items(production) .into_iter() .filter(|function| { let header = function_header(function); @@ -2168,6 +2454,87 @@ pub async fn publish_nip43_membership_locked(tx: &mut AdmittedTx) { ); } +#[test] +fn event_floor_policy_fails_closed_on_sql_it_cannot_attribute() { + let dir = std::env::temp_dir().join(format!("buzz-floor-policy-{}", uuid::Uuid::new_v4())); + std::fs::create_dir_all(&dir).expect("fixture dir"); + std::fs::write( + dir.join("insert_event.sql"), + "INSERT INTO \"events\" (id) VALUES ($1)", + ) + .expect("sql fixture"); + std::fs::write(dir.join("notes.md"), "Prose that quotes nothing.").expect("md fixture"); + let path = dir.join("writer.rs"); + let source = r##" +const LEADING_INSERT: &str = "INSERT INTO events (id) VALUES ($1)"; +const USER_AGENT: &str = concat!("buzz-agent/", env!("CARGO_PKG_VERSION")); +static NOTES: &str = include_str!("notes.md"); +async fn joined_insert(tx: &mut AdmittedTx) { + sqlx::query(concat!("INSERT INTO ", "events (id) VALUES ($1)")); +} +async fn guarded_joined_insert(tx: &mut AdmittedTx) { + crate::replica_fence::guard_event_created_at_floor(tx, created_at, channel).await?; + sqlx::query(concat!("INSERT INTO ", r#"events"#, " (id) VALUES ($1)")); +} +const TRAILING_REWRITE: &str = "UPDATE events SET created_at = $1 WHERE id = $2"; +async fn joined_rewrite(tx: &mut AdmittedTx) { + crate::replica_fence::guard_event_created_at_floor(tx, created_at, channel).await?; + sqlx::query(concat!("UPDATE events ", "SET created_at = $1 WHERE id = $2")); +} +async fn included_insert(tx: &mut AdmittedTx) { + sqlx::query(include_str!("insert_event.sql")); +} +async fn guarded_included_insert(tx: &mut AdmittedTx) { + crate::replica_fence::guard_event_created_at_floor(tx, created_at, channel).await?; + sqlx::query(include_str!("insert_event.sql")); +} +async fn opaque_include(tx: &mut AdmittedTx) { + sqlx::query(include_str!(concat!(env!("OUT_DIR"), "/insert.sql"))); +} +async fn opaque_concat(tx: &mut AdmittedTx) { + sqlx::query(concat!("INSERT INTO ", env!("EVENTS_TABLE"))); +} +async fn two_phase(tx: &mut AdmittedTx) { + sqlx::query("PREPARE TRANSACTION 'fence'"); +} +"##; + let violations = file_event_floor_policy_violations(&path, source); + std::fs::remove_dir_all(&dir).expect("remove fixture dir"); + assert_eq!( + violations, + [ + r#"include_str!(concat!(env!("OUT_DIR"), "/insert.sql")) is opaque"#, + r#"concat!("INSERT INTO ", env!("EVENTS_TABLE")) spells SQL around a non-literal argument"#, + "event-write SQL outside any function body", + "PREPARE TRANSACTION in a production writer", + "async fn joined_insert(tx: &mut AdmittedTx) {", + "async fn joined_rewrite(tx: &mut AdmittedTx) {", + "async fn included_insert(tx: &mut AdmittedTx) {", + ] + ); + + // Each item-level form is caught on its own, wherever it sits. + for item in [ + "const LEADING: &str = \"INSERT INTO events (id) VALUES ($1)\";\nfn f() {}\n", + "fn f() {}\nconst TRAILING: &str = \"UPDATE events SET channel_id = $1\";\nfn g() {}\n", + "fn f() {}\nstatic INCLUDED: &str = include_str!(\"insert_event.sql\");\n", + ] { + std::fs::create_dir_all(&dir).expect("fixture dir"); + std::fs::write( + dir.join("insert_event.sql"), + "insert into events (id) values ($1)", + ) + .expect("sql fixture"); + let found = file_event_floor_policy_violations(&path, item); + std::fs::remove_dir_all(&dir).expect("remove fixture dir"); + assert_eq!( + found, + ["event-write SQL outside any function body"], + "item-level SQL in `{item}`" + ); + } +} + #[test] fn every_production_channel_event_insert_checks_the_floor_and_never_rewrites_it() { let files = collect_production_rs_files(); @@ -2181,8 +2548,9 @@ fn every_production_channel_event_insert_checks_the_floor_and_never_rewrites_it( let mut guarded_writers = 0; let mut violations = Vec::new(); for path in files { - let production = strip_cfg_test_items(&std::fs::read_to_string(&path).expect("source")); - guarded_writers += function_slices(&production) + let source = std::fs::read_to_string(&path).expect("source"); + let (production, _) = sql_visible_source(&path, &strip_cfg_test_items(&source)); + guarded_writers += function_items(&production) .into_iter() .filter(|function| { inserts_events_row(function) @@ -2191,7 +2559,7 @@ fn every_production_channel_event_insert_checks_the_floor_and_never_rewrites_it( }) .count(); violations.extend( - event_floor_policy_violations(&production) + file_event_floor_policy_violations(&path, &source) .into_iter() .map(|header| format!("{}: {header}", path.display())), ); From 2497025ec8b510d2469774c5ed20788d64724aff Mon Sep 17 00:00:00 2001 From: Elrond <28d6302a099e5225b02c4155ac4236e4912603df2ab08dbfc2f4fef08ce598c8@buzz.block.builderlab.xyz> Date: Fri, 16 Oct 2026 10:26:30 -0400 Subject: [PATCH 09/10] buzz-db: check floor SQL per literal, with placeholders for what it cannot read Review of 8752ce124 found the opaque-concat rule red on production: the two retention deletes concat a predicate macro into their WHERE, and any "update " around a non-literal argument was a violation. Replace that heuristic with one detector. concat! now becomes a single literal with an {opaque} placeholder per non-literal argument, and every SQL literal is checked on its own: a placeholder (from concat! or format!) in a write target or SET column, a floor-column SET list whose UPDATE is in another literal, an INSERT INTO with no table after it or an events INSERT with a placeholder tail, events written by MERGE INTO or COPY, and a dangling PREPARE are violations. A placeholder in a WHERE or a value is allowed, so the retention deletes and the tenant-severing UPDATE {table} SET community_id = NULL pass. Also recognize concat!/include_str! with [] and {} delimiters, and ban sqlx query_file*! macros, which load SQL the scan cannot see. Document that the prepared-transaction count is cluster-wide on purpose, like the activity scan. Signed-off-by: Elrond <28d6302a099e5225b02c4155ac4236e4912603df2ab08dbfc2f4fef08ce598c8@buzz.block.builderlab.xyz> --- crates/buzz-db/src/runtime/replica_fence.rs | 5 +- crates/buzz-db/tests/observability_source.rs | 381 +++++++++++++------ 2 files changed, 276 insertions(+), 110 deletions(-) diff --git a/crates/buzz-db/src/runtime/replica_fence.rs b/crates/buzz-db/src/runtime/replica_fence.rs index 9a9e378db46..b5ede15775f 100644 --- a/crates/buzz-db/src/runtime/replica_fence.rs +++ b/crates/buzz-db/src/runtime/replica_fence.rs @@ -451,7 +451,10 @@ async fn sample_writer(writer: &PgPool) -> Result { // `pg_prepared_xacts` does not expose, so `prepared` no longer bounds // their rows. No supported writer issues PREPARE TRANSACTION (the // source policy enforces it) and `max_prepared_transactions` defaults - // to 0, so any prepared transaction is out-of-band: fail closed. + // to 0, so any prepared transaction is out-of-band: fail closed. The + // count is cluster-wide, like the activity scan: a prepared + // transaction in any database on the writer is outside the supported + // deploy, so it closes the fence rather than being scoped away. let row = sqlx::query( r#" SELECT diff --git a/crates/buzz-db/tests/observability_source.rs b/crates/buzz-db/tests/observability_source.rs index 939384e325f..65c8d6be3af 100644 --- a/crates/buzz-db/tests/observability_source.rs +++ b/crates/buzz-db/tests/observability_source.rs @@ -2011,7 +2011,8 @@ fn is_channel_null_event_writer(function: &str) -> bool { } /// Whether a `SET` list assigns `created_at` or `channel_id`, including the -/// row form `(a, b) = (...)`. Stops at the first `WHERE`. +/// row form `(a, b) = (...)`, or a column it cannot read (a `{…}` +/// placeholder). Stops at the first `WHERE`. fn assigns_event_floor_column(assignments: &str) -> bool { let assignments = assignments.split(" where ").next().expect("first part"); let mut depth = 0usize; @@ -2031,36 +2032,62 @@ fn assigns_event_floor_column(assignments: &str) -> bool { .trim_start_matches('(') .trim_end_matches(')') .split(',') - .any(|column| matches!(column.trim(), "created_at" | "channel_id")) + .map(str::trim) + .any(|column| { + matches!(column, "created_at" | "channel_id") || column.starts_with('{') + }) }) }) } -fn updates_event_floor_columns(function: &str) -> bool { +/// The table a write targets, without its schema (a literal or a `{…}` +/// placeholder; any schema may hold an `events` table). +fn unqualified_table(target: &str) -> &str { + target.rsplit_once('.').map_or(target, |(_, table)| table) +} + +/// Whether one SQL literal (normalized) assigns a floor column through a +/// direct `UPDATE` of `events`, or of a target it does not spell: a +/// placeholder, or a `SET` list whose `UPDATE` lies in another literal. +fn literal_updates_event_floor_columns(sql: &str) -> bool { // `SET(` with no space is still a row assignment. - let normalized = normalized_sql_source(function).replace(" set(", " set ("); - let direct_update = normalized.match_indices("update ").any(|(index, marker)| { - let Some((target, assignments)) = normalized[index + marker.len()..].split_once(" set ") - else { + let sql = format!(" {} ", sql.trim()).replace(" set(", " set ("); + sql.match_indices(" set ").any(|(index, marker)| { + if !assigns_event_floor_column(&sql[index + marker.len()..]) { return false; + } + let before = &sql[..index]; + if before.ends_with(" do update") { + // The upsert check resolves the INSERT, which may be in another + // literal of the same builder. + return false; + } + let Some(update) = before.rfind(" update ") else { + return true; }; - let mut target = target.split_whitespace(); + let mut target = before[update + " update ".len()..].split_whitespace(); let mut table = target.next(); if table == Some("only") { table = target.next(); } - if !matches!(table, Some("events" | "public.events")) { - return false; - } // The target may have an alias, with or without AS. let alias = target.collect::>(); - if !matches!(alias.as_slice(), [] | [_] | ["as", _]) { - return false; - } - assigns_event_floor_column(assignments) - }); + let (Some(table), [] | [_] | ["as", _]) = (table, alias.as_slice()) else { + return true; + }; + let table = unqualified_table(table); + table == "events" || table.contains('{') + }) +} + +fn updates_event_floor_columns(function: &str) -> bool { + let direct_update = source_spans(function) + .iter() + .filter_map(|span| span.string_body(function)) + .any(|body| literal_updates_event_floor_columns(&normalized_sql_source(body))); // `ON CONFLICT ... DO UPDATE SET` names no table: it updates the target of // the INSERT it belongs to, so an events upsert can re-home an existing row. + let normalized = normalized_sql_source(function).replace(" set(", " set ("); let events_upsert = normalized .match_indices("do update set ") .any(|(index, marker)| { @@ -2077,6 +2104,43 @@ fn updates_event_floor_columns(function: &str) -> bool { direct_update || events_upsert } +/// Writes one SQL literal (normalized) hides from the per-function checks: +/// an INSERT whose target or tail is not literal, `events` written by a +/// statement other than INSERT, or a PREPARE. +fn literal_sql_violations(sql: &str) -> Vec<&'static str> { + let sql = format!(" {} ", sql.trim()); + let mut violations = Vec::new(); + for (index, _) in sql.match_indices(" into ") { + let rest = &sql[index + " into ".len()..]; + let target = rest.split([' ', '(']).next().expect("first part"); + let table = unqualified_table(target); + if sql[..index].ends_with(" insert") { + if target.is_empty() || table.contains('{') { + violations.push("INSERT target is not a literal table"); + } else if table == "events" && rest.contains('{') { + violations.push("events INSERT with a non-literal part"); + } + } else if table == "events" { + violations.push("events written by a statement other than INSERT"); + } + } + if sql.match_indices(" copy ").any(|(index, marker)| { + let target = sql[index + marker.len()..].split([' ', '(']).next(); + target.is_some_and(|target| unqualified_table(target) == "events") + }) { + violations.push("events written by a statement other than INSERT"); + } + let words = sql.split_whitespace().collect::>(); + if words + .windows(2) + .any(|pair| pair[0] == "prepare" && (pair[1] == "transaction" || pair[1].starts_with('{'))) + || words.last() == Some(&"prepare") + { + violations.push("PREPARE TRANSACTION in a production writer"); + } + violations +} + /// The offset of the delimiter that closes the bracket opened at `open` in /// `masked` (comments and literals already blanked). fn matching_close(masked: &str, open: usize) -> Option { @@ -2098,7 +2162,22 @@ fn matching_close(masked: &str, open: usize) -> Option { /// `text` as a plain Rust string literal. fn plain_literal(text: &str) -> String { - format!("\"{}\"", text.replace('\\', "\\\\").replace('"', "\\\"")) + format!("\"{}\"", plain_body(text)) +} + +/// Raw `text` escaped as the body of a plain string literal. +fn plain_body(text: &str) -> String { + text.replace('\\', "\\\\").replace('"', "\\\"") +} + +/// A string literal's body, as written in a plain (escaped) literal. +fn plain_string_body(span: &SourceSpan, source: &str) -> Option { + let body = span.string_body(source)?; + Some(if matches!(span.kind, SpanKind::Str) { + body.to_owned() + } else { + plain_body(body) + }) } /// The string literal that is the whole of `arg`, if it is one. @@ -2118,43 +2197,48 @@ fn sole_string_literal(arg: &str) -> Option { }) } -/// Whether literal text spells event-write SQL the floor policy must see. -fn spells_event_write_sql(text: &str) -> bool { - let normalized = normalized_sql_source(text); - normalized.contains("insert into") || normalized.contains("update ") +/// The opening delimiter of each `name!` macro call in `masked` (comments +/// and literals blanked), last first, with any path prefix (`::core::`) and +/// any of `(`, `[` or `{`: `(start of name, delimiter offset)`. +fn macro_calls(masked: &str, name: &str) -> Vec<(usize, usize)> { + let mut calls = masked + .match_indices(name) + .filter_map(|(at, _)| { + let starts_token = !masked[..at] + .chars() + .next_back() + .is_some_and(|ch| ch.is_ascii_alphanumeric() || ch == '_'); + let after = masked[at + name.len()..].strip_prefix('!')?; + let open = at + name.len() + 1 + (after.len() - after.trim_start().len()); + (starts_token && matches!(masked.as_bytes().get(open), Some(b'(' | b'[' | b'{'))) + .then_some((at, open)) + }) + .collect::>(); + calls.reverse(); + calls } /// Bring SQL that per-function literal scans cannot see into view, so the /// floor policy fails closed on it now that no trigger backs it up: each /// `include_str!("…")` becomes one literal holding the included file -/// (resolved against `path`), and the literal arguments of each `concat!` are -/// joined into one literal. Forms that stay opaque are violations: an -/// `include_str!` whose path is not one literal or cannot be read, and a -/// `concat!` that spells SQL around a non-literal argument. +/// (resolved against `path`), and each `concat!` becomes one literal of its +/// arguments, with `{opaque}` standing in for any non-literal argument. The +/// SQL checks treat a placeholder in a write target or a `SET` column as +/// unattributable. An `include_str!` whose path is not one literal or +/// cannot be read is a violation, as is any `query_file` macro. fn sql_visible_source(path: &std::path::Path, source: &str) -> (String, Vec) { let mut violations = Vec::new(); let mut text = source.to_owned(); - for (macro_name, expand_include) in [("include_str!(", true), ("concat!(", false)] { + for (macro_name, expand_include) in [("include_str", true), ("concat", false)] { // Last occurrence first: a rewrite never moves an earlier offset, and // an inner `concat!` is joined before the one around it. loop { let masked = mask_comments_and_literals(&text); - let Some(at) = masked - .match_indices(macro_name) - .map(|(at, _)| at) - .filter(|at| { - !masked[..*at] - .chars() - .next_back() - .is_some_and(|ch| ch.is_ascii_alphanumeric() || ch == '_') - }) - .last() - else { + let Some(&(at, open)) = macro_calls(&masked, macro_name).first() else { break; }; - let open = at + macro_name.len() - 1; let Some(close) = matching_close(&masked, open) else { - violations.push(format!("{macro_name}…) is unterminated")); + violations.push(format!("{macro_name}!…) is unterminated")); break; }; let args = &text[open + 1..close]; @@ -2174,9 +2258,7 @@ fn sql_visible_source(path: &std::path::Path, source: &str) -> (String, Vec (String, Vec { - let body = span.string_body(arg).unwrap_or_default(); - let body = if matches!(span.kind, SpanKind::Str) { - body.to_owned() - } else { - body.replace('\\', "\\\\").replace('"', "\\\"") - }; - joined.push_str(&body); - } - None => { - opaque = true; - if !joined.is_empty() { - pieces.push(format!("\"{joined}\"")); - } - joined.clear(); - pieces.push(arg.trim().to_owned()); - } + match sole_string_literal(arg) + .and_then(|span| plain_string_body(&span, arg)) + { + Some(body) => joined.push_str(&body), + None => joined.push_str("{opaque}"), } } _ => {} } } - if !joined.is_empty() { - pieces.push(format!("\"{joined}\"")); - } - if opaque && spells_event_write_sql(args) { - violations.push(format!( - "concat!({}) spells SQL around a non-literal argument", - args.trim() - )); - } - if opaque { - // Keep the opaque form visible to the per-function scan but - // drop the macro name so this loop terminates. - format!("concat_opaque!({})", pieces.join(", ")) - } else { - pieces.pop().unwrap_or_else(|| plain_literal("")) - } + format!("\"{joined}\"") }; text.replace_range(at..=close, &replacement); } } + // sqlx's `query_file*!` macros load SQL from a file at compile time; + // nothing uses them, so ban them rather than resolve them. + let masked = mask_comments_and_literals(&text); + for (at, _) in masked.match_indices("query_file") { + let starts_token = !masked[..at] + .chars() + .next_back() + .is_some_and(|ch| ch.is_ascii_alphanumeric() || ch == '_'); + let name_len = masked[at..] + .find(|ch: char| !(ch.is_ascii_alphanumeric() || ch == '_')) + .unwrap_or(masked.len() - at); + if starts_token && masked[at + name_len..].trim_start().starts_with('!') { + violations.push(format!( + "{}! loads SQL from a file", + &masked[at..at + name_len] + )); + } + } (text, violations) } @@ -2266,8 +2338,9 @@ fn function_items(production: &str) -> Vec<&str> { .collect() } -/// String literal bodies outside every function body: consts, statics and -/// macro bodies. A function's floor guard cannot be attributed to SQL there. +/// String literals outside every function body (consts, statics and macro +/// bodies), as plain literals one per line. A function's floor guard cannot +/// be attributed to SQL there. fn item_level_literals(production: &str) -> String { let base = production.as_ptr() as usize; let functions: Vec<(usize, usize)> = function_items(production) @@ -2285,9 +2358,8 @@ fn item_level_literals(production: &str) -> String { { continue; } - if let Some(body) = span.string_body(production) { - literals.push_str(body); - literals.push('\n'); + if let Some(body) = plain_string_body(&span, production) { + literals.push_str(&format!("\"{body}\"\n")); } } literals @@ -2297,20 +2369,21 @@ fn item_level_literals(production: &str) -> String { /// [`sql_visible_source`] has exposed included and concatenated SQL. fn file_event_floor_policy_violations(path: &std::path::Path, source: &str) -> Vec { let (production, mut violations) = sql_visible_source(path, &strip_cfg_test_items(source)); - let item_level = item_level_literals(&production); - if inserts_events_row(&item_level) || updates_event_floor_columns(&item_level) { - violations.push("event-write SQL outside any function body".to_owned()); - } // The fence cannot bound a prepared transaction's rows without the // retired trigger (see `replica_fence::sample_writer`), so no supported // writer may prepare one. - let literals: String = source_spans(&production) - .iter() - .filter_map(|span| span.string_body(&production)) - .collect::>() - .join("\n"); - if normalized_sql_source(&literals).contains("prepare transaction") { - violations.push("PREPARE TRANSACTION in a production writer".to_owned()); + for span in source_spans(&production) { + if let Some(body) = span.string_body(&production) { + for violation in literal_sql_violations(&normalized_sql_source(body)) { + if !violations.iter().any(|seen| seen == violation) { + violations.push(violation.to_owned()); + } + } + } + } + let item_level = item_level_literals(&production); + if inserts_events_row(&item_level) || updates_event_floor_columns(&item_level) { + violations.push("event-write SQL outside any function body".to_owned()); } violations.extend(event_floor_policy_violations(&production)); violations @@ -2499,37 +2572,127 @@ async fn two_phase(tx: &mut AdmittedTx) { } "##; let violations = file_event_floor_policy_violations(&path, source); + + // One writer per shape: the SQL is split, assembled, loaded or spelled so + // the per-literal checks cannot read it whole. + const WRITER: &str = "async fn writer(tx: &mut AdmittedTx) {"; + const OPAQUE_INSERT: &str = "INSERT target is not a literal table"; + let shapes: [(&str, &[&str]); 19] = [ + // Controls: an opaque WHERE predicate (the retention deletes), a + // placeholder table with no floor column (tenant severing), and a + // builder whose upsert continues in another literal. + ( + r#"sqlx::query(concat!("UPDATE events SET deleted_at = NOW() WHERE NOT ", pred!(), ""))"#, + &[], + ), + ( + r#"format!("UPDATE {table} SET community_id = NULL WHERE community_id = $1")"#, + &[], + ), + ( + r#"QueryBuilder::new("INSERT INTO listeners (a) ").push(" ON CONFLICT (a) DO UPDATE SET created_at = EXCLUDED.created_at")"#, + &[], + ), + // A non-literal `concat!` argument as the target or a SET column. + ( + r#"concat!("UPDATE ", table!(), " SET created_at = $1")"#, + &[WRITER], + ), + ( + r#"concat!("UPDATE events SET ", cols!(), " = $1")"#, + &[WRITER], + ), + // Every macro delimiter. + ( + r#"concat!["INSERT INTO ", "events (id) VALUES ($1)"]"#, + &[WRITER], + ), + ( + r#"::core::concat!{"INSERT INTO ", "events (id) VALUES ($1)"}"#, + &[WRITER], + ), + (r#"include_str!["insert_event.sql"]"#, &[WRITER]), + (r#"include_str! {"insert_event.sql"}"#, &[WRITER]), + // Compile-time file SQL is banned outright. + ( + r#"sqlx::query_file!("insert_event.sql")"#, + &["query_file! loads SQL from a file"], + ), + ( + r#"sqlx::query_file_as!(Row, "insert_event.sql")"#, + &["query_file_as! loads SQL from a file"], + ), + // SQL assembled at runtime. + ( + r#"format!("{}{}", "INSERT INTO ", "events (id) VALUES ($1)")"#, + &[OPAQUE_INSERT], + ), + ( + r#"format!("INSERT INTO {t} (id) VALUES ($1)")"#, + &[OPAQUE_INSERT], + ), + ( + r#"["INSERT INTO ", "events (id) VALUES ($1)"].concat()"#, + &[OPAQUE_INSERT], + ), + ( + r#"format!("INSERT INTO events (id) VALUES ($1) {suffix}")"#, + &["events INSERT with a non-literal part", WRITER], + ), + ( + r#"format!("{}{}", "UPDATE ", "events SET created_at = $1")"#, + &[WRITER], + ), + ( + r#"format!("UPDATE public.events SET {column} = $1")"#, + &[WRITER], + ), + ( + r#"sqlx::query("MERGE INTO events e USING src ON e.id = src.id WHEN MATCHED THEN DO NOTHING")"#, + &["events written by a statement other than INSERT"], + ), + ( + r#"format!("{} TRANSACTION 'x'", "PREPARE")"#, + &["PREPARE TRANSACTION in a production writer"], + ), + ]; + let shape_violations = shapes.map(|(shape, _)| { + file_event_floor_policy_violations(&path, &format!("{WRITER}\n {shape};\n}}\n")) + }); + + // Each item-level form is caught on its own, wherever it sits. + std::fs::write( + dir.join("insert_event.sql"), + "insert into events (id) values ($1)", + ) + .expect("sql fixture"); + let items = [ + "const LEADING: &str = \"INSERT INTO events (id) VALUES ($1)\";\nfn f() {}\n", + "fn f() {}\nconst TRAILING: &str = \"UPDATE events SET channel_id = $1\";\nfn g() {}\n", + "fn f() {}\nstatic INCLUDED: &str = include_str!(\"insert_event.sql\");\n", + ]; + let item_violations = items.map(|item| file_event_floor_policy_violations(&path, item)); std::fs::remove_dir_all(&dir).expect("remove fixture dir"); + assert_eq!( violations, [ r#"include_str!(concat!(env!("OUT_DIR"), "/insert.sql")) is opaque"#, - r#"concat!("INSERT INTO ", env!("EVENTS_TABLE")) spells SQL around a non-literal argument"#, - "event-write SQL outside any function body", + OPAQUE_INSERT, "PREPARE TRANSACTION in a production writer", + "event-write SQL outside any function body", "async fn joined_insert(tx: &mut AdmittedTx) {", "async fn joined_rewrite(tx: &mut AdmittedTx) {", "async fn included_insert(tx: &mut AdmittedTx) {", ] ); - - // Each item-level form is caught on its own, wherever it sits. - for item in [ - "const LEADING: &str = \"INSERT INTO events (id) VALUES ($1)\";\nfn f() {}\n", - "fn f() {}\nconst TRAILING: &str = \"UPDATE events SET channel_id = $1\";\nfn g() {}\n", - "fn f() {}\nstatic INCLUDED: &str = include_str!(\"insert_event.sql\");\n", - ] { - std::fs::create_dir_all(&dir).expect("fixture dir"); - std::fs::write( - dir.join("insert_event.sql"), - "insert into events (id) values ($1)", - ) - .expect("sql fixture"); - let found = file_event_floor_policy_violations(&path, item); - std::fs::remove_dir_all(&dir).expect("remove fixture dir"); + for ((shape, expected), found) in shapes.iter().zip(&shape_violations) { + assert_eq!(found, expected, "shape `{shape}`"); + } + for (item, found) in items.iter().zip(&item_violations) { assert_eq!( found, - ["event-write SQL outside any function body"], + &["event-write SQL outside any function body"], "item-level SQL in `{item}`" ); } From 29628c53a107e309f38cf4653fa2778f11ada1c2 Mon Sep 17 00:00:00 2001 From: Elrond <28d6302a099e5225b02c4155ac4236e4912603df2ab08dbfc2f4fef08ce598c8@buzz.block.builderlab.xyz> Date: Fri, 16 Oct 2026 10:35:14 -0400 Subject: [PATCH 10/10] buzz-db: treat a SET list split across literals as a floor write The per-literal UPDATE check only saw the SET list inside the literal that holds SET, so format!("UPDATE events SET {}", ..), a split [..].concat(), and QueryBuilder pushes after SET slipped through. A SET list that is empty or holds an item with no '=' is now unattributable when its target is events, a placeholder, or DO UPDATE. Prose that merely contains update/set is still ignored. Signed-off-by: Elrond <28d6302a099e5225b02c4155ac4236e4912603df2ab08dbfc2f4fef08ce598c8@buzz.block.builderlab.xyz> --- crates/buzz-db/tests/observability_source.rs | 55 +++++++++++++++++--- 1 file changed, 49 insertions(+), 6 deletions(-) diff --git a/crates/buzz-db/tests/observability_source.rs b/crates/buzz-db/tests/observability_source.rs index 65c8d6be3af..784363174af 100644 --- a/crates/buzz-db/tests/observability_source.rs +++ b/crates/buzz-db/tests/observability_source.rs @@ -2040,6 +2040,25 @@ fn assigns_event_floor_column(assignments: &str) -> bool { }) } +/// Whether a `SET` list (up to the first `WHERE`) is missing from its +/// literal: empty, or holding an item with no `=`, such as a `{…}` +/// placeholder or a column whose value follows in another literal. +fn set_list_is_unreadable(assignments: &str) -> bool { + let assignments = assignments.split(" where ").next().expect("first part"); + let mut depth = 0usize; + assignments.trim().is_empty() + || assignments + .split(|ch| { + match ch { + '(' => depth += 1, + ')' => depth = depth.saturating_sub(1), + _ => {} + } + ch == ',' && depth == 0 + }) + .any(|assignment| !assignment.contains('=')) +} + /// The table a write targets, without its schema (a literal or a `{…}` /// placeholder; any schema may hold an `events` table). fn unqualified_table(target: &str) -> &str { @@ -2049,21 +2068,26 @@ fn unqualified_table(target: &str) -> &str { /// Whether one SQL literal (normalized) assigns a floor column through a /// direct `UPDATE` of `events`, or of a target it does not spell: a /// placeholder, or a `SET` list whose `UPDATE` lies in another literal. +/// A `SET` list it cannot read counts as assigning one. fn literal_updates_event_floor_columns(sql: &str) -> bool { // `SET(` with no space is still a row assignment. let sql = format!(" {} ", sql.trim()).replace(" set(", " set ("); sql.match_indices(" set ").any(|(index, marker)| { - if !assigns_event_floor_column(&sql[index + marker.len()..]) { + let assignments = &sql[index + marker.len()..]; + // A SET list continued in another literal or argument cannot be read. + let unreadable = set_list_is_unreadable(assignments); + if !unreadable && !assigns_event_floor_column(assignments) { return false; } let before = &sql[..index]; if before.ends_with(" do update") { // The upsert check resolves the INSERT, which may be in another - // literal of the same builder. - return false; + // literal of the same builder, but only from a readable SET list. + return unreadable; } let Some(update) = before.rfind(" update ") else { - return true; + // A bare SET (Redis `SET`, prose) is not SQL unless it assigns. + return !unreadable; }; let mut target = before[update + " update ".len()..].split_whitespace(); let mut table = target.next(); @@ -2073,7 +2097,9 @@ fn literal_updates_event_floor_columns(sql: &str) -> bool { // The target may have an alias, with or without AS. let alias = target.collect::>(); let (Some(table), [] | [_] | ["as", _]) = (table, alias.as_slice()) else { - return true; + // An unreadable target is a violation only when it assigns a + // floor column; prose ("update … is set") assigns nothing. + return !unreadable; }; let table = unqualified_table(table); table == "events" || table.contains('{') @@ -2577,7 +2603,7 @@ async fn two_phase(tx: &mut AdmittedTx) { // the per-literal checks cannot read it whole. const WRITER: &str = "async fn writer(tx: &mut AdmittedTx) {"; const OPAQUE_INSERT: &str = "INSERT target is not a literal table"; - let shapes: [(&str, &[&str]); 19] = [ + let shapes: [(&str, &[&str]); 23] = [ // Controls: an opaque WHERE predicate (the retention deletes), a // placeholder table with no floor column (tenant severing), and a // builder whose upsert continues in another literal. @@ -2647,6 +2673,23 @@ async fn two_phase(tx: &mut AdmittedTx) { r#"format!("UPDATE public.events SET {column} = $1")"#, &[WRITER], ), + // A SET list continued in another literal or argument. + ( + r#"format!("UPDATE events SET {}", "created_at = $1")"#, + &[WRITER], + ), + ( + r#"["UPDATE events SET ", "channel_id = $1"].concat()"#, + &[WRITER], + ), + ( + r#"QueryBuilder::new("UPDATE events SET ").push("created_at = ")"#, + &[WRITER], + ), + ( + r#"QueryBuilder::new("INSERT INTO events (id) ").push(" ON CONFLICT (id) DO UPDATE SET ").push("channel_id = EXCLUDED.channel_id")"#, + &[WRITER], + ), ( r#"sqlx::query("MERGE INTO events e USING src ON e.id = src.id WHEN MATCHED THEN DO NOTHING")"#, &["events written by a statement other than INSERT"],