diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index e0875618dd0..101542f0eaf 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -503,24 +503,29 @@ shape is an explicit transition, not a flag day: transitions take the matching exclusive lock. A supported serving transaction never spans communities; unmigrated paths still rely on trigger/function backstops. -- **Replica floor (transition foundation):** runtime now provides a shared - replica-floor lock helper for compliant channel-event writers, and the writer - probe handshake takes the exclusive counterpart before sampling `S`, scanning - activity, and committing the heartbeat token last. `probe_once` remains the - sole token/fence-wall publication path; there is no separately persisted active - floor cutoff. Commit-time trigger+GUC enforcement remains authoritative for all - paths. -- **Dual enforcement (current):** application-owned lock+precheck paths run in - front of the existing trigger/function enforcement; commit-time trigger checks - remain authoritative during this phase. Removing those backstops is a separate - gated migration, not part of the lock foundation. +- **Replica floor:** supported channel-event insertions check `created_at` + against `clock_timestamp()` on their admitted transaction using the shared + 960-second floor. The ordered heartbeat handshake and activity scan retain + the same fence-wall calculation. Migration 0062 retires only the deferred + floor trigger/function; writer connections no longer arm its GUC and probe + startup no longer requires those objects. +- **Floor retirement deploy gate:** phase 1 must be deployed to every writer + pod/job, with no rollback expected, before applying the destructive contract. + Drain old connections/in-flight transactions, stop out-of-band writers, and + hold the replica breaker closed through the drop until replay catches up. + Already-running old pods do not reverify the trigger on each heartbeat. +- **Dual enforcement (community fence transition):** application-owned + lock+precheck paths run in front of the existing community trigger/function + enforcement. Those backstops remain in place until their own separately gated + retirement. - **Role separation:** relay/runtime code owns admission and lock protocols; operator maintenance/backfill workflows must either use those protocols or run under explicit reviewed procedures that keep routing fences closed. - **Unsupported shape:** direct owner SQL that bypasses supported transaction and lock protocols is not a supported write path. -- **Fleet gates and reconciliation:** startup/fence probes verify guard catalog + - behavior, pgschema bootstraps run reconciliation, and lock/pool metrics remain +- **Fleet gates and reconciliation:** retained guards keep their catalog and + behavior checks; heartbeat/epoch and activity checks still fail closed. + pgschema bootstraps run reconciliation, and lock/pool metrics remain the operational evidence path for rollout safety. **Does NOT:** cache queries, implement connection pooling logic (delegated to sqlx), or make network calls outside Postgres. diff --git a/crates/buzz-db/src/error.rs b/crates/buzz-db/src/error.rs index 8c051a7a793..a7eae563108 100644 --- a/crates/buzz-db/src/error.rs +++ b/crates/buzz-db/src/error.rs @@ -81,6 +81,13 @@ pub enum DbError { #[error("invalid timestamp: {0}")] InvalidTimestamp(i64), + /// A channel event is too old for replica-fence completeness. Retrying the + /// same signed event cannot make its timestamp newer. Like the mixed-version + /// trigger's SQLx error, this remains a database error at the relay boundary + /// (WS OK false / HTTP 500), rather than a different ingest rejection class. + #[error("database error: events.created_at is below the replica-fence floor")] + CreatedAtFloorRejected, + /// A roster mutation would remove the last effective relay Operator, /// leaving no one able to administer the deployment through the API. /// The transaction is rolled back and the caller must add a replacement diff --git a/crates/buzz-db/src/lib.rs b/crates/buzz-db/src/lib.rs index 2f306c35d08..3d091bc4e80 100644 --- a/crates/buzz-db/src/lib.rs +++ b/crates/buzz-db/src/lib.rs @@ -45,15 +45,15 @@ pub const DB_POOL_ACQUIRE_RAW_SERIES_PER_POD: usize = runtime::observability::POOL_ACQUIRE_RAW_SERIES_PER_POD; /// Valid database connection role/step pairs with a start counter. -pub const DB_CONNECTION_STARTED_STEPS: [(DbPoolRole, DbConnectionStep); 4] = +pub const DB_CONNECTION_STARTED_STEPS: [(DbPoolRole, DbConnectionStep); 3] = runtime::CONNECTION_STARTED_STEPS; /// Valid database connection role/step pairs with a duration histogram. -pub const DB_CONNECTION_DURATION_STEPS: [(DbPoolRole, DbConnectionStep); 4] = +pub const DB_CONNECTION_DURATION_STEPS: [(DbPoolRole, DbConnectionStep); 3] = runtime::CONNECTION_DURATION_STEPS; /// Valid database connection role/step/outcome terminal combinations. -pub const DB_CONNECTION_TERMINALS: [(DbPoolRole, DbConnectionStep, DbConnectionOutcome); 15] = +pub const DB_CONNECTION_TERMINALS: [(DbPoolRole, DbConnectionStep, DbConnectionOutcome); 12] = runtime::CONNECTION_TERMINALS; /// Raw Prometheus series ceiling per pod for connection-step telemetry. diff --git a/crates/buzz-db/src/runtime/connection_observability.rs b/crates/buzz-db/src/runtime/connection_observability.rs index a1ab04022fd..710d3e04a50 100644 --- a/crates/buzz-db/src/runtime/connection_observability.rs +++ b/crates/buzz-db/src/runtime/connection_observability.rs @@ -17,8 +17,6 @@ pub enum DbConnectionStep { WriterPool, /// A physical connection has completed DNS/network/TLS/authentication. PhysicalConnect, - /// Install the created-at replica-fence floor. - CreatedAtFloor, /// Install lock, idle-transaction, and statement timeouts. SessionTimeouts, /// Verify READ COMMITTED transaction isolation. @@ -29,10 +27,9 @@ pub enum DbConnectionStep { impl DbConnectionStep { /// Complete metric-label vocabulary. - pub const ALL: [Self; 6] = [ + pub const ALL: [Self; 5] = [ Self::WriterPool, Self::PhysicalConnect, - Self::CreatedAtFloor, Self::SessionTimeouts, Self::Isolation, Self::Ready, @@ -43,7 +40,6 @@ impl DbConnectionStep { match self { Self::WriterPool => "writer_pool", Self::PhysicalConnect => "physical_connect", - Self::CreatedAtFloor => "created_at_floor", Self::SessionTimeouts => "session_timeouts", Self::Isolation => "isolation", Self::Ready => "ready", @@ -77,18 +73,17 @@ impl DbConnectionOutcome { } /// Valid role/step pairs with an explicit start counter. -pub const CONNECTION_STARTED_STEPS: [(DbPoolRole, DbConnectionStep); 4] = [ +pub const CONNECTION_STARTED_STEPS: [(DbPoolRole, DbConnectionStep); 3] = [ (DbPoolRole::Writer, DbConnectionStep::WriterPool), - (DbPoolRole::Writer, DbConnectionStep::CreatedAtFloor), (DbPoolRole::Writer, DbConnectionStep::SessionTimeouts), (DbPoolRole::Writer, DbConnectionStep::Isolation), ]; /// Valid role/step pairs with a duration histogram. -pub const CONNECTION_DURATION_STEPS: [(DbPoolRole, DbConnectionStep); 4] = CONNECTION_STARTED_STEPS; +pub const CONNECTION_DURATION_STEPS: [(DbPoolRole, DbConnectionStep); 3] = CONNECTION_STARTED_STEPS; /// Valid role/step/outcome terminal combinations. -pub const CONNECTION_TERMINALS: [(DbPoolRole, DbConnectionStep, DbConnectionOutcome); 15] = [ +pub const CONNECTION_TERMINALS: [(DbPoolRole, DbConnectionStep, DbConnectionOutcome); 12] = [ ( DbPoolRole::Writer, DbConnectionStep::WriterPool, @@ -114,21 +109,6 @@ pub const CONNECTION_TERMINALS: [(DbPoolRole, DbConnectionStep, DbConnectionOutc DbConnectionStep::PhysicalConnect, DbConnectionOutcome::Succeeded, ), - ( - DbPoolRole::Writer, - DbConnectionStep::CreatedAtFloor, - DbConnectionOutcome::Succeeded, - ), - ( - DbPoolRole::Writer, - DbConnectionStep::CreatedAtFloor, - DbConnectionOutcome::Failed, - ), - ( - DbPoolRole::Writer, - DbConnectionStep::CreatedAtFloor, - DbConnectionOutcome::Cancelled, - ), ( DbPoolRole::Writer, DbConnectionStep::SessionTimeouts, @@ -166,8 +146,8 @@ pub const CONNECTION_TERMINALS: [(DbPoolRole, DbConnectionStep, DbConnectionOutc ), ]; -/// Four start counters + four 13-series histograms + fifteen terminal counters. -pub const CONNECTION_RAW_SERIES_PER_POD: usize = 4 + (4 * 13) + 15; +/// Three start counters + three 13-series histograms + twelve terminal counters. +pub const CONNECTION_RAW_SERIES_PER_POD: usize = 3 + (3 * 13) + 12; pub(crate) struct DbConnectionStepAttempt { pool_role: DbPoolRole, @@ -282,13 +262,12 @@ mod tests { [ "writer_pool", "physical_connect", - "created_at_floor", "session_timeouts", "isolation", "ready", ] ); - assert_eq!(CONNECTION_RAW_SERIES_PER_POD, 71); + assert_eq!(CONNECTION_RAW_SERIES_PER_POD, 54); } #[test] @@ -297,7 +276,7 @@ mod tests { let snapshotter = recorder.snapshotter(); let _guard = metrics::set_default_local_recorder(&recorder); let attempt = - DbConnectionStepAttempt::start(DbPoolRole::Writer, DbConnectionStep::CreatedAtFloor); + DbConnectionStepAttempt::start(DbPoolRole::Writer, DbConnectionStep::SessionTimeouts); drop(attempt); let metrics = snapshotter.snapshot().into_vec(); @@ -309,7 +288,7 @@ mod tests { .map(|label| (label.key(), label.value())) .collect::>(); assert_eq!(labels.get("pool_role"), Some(&"writer")); - assert_eq!(labels.get("step"), Some(&"created_at_floor")); + assert_eq!(labels.get("step"), Some(&"session_timeouts")); assert!(!labels.contains_key("reason")); assert!(!labels.contains_key("connection_ordinal")); match value { diff --git a/crates/buzz-db/src/runtime/migration.rs b/crates/buzz-db/src/runtime/migration.rs index b6ead54e06f..a0759d99c7d 100644 --- a/crates/buzz-db/src/runtime/migration.rs +++ b/crates/buzz-db/src/runtime/migration.rs @@ -50,14 +50,6 @@ pub(crate) async fn run_migrations_through(pool: &PgPool, target: i64) -> Result async fn run_migrations_locked(conn: &mut PgConnection) -> Result<()> { reject_legacy_nip_rs_cardinality_ambiguity(conn).await?; MIGRATOR.run(&mut *conn).await?; - // The replica-fence proof (see `replica_fence`) requires the commit-time - // `created_at` floor trigger from migration 0021 — correctly shaped — on - // the `events` parent and every partition. `CREATE TABLE .. PARTITION OF` - // clones parent triggers, but a partition attached with `ATTACH - // PARTITION` or created by an older code path would silently escape the - // guard, so migration fails closed if any is missing. (The fence probe - // re-runs this same check at startup on non-migrating relays.) - crate::replica_fence::verify_floor_guard_catalog(&mut *conn).await?; crate::channel::verify_channel_roster_fence_catalog(&mut *conn).await?; Ok(()) } @@ -705,10 +697,11 @@ mod postgres_tests { let mut migrations: Vec<_> = MIGRATOR.iter().collect(); migrations.sort_by_key(|migration| migration.version); - assert_eq!(migrations.len(), 60); + assert_eq!(migrations.len(), 61); assert_eq!(migrations[58].version, 59); assert!(migrations[58].sql.as_str().contains("buzz.push_enabled")); assert_eq!(migrations[59].version, 60); + assert_eq!(migrations[60].version, 62); assert!(migrations[59] .sql .as_str() @@ -1044,12 +1037,6 @@ mod postgres_tests { .as_str() .contains("CREATE TABLE join_policy_acceptances")); - // Replica-fence commit-time floor guard on channel-bearing events. - assert_eq!(migrations[20].version, 21); - assert!(migrations[20] - .sql - .as_str() - .contains("events_created_at_floor_guard")); assert!(!migrations[0] .sql .as_str() diff --git a/crates/buzz-db/src/runtime/mod.rs b/crates/buzz-db/src/runtime/mod.rs index 266003f5a09..0e7178ae874 100644 --- a/crates/buzz-db/src/runtime/mod.rs +++ b/crates/buzz-db/src/runtime/mod.rs @@ -356,8 +356,8 @@ pub(crate) mod route_proof { use uuid::Uuid; /// Proof that a query/page can only return rows with - /// `channel_id IS NOT NULL` — the domain of the commit-time floor guard - /// (migration 0021). `channel_ids` (retains channel-NULL rows) and + /// `channel_id IS NOT NULL` — the domain of the application floor check + /// (see [`replica_fence`]). `channel_ids` (retains channel-NULL rows) and /// `global_only = false` are explicitly NOT proofs. /// /// Each constructor keys off *how* its path proves channel-bearing-ness: @@ -415,18 +415,18 @@ pub(crate) enum RoutePredicate { /// read budget `B` (default off). Bounds TIME — the page misses at most /// the freshest `B` of writes. Sound for ANY query shape, including /// global (channel-NULL) rows: it relies only on heartbeat commit order, - /// not the floor guard. + /// not the floor check. Bounded, /// Completeness: the proved wall must cover the page's upper bound. /// Bounds CONTENT — every row at/below `upper` is present, meaningful /// even when the cursor is hours old, where `B`-freshness says nothing. - /// Sound ONLY on the floor guard's domain (channel-bearing rows), hence + /// Sound ONLY on the application floor's domain (channel-bearing rows), hence /// the proof token. `upper` is non-optional: the no-upper-bound /// post-verifying case is [`RoutePredicate::CoveredPostVerified`]. /// /// Bounds INSERT-completeness only — "no missing rows", not "no extra /// rows". Soft deletes are `UPDATE .. SET deleted_at` commits outside - /// the floor guard and never touch `created_at`, so a covered page can + /// the floor check and never touch `created_at`, so a covered page can /// briefly serve a row the writer already excludes; deletion visibility /// is bounded by replication lag under `FENCE_STALENESS` (30s), not by /// `upper` or `B`. Do not extend the covered arm to a surface that @@ -708,10 +708,6 @@ impl Db { /// When `config.read_database_url` is set, a second pool with the same /// sizing is connected to it for lag-tolerant reads (see [`Db::read`]). /// - /// The writer pool arms the commit-time `created_at` floor guard - /// (migration 0021) on every connection by setting the - /// `buzz.created_at_floor` GUC — this is what makes the replica fence - /// proof hold for every insert path that goes through this pool. pub async fn new(config: &DbConfig) -> Result { let pool = Self::connect_writer_pool(config).await?; let read_max_connections = config @@ -742,11 +738,10 @@ impl Db { /// Connect the writer pool with all session-level safety premises. /// - /// SQLx stores one `after_connect` hook, so the floor guard and transaction - /// isolation assertion must remain in this single closure. Registering a - /// second hook replaces the first and silently disarms the floor trigger. + /// SQLx stores one `after_connect` hook, so session timeouts and the + /// transaction isolation assertion must remain in this single closure. /// Additional writer pools, including the relay audit pool, must use this - /// constructor so they inherit the timeout, floor-guard, and isolation + /// constructor so they inherit the timeout and isolation /// policy installed by [`Db::new`]. pub async fn connect_writer_pool(config: &DbConfig) -> Result { use connection_observability::{ @@ -771,22 +766,6 @@ impl Db { // point at which DNS/network/TLS/authentication succeeded. record_milestone(DbPoolRole::Writer, DbConnectionStep::PhysicalConnect); - // `SET` cannot take bind parameters; `set_config` can. - let floor = DbConnectionStepAttempt::start( - DbPoolRole::Writer, - DbConnectionStep::CreatedAtFloor, - ); - if let Err(error) = - sqlx::query("SELECT set_config('buzz.created_at_floor', $1, false)") - .bind(replica_fence::CREATED_AT_FLOOR_SECS.to_string()) - .execute(&mut *conn) - .await - { - floor.fail(); - return Err(error); - } - floor.succeed(); - // `lock_timeout` fails the waiting statement; it does not // cancel the holder. `idle_in_transaction_session_timeout` // reaps only holders idling inside an open transaction, @@ -893,9 +872,8 @@ impl Db { /// are dialed only on first acquire; the ~10-minute reaper never tops /// the pool back up, which is fine — routed reads re-fill it on demand. /// - /// No floor guard or writer-isolation assertion: replica sessions are - /// read-only, so the commit-time trigger from migration 0021 never fires - /// here and the write fence that depends on READ COMMITTED is never reached. + /// No writer-isolation assertion: replica sessions are read-only, so the + /// write fence that depends on READ COMMITTED is never reached here. fn connect_read_pool(config: &DbConfig, url: &str, max_connections: u32) -> Result { Ok(PgPoolOptions::new() .max_connections(max_connections) @@ -1015,43 +993,19 @@ impl Db { &self.fence } - /// Verify the floor guard end-to-end, then spawn the background fence - /// probe. Returns `Ok(false)` when no replica is configured. - /// - /// Ordering matters (Perci, PR #2084 review): this must run **after** - /// the migration decision. On a relay with `BUZZ_AUTO_MIGRATE` off, the - /// writer pool arms the GUC regardless, but if migration 0021 has not - /// been applied there is no trigger enforcing it — and a heartbeat probe - /// would open the fence over an unenforced floor. So the probe is gated - /// on an unconditional two-part verification against the live schema: - /// catalog shape ([`replica_fence::verify_floor_guard_catalog`]) and - /// observed semantics through this exact pool - /// ([`replica_fence::verify_floor_guard_behavior`]). - /// - /// On any verification failure the probe is never spawned and the fence - /// stays closed: every cursor page routes to the writer. The relay keeps - /// serving — degraded capacity, never holes. - pub async fn spawn_fence_probe(&self) -> Result { + /// Spawn the background fence probe after the migration decision. + /// Returns `false` when no replica is configured. Probe failures keep + /// the fence closed and cursor reads on the writer. Application floor + /// coverage is enforced by the supported-writer contract and fleet gate. + pub fn spawn_fence_probe(&self) -> bool { if self.read_pool.is_none() { - return Ok(false); + return false; } - self.verify_replica_fence_at_boot().await?; tokio::spawn(replica_fence::run_probe( self.pool.clone(), std::sync::Arc::clone(&self.fence), )); - Ok(true) - } - - /// Verify replica-fence catalog shape and behavior through attributed - /// writer/bootstrap acquisitions without starting the recurring probe. - pub(crate) async fn verify_replica_fence_at_boot(&self) -> Result<()> { - let mut connection = - observability::acquire_writer(&self.pool, observability::WriterOperation::Bootstrap) - .await?; - replica_fence::verify_floor_guard_catalog(&mut *connection).await?; - drop(connection); - replica_fence::verify_floor_guard_behavior(&self.pool).await + true } /// The pool for lag-tolerant reads: the read replica when configured, @@ -1613,3 +1567,7 @@ mod pool_role_tests { #[cfg(test)] #[path = "tests.rs"] mod postgres_tests; + +#[cfg(test)] +#[path = "tests/floor_retirement.rs"] +mod floor_retirement_postgres_tests; diff --git a/crates/buzz-db/src/runtime/observability.rs b/crates/buzz-db/src/runtime/observability.rs index d487b9a3621..d6fdd3a4087 100644 --- a/crates/buzz-db/src/runtime/observability.rs +++ b/crates/buzz-db/src/runtime/observability.rs @@ -1478,11 +1478,14 @@ mod tests { let test_scope = CommunityId::from_uuid(Uuid::new_v4()); let query = crate::EventQuery::for_community(test_scope); - routed_db.read_pool_boot_ping_for_tests().await; - routed_db - .verify_replica_fence_at_boot() + writer_db + .ensure_configured_community_for_bootstrap(&format!( + "pool-operation-bootstrap-{}.invalid", + Uuid::new_v4().simple() + )) .await - .expect("real startup fence verification succeeds"); + .expect("real community bootstrap succeeds"); + routed_db.read_pool_boot_ping_for_tests().await; let _ = crate::replica_fence::probe_once(&writer_pool, routed_db.fence()).await; routed_db.fence().force_open_for_tests(Utc::now()); assert_eq!( diff --git a/crates/buzz-db/src/runtime/replica_fence.rs b/crates/buzz-db/src/runtime/replica_fence.rs index 520ff9e8698..b5ede15775f 100644 --- a/crates/buzz-db/src/runtime/replica_fence.rs +++ b/crates/buzz-db/src/runtime/replica_fence.rs @@ -3,12 +3,11 @@ //! A read replica may serve a cursor page only when every row the page could //! contain is provably present on the replica. The proof rests on two parts: //! -//! 1. **Commit-time floor** (migration 0021): a deferred constraint trigger -//! aborts, at COMMIT, any transaction inserting a channel-bearing `events` -//! row with `created_at` more than `floor` seconds before commit time -//! (`clock_timestamp()`, evaluated inside commit processing). Enforcement -//! is armed per session via the `buzz.created_at_floor` GUC, which the -//! relay's writer pool sets on every connection. +//! 1. **Insert-time floor**: supported channel-event writers check `created_at` +//! against database `clock_timestamp()` on their admitted transaction for +//! each new row. Duplicate INSERTs need no floor check. This is the only +//! guard: migration 0062 retired 0021's deferred trigger and the +//! writer-pool `buzz.created_at_floor` GUC. //! 2. **Ordered heartbeat handshake** (this module): on one pinned writer //! connection, one transaction first takes the exclusive replica-floor //! advisory lock (draining compliant shared-lock channel writers), then @@ -24,8 +23,9 @@ //! (a) finished before the activity scan — its commit precedes `M`'s //! commit, so the replica session has replayed it; //! (b) open at the activity scan — represented by `xact_start`, so it is -//! bounded by the `oldest_xact_start` term; -//! (c) started after the activity scan — its deferred floor guard runs +//! bounded by the `oldest_xact_start` term (a prepared transaction has +//! no `xact_start` to read, so one existing fails the sample closed); +//! (c) started after the activity scan — its insert-time app guard runs //! after `S`, so it cannot commit a row with //! `created_at < S - floor`. //! There is no fourth bucket. Each committed token `M` therefore proves a @@ -53,9 +53,9 @@ //! mismatch, or an observed token below every retained entry all route the //! request back to the writer — degraded capacity, never holes. //! -//! Operational bypasses (sessions without the GUC, `session_replication_role -//! = replica` restores) are outside the proof by design and require holding -//! the fence closed for their duration; see `migrations/0021`. +//! Raw SQL and `session_replication_role = replica` restores bypass the +//! application floor check. These operational bypasses are outside the proof +//! and require holding the fence closed. use std::collections::VecDeque; use std::sync::{Arc, Mutex}; @@ -67,25 +67,56 @@ use uuid::Uuid; use buzz_datastore_tracing::datastore_span; -/// Seconds of `created_at` history the commit-time floor guard tolerates. +/// Seconds of `created_at` history the application floor check tolerates. /// /// Must exceed the relay's ingest envelope (±900 s) by enough slack that a -/// legitimately accepted event still commits within the floor even under -/// slow validation/lock waits. The writer pool arms the guard with this value -/// and the fence subtracts it; the two uses must never diverge. +/// legitimately accepted event still passes the check after slow +/// validation/lock waits. Application checks and the fence wall use this +/// same value; the two uses must never diverge. pub const CREATED_AT_FLOOR_SECS: i64 = 960; +/// Enforce the channel-event floor on the admitted INSERT connection. +/// +/// One SELECT keeps the five existing INSERT shapes intact. Ordinary/thread +/// writers check after INSERT only when it inserted a row, before follow-ups; +/// replacements check after replay exits, before superseding the old head. +/// The caller must propagate failure and roll back the write transaction. Its +/// database clock is sampled after BEGIN: transactions open at the probe scan +/// are bounded by xact_start; transactions starting later sample after S. No +/// commit hook, candidate tracking or ordering against TTL refresh is needed. +/// NULL-channel rows never enter covered channel windows and need no query. +pub(crate) async fn guard_event_created_at_floor( + tx: &mut crate::AdmittedTx, + created_at: DateTime, + channel_id: Option, +) -> crate::Result<()> { + if channel_id.is_none() { + return Ok(()); + } + let below_floor: bool = sqlx::query_scalar( + "SELECT $1::timestamptz < clock_timestamp() - make_interval(secs => $2::double precision)", + ) + .bind(created_at) + .bind(CREATED_AT_FLOOR_SECS as f64) + .fetch_one(tx.conn()) + .await?; + if below_floor { + return Err(crate::DbError::CreatedAtFloorRejected); + } + Ok(()) +} + /// Deployment-global advisory-lock key ordering compliant channel-event write /// transactions against the writer probe handshake. /// /// Shared holders are channel-event write transactions validating `created_at` -/// under the armed floor contract; the probe takes the exclusive lock before -/// sampling and publishing its heartbeat token. +/// under the application floor contract; the probe takes the exclusive lock +/// before sampling and publishing its heartbeat token. pub const REPLICA_FLOOR_LOCK_KEY: i64 = 0x62757a7a666c6f72; /// Safety margin subtracted from the fence on top of the floor. /// -/// All proof timestamps (`clock_timestamp()`, `xact_start`, the guard's +/// All proof timestamps (`clock_timestamp()`, `xact_start`, the floor check's /// clock) come from the writer host, so this only needs to absorb /// `created_at` second-truncation and scheduling noise, not clock skew /// between machines. @@ -319,208 +350,6 @@ impl ReplicaFence { } } -/// Catalog-level verification that the commit-time floor guard (migration -/// 0021) is present and correctly shaped on the `events` parent AND every -/// partition: right function, `DEFERRABLE INITIALLY DEFERRED`, row-level, -/// AFTER, firing on both INSERT and UPDATE (an UPDATE can move an exempt -/// channel-NULL row into the guarded set, or rewrite `created_at` downward). -/// -/// This is a name-and-shape check only; it cannot detect a sabotaged -/// function body. [`verify_floor_guard_behavior`] proves the semantics. -/// -/// Generic over the executor so the migration path can run it on the -/// lock-holding connection while the startup probe keeps using the pool. -#[datastore_span(name = "replica_fence_verify_catalog", system = "postgresql")] -pub async fn verify_floor_guard_catalog<'e>( - executor: impl sqlx::PgExecutor<'e>, -) -> crate::Result<()> { - // tgtype bits: 1 = ROW, 2 = BEFORE, 4 = INSERT, 16 = UPDATE, 64 = INSTEAD. - // Required: ROW + INSERT + UPDATE set, BEFORE + INSTEAD clear. - let missing: Vec = sqlx::query_scalar( - r#" - SELECT c.relname::text - FROM ( - SELECT 'events'::regclass AS oid - UNION ALL - SELECT inhrelid FROM pg_inherits WHERE inhparent = 'events'::regclass - ) rels - JOIN pg_class c ON c.oid = rels.oid - WHERE NOT EXISTS ( - SELECT 1 FROM pg_trigger t - WHERE t.tgrelid = rels.oid - AND t.tgname = 'events_created_at_floor' - AND t.tgfoid = 'events_created_at_floor_guard'::regproc - AND t.tgdeferrable - AND t.tginitdeferred - AND t.tgtype & 1 = 1 -- row-level - AND t.tgtype & 2 = 0 -- AFTER, not BEFORE - AND t.tgtype & 64 = 0 -- not INSTEAD OF - AND t.tgtype & 4 = 4 -- fires on INSERT - AND t.tgtype & 16 = 16 -- fires on UPDATE - ) - "#, - ) - .fetch_all(executor) - .await?; - if !missing.is_empty() { - return Err(crate::error::DbError::InvalidData(format!( - "created_at floor guard trigger missing or mis-shaped on: {} \ - (replica fence must stay closed)", - missing.join(", ") - ))); - } - Ok(()) -} - -/// Behavioral verification of the floor guard, end-to-end through the armed -/// pool. A catalog check cannot detect a no-op function body or an unarmed -/// pool; this proves the semantics the fence proof cites, inside one -/// rolled-back transaction: -/// -/// 1. the pool's session GUC equals [`CREATED_AT_FLOOR_SECS`] (arming); -/// 2. an old channel-bearing INSERT raises `check_violation` (23514); -/// 3. a fresh channel-bearing INSERT commits; -/// 4. rewriting a fresh row's `created_at` below the floor raises; -/// 5. an old channel-NULL INSERT is exempt, but flipping its `channel_id` -/// on raises (the `UPDATE OF` arm). -/// -/// `SET CONSTRAINTS ALL IMMEDIATE` makes the deferred trigger fire per -/// statement so each adversary is observable under a savepoint; deferral to -/// COMMIT is separately pinned by the held-transaction fixture. -#[datastore_span(name = "replica_fence_verify_behavior", system = "postgresql")] -pub async fn verify_floor_guard_behavior(pool: &PgPool) -> crate::Result<()> { - use crate::error::DbError; - - let expect_violation = |res: Result, - what: &str| - -> crate::Result<()> { - match res { - Err(sqlx::Error::Database(e)) if e.code().as_deref() == Some("23514") => Ok(()), - Ok(_) => Err(DbError::InvalidData(format!( - "floor guard is inert: {what} was accepted (replica fence must stay closed)" - ))), - Err(e) => Err(DbError::InvalidData(format!( - "floor guard verification failed unexpectedly on {what}: {e}" - ))), - } - }; - - let connection = crate::observability::acquire_writer( - pool, - crate::observability::WriterOperation::Bootstrap, - ) - .await?; - let mut tx = sqlx::Transaction::begin(connection, None).await?; - - // 1. Pool arming (Perci: assert the effective value, not the intent). - let armed: String = sqlx::query_scalar("SHOW buzz.created_at_floor") - .fetch_one(&mut *tx) - .await - .map_err(|e| { - DbError::InvalidData(format!( - "buzz.created_at_floor GUC not set on this pool: {e}" - )) - })?; - if armed != CREATED_AT_FLOOR_SECS.to_string() { - return Err(DbError::InvalidData(format!( - "buzz.created_at_floor is '{armed}', expected '{CREATED_AT_FLOOR_SECS}': \ - pool is not armed" - ))); - } - - sqlx::query("SET CONSTRAINTS ALL IMMEDIATE") - .execute(&mut *tx) - .await?; - - // Scratch community satisfying the FK; the whole transaction rolls back. - let community = uuid::Uuid::new_v4(); - sqlx::query("INSERT INTO communities (id, host) VALUES ($1, $2)") - .bind(community) - .bind(format!("fence-verify-{}.invalid", community.simple())) - .execute(&mut *tx) - .await?; - let channel = uuid::Uuid::new_v4(); - - let insert = |tx_id: [u8; 32], age_secs: i64, ch: Option| { - sqlx::query( - "INSERT INTO events (community_id, id, pubkey, created_at, kind, tags, \ - content, sig, received_at, channel_id) \ - VALUES ($1, $2, $3, clock_timestamp() - make_interval(secs => $4::double precision), \ - 9, '[]', 'fence-verify', $5, NOW(), $6)", - ) - .bind(community) - .bind(tx_id.to_vec()) - .bind(vec![0u8; 32]) - .bind(age_secs as f64) - .bind(vec![0u8; 64]) - .bind(ch) - }; - let old_age = CREATED_AT_FLOOR_SECS + 60; - - // 2. Old channel-bearing insert must raise. - sqlx::query("SAVEPOINT floor_probe") - .execute(&mut *tx) - .await?; - let res = insert(rand_id(), old_age, Some(channel)) - .execute(&mut *tx) - .await; - expect_violation(res, "an old channel-bearing INSERT")?; - sqlx::query("ROLLBACK TO SAVEPOINT floor_probe") - .execute(&mut *tx) - .await?; - - // 3. Fresh channel-bearing insert must pass. - let fresh_id = rand_id(); - insert(fresh_id, 0, Some(channel)).execute(&mut *tx).await?; - - // 4. Rewriting created_at below the floor must raise (in- or - // cross-partition, either arm of the guard catches the NEW row). - sqlx::query("SAVEPOINT floor_probe") - .execute(&mut *tx) - .await?; - let res = sqlx::query( - "UPDATE events SET created_at = clock_timestamp() - make_interval(secs => $1::double precision) \ - WHERE community_id = $2 AND id = $3", - ) - .bind(old_age as f64) - .bind(community) - .bind(fresh_id.to_vec()) - .execute(&mut *tx) - .await; - expect_violation(res, "rewriting created_at below the floor")?; - sqlx::query("ROLLBACK TO SAVEPOINT floor_probe") - .execute(&mut *tx) - .await?; - - // 5. Old channel-NULL insert is exempt; flipping channel_id on must raise. - let null_id = rand_id(); - insert(null_id, old_age, None).execute(&mut *tx).await?; - sqlx::query("SAVEPOINT floor_probe") - .execute(&mut *tx) - .await?; - let res = sqlx::query("UPDATE events SET channel_id = $1 WHERE community_id = $2 AND id = $3") - .bind(channel) - .bind(community) - .bind(null_id.to_vec()) - .execute(&mut *tx) - .await; - expect_violation(res, "moving an old channel-NULL row into a channel")?; - sqlx::query("ROLLBACK TO SAVEPOINT floor_probe") - .execute(&mut *tx) - .await?; - - tx.rollback().await?; - Ok(()) -} - -fn rand_id() -> [u8; 32] { - let mut id = [0u8; 32]; - for chunk in id.chunks_mut(16) { - chunk.copy_from_slice(&uuid::Uuid::new_v4().into_bytes()[..chunk.len()]); - } - id -} - /// One writer-side sample of the ordered handshake. #[derive(Debug)] struct WriterSample { @@ -555,6 +384,17 @@ pub enum ProbeError { /// Number of other client backends with masked/unknown state. masked: i64, }, + /// A prepared (2PC) transaction exists. Its rows were floor-checked at + /// insert time, which `pg_prepared_xacts` cannot bound, and no supported + /// writer prepares transactions — treat it as out-of-band. + #[error( + "{prepared} prepared transaction(s) in pg_prepared_xacts — out-of-band writer; \ + replica floor cannot bound their rows" + )] + PreparedTransactions { + /// Number of prepared transactions in the cluster. + prepared: i64, + }, /// The single heartbeat row (migration 0026) is missing on the writer. #[error("replica_heartbeat row missing on the writer — migration 0026 not applied?")] HeartbeatRowMissing, @@ -601,23 +441,27 @@ async fn sample_writer(writer: &PgPool) -> Result { // conservative direction (a long autovacuum merely holds the fence // back), and it keeps the classification simple. // Scope is every other backend regardless of role or application - // name: an admin psql transaction writes under the same trigger and - // must be representable in the oldest-xact term. + // name: every open transaction must be representable in the + // oldest-xact term. Out-of-band writes require a closed breaker. // // Prepared transactions (2PC) are a bucket of their own: while // prepared they have left `pg_stat_activity` but can still commit - // after the token. Their deferred floor guard already ran at PREPARE, - // so `pg_prepared_xacts.prepared` bounds their rows exactly like - // `xact_start`; fold it into the same minimum. + // after the token. Only the retired deferred trigger checked their + // rows at PREPARE; the insert-time app guard runs earlier, at a clock + // `pg_prepared_xacts` does not expose, so `prepared` no longer bounds + // their rows. No supported writer issues PREPARE TRANSACTION (the + // source policy enforces it) and `max_prepared_transactions` defaults + // to 0, so any prepared transaction is out-of-band: fail closed. The + // count is cluster-wide, like the activity scan: a prepared + // transaction in any database on the writer is outside the supported + // deploy, so it closes the fence rather than being scoped away. let row = sqlx::query( r#" SELECT - least( - (SELECT min(xact_start) - FROM pg_stat_activity - WHERE pid <> pg_backend_pid()), - (SELECT min(prepared) FROM pg_prepared_xacts) - ) AS oldest_xact_start, + (SELECT min(xact_start) + FROM pg_stat_activity + WHERE pid <> pg_backend_pid()) AS oldest_xact_start, + (SELECT count(*) FROM pg_prepared_xacts) AS prepared, (SELECT count(*) FROM pg_stat_activity WHERE pid <> pg_backend_pid() @@ -634,6 +478,10 @@ async fn sample_writer(writer: &PgPool) -> Result { if masked > 0 { return Err(ProbeError::MaskedActivity { masked }); } + let prepared: i64 = row.get("prepared"); + if prepared > 0 { + return Err(ProbeError::PreparedTransactions { prepared }); + } let oldest_xact_start: Option> = row.get("oldest_xact_start"); // 3. Token commit LAST, on the same pinned connection. The single-row @@ -1150,6 +998,62 @@ mod postgres_tests { .expect("drop role"); } + /// With the floor trigger retired, nothing bounds a prepared (2PC) + /// transaction's rows by `pg_prepared_xacts.prepared`, so the sample must + /// fail closed while one exists. On a server with the default + /// `max_prepared_transactions = 0` (CI, compose, chart) the test instead + /// pins that 2PC is unavailable, which is the supported-writer exclusion. + #[tokio::test] + #[ignore = "requires Postgres"] + async fn cluster_global_sample_writer_fails_closed_on_prepared_transaction() { + let pool = PgPool::connect(&crate::test_support::database_url()) + .await + .expect("connect"); + let max_prepared: String = sqlx::query_scalar("SHOW max_prepared_transactions") + .fetch_one(&pool) + .await + .expect("show max_prepared_transactions"); + + let gid = format!("fence_probe_{}", uuid::Uuid::new_v4().simple()); + let mut conn = pool.acquire().await.expect("acquire"); + sqlx::query("BEGIN") + .execute(&mut *conn) + .await + .expect("begin"); + sqlx::query("SELECT 1") + .execute(&mut *conn) + .await + .expect("touch tx"); + let prepare = sqlx::query(sqlx::AssertSqlSafe(format!("PREPARE TRANSACTION '{gid}'"))) + .execute(&mut *conn) + .await; + + if max_prepared == "0" { + prepare.expect_err("PREPARE TRANSACTION must fail when 2PC is disabled"); + sqlx::query("ROLLBACK") + .execute(&mut *conn) + .await + .expect("rollback"); + return; + } + + prepare.expect("prepare transaction"); + drop(conn); + let result = sample_writer(&pool).await; + sqlx::query(sqlx::AssertSqlSafe(format!("ROLLBACK PREPARED '{gid}'"))) + .execute(&pool) + .await + .expect("rollback prepared"); + let err = result.expect_err("a prepared transaction must fail the sample closed"); + assert!( + matches!(err, ProbeError::PreparedTransactions { prepared } if prepared >= 1), + "expected PreparedTransactions, got {err:?}" + ); + sample_writer(&pool) + .await + .expect("sample succeeds once no transaction is prepared"); + } + /// The Aurora PostgreSQL identity function name is exact — the /// MySQL-family near-miss (`aurora_server_id`) would make the /// capability probe cache a permanent false on real Aurora and diff --git a/crates/buzz-db/src/runtime/tests.rs b/crates/buzz-db/src/runtime/tests.rs index 2e0e260bd26..9c66f7fe44f 100644 --- a/crates/buzz-db/src/runtime/tests.rs +++ b/crates/buzz-db/src/runtime/tests.rs @@ -908,27 +908,27 @@ fn signed_event_at(keys: &nostr::Keys, content: &str, secs: u64) -> nostr::Event .expect("sign event") } +// Routing tests intentionally model stored history and divergent replicas. +// Raw fixture SQL keeps them independent of serving-time floor admission. async fn insert_top_level(pool: &PgPool, community: Uuid, channel: Uuid, ev: &nostr::Event) { - let ts = chrono::DateTime::from_timestamp(ev.created_at.as_secs() as i64, 0).expect("valid ts"); - event::insert_event_with_thread_metadata( + crate::test_support::insert_historical_event( pool, CommunityId::from_uuid(community), ev, - Some(channel), - Some(event::ThreadMetadataParams { - event_id: ev.id.as_bytes(), - event_created_at: ts, - channel_id: channel, - parent_event_id: None, - parent_event_created_at: None, - root_event_id: None, - root_event_created_at: None, - depth: 0, - broadcast: true, - }), + channel, + ) + .await; + sqlx::query( + "INSERT INTO thread_metadata (community_id,event_id,event_created_at,channel_id,depth,broadcast) \ + VALUES ($1,$2,to_timestamp($3),$4,0,true)", ) + .bind(community) + .bind(ev.id.as_bytes().as_slice()) + .bind(ev.created_at.as_secs() as f64) + .bind(channel) + .execute(pool) .await - .expect("insert top-level event"); + .expect("seed top-level metadata"); } async fn insert_thread_reply( @@ -938,29 +938,37 @@ async fn insert_thread_reply( root: &nostr::Event, reply: &nostr::Event, ) { - let reply_ts = - chrono::DateTime::from_timestamp(reply.created_at.as_secs() as i64, 0).expect("valid ts"); - let root_ts = - chrono::DateTime::from_timestamp(root.created_at.as_secs() as i64, 0).expect("valid ts"); - event::insert_event_with_thread_metadata( + crate::test_support::insert_historical_event( pool, CommunityId::from_uuid(community), reply, - Some(channel), - Some(event::ThreadMetadataParams { - event_id: reply.id.as_bytes(), - event_created_at: reply_ts, - channel_id: channel, - parent_event_id: Some(root.id.as_bytes()), - parent_event_created_at: Some(root_ts), - root_event_id: Some(root.id.as_bytes()), - root_event_created_at: Some(root_ts), - depth: 1, - broadcast: false, - }), + channel, + ) + .await; + sqlx::query( + "INSERT INTO thread_metadata \ + (community_id,event_id,event_created_at,channel_id,parent_event_id,parent_event_created_at, \ + root_event_id,root_event_created_at,depth,broadcast) \ + VALUES ($1,$2,to_timestamp($3),$4,$5,to_timestamp($6),$5,to_timestamp($6),1,false)", + ) + .bind(community) + .bind(reply.id.as_bytes().as_slice()) + .bind(reply.created_at.as_secs() as f64) + .bind(channel) + .bind(root.id.as_bytes().as_slice()) + .bind(root.created_at.as_secs() as f64) + .execute(pool) + .await + .expect("seed reply metadata"); + sqlx::query( + "UPDATE thread_metadata SET reply_count=reply_count+1,descendant_count=descendant_count+1, \ + last_reply_at=now() WHERE community_id=$1 AND event_id=$2", ) + .bind(community) + .bind(root.id.as_bytes().as_slice()) + .execute(pool) .await - .expect("insert reply"); + .expect("seed root counters"); } /// Composite thread cursor: 8-byte BE seconds + raw event id. @@ -1925,7 +1933,7 @@ async fn routed_fallback_spends_one_acquire_budget_when_aurora_cache_is_cold() { format!("{}/{}", &base[..idx], wname) }; - // `Db::new` so the writer arms the floor guard and the reader is the + // `Db::new` so the writer uses production session policy and the reader is the // real lazy `connect_read_pool` pool (min_connections=0, 150ms // acquire timeout). Reader is sized 1 so holding one connection // saturates it. @@ -2233,9 +2241,8 @@ async fn routed_reads_are_confined_to_the_requested_community() { } /// D4: a LAZY reader pool (connect_lazy, min_connections=0, never yet -/// used) must still let [`Db::spawn_fence_probe`] verify the writer's -/// floor guard and spawn — reader-down or reader-idle at boot must not -/// disable fence probing. +/// used) must still let [`Db::spawn_fence_probe`] spawn — reader-down or +/// reader-idle at boot must not disable fence probing. #[tokio::test] #[ignore = "requires Postgres"] async fn lazy_reader_pool_still_spawns_fence_probe() { @@ -2250,11 +2257,7 @@ async fn lazy_reader_pool_still_spawns_fence_probe() { let idx = base.rfind('/').expect("db url has a path segment"); format!("{}/{}", &base[..idx], wname) }; - // `Db::new` (not `from_pools`) so the WRITER pool arms the - // `buzz.created_at_floor` GUC — `spawn_fence_probe` verifies the - // floor guard on a writer connection, and `create_scratch_db`'s - // plain `PgPool::connect` never arms it. The reader is still the - // lazy `connect_read_pool` pool this test is about. + // Use the production writer policy and lazy `connect_read_pool`. let db = Db::new(&DbConfig { database_url: writer_url.clone(), read_database_url: Some(writer_url), @@ -2262,12 +2265,9 @@ async fn lazy_reader_pool_still_spawns_fence_probe() { ..DbConfig::default() }) .await - .expect("connect armed Db with lazy reader"); + .expect("connect Db with lazy reader"); - let spawned = db - .spawn_fence_probe() - .await - .expect("floor-guard verification must pass on the migrated writer"); + let spawned = db.spawn_fence_probe(); assert!(spawned, "a configured (lazy) reader must spawn the probe"); drop_scratch_db(&admin, db.pool.clone(), &wname).await; @@ -2560,100 +2560,6 @@ async fn thread_full_replica_page_above_fence_is_reverified_on_writer() { drop_scratch_db(&admin, writer, &wname).await; } -/// Commit-time floor guard (migration 0021), exact held-transaction -/// adversary: a channel-bearing row whose `created_at` is older than the -/// floor at COMMIT time must abort the transaction — the guard runs -/// inside commit processing with `clock_timestamp()`, so holding the -/// transaction open cannot outrun it. channel_id-NULL rows are -/// structurally exempt, and sessions without the GUC are unaffected. -#[tokio::test] -#[ignore = "requires Postgres"] -async fn created_at_floor_guard_aborts_old_channel_rows_at_commit() { - let admin = PgPool::connect(&admin_url().await) - .await - .expect("connect admin"); - let (pool, name) = create_scratch_db(&admin, "floor_guard").await; - - let author = nostr::Keys::generate(); - let community = Uuid::new_v4(); - let channel = Uuid::new_v4(); - seed_community_channel(&pool, community, channel, &author).await; - - let insert_raw = |ev: nostr::Event, channel_id: Option| { - let pool = pool.clone(); - async move { - let mut tx = pool.begin().await.expect("begin"); - // Arm the guard for this transaction only (the relay's - // writer pool arms it per connection; tests are explicit). - sqlx::query("SELECT set_config('buzz.created_at_floor', $1, true)") - .bind(crate::replica_fence::CREATED_AT_FLOOR_SECS.to_string()) - .execute(&mut *tx) - .await - .expect("arm guard"); - sqlx::query( - "INSERT INTO events (community_id, id, pubkey, created_at, kind, tags, \ - content, sig, received_at, channel_id) \ - VALUES ($1, $2, $3, to_timestamp($4), 9, '[]', $5, $6, NOW(), $7)", - ) - .bind(community) - .bind(ev.id.as_bytes().as_slice()) - .bind(ev.pubkey.to_bytes().as_slice()) - .bind(ev.created_at.as_secs() as f64) - .bind(&ev.content) - .bind(ev.sig.serialize().as_slice()) - .bind(channel_id) - .execute(&mut *tx) - .await - .expect("insert inside tx (guard is deferred to commit)"); - // Hold the transaction "open" past the insert, then commit — - // the deferred guard must still see the stale created_at. - sqlx::query("SELECT pg_sleep(0.05)") - .execute(&mut *tx) - .await - .expect("hold tx"); - tx.commit().await - } - }; - - let now_secs = chrono::Utc::now().timestamp() as u64; - let floor = crate::replica_fence::CREATED_AT_FLOOR_SECS as u64; - - // Old channel-bearing row → COMMIT aborts with check_violation. - let old = signed_event_at(&author, "old-held-tx", now_secs - floor - 60); - let err = insert_raw(old, Some(channel)) - .await - .expect_err("below-floor channel row must abort at COMMIT"); - let code = match &err { - sqlx::Error::Database(db_err) => db_err.code().map(|c| c.to_string()), - other => panic!("expected database error, got {other:?}"), - }; - assert_eq!( - code.as_deref(), - Some("23514"), - "guard raises check_violation" - ); - - // Fresh channel-bearing row → commits. - let fresh = signed_event_at(&author, "fresh", now_secs); - insert_raw(fresh, Some(channel)) - .await - .expect("fresh row commits under the armed guard"); - - // Old row WITHOUT a channel (push lease / profile shapes) → - // structurally exempt, commits. - let old_global = signed_event_at(&author, "old-global", now_secs - floor - 60); - insert_raw(old_global, None) - .await - .expect("channel_id-NULL rows are exempt from the floor"); - - // Unarmed session (no GUC) → guard inert; backfills stay possible - // (and must hold the fence closed, per the migration header). - let old_backfill = signed_event_at(&author, "old-backfill", now_secs - floor - 60); - insert_top_level(&pool, community, channel, &old_backfill).await; - - drop_scratch_db(&admin, pool, &name).await; -} - #[test] fn writer_pool_safety_hook_is_single_and_composed() { let source = include_str!("mod.rs"); @@ -2667,13 +2573,10 @@ fn writer_pool_safety_hook_is_single_and_composed() { 1, "SQLx replaces after_connect hooks; writer safety must use exactly one" ); - assert!(connect_pool.contains("buzz.created_at_floor")); assert!(connect_pool.contains("SHOW transaction_isolation")); assert!(connect_pool.contains("'lock_timeout'")); assert!(connect_pool.contains("'idle_in_transaction_session_timeout'")); assert!(connect_pool.contains("'statement_timeout'")); - assert!(!connect_pool.contains("arm_floor_guard")); - assert!(!connect_pool.contains("_arm_floor_guard")); assert!(!connect_pool.contains("allow(unused_variables)")); let reader_doc = source @@ -2730,7 +2633,6 @@ async fn writer_pool_metrics_record_every_initial_connection_ready() { 2 ); for step in [ - DbConnectionStep::CreatedAtFloor, DbConnectionStep::SessionTimeouts, DbConnectionStep::Isolation, ] { @@ -3082,106 +2984,6 @@ async fn session_timeouts_install_through_db_new_and_bound_lock_waits() { drop_scratch_db(&admin, db.pool.clone(), &name).await; } -/// The armed writer pool (`Db::new`) must enforce the floor end-to-end -/// through the public insert APIs, and the session GUC must be verifiably -/// set on pooled connections. -#[tokio::test] -#[ignore = "requires Postgres"] -async fn armed_pool_rejects_old_channel_inserts_through_public_api() { - let admin = PgPool::connect(&admin_url().await) - .await - .expect("connect admin"); - let (seed_pool, name) = create_scratch_db(&admin, "floor_pool").await; - - let author = nostr::Keys::generate(); - let community = Uuid::new_v4(); - let channel = Uuid::new_v4(); - seed_community_channel(&seed_pool, community, channel, &author).await; - - // Connect a Db the production way: after_connect arms the guard. - let base = admin_url().await; - let idx = base.rfind('/').expect("db url has a path segment"); - let scratch_url = format!("{}/{}", &base[..idx], name); - let db = Db::new(&DbConfig { - database_url: scratch_url, - max_connections: 2, - ..DbConfig::default() - }) - .await - .expect("connect armed Db"); - let cid = CommunityId::from_uuid(community); - - // Perci nit: assert the effective session value, not the intent. - let effective: String = sqlx::query_scalar("SHOW buzz.created_at_floor") - .fetch_one(&db.pool) - .await - .expect("SHOW guard GUC"); - assert_eq!( - effective, - crate::replica_fence::CREATED_AT_FLOOR_SECS.to_string(), - "writer pool must arm the floor guard on every connection" - ); - let isolation: String = sqlx::query_scalar("SHOW transaction_isolation") - .fetch_one(&db.pool) - .await - .expect("SHOW writer isolation"); - assert_eq!( - isolation, "read committed", - "the same writer after_connect hook must enforce the isolation premise" - ); - - let now_secs = chrono::Utc::now().timestamp() as u64; - let floor = crate::replica_fence::CREATED_AT_FLOOR_SECS as u64; - - // insert_event (single INSERT, autocommit): old channel row rejected. - let old = signed_event_at(&author, "old-direct", now_secs - floor - 60); - let err = event::insert_event(&db.pool, cid, &old, Some(channel)) - .await - .expect_err("armed pool must reject below-floor channel inserts"); - assert!( - err.to_string().contains("below the replica-fence floor"), - "unexpected error: {err}" - ); - - // insert_event_with_thread_metadata (multi-statement tx): same. - let old2 = signed_event_at(&author, "old-thread-meta", now_secs - floor - 90); - let ts = - chrono::DateTime::from_timestamp(old2.created_at.as_secs() as i64, 0).expect("valid ts"); - let err = event::insert_event_with_thread_metadata( - &db.pool, - cid, - &old2, - Some(channel), - Some(event::ThreadMetadataParams { - event_id: old2.id.as_bytes(), - event_created_at: ts, - channel_id: channel, - parent_event_id: None, - parent_event_created_at: None, - root_event_id: None, - root_event_created_at: None, - depth: 0, - broadcast: true, - }), - ) - .await - .expect_err("armed pool must reject below-floor thread-metadata inserts"); - assert!( - err.to_string().contains("below the replica-fence floor"), - "unexpected error: {err}" - ); - - // Fresh events pass through both APIs. - let fresh = signed_event_at(&author, "fresh-direct", now_secs); - event::insert_event(&db.pool, cid, &fresh, Some(channel)) - .await - .expect("fresh insert passes the armed guard"); - - drop_scratch_db(&admin, seed_pool, &name).await; - // db pool still holds connections to the dropped DB; close it. - db.pool.close().await; -} - /// A writer transaction holding the shared replica-floor advisory lock, the /// shape a floor-compliant writer takes before the exclusive probe can run. async fn begin_replica_floor_locked_writer(db: &Db) -> sqlx::Transaction<'static, sqlx::Postgres> { @@ -3189,7 +2991,7 @@ async fn begin_replica_floor_locked_writer(db: &Db) -> sqlx::Transaction<'static .pool .begin() .await - .expect("begin floor-guarded writer tx"); + .expect("begin floor-compliant writer tx"); sqlx::query("SELECT pg_advisory_xact_lock_shared($1)") .bind(crate::replica_fence::REPLICA_FLOOR_LOCK_KEY) .execute(&mut *tx) @@ -3317,201 +3119,6 @@ async fn replica_floor_probe_waits_for_shared_writer_and_records_after_release() drop_scratch_db(&admin, seed_pool, &name).await; } -/// `spawn_fence_probe` must verify the floor guard before letting the -/// probe run — catalog shape AND observed behavior — and refuse on -/// sabotage. This is the production gate for a relay running with -/// `BUZZ_AUTO_MIGRATE` off: an armed GUC with no enforcing trigger must -/// never yield an open fence. -#[tokio::test] -#[ignore = "requires Postgres"] -async fn fence_probe_refuses_to_start_without_verified_floor_guard() { - let admin = PgPool::connect(&admin_url().await) - .await - .expect("connect admin"); - let (seed_pool, wname) = create_scratch_db(&admin, "fence_gate_w").await; - let (replica_pool, rname) = create_scratch_db(&admin, "fence_gate_r").await; - seed_pool.close().await; - replica_pool.close().await; - - let base = admin_url().await; - let idx = base.rfind('/').expect("db url has a path segment"); - let writer_url = format!("{}/{}", &base[..idx], wname); - let replica_url = format!("{}/{}", &base[..idx], rname); - - // Healthy schema: verification passes, probe starts. A SEPARATE Db - // instance, because its background probe legitimately opens its own - // fence (the heartbeat probe is writer-side only) — the refusal - // assertions below must run against a fence whose spawns were all - // refused. - let db_healthy = Db::new(&DbConfig { - database_url: writer_url.clone(), - read_database_url: Some(replica_url.clone()), - max_connections: 2, - ..DbConfig::default() - }) - .await - .expect("connect armed Db with replica"); - assert!( - db_healthy - .spawn_fence_probe() - .await - .expect("verification passes"), - "probe must start on a verified schema" - ); - - let db = Db::new(&DbConfig { - database_url: writer_url, - read_database_url: Some(replica_url), - max_connections: 2, - ..DbConfig::default() - }) - .await - .expect("connect armed Db with replica"); - - // Sabotage A: catalog-shaped no-op — same trigger, gutted function - // body. Catalog check alone would pass; behavior check must refuse. - sqlx::query( - "CREATE OR REPLACE FUNCTION events_created_at_floor_guard() RETURNS trigger \ - LANGUAGE plpgsql AS $$ BEGIN RETURN NULL; END $$", - ) - .execute(&db.pool) - .await - .expect("gut the guard function"); - let err = db - .spawn_fence_probe() - .await - .expect_err("inert guard body must refuse the probe"); - assert!( - err.to_string().contains("floor guard is inert"), - "unexpected error: {err}" - ); - - // Sabotage B: trigger dropped entirely (the BUZZ_AUTO_MIGRATE=off / - // 0021-unapplied shape). Catalog check must refuse. - sqlx::query("DROP TRIGGER events_created_at_floor ON events") - .execute(&db.pool) - .await - .expect("drop the guard trigger"); - let err = db - .spawn_fence_probe() - .await - .expect_err("missing trigger must refuse the probe"); - assert!( - err.to_string().contains("missing or mis-shaped"), - "unexpected error: {err}" - ); - - // In both refusal states the fence never opened. - assert!( - db.fence().verified_through().is_none(), - "fence must remain closed when verification refuses the probe" - ); - - db_healthy.pool.close().await; - if let Some(rp) = &db_healthy.read_pool { - rp.close().await; - } - db.pool.close().await; - if let Some(rp) = &db.read_pool { - rp.close().await; - } - let _ = sqlx::query(sqlx::AssertSqlSafe(format!( - "DROP DATABASE IF EXISTS {wname} WITH (FORCE)" - ))) - .execute(&admin) - .await; - let _ = sqlx::query(sqlx::AssertSqlSafe(format!( - "DROP DATABASE IF EXISTS {rname} WITH (FORCE)" - ))) - .execute(&admin) - .await; -} - -/// The `UPDATE OF` arm of the floor guard (Perci's second structural -/// hole): an old row legitimately admitted with `channel_id` NULL must -/// not be movable into keyset windows, and a channel row's `created_at` -/// must not be movable below the fence — through raw SQL, at COMMIT. -#[tokio::test] -#[ignore = "requires Postgres"] -async fn floor_guard_blocks_updates_that_move_rows_below_the_fence() { - let admin = PgPool::connect(&admin_url().await) - .await - .expect("connect admin"); - let (pool, name) = create_scratch_db(&admin, "floor_upd").await; - - let author = nostr::Keys::generate(); - let community = Uuid::new_v4(); - let channel = Uuid::new_v4(); - seed_community_channel(&pool, community, channel, &author).await; - - let now_secs = chrono::Utc::now().timestamp() as u64; - let floor = crate::replica_fence::CREATED_AT_FLOOR_SECS as u64; - - // Seed via unarmed session: one old channel-NULL row, one fresh - // channel row. - let old_null = signed_event_at(&author, "old-null", now_secs - floor - 120); - insert_top_level(&pool, community, channel, &old_null).await; - sqlx::query("UPDATE events SET channel_id = NULL WHERE community_id = $1 AND id = $2") - .bind(community) - .bind(old_null.id.as_bytes().as_slice()) - .execute(&pool) - .await - .expect("detach channel (unarmed seed)"); - let fresh = signed_event_at(&author, "fresh-row", now_secs); - insert_top_level(&pool, community, channel, &fresh).await; - - // Armed transaction, deferred to COMMIT (the production shape). - let run_armed_update = |sql: &'static str, id: Vec, age: Option| { - let pool = pool.clone(); - async move { - let mut tx = pool.begin().await.expect("begin"); - sqlx::query("SELECT set_config('buzz.created_at_floor', $1, true)") - .bind(crate::replica_fence::CREATED_AT_FLOOR_SECS.to_string()) - .execute(&mut *tx) - .await - .expect("arm guard"); - let q = sqlx::query(sql).bind(community).bind(id); - let q = match age { - Some(a) => q.bind(a as f64), - None => q, - }; - q.execute(&mut *tx) - .await - .expect("update inside tx (deferred)"); - tx.commit().await - } - }; - - // channel-NULL → channel-bearing on an old row: COMMIT must abort. - let err = run_armed_update( - "UPDATE events SET channel_id = community_id WHERE community_id = $1 AND id = $2", - old_null.id.as_bytes().to_vec(), - None, - ) - .await - .expect_err("moving an old channel-NULL row into a channel must abort at COMMIT"); - assert!( - matches!(&err, sqlx::Error::Database(e) if e.code().as_deref() == Some("23514")), - "unexpected error: {err}" - ); - - // created_at rewrite below the floor on a channel row: COMMIT must abort. - let err = run_armed_update( - "UPDATE events SET created_at = clock_timestamp() - make_interval(secs => $3::double precision) \ - WHERE community_id = $1 AND id = $2", - fresh.id.as_bytes().to_vec(), - Some(floor + 120), - ) - .await - .expect_err("rewriting created_at below the floor must abort at COMMIT"); - assert!( - matches!(&err, sqlx::Error::Database(e) if e.code().as_deref() == Some("23514")), - "unexpected error: {err}" - ); - - drop_scratch_db(&admin, pool, &name).await; -} - // ---- Statement-deadline cancellation vs. replica fallback ---------------- // // A replica read cancelled by `statement_timeout` (SQLSTATE 57014) must be diff --git a/crates/buzz-db/src/runtime/tests/floor_retirement.rs b/crates/buzz-db/src/runtime/tests/floor_retirement.rs new file mode 100644 index 00000000000..fbf3e20c5ea --- /dev/null +++ b/crates/buzz-db/src/runtime/tests/floor_retirement.rs @@ -0,0 +1,141 @@ +//! Contract-migration coverage; application insert coverage belongs to phase 1. + +use super::*; + +async fn floor_objects(pool: &PgPool) -> (i64, i64) { + sqlx::query_as( + "SELECT (SELECT count(*) FROM pg_trigger WHERE tgname = 'events_created_at_floor'), \ + (SELECT count(*) FROM pg_proc p JOIN pg_namespace n ON n.oid = p.pronamespace \ + WHERE n.nspname = 'public' AND p.proname = 'events_created_at_floor_guard')", + ) + .fetch_one(pool) + .await + .expect("read floor catalog") +} + +async fn retained_guards(pool: &PgPool) -> Vec<(String, String)> { + sqlx::query_as( + "SELECT c.relname, pg_get_triggerdef(t.oid) FROM pg_trigger t \ + JOIN pg_class c ON c.oid = t.tgrelid \ + JOIN pg_namespace n ON n.oid = c.relnamespace \ + WHERE n.nspname = 'public' AND t.tgname <> 'events_created_at_floor' \ + UNION ALL \ + SELECT c.relname, pg_get_constraintdef(k.oid) FROM pg_constraint k \ + JOIN pg_class c ON c.oid = k.conrelid \ + JOIN pg_namespace n ON n.oid = c.relnamespace \ + WHERE n.nspname = 'public' AND k.contype = 'f' \ + ORDER BY 1, 2", + ) + .fetch_all(pool) + .await + .expect("read retained triggers and foreign keys") +} + +#[tokio::test] +#[ignore = "requires Postgres"] +async fn migration_schema_floor_retirement_drops_parent_and_partition_clones() { + let pool = PgPool::connect(&crate::test_support::database_url()) + .await + .expect("connect isolated migration database"); + crate::migration::run_migrations_through(&pool, 60) + .await + .expect("apply pre-contract schema"); + let partitions: i64 = + sqlx::query_scalar("SELECT count(*) FROM pg_partition_tree('events'::regclass)") + .fetch_one(&pool) + .await + .expect("count parent and partitions"); + assert_eq!(floor_objects(&pool).await, (partitions, 1)); + let before = retained_guards(&pool).await; + crate::migration::run_migrations(&pool) + .await + .expect("apply contract migration and retained startup checks"); + assert_eq!(floor_objects(&pool).await, (0, 0)); + assert_eq!(retained_guards(&pool).await, before); + pool.close().await; +} + +#[tokio::test] +#[ignore = "requires Postgres"] +async fn migration_schema_floor_retirement_cleans_child_copies_idempotently() { + let pool = PgPool::connect(&crate::test_support::database_url()) + .await + .expect("connect isolated migration database"); + crate::migration::run_migrations_through(&pool, 60) + .await + .expect("apply pre-contract schema"); + // Model independently attached copies after the parent guard was removed. + sqlx::raw_sql( + "DROP TRIGGER events_created_at_floor ON events; \ + CREATE CONSTRAINT TRIGGER events_created_at_floor \ + AFTER INSERT OR UPDATE OF created_at, channel_id ON events_p_past \ + DEFERRABLE INITIALLY DEFERRED FOR EACH ROW \ + EXECUTE FUNCTION events_created_at_floor_guard();", + ) + .execute(&pool) + .await + .expect("install independent child copy"); + assert_eq!(floor_objects(&pool).await, (1, 1)); + let before = retained_guards(&pool).await; + crate::migration::run_migrations(&pool) + .await + .expect("retire independent copy"); + assert_eq!(floor_objects(&pool).await, (0, 0)); + let mut tx = pool.begin().await.expect("begin idempotence check"); + sqlx::raw_sql(include_str!( + "../../../../../migrations/0062_retire_created_at_floor.sql" + )) + .execute(&mut *tx) + .await + .expect("repeat contract SQL after retirement"); + tx.commit().await.expect("commit repeated retirement"); + assert_eq!(floor_objects(&pool).await, (0, 0)); + assert_eq!(retained_guards(&pool).await, before); + pool.close().await; +} + +#[tokio::test] +#[ignore = "requires Postgres"] +async fn floor_retirement_desired_state_has_no_floor_objects() { + // The nextest wrapper bootstraps this database through real pgschema apply + // and reconciliation; desired-state/migration parity is checked by + // `admin_schema_parity_between_desired_state_and_migrations`. + let pool = PgPool::connect(&crate::test_support::database_url()) + .await + .expect("connect desired-state database"); + assert_eq!(floor_objects(&pool).await, (0, 0)); + pool.close().await; +} + +#[tokio::test] +#[ignore = "requires Postgres"] +async fn floor_retirement_writer_and_probe_run_on_retired_schema() { + let url = crate::test_support::database_url(); + let db = Db::new(&DbConfig { + database_url: url.clone(), + read_database_url: Some(url), + max_connections: 2, + min_connections: 1, + ..DbConfig::default() + }) + .await + .expect("connect production writer with retired schema"); + let armed: Option = + sqlx::query_scalar("SELECT nullif(current_setting('buzz.created_at_floor', true), '')") + .fetch_one(&db.pool) + .await + .expect("inspect writer session setting"); + assert_eq!(armed, None, "writer must no longer arm the retired guard"); + assert_eq!(floor_objects(&db.pool).await, (0, 0)); + assert!(db.spawn_fence_probe()); + crate::replica_fence::probe_once(&db.pool, db.fence()) + .await + .expect("retired schema still supports the ordered heartbeat"); + + db.pool.close().await; + db.read_pool + .as_ref() + .expect("reader configured") + .close() + .await; +} diff --git a/crates/buzz-db/src/runtime/tests/thread_window_postgres_tests.rs b/crates/buzz-db/src/runtime/tests/thread_window_postgres_tests.rs index beda4b199fc..1f0d1927e18 100644 --- a/crates/buzz-db/src/runtime/tests/thread_window_postgres_tests.rs +++ b/crates/buzz-db/src/runtime/tests/thread_window_postgres_tests.rs @@ -87,9 +87,7 @@ async fn thread_window_upper_fence_terminal_snapshot_and_fallback() { .sign_with_keys(&keys) .unwrap(); for pool in [&writer, &replica] { - event::insert_event(pool, community, &edit, Some(channel)) - .await - .unwrap(); + crate::test_support::insert_historical_event(pool, community, &edit, channel).await; event::insert_event(pool, community, &deletion, None) .await .unwrap(); diff --git a/crates/buzz-db/src/store/channel_members.rs b/crates/buzz-db/src/store/channel_members.rs index 1994de14ed4..82bf9d52822 100644 --- a/crates/buzz-db/src/store/channel_members.rs +++ b/crates/buzz-db/src/store/channel_members.rs @@ -396,6 +396,12 @@ impl LockedMemberSnapshot { )); } } + crate::replica_fence::guard_event_created_at_floor( + &mut self.tx, + created_at, + Some(channel_id), + ) + .await?; sqlx::query("UPDATE events SET deleted_at = NOW() WHERE community_id = $1 AND kind = $2 AND pubkey = $3 AND channel_id = $4 AND deleted_at IS NULL") .bind(community_id.as_uuid()).bind(kind).bind(pubkey.as_slice()).bind(channel_id) .execute(self.tx.conn()).await?; diff --git a/crates/buzz-db/src/store/event.rs b/crates/buzz-db/src/store/event.rs index 183585e2785..341a9a8da8c 100644 --- a/crates/buzz-db/src/store/event.rs +++ b/crates/buzz-db/src/store/event.rs @@ -439,7 +439,8 @@ pub async fn insert_event( /// /// This is the transaction-composition seam for callers that must keep the /// event insert open while performing related work. The caller owns commit or -/// rollback. +/// rollback. On error, roll back or drop the transaction: the event row may +/// already have been inserted before the app floor check rejected it. pub async fn insert_event_in_transaction( tx: &mut AdmittedTx, event: &Event, @@ -504,6 +505,7 @@ async fn insert_event_on( let was_inserted = result.rows_affected() > 0; if was_inserted { + crate::replica_fence::guard_event_created_at_floor(tx, created_at, channel_id).await?; crate::store::event_follow_up::after_admitted_insert( tx, id_bytes.as_slice(), @@ -1756,6 +1758,7 @@ pub(crate) async fn insert_event_with_thread_metadata_tx( let was_inserted = result.rows_affected() > 0; if was_inserted { + crate::replica_fence::guard_event_created_at_floor(tx, created_at, channel_id).await?; crate::store::event_follow_up::after_admitted_insert( tx, id_bytes.as_slice(), @@ -2319,7 +2322,7 @@ impl Db { /// /// Counts route on the BOUNDED arm only, never covered: the covered /// arm bounds insert-completeness but not deletion visibility (soft - /// deletes are UPDATEs outside the floor guard), and a count has no + /// deletes are UPDATEs outside the floor check), and a count has no /// downstream per-row re-filter to absorb extra rows — a silently /// inflated number for up to `FENCE_STALENESS` is a different product /// statement than a page briefly showing a deleted row. `Bounded` ties @@ -3871,16 +3874,21 @@ mod postgres_tests { // ─── canvas CAS tests ───────────────────────────────────────────────────── - fn make_canvas_event_at(content: &str, created_at: u64) -> nostr::Event { - make_event_at(buzz_core::kind::KIND_CANVAS as u16, content, created_at) + fn make_canvas_event_at(base: u64, content: &str, created_at: u64) -> nostr::Event { + // Each test owns its fresh base; logical seconds and ties stay explicit. + make_event_at( + buzz_core::kind::KIND_CANVAS as u16, + content, + base + created_at, + ) } /// Build two canvas events sharing `created_at`, returned `(lower, higher)` /// by event id. Regenerates until the ids differ (always, since keys differ) /// so tests can assert deterministic head ordering on the id tiebreak. - fn same_second_ordered_pair(created_at: u64) -> (nostr::Event, nostr::Event) { - let a = make_canvas_event_at("# A", created_at); - let b = make_canvas_event_at("# B", created_at); + fn same_second_ordered_pair(base: u64, created_at: u64) -> (nostr::Event, nostr::Event) { + let a = make_canvas_event_at(base, "# A", created_at); + let b = make_canvas_event_at(base, "# B", created_at); if a.id.as_bytes() <= b.id.as_bytes() { (a, b) } else { @@ -3888,6 +3896,20 @@ mod postgres_tests { } } + #[test] + fn canvas_fixture_base_is_owned_by_each_test() { + let first_base = 1_700_000_000; + let second_base = first_base + crate::replica_fence::CREATED_AT_FLOOR_SECS as u64 * 2; + let first = make_canvas_event_at(first_base, "first test", 1000); + let second = make_canvas_event_at(second_base, "later test", 1000); + assert_eq!(first.created_at.as_secs(), first_base + 1000); + assert_eq!(second.created_at.as_secs(), second_base + 1000); + let (lower, higher) = same_second_ordered_pair(second_base, 1000); + assert_eq!(lower.created_at, higher.created_at); + assert_eq!(lower.created_at.as_secs(), second_base + 1000); + assert!(lower.id.as_bytes() < higher.id.as_bytes()); + } + #[tokio::test] async fn canvas_head_checked_rejects_non_canvas_kinds_before_opening_a_transaction() { // A lazy pool that is never connected: the kind check must reject @@ -3915,10 +3937,11 @@ mod postgres_tests { #[tokio::test] #[ignore = "requires Postgres"] async fn channel_head_checked_expect_no_head_creates_first_canvas() { + let canvas_base = nostr::Timestamp::now().as_secs() - 1000; let pool = setup_pool().await; let community = CommunityId::from_uuid(make_test_community(&pool).await); let channel = make_test_channel(&pool, community.as_uuid().to_owned(), None).await; - let event = make_canvas_event_at("# First", 1000); + let event = make_canvas_event_at(canvas_base, "# First", 1000); let (_, status) = insert_canvas_head_checked( &pool, @@ -3935,10 +3958,11 @@ mod postgres_tests { #[tokio::test] #[ignore = "requires Postgres"] async fn channel_head_checked_expect_no_head_rejects_when_head_exists() { + let canvas_base = nostr::Timestamp::now().as_secs() - 1000; let pool = setup_pool().await; let community = CommunityId::from_uuid(make_test_community(&pool).await); let channel = make_test_channel(&pool, community.as_uuid().to_owned(), None).await; - let first = make_canvas_event_at("# First", 1000); + let first = make_canvas_event_at(canvas_base, "# First", 1000); insert_canvas_head_checked( &pool, community, @@ -3949,7 +3973,7 @@ mod postgres_tests { .await .expect("first canvas"); - let second = make_canvas_event_at("# Racing create", 1001); + let second = make_canvas_event_at(canvas_base, "# Racing create", 1001); let (_, status) = insert_canvas_head_checked( &pool, community, @@ -3975,10 +3999,11 @@ mod postgres_tests { #[tokio::test] #[ignore = "requires Postgres"] async fn channel_head_checked_expected_head_matches_and_advances() { + let canvas_base = nostr::Timestamp::now().as_secs() - 1000; let pool = setup_pool().await; let community = CommunityId::from_uuid(make_test_community(&pool).await); let channel = make_test_channel(&pool, community.as_uuid().to_owned(), None).await; - let first = make_canvas_event_at("# First", 1000); + let first = make_canvas_event_at(canvas_base, "# First", 1000); insert_canvas_head_checked( &pool, community, @@ -3989,7 +4014,7 @@ mod postgres_tests { .await .expect("first canvas"); - let second = make_canvas_event_at("# Second", 1001); + let second = make_canvas_event_at(canvas_base, "# Second", 1001); let head = first.id.as_bytes(); let (_, status) = insert_canvas_head_checked( &pool, @@ -4006,10 +4031,11 @@ mod postgres_tests { #[tokio::test] #[ignore = "requires Postgres"] async fn channel_head_checked_expected_head_mismatch_rejects() { + let canvas_base = nostr::Timestamp::now().as_secs() - 1000; let pool = setup_pool().await; let community = CommunityId::from_uuid(make_test_community(&pool).await); let channel = make_test_channel(&pool, community.as_uuid().to_owned(), None).await; - let first = make_canvas_event_at("# First", 1000); + let first = make_canvas_event_at(canvas_base, "# First", 1000); insert_canvas_head_checked( &pool, community, @@ -4020,7 +4046,7 @@ mod postgres_tests { .await .expect("first canvas"); - let stale = make_canvas_event_at("# Stale edit", 1002); + let stale = make_canvas_event_at(canvas_base, "# Stale edit", 1002); let wrong_head = [0u8; 32]; let (_, status) = insert_canvas_head_checked( &pool, @@ -4037,10 +4063,11 @@ mod postgres_tests { #[tokio::test] #[ignore = "requires Postgres"] async fn channel_head_checked_expected_head_missing_rejects() { + let canvas_base = nostr::Timestamp::now().as_secs() - 1000; let pool = setup_pool().await; let community = CommunityId::from_uuid(make_test_community(&pool).await); let channel = make_test_channel(&pool, community.as_uuid().to_owned(), None).await; - let event = make_canvas_event_at("# Edit with no head", 1000); + let event = make_canvas_event_at(canvas_base, "# Edit with no head", 1000); let some_head = [1u8; 32]; let (_, status) = insert_canvas_head_checked( @@ -4058,10 +4085,11 @@ mod postgres_tests { #[tokio::test] #[ignore = "requires Postgres"] async fn channel_head_checked_replay_of_head_is_duplicate() { + let canvas_base = nostr::Timestamp::now().as_secs() - 1000; let pool = setup_pool().await; let community = CommunityId::from_uuid(make_test_community(&pool).await); let channel = make_test_channel(&pool, community.as_uuid().to_owned(), None).await; - let event = make_canvas_event_at("# First", 1000); + let event = make_canvas_event_at(canvas_base, "# First", 1000); insert_canvas_head_checked( &pool, community, @@ -4093,10 +4121,11 @@ mod postgres_tests { #[tokio::test] #[ignore = "requires Postgres"] async fn channel_head_checked_replay_skips_stale_precondition() { + let canvas_base = nostr::Timestamp::now().as_secs() - 1000; let pool = setup_pool().await; let community = CommunityId::from_uuid(make_test_community(&pool).await); let channel = make_test_channel(&pool, community.as_uuid().to_owned(), None).await; - let event = make_canvas_event_at("# First", 1000); + let event = make_canvas_event_at(canvas_base, "# First", 1000); insert_canvas_head_checked( &pool, community, @@ -4128,11 +4157,12 @@ mod postgres_tests { #[tokio::test] #[ignore = "requires Postgres"] async fn channel_head_checked_rejects_same_second_higher_id() { + let canvas_base = nostr::Timestamp::now().as_secs() - 1000; let pool = setup_pool().await; let community = CommunityId::from_uuid(make_test_community(&pool).await); let channel = make_test_channel(&pool, community.as_uuid().to_owned(), None).await; - let (lower, higher) = same_second_ordered_pair(1000); + let (lower, higher) = same_second_ordered_pair(canvas_base, 1000); insert_canvas_head_checked( &pool, community, @@ -4171,11 +4201,12 @@ mod postgres_tests { #[tokio::test] #[ignore = "requires Postgres"] async fn channel_head_checked_accepts_same_second_lower_id() { + let canvas_base = nostr::Timestamp::now().as_secs() - 1000; let pool = setup_pool().await; let community = CommunityId::from_uuid(make_test_community(&pool).await); let channel = make_test_channel(&pool, community.as_uuid().to_owned(), None).await; - let (lower, higher) = same_second_ordered_pair(1000); + let (lower, higher) = same_second_ordered_pair(canvas_base, 1000); // Seed the higher-id event as the head first. insert_canvas_head_checked( &pool, @@ -4205,10 +4236,11 @@ mod postgres_tests { #[tokio::test] #[ignore = "requires Postgres"] async fn channel_head_checked_rejects_behind_clock_writer() { + let canvas_base = nostr::Timestamp::now().as_secs() - 1000; let pool = setup_pool().await; let community = CommunityId::from_uuid(make_test_community(&pool).await); let channel = make_test_channel(&pool, community.as_uuid().to_owned(), None).await; - let head = make_canvas_event_at("# Head", 2000); + let head = make_canvas_event_at(canvas_base, "# Head", 2000); insert_canvas_head_checked( &pool, community, @@ -4220,7 +4252,7 @@ mod postgres_tests { .expect("head at t=2000"); // Writer's clock is behind — created_at earlier than head. - let behind = make_canvas_event_at("# Behind clock", 1100); + let behind = make_canvas_event_at(canvas_base, "# Behind clock", 1100); let (_, status) = insert_canvas_head_checked( &pool, community, @@ -4304,12 +4336,13 @@ mod postgres_tests { #[tokio::test] #[ignore = "requires Postgres"] async fn channel_head_checked_deleted_replay_returns_revision_mismatch() { + let canvas_base = nostr::Timestamp::now().as_secs() - 1000; let pool = setup_pool().await; let community = CommunityId::from_uuid(make_test_community(&pool).await); let channel = make_test_channel(&pool, community.as_uuid().to_owned(), None).await; // H: seed the initial head. - let head = make_canvas_event_at("# Head", 1000); + let head = make_canvas_event_at(canvas_base, "# Head", 1000); insert_canvas_head_checked( &pool, community, @@ -4322,7 +4355,7 @@ mod postgres_tests { let head_id = head.id.to_bytes().to_vec(); // A: insert a second revision. - let a = make_canvas_event_at("# A", 1001); + let a = make_canvas_event_at(canvas_base, "# A", 1001); insert_canvas_head_checked( &pool, community, @@ -4427,11 +4460,12 @@ mod postgres_tests { #[tokio::test] #[ignore = "requires Postgres"] async fn channel_head_checked_concurrent_authors_only_one_advances() { + let canvas_base = nostr::Timestamp::now().as_secs() - 1000; let pool = setup_pool().await; let community = CommunityId::from_uuid(make_test_community(&pool).await); let channel = make_test_channel(&pool, community.as_uuid().to_owned(), None).await; - let base = make_canvas_event_at("# Base", 1000); + let base = make_canvas_event_at(canvas_base, "# Base", 1000); insert_canvas_head_checked( &pool, community, @@ -4456,8 +4490,8 @@ mod postgres_tests { // Both edits assert `base` as their head and are timestamped strictly // ahead of it (writer discipline), so neither trips SupersedeFailed — // only the advisory-lock serialization decides the winner. - let a = make_canvas_event_at("# Author A", 1001); - let b = make_canvas_event_at("# Author B", 1002); + let a = make_canvas_event_at(canvas_base, "# Author A", 1001); + let b = make_canvas_event_at(canvas_base, "# Author B", 1002); let (pool_a, pool_b) = (pool.clone(), pool.clone()); let (id_a, id_b) = (base_id.clone(), base_id.clone()); @@ -4568,6 +4602,7 @@ mod postgres_tests { #[tokio::test] #[ignore = "requires Postgres"] async fn channel_head_checked_concurrent_first_create_only_one_wins() { + let canvas_base = nostr::Timestamp::now().as_secs() - 1000; let pool = setup_pool().await; let community = CommunityId::from_uuid(make_test_community(&pool).await); let channel = make_test_channel(&pool, community.as_uuid().to_owned(), None).await; @@ -4581,8 +4616,8 @@ mod postgres_tests { .await .expect("blocker acquires key"); - let a = make_canvas_event_at("# First A", 1000); - let b = make_canvas_event_at("# First B", 1001); + let a = make_canvas_event_at(canvas_base, "# First A", 1000); + let b = make_canvas_event_at(canvas_base, "# First B", 1001); let (pool_a, pool_b) = (pool.clone(), pool.clone()); let ta = tokio::spawn(async move { @@ -4689,6 +4724,7 @@ mod postgres_tests { #[tokio::test] #[ignore = "requires Postgres"] async fn db_untagged_canvas_append_serializes_on_advisory_key() { + let canvas_base = nostr::Timestamp::now().as_secs() - 1000; let pool = setup_pool().await; let db = crate::Db::from_pool(pool.clone()); let community = CommunityId::from_uuid(make_test_community(&pool).await); @@ -4702,7 +4738,7 @@ mod postgres_tests { .await .expect("holder acquires key"); - let event = make_canvas_event_at("# Untagged", 1000); + let event = make_canvas_event_at(canvas_base, "# Untagged", 1000); let write_task = tokio::spawn(async move { db.insert_event_with_thread_metadata(community, &event, Some(channel), None) .await @@ -4744,13 +4780,14 @@ mod postgres_tests { #[tokio::test] #[ignore = "requires Postgres"] async fn channel_head_canvas_deletion_serializes_on_advisory_key() { + let canvas_base = nostr::Timestamp::now().as_secs() - 1000; let pool = setup_pool().await; let community = CommunityId::from_uuid(make_test_community(&pool).await); let channel = make_test_channel(&pool, community.as_uuid().to_owned(), None).await; let lock_key = canvas_lock_key(community, channel); // Insert a canvas event to delete. - let event = make_canvas_event_at("# To delete", 1000); + let event = make_canvas_event_at(canvas_base, "# To delete", 1000); insert_canvas_head_checked( &pool, community, diff --git a/crates/buzz-db/src/store/event_floor_postgres_tests.rs b/crates/buzz-db/src/store/event_floor_postgres_tests.rs new file mode 100644 index 00000000000..bf7451763fb --- /dev/null +++ b/crates/buzz-db/src/store/event_floor_postgres_tests.rs @@ -0,0 +1,378 @@ +//! App-floor regressions with no enabled floor-trigger clone in the +//! nextest-provided per-test database, so the DB backstop cannot mask a bypass. + +use buzz_core::CommunityId; +use chrono::{DateTime, Utc}; +use nostr::{Event, EventBuilder, Keys, Kind, Tag, Timestamp}; +use sqlx::postgres::PgPoolOptions; +use uuid::Uuid; + +use crate::replica_fence::{ + probe_once, ReplicaFence, CREATED_AT_FLOOR_SECS, FENCE_CLOCK_MARGIN_SECS, +}; +use crate::{event, AdmittedTx, Db, DbError}; + +async fn fixture() -> (Db, CommunityId, Uuid, Keys) { + let pool = PgPoolOptions::new() + .max_connections(4) + .connect(&crate::test_support::database_url()) + .await + .expect("isolated test database"); + let enabled: i64 = sqlx::query_scalar( + "SELECT count(*) FROM pg_trigger \ + WHERE tgname = 'events_created_at_floor' AND tgenabled <> 'D' \ + AND tgrelid IN (SELECT relid FROM pg_partition_tree('events'))", + ) + .fetch_one(&pool) + .await + .expect("check every trigger clone"); + assert_eq!(enabled, 0, "no partition may mask a missing app check"); + let community = CommunityId::from_uuid(Uuid::new_v4()); + sqlx::query("INSERT INTO communities (id, host) VALUES ($1, $2)") + .bind(community.as_uuid()) + .bind(format!("floor-{}.invalid", community.as_uuid())) + .execute(&pool) + .await + .expect("community"); + let channel = Uuid::new_v4(); + let keys = Keys::generate(); + sqlx::query( + "INSERT INTO channels (id, community_id, name, created_by) VALUES ($1,$2,'floor',$3)", + ) + .bind(channel) + .bind(community.as_uuid()) + .bind(keys.public_key().to_bytes().as_slice()) + .execute(&pool) + .await + .expect("channel"); + (Db::from_pool(pool), community, channel, keys) +} + +async fn database_clock(db: &Db) -> DateTime { + sqlx::query_scalar("SELECT clock_timestamp()") + .fetch_one(db.pool()) + .await + .expect("database clock") +} + +fn signed(keys: &Keys, kind: u16, channel: Uuid, time: DateTime) -> Event { + EventBuilder::new(Kind::Custom(kind), "floor-test") + .tags([Tag::parse(["d", &channel.to_string()]).expect("coordinate")]) + .custom_created_at(Timestamp::from(time.timestamp() as u64)) + .sign_with_keys(keys) + .expect("signed event") +} + +async fn count(db: &Db, community: CommunityId, event: &Event) -> i64 { + sqlx::query_scalar("SELECT count(*) FROM events WHERE community_id=$1 AND id=$2") + .bind(community.as_uuid()) + .bind(event.id.as_bytes().as_slice()) + .fetch_one(db.pool()) + .await + .expect("event count") +} + +async fn begin(db: &Db, community: CommunityId) -> AdmittedTx { + db.begin_event_write_transaction(community) + .await + .expect("admission") +} + +fn rejected(result: crate::Result) { + assert!( + matches!(result, Err(DbError::CreatedAtFloorRejected)), + "expected the typed app-floor rejection, got {result:?}" + ); +} + +#[tokio::test] +#[ignore = "requires Postgres"] +async fn insert_event_floor_rejects_old_and_accepts_fresh() { + let (db, community, channel, keys) = fixture().await; + let now = database_clock(&db).await; + let old = signed( + &keys, + 9, + channel, + now - chrono::Duration::seconds(CREATED_AT_FLOOR_SECS + 60), + ); + rejected(event::insert_event(db.pool(), community, &old, Some(channel)).await); + assert_eq!(count(&db, community, &old).await, 0); + let fresh = signed(&keys, 9, channel, now); + assert!( + event::insert_event(db.pool(), community, &fresh, Some(channel)) + .await + .expect("fresh") + .1 + ); + assert_eq!(count(&db, community, &fresh).await, 1); + // Channel-NULL history is structurally outside covered windows. + assert!( + event::insert_event(db.pool(), community, &old, None) + .await + .expect("NULL-channel history") + .1 + ); +} + +#[tokio::test] +#[ignore = "requires Postgres"] +async fn insert_event_floor_accepts_expired_duplicate() { + let (db, community, channel, keys) = fixture().await; + let old = signed( + &keys, + 9, + channel, + database_clock(&db).await - chrono::Duration::seconds(CREATED_AT_FLOOR_SECS + 60), + ); + crate::test_support::insert_historical_event(db.pool(), community, &old, channel).await; + let (_, inserted) = event::insert_event(db.pool(), community, &old, Some(channel)) + .await + .expect("expired duplicate is idempotent"); + assert!(!inserted); + assert_eq!(count(&db, community, &old).await, 1); +} + +#[tokio::test] +#[ignore = "requires Postgres"] +async fn thread_metadata_event_floor_accepts_expired_duplicate() { + let (db, community, channel, keys) = fixture().await; + let time = database_clock(&db).await - chrono::Duration::seconds(CREATED_AT_FLOOR_SECS + 60); + let old = signed(&keys, 9, channel, time); + crate::test_support::insert_historical_event(db.pool(), community, &old, channel).await; + let mut tx = begin(&db, community).await; + let meta = event::ThreadMetadataParams { + event_id: old.id.as_bytes(), + event_created_at: DateTime::from_timestamp(old.created_at.as_secs() as i64, 0) + .expect("timestamp"), + channel_id: channel, + parent_event_id: None, + parent_event_created_at: None, + root_event_id: None, + root_event_created_at: None, + depth: 0, + broadcast: true, + }; + let (_, inserted) = + event::insert_event_with_thread_metadata_tx(&mut tx, &old, Some(channel), Some(meta)) + .await + .expect("expired duplicate is idempotent"); + assert!(!inserted); + tx.commit().await.expect("commit duplicate result"); + assert_eq!(count(&db, community, &old).await, 1); + let metadata: i64 = sqlx::query_scalar( + "SELECT count(*) FROM thread_metadata WHERE community_id=$1 AND event_id=$2", + ) + .bind(community.as_uuid()) + .bind(old.id.as_bytes().as_slice()) + .fetch_one(db.pool()) + .await + .expect("metadata count"); + assert_eq!(metadata, 0, "a duplicate must not run thread follow-ups"); +} + +#[tokio::test] +#[ignore = "requires Postgres"] +async fn thread_metadata_event_floor_rejects_old_and_accepts_fresh() { + let (db, community, channel, keys) = fixture().await; + let now = database_clock(&db).await; + for (time, old) in [ + ( + now - chrono::Duration::seconds(CREATED_AT_FLOOR_SECS + 60), + true, + ), + (now, false), + ] { + let event = signed(&keys, 9, channel, time); + let meta = event::ThreadMetadataParams { + event_id: event.id.as_bytes().as_slice(), + event_created_at: time, + channel_id: channel, + parent_event_id: None, + parent_event_created_at: None, + root_event_id: None, + root_event_created_at: None, + depth: 0, + broadcast: false, + }; + let mut tx = begin(&db, community).await; + let result = + event::insert_event_with_thread_metadata_tx(&mut tx, &event, Some(channel), Some(meta)) + .await; + if old { + rejected(result); + tx.rollback().await.expect("rollback refusal"); + assert_eq!(count(&db, community, &event).await, 0); + } else { + assert!(result.expect("fresh").1); + tx.commit().await.expect("commit"); + assert_eq!(count(&db, community, &event).await, 1); + } + } +} + +#[tokio::test] +#[ignore = "requires Postgres"] +async fn parameterized_event_floor_rejects_old_and_accepts_fresh() { + let (db, community, channel, keys) = fixture().await; + let now = database_clock(&db).await; + let old = signed( + &keys, + 30078, + channel, + now - chrono::Duration::seconds(CREATED_AT_FLOOR_SECS + 60), + ); + rejected( + db.replace_parameterized_event(community, &old, &channel.to_string(), Some(channel)) + .await, + ); + assert_eq!(count(&db, community, &old).await, 0); + let fresh = signed(&keys, 30078, channel, now); + assert!( + db.replace_parameterized_event(community, &fresh, &channel.to_string(), Some(channel)) + .await + .expect("fresh") + .1 + ); + assert_eq!(count(&db, community, &fresh).await, 1); +} + +#[tokio::test] +#[ignore = "requires Postgres"] +async fn addressable_event_floor_rejects_old_and_accepts_fresh() { + let (db, community, channel, keys) = fixture().await; + let now = database_clock(&db).await; + let old = signed( + &keys, + 39000, + channel, + now - chrono::Duration::seconds(CREATED_AT_FLOOR_SECS + 60), + ); + rejected( + db.replace_addressable_event(community, &old, Some(channel)) + .await, + ); + assert_eq!(count(&db, community, &old).await, 0); + let fresh = signed(&keys, 39000, channel, now); + assert!( + db.replace_addressable_event(community, &fresh, Some(channel)) + .await + .expect("fresh") + .1 + ); + assert_eq!(count(&db, community, &fresh).await, 1); +} + +#[tokio::test] +#[ignore = "requires Postgres"] +async fn member_snapshot_event_floor_rejects_old_and_accepts_fresh() { + let (db, community, channel, keys) = fixture().await; + let now = database_clock(&db).await; + let old = signed( + &keys, + 39002, + channel, + now - chrono::Duration::seconds(CREATED_AT_FLOOR_SECS + 60), + ); + let mut snapshot = db + .lock_member_snapshot(community, channel, &keys.public_key().to_bytes()) + .await + .expect("snapshot"); + rejected(snapshot.replace_member_event(&old).await); + // A refusal must leave the held snapshot usable and write no event. + assert_eq!(count(&db, community, &old).await, 0); + let fresh = signed(&keys, 39002, channel, now); + assert!( + snapshot + .replace_member_event(&fresh) + .await + .expect("fresh") + .1 + ); + snapshot.release().await.expect("commit snapshot"); + assert_eq!(count(&db, community, &fresh).await, 1); +} + +#[tokio::test] +#[ignore = "requires Postgres"] +async fn cluster_global_event_floor_held_transaction_cannot_write_below_probe_wall() { + let (db, community, channel, keys) = fixture().await; + let mut tx = begin(&db, community).await; + let start: DateTime = sqlx::query_scalar("SELECT now()") + .fetch_one(tx.conn()) + .await + .expect("transaction start"); + let entry = probe_once(db.pool(), &ReplicaFence::new()) + .await + .expect("heartbeat while writer is open"); + assert!( + entry.fence_wall + <= start - chrono::Duration::seconds(CREATED_AT_FLOOR_SECS + FENCE_CLOCK_MARGIN_SECS) + ); + let below_wall = signed( + &keys, + 9, + channel, + entry.fence_wall - chrono::Duration::seconds(1), + ); + sqlx::query("SAVEPOINT rejected_floor_insert") + .execute(tx.conn()) + .await + .expect("isolate rejected insert"); + rejected(event::insert_event_in_transaction(&mut tx, &below_wall, Some(channel)).await); + // The check runs after INSERT. Roll back that write before continuing in + // the same held transaction, just as a caller must on any insert failure. + sqlx::query("ROLLBACK TO SAVEPOINT rejected_floor_insert") + .execute(tx.conn()) + .await + .expect("roll back rejected row"); + sqlx::query("RELEASE SAVEPOINT rejected_floor_insert") + .execute(tx.conn()) + .await + .expect("release rejection savepoint"); + // Sample just after the probe; a near-floor accepted insert is above its + // wall even when this transaction commits after the heartbeat. + let fresh = signed( + &keys, + 9, + channel, + database_clock(&db).await - chrono::Duration::seconds(CREATED_AT_FLOOR_SECS - 5), + ); + let created_at = + DateTime::from_timestamp(fresh.created_at.as_secs() as i64, 0).expect("timestamp"); + assert!(created_at > entry.fence_wall); + assert!( + event::insert_event_in_transaction(&mut tx, &fresh, Some(channel)) + .await + .expect("near floor") + .1 + ); + tx.commit().await.expect("commit after probe"); + assert_eq!(count(&db, community, &below_wall).await, 0); + assert_eq!(count(&db, community, &fresh).await, 1); +} + +#[tokio::test] +#[ignore = "requires Postgres"] +async fn event_floor_samples_statement_clock_after_transaction_start() { + let (db, community, channel, keys) = fixture().await; + let mut tx = begin(&db, community).await; + // The smallest integral event timestamp inside the floor at BEGIN. + let secs: i64 = + sqlx::query_scalar("SELECT ceil(extract(epoch FROM now()) - $1::double precision)::bigint") + .bind(CREATED_AT_FLOOR_SECS as f64) + .fetch_one(tx.conn()) + .await + .expect("BEGIN boundary"); + let time = DateTime::from_timestamp(secs, 0).expect("timestamp"); + // Wait on the DB clock until this exact boundary expires (at most 1.1 s), + // not an arbitrary delay and not a Rust clock. now() remains frozen inside. + tokio::time::timeout(std::time::Duration::from_secs(3), + sqlx::query("SELECT pg_sleep(GREATEST(0, extract(epoch FROM ($1::timestamptz + make_interval(secs => $2::double precision) - clock_timestamp()))) + 0.1)") + .bind(time).bind(CREATED_AT_FLOOR_SECS as f64).execute(tx.conn())) + .await.expect("bounded boundary wait").expect("database wait"); + let event = signed(&keys, 9, channel, time); + rejected(event::insert_event_in_transaction(&mut tx, &event, Some(channel)).await); + tx.rollback().await.expect("rollback"); + assert_eq!(count(&db, community, &event).await, 0); +} diff --git a/crates/buzz-db/src/store/mod.rs b/crates/buzz-db/src/store/mod.rs index efa46f17485..756a276c207 100644 --- a/crates/buzz-db/src/store/mod.rs +++ b/crates/buzz-db/src/store/mod.rs @@ -26,6 +26,9 @@ pub(crate) mod event_follow_up; #[cfg(test)] mod event_follow_up_postgres_tests; +#[cfg(test)] +mod event_floor_postgres_tests; + /// Home feed queries. pub mod feed; /// Git repository name registry (NIP-34 kind:30617). diff --git a/crates/buzz-db/src/store/personal_read/participation_postgres_tests.rs b/crates/buzz-db/src/store/personal_read/participation_postgres_tests.rs index 5a3d3a0acad..2800b9b1804 100644 --- a/crates/buzz-db/src/store/personal_read/participation_postgres_tests.rs +++ b/crates/buzz-db/src/store/personal_read/participation_postgres_tests.rs @@ -70,10 +70,8 @@ impl World { .custom_created_at(Timestamp::from(at)) .sign_with_keys(author) .unwrap(); - self.db - .insert_event(self.community, &event, Some(channel)) - .await - .unwrap(); + crate::test_support::insert_historical_event(&self.pool, self.community, &event, channel) + .await; event } diff --git a/crates/buzz-db/src/store/personal_read/postgres_tests.rs b/crates/buzz-db/src/store/personal_read/postgres_tests.rs index cba0d992a47..7678f4c13cb 100644 --- a/crates/buzz-db/src/store/personal_read/postgres_tests.rs +++ b/crates/buzz-db/src/store/personal_read/postgres_tests.rs @@ -164,8 +164,9 @@ async fn personal_read_sidebar_author_window_excludes_expired_and_late_old_messa "an empty author-time window has nothing unread" ); assert_eq!(row.latest_id.as_deref(), Some(latest.id.to_hex().as_str())); - // Acceptance time is irrelevant: a message accepted now with an author - // time beyond the horizon stays out; one authored now is counted. + // Stored historical data with an author time beyond the horizon stays out; + // one authored now is counted. The serving writer now rejects the old + // timestamp, so seed the historical row through unarmed fixture SQL. let now = nostr::Timestamp::now().as_secs(); for (age, unread, mentions) in [(40 * 86400, false, 0), (0, true, 1)] { let event = EventBuilder::new(Kind::Custom(9), "accepted now") @@ -173,9 +174,7 @@ async fn personal_read_sidebar_author_window_excludes_expired_and_late_old_messa .custom_created_at(nostr::Timestamp::from(now - age)) .sign_with_keys(&Keys::generate()) .unwrap(); - db.insert_event(community, &event, Some(channel)) - .await - .unwrap(); + crate::test_support::insert_historical_event(&pool, community, &event, channel).await; let row = &sidebar(&db, community, &actor).await.channels[0]; assert_eq!((row.unread, row.mentions), (unread, mentions), "age={age}"); } diff --git a/crates/buzz-db/src/store/personal_read/threads_postgres_tests.rs b/crates/buzz-db/src/store/personal_read/threads_postgres_tests.rs index 2bb86aab8da..09786f55856 100644 --- a/crates/buzz-db/src/store/personal_read/threads_postgres_tests.rs +++ b/crates/buzz-db/src/store/personal_read/threads_postgres_tests.rs @@ -32,9 +32,7 @@ async fn post_as( .custom_created_at(nostr::Timestamp::from(at)) .sign_with_keys(author) .unwrap(); - db.insert_event(community, &event, Some(channel)) - .await - .unwrap(); + crate::test_support::insert_historical_event(&db.pool, community, &event, channel).await; event } diff --git a/crates/buzz-db/src/store/relay_members.rs b/crates/buzz-db/src/store/relay_members.rs index 6b21c7ce01a..61d7104da14 100644 --- a/crates/buzz-db/src/store/relay_members.rs +++ b/crates/buzz-db/src/store/relay_members.rs @@ -10,7 +10,6 @@ use buzz_core::StoredEvent; use buzz_datastore_tracing::datastore_span; use chrono::{DateTime, Utc}; use sqlx::{PgPool, Row as _}; -use uuid::Uuid; use crate::error::{DbError, Result}; use crate::{observability, replaceable, CommunityId, Db, RouteDecision, RoutePredicate}; @@ -1372,7 +1371,7 @@ impl Db { let insert_result = sqlx::query( "INSERT INTO events (community_id, id, pubkey, created_at, kind, tags, content, sig, received_at, channel_id, d_tag) \ - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) \ + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, NULL, $10) \ ON CONFLICT DO NOTHING", ) .bind(community_id.as_uuid()) @@ -1384,7 +1383,6 @@ impl Db { .bind(&event.content) .bind(sig_bytes.as_slice()) .bind(received_at) - .bind::>(None) .bind(d_tag.as_deref()) .execute(tx.conn()) .await?; diff --git a/crates/buzz-db/src/store/replaceable.rs b/crates/buzz-db/src/store/replaceable.rs index 1bce5766a46..8dccadcb5c2 100644 --- a/crates/buzz-db/src/store/replaceable.rs +++ b/crates/buzz-db/src/store/replaceable.rs @@ -251,6 +251,8 @@ async fn replace_parameterized_event_in_transaction_impl( )); } + crate::replica_fence::guard_event_created_at_floor(tx, created_at, channel_id).await?; + // Everything from the supersede through the mention index runs under one // savepoint on the admitted transaction. A duplicate insert or any failure // (including mention indexing) rolls the whole replacement back and leaves @@ -485,6 +487,9 @@ impl Db { } } + crate::replica_fence::guard_event_created_at_floor(&mut tx, created_at, channel_id) + .await?; + // Soft-delete the old event (if any). IS NOT DISTINCT FROM for NULL safety. sqlx::query( "UPDATE events SET deleted_at = NOW() \ diff --git a/crates/buzz-db/src/test_support.rs b/crates/buzz-db/src/test_support.rs index e94b8b14491..ec1e816e391 100644 --- a/crates/buzz-db/src/test_support.rs +++ b/crates/buzz-db/src/test_support.rs @@ -8,6 +8,54 @@ pub(crate) fn database_url() -> String { .unwrap_or_else(|_| DEFAULT_DATABASE_URL.to_owned()) } +/// Seed historical read fixtures through raw SQL on an unarmed test pool. +/// Supported serving APIs reject these old timestamps; this represents data +/// already in storage, while retaining admission, mentions and event follow-ups. +#[cfg(test)] // Keep raw fixture SQL visibly test-only to the source-policy scan. +pub(crate) async fn insert_historical_event( + pool: &sqlx::PgPool, + community: buzz_core::CommunityId, + event: &nostr::Event, + channel: uuid::Uuid, +) { + let mut tx = crate::begin_community_write_transaction( + pool, + community, + crate::observability::WriterOperation::EventWrite, + ) + .await + .expect("admit historical fixture"); + sqlx::query( + "INSERT INTO events (community_id,id,pubkey,created_at,kind,tags,content,sig,channel_id,d_tag) \ + VALUES ($1,$2,$3,to_timestamp($4),$5,$6,$7,$8,$9,$10)", + ) + .bind(community.as_uuid()) + .bind(event.id.as_bytes().as_slice()) + .bind(event.pubkey.to_bytes().as_slice()) + .bind(event.created_at.as_secs() as f64) + .bind(i32::from(event.kind.as_u16())) + .bind(serde_json::to_value(&event.tags).expect("fixture tags")) + .bind(&event.content) + .bind(event.sig.serialize().as_slice()) + .bind(channel) + .bind(crate::event::extract_d_tag(event)) + .execute(tx.conn()) + .await + .expect("insert historical fixture"); + crate::store::event_follow_up::after_admitted_insert( + &mut tx, + event.id.as_bytes(), + i32::from(event.kind.as_u16()), + Some(channel), + ) + .await + .expect("fixture follow-ups"); + crate::insert_mentions_in_transaction(&mut tx, event, Some(channel)) + .await + .expect("fixture mentions"); + tx.commit().await.expect("commit historical fixture"); +} + /// Move a community fixture into a deletion lifecycle state the way the /// executor does: the tombstone trigger only accepts lifecycle changes from a /// transaction carrying the executor GUCs for that community, including its diff --git a/crates/buzz-db/tests/observability_source.rs b/crates/buzz-db/tests/observability_source.rs index 4d4edcf8e98..784363174af 100644 --- a/crates/buzz-db/tests/observability_source.rs +++ b/crates/buzz-db/tests/observability_source.rs @@ -444,22 +444,11 @@ fn p0_pool_acquisitions_use_typed_operation_pairs_without_other() { let runtime = include_str!("../src/runtime/mod.rs"); assert!(runtime.contains("observability::refresh_pool_waiters(self.read_pool.is_some())")); - assert!(runtime.contains("self.verify_replica_fence_at_boot().await?")); - let fence_boot = runtime - .split_once("pub(crate) async fn verify_replica_fence_at_boot") - .expect("runtime must expose attributed boot fence verification") - .1 - .split_once("/// The pool for lag-tolerant reads") - .expect("boot fence verification must precede routed-read plumbing") - .0; - assert!(fence_boot.contains("WriterOperation::Bootstrap")); - let replica_fence = include_str!("../src/runtime/replica_fence.rs"); let replica_fence_production = replica_fence .split("\n#[cfg(test)]") .next() .expect("replica-fence production source"); - assert!(replica_fence_production.contains("WriterOperation::Bootstrap")); assert!(replica_fence_production.contains("WriterOperation::Maintenance")); assert!(!replica_fence_production.contains("pool.begin().await")); assert!(!replica_fence_production.contains("writer.acquire().await")); @@ -915,8 +904,7 @@ fn has_any_tenant_local_chokepoint(source: &str) -> bool { const ADMITTED_TX_SIGNATURE_MARKERS: [&str; 2] = ["&mut AdmittedTx", "&mut crate::AdmittedTx"]; // Narrow reviewed exceptions for non-serving verification probes only. -const GUARDED_WRITE_FUNCTION_EXCEPTIONS: [&str; 3] = [ - "pub async fn verify_floor_guard_behavior(", +const GUARDED_WRITE_FUNCTION_EXCEPTIONS: [&str; 2] = [ "pub async fn verify_channel_roster_fence_behavior(", "pub async fn backfill_d_tags(&self) -> Result {", ]; @@ -1422,10 +1410,8 @@ fn serving_table_writes_expose_syntactic_chokepoint_or_guarded_tx_routes() { // Rolled-back verification probes write `events` rows that never commit, so // they owe no push job or TTL refresh. -const EVENT_INSERT_FOLLOW_UP_EXCEPTIONS: [&str; 2] = [ - "pub async fn verify_floor_guard_behavior(", - "pub async fn verify_channel_roster_fence_behavior(", -]; +const EVENT_INSERT_FOLLOW_UP_EXCEPTIONS: [&str; 1] = + ["pub async fn verify_channel_roster_fence_behavior("]; /// `source` with every `#[cfg(test)]` item removed, wherever it sits in the /// file. Each item runs to the `;` that ends it or to the brace that closes @@ -1497,21 +1483,60 @@ fn raw_literal_prefix_starts_token(bytes: &[u8], index: usize) -> bool { fn mask_comments_and_literals(source: &str) -> String { let bytes = source.as_bytes(); let mut masked = bytes.to_vec(); - let mut blank = |from: usize, to: usize| { - for byte in &mut masked[from..to.min(bytes.len())] { + for span in source_spans(source) { + for byte in &mut masked[span.start..span.end] { if *byte != b'\n' { *byte = b' '; } } - }; + } + String::from_utf8(masked).expect("masking replaces whole ASCII-delimited spans") +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum SpanKind { + Comment, + /// A plain (`"…"`) string literal; `start` is its opening quote. + Str, + /// A raw string literal; `start` is its `r`, followed by `hashes` `#`s. + RawStr { + hashes: usize, + }, + Char, +} + +struct SourceSpan { + start: usize, + end: usize, + kind: SpanKind, +} + +impl SourceSpan { + /// The text between a string literal's delimiters, escapes left as written. + fn string_body<'a>(&self, source: &'a str) -> Option<&'a str> { + match self.kind { + SpanKind::Str => source.get(self.start + 1..self.end.saturating_sub(1)), + SpanKind::RawStr { hashes } => { + source.get(self.start + 2 + hashes..self.end.saturating_sub(1 + hashes)) + } + SpanKind::Comment | SpanKind::Char => None, + } + } +} + +/// Every comment, string, raw string, and char literal in `source`, in order. +fn source_spans(source: &str) -> Vec { + let bytes = source.as_bytes(); + let mut spans = Vec::new(); let mut index = 0usize; while index < bytes.len() { let start = index; - match bytes[index] { + let kind = match bytes[index] { b'/' if bytes.get(index + 1) == Some(&b'/') => { while index < bytes.len() && bytes[index] != b'\n' { index += 1; } + SpanKind::Comment } b'/' if bytes.get(index + 1) == Some(&b'*') => { let mut depth = 0usize; @@ -1529,6 +1554,7 @@ fn mask_comments_and_literals(source: &str) -> String { index += 1; } } + SpanKind::Comment } b'r' if matches!(bytes.get(index + 1), Some(b'"' | b'#')) && raw_literal_prefix_starts_token(bytes, index) @@ -1550,6 +1576,7 @@ fn mask_comments_and_literals(source: &str) -> String { index = source[body..] .find(&close) .map_or(bytes.len(), |offset| body + offset + close.len()); + SpanKind::RawStr { hashes } } b'"' => { index += 1; @@ -1557,18 +1584,24 @@ fn mask_comments_and_literals(source: &str) -> String { index += if bytes[index] == b'\\' { 2 } else { 1 }; } index += 1; + SpanKind::Str } b'\'' if char_literal_end(source, index).is_some() => { index = char_literal_end(source, index).expect("guard matched"); + SpanKind::Char } _ => { index += 1; continue; } - } - blank(start, index); + }; + spans.push(SourceSpan { + start, + end: index.min(bytes.len()), + kind, + }); } - String::from_utf8(masked).expect("masking replaces whole ASCII-delimited spans") + spans } /// The end (exclusive) of the char literal whose opening quote is at `start`, @@ -1626,14 +1659,19 @@ fn masker_blanks_every_literal_form_and_keeps_code() { } } -/// Whether `source` inserts into `events` itself (not `event_mentions` or any -/// other `events_*` table), in any letter case, across line breaks and string -/// continuations, `\n`/`\t`/`\r` escapes, and with or without a `public.` -/// schema prefix. -fn inserts_events_row(source: &str) -> bool { +/// Normalize SQL case, quoted identifiers, whitespace and Rust string escapes for +/// the event INSERT and UPDATE policies. +fn normalized_sql_source(source: &str) -> String { let mut normalized = String::with_capacity(source.len()); let mut chars = source.chars().peekable(); while let Some(ch) = chars.next() { + if ch == '"' { + continue; + } + if ch == '\\' && chars.peek() == Some(&'"') { + chars.next(); + continue; + } if ch.is_whitespace() || ch == '\\' { if ch == '\\' && matches!(chars.peek(), Some('n' | 't' | 'r')) { chars.next(); @@ -1645,6 +1683,13 @@ fn inserts_events_row(source: &str) -> bool { normalized.push(ch.to_ascii_lowercase()); } } + normalized +} + +/// Whether `source` inserts into `events` itself (not `event_mentions` or any +/// other `events_*` table), with or without a `public.` schema prefix. +fn inserts_events_row(source: &str) -> bool { + let normalized = normalized_sql_source(source); ["insert into events", "insert into public.events"] .iter() .any(|marker| { @@ -1693,6 +1738,9 @@ pub(crate) async fn unhooked_writer(tx: &mut AdmittedTx) { .await .expect("write"); } +pub(crate) async fn quoted_writer(tx: &mut AdmittedTx) { + sqlx::query("INSERT INTO \"public\".\"events\" (id) VALUES ($1)"); +} pub(crate) async fn push_only_writer(tx: &mut AdmittedTx) { sqlx::query("INSERT INTO events \ (community_id, id) VALUES ($1, $2)") @@ -1728,7 +1776,7 @@ pub(crate) async fn mention_writer(tx: &mut AdmittedTx) { .await .expect("write"); } -pub async fn verify_floor_guard_behavior(pool: &PgPool) { +pub async fn verify_channel_roster_fence_behavior(pool: &PgPool) { sqlx::query("INSERT INTO events (community_id, id) VALUES ($1, $2)") .execute(pool) .await @@ -1857,6 +1905,7 @@ mod tests { event_insert_follow_up_violations(&strip_cfg_test_items(source)), [ "pub(crate) async fn unhooked_writer(tx: &mut AdmittedTx) {", + "pub(crate) async fn quoted_writer(tx: &mut AdmittedTx) {", "pub(crate) async fn push_only_writer(tx: &mut AdmittedTx) {", "pub(crate) async fn lowercase_writer(tx: &mut AdmittedTx) {", "pub(crate) async fn continued_writer(tx: &mut AdmittedTx) {", @@ -1877,8 +1926,7 @@ mod tests { ); } -#[test] -fn every_production_event_insert_runs_the_follow_up_hook() { +fn collect_production_rs_files() -> Vec { use std::path::{Path, PathBuf}; fn collect_rs_files(dir: &Path, out: &mut Vec) { @@ -1903,6 +1951,12 @@ fn every_production_event_insert_runs_the_follow_up_hook() { } } + files +} + +#[test] +fn every_production_event_insert_runs_the_follow_up_hook() { + let files = collect_production_rs_files(); let mut hooked_writers = 0usize; let mut violations = Vec::new(); for path in files { @@ -1935,3 +1989,927 @@ fn every_production_event_insert_runs_the_follow_up_hook() { "expected at least seven hooked event writers, found {hooked_writers}" ); } + +// This startup probe always rolls back; it deliberately exercises the roster +// DB backstop rather than the app check. The two serving writers are exempt only +// while their INSERT remains structurally channel-NULL (checked below). +const EVENT_FLOOR_PROBE_EXCEPTIONS: [&str; 1] = + ["pub async fn verify_channel_roster_fence_behavior("]; + +fn is_channel_null_event_writer(function: &str) -> bool { + let header = function_header(function); + let sql = normalized_sql_source(function).replace(' ', ""); + if header.starts_with("pub async fn publish_nip43_membership_locked(") { + // Kind 13534: channel_id is the tenth value, a literal NULL. + return sql.contains("values($1,$2,$3,$4,$5,$6,$7,$8,$9,null,$10)"); + } + if header.starts_with("pub async fn accept_lease_event(") { + // Kind 30350: channel_id is the literal NULL after received_at. + return sql.contains("now(),null,$8)"); + } + false +} + +/// Whether a `SET` list assigns `created_at` or `channel_id`, including the +/// row form `(a, b) = (...)`, or a column it cannot read (a `{…}` +/// placeholder). Stops at the first `WHERE`. +fn assigns_event_floor_column(assignments: &str) -> bool { + let assignments = assignments.split(" where ").next().expect("first part"); + let mut depth = 0usize; + assignments + .split(|ch| { + match ch { + '(' => depth += 1, + ')' => depth = depth.saturating_sub(1), + _ => {} + } + ch == ',' && depth == 0 + }) + .any(|assignment| { + assignment.split_once('=').is_some_and(|(columns, _)| { + columns + .trim() + .trim_start_matches('(') + .trim_end_matches(')') + .split(',') + .map(str::trim) + .any(|column| { + matches!(column, "created_at" | "channel_id") || column.starts_with('{') + }) + }) + }) +} + +/// Whether a `SET` list (up to the first `WHERE`) is missing from its +/// literal: empty, or holding an item with no `=`, such as a `{…}` +/// placeholder or a column whose value follows in another literal. +fn set_list_is_unreadable(assignments: &str) -> bool { + let assignments = assignments.split(" where ").next().expect("first part"); + let mut depth = 0usize; + assignments.trim().is_empty() + || assignments + .split(|ch| { + match ch { + '(' => depth += 1, + ')' => depth = depth.saturating_sub(1), + _ => {} + } + ch == ',' && depth == 0 + }) + .any(|assignment| !assignment.contains('=')) +} + +/// The table a write targets, without its schema (a literal or a `{…}` +/// placeholder; any schema may hold an `events` table). +fn unqualified_table(target: &str) -> &str { + target.rsplit_once('.').map_or(target, |(_, table)| table) +} + +/// Whether one SQL literal (normalized) assigns a floor column through a +/// direct `UPDATE` of `events`, or of a target it does not spell: a +/// placeholder, or a `SET` list whose `UPDATE` lies in another literal. +/// A `SET` list it cannot read counts as assigning one. +fn literal_updates_event_floor_columns(sql: &str) -> bool { + // `SET(` with no space is still a row assignment. + let sql = format!(" {} ", sql.trim()).replace(" set(", " set ("); + sql.match_indices(" set ").any(|(index, marker)| { + let assignments = &sql[index + marker.len()..]; + // A SET list continued in another literal or argument cannot be read. + let unreadable = set_list_is_unreadable(assignments); + if !unreadable && !assigns_event_floor_column(assignments) { + return false; + } + let before = &sql[..index]; + if before.ends_with(" do update") { + // The upsert check resolves the INSERT, which may be in another + // literal of the same builder, but only from a readable SET list. + return unreadable; + } + let Some(update) = before.rfind(" update ") else { + // A bare SET (Redis `SET`, prose) is not SQL unless it assigns. + return !unreadable; + }; + let mut target = before[update + " update ".len()..].split_whitespace(); + let mut table = target.next(); + if table == Some("only") { + table = target.next(); + } + // The target may have an alias, with or without AS. + let alias = target.collect::>(); + let (Some(table), [] | [_] | ["as", _]) = (table, alias.as_slice()) else { + // An unreadable target is a violation only when it assigns a + // floor column; prose ("update … is set") assigns nothing. + return !unreadable; + }; + let table = unqualified_table(table); + table == "events" || table.contains('{') + }) +} + +fn updates_event_floor_columns(function: &str) -> bool { + let direct_update = source_spans(function) + .iter() + .filter_map(|span| span.string_body(function)) + .any(|body| literal_updates_event_floor_columns(&normalized_sql_source(body))); + // `ON CONFLICT ... DO UPDATE SET` names no table: it updates the target of + // the INSERT it belongs to, so an events upsert can re-home an existing row. + let normalized = normalized_sql_source(function).replace(" set(", " set ("); + let events_upsert = normalized + .match_indices("do update set ") + .any(|(index, marker)| { + normalized[..index] + .rfind("insert into ") + .is_some_and(|insert| { + let table = normalized[insert + "insert into ".len()..] + .split(|ch: char| !(ch.is_ascii_alphanumeric() || ch == '_' || ch == '.')) + .next(); + matches!(table, Some("events" | "public.events")) + }) + && assigns_event_floor_column(&normalized[index + marker.len()..]) + }); + direct_update || events_upsert +} + +/// Writes one SQL literal (normalized) hides from the per-function checks: +/// an INSERT whose target or tail is not literal, `events` written by a +/// statement other than INSERT, or a PREPARE. +fn literal_sql_violations(sql: &str) -> Vec<&'static str> { + let sql = format!(" {} ", sql.trim()); + let mut violations = Vec::new(); + for (index, _) in sql.match_indices(" into ") { + let rest = &sql[index + " into ".len()..]; + let target = rest.split([' ', '(']).next().expect("first part"); + let table = unqualified_table(target); + if sql[..index].ends_with(" insert") { + if target.is_empty() || table.contains('{') { + violations.push("INSERT target is not a literal table"); + } else if table == "events" && rest.contains('{') { + violations.push("events INSERT with a non-literal part"); + } + } else if table == "events" { + violations.push("events written by a statement other than INSERT"); + } + } + if sql.match_indices(" copy ").any(|(index, marker)| { + let target = sql[index + marker.len()..].split([' ', '(']).next(); + target.is_some_and(|target| unqualified_table(target) == "events") + }) { + violations.push("events written by a statement other than INSERT"); + } + let words = sql.split_whitespace().collect::>(); + if words + .windows(2) + .any(|pair| pair[0] == "prepare" && (pair[1] == "transaction" || pair[1].starts_with('{'))) + || words.last() == Some(&"prepare") + { + violations.push("PREPARE TRANSACTION in a production writer"); + } + violations +} + +/// The offset of the delimiter that closes the bracket opened at `open` in +/// `masked` (comments and literals already blanked). +fn matching_close(masked: &str, open: usize) -> Option { + let mut depth = 0usize; + for (index, byte) in masked.bytes().enumerate().skip(open) { + match byte { + b'(' | b'[' | b'{' => depth += 1, + b')' | b']' | b'}' => { + depth = depth.checked_sub(1)?; + if depth == 0 { + return Some(index); + } + } + _ => {} + } + } + None +} + +/// `text` as a plain Rust string literal. +fn plain_literal(text: &str) -> String { + format!("\"{}\"", plain_body(text)) +} + +/// Raw `text` escaped as the body of a plain string literal. +fn plain_body(text: &str) -> String { + text.replace('\\', "\\\\").replace('"', "\\\"") +} + +/// A string literal's body, as written in a plain (escaped) literal. +fn plain_string_body(span: &SourceSpan, source: &str) -> Option { + let body = span.string_body(source)?; + Some(if matches!(span.kind, SpanKind::Str) { + body.to_owned() + } else { + plain_body(body) + }) +} + +/// The string literal that is the whole of `arg`, if it is one. +fn sole_string_literal(arg: &str) -> Option { + let trimmed = arg.trim(); + let offset = arg.len() - arg.trim_start().len(); + let mut spans = source_spans(trimmed).into_iter(); + let span = spans.next()?; + (spans.next().is_none() + && span.start == 0 + && span.end == trimmed.len() + && matches!(span.kind, SpanKind::Str | SpanKind::RawStr { .. })) + .then_some(SourceSpan { + start: span.start + offset, + end: span.end + offset, + kind: span.kind, + }) +} + +/// The opening delimiter of each `name!` macro call in `masked` (comments +/// and literals blanked), last first, with any path prefix (`::core::`) and +/// any of `(`, `[` or `{`: `(start of name, delimiter offset)`. +fn macro_calls(masked: &str, name: &str) -> Vec<(usize, usize)> { + let mut calls = masked + .match_indices(name) + .filter_map(|(at, _)| { + let starts_token = !masked[..at] + .chars() + .next_back() + .is_some_and(|ch| ch.is_ascii_alphanumeric() || ch == '_'); + let after = masked[at + name.len()..].strip_prefix('!')?; + let open = at + name.len() + 1 + (after.len() - after.trim_start().len()); + (starts_token && matches!(masked.as_bytes().get(open), Some(b'(' | b'[' | b'{'))) + .then_some((at, open)) + }) + .collect::>(); + calls.reverse(); + calls +} + +/// Bring SQL that per-function literal scans cannot see into view, so the +/// floor policy fails closed on it now that no trigger backs it up: each +/// `include_str!("…")` becomes one literal holding the included file +/// (resolved against `path`), and each `concat!` becomes one literal of its +/// arguments, with `{opaque}` standing in for any non-literal argument. The +/// SQL checks treat a placeholder in a write target or a `SET` column as +/// unattributable. An `include_str!` whose path is not one literal or +/// cannot be read is a violation, as is any `query_file` macro. +fn sql_visible_source(path: &std::path::Path, source: &str) -> (String, Vec) { + let mut violations = Vec::new(); + let mut text = source.to_owned(); + for (macro_name, expand_include) in [("include_str", true), ("concat", false)] { + // Last occurrence first: a rewrite never moves an earlier offset, and + // an inner `concat!` is joined before the one around it. + loop { + let masked = mask_comments_and_literals(&text); + let Some(&(at, open)) = macro_calls(&masked, macro_name).first() else { + break; + }; + let Some(close) = matching_close(&masked, open) else { + violations.push(format!("{macro_name}!…) is unterminated")); + break; + }; + let args = &text[open + 1..close]; + let replacement = if expand_include { + let included = sole_string_literal(args) + .and_then(|span| span.string_body(args).map(str::to_owned)) + .and_then(|relative| { + let resolved = path.parent()?.join(&relative); + std::fs::read_to_string(resolved).ok() + }); + match included { + Some(contents) => plain_literal(&contents), + None => { + violations.push(format!("include_str!({}) is opaque", args.trim())); + plain_literal("") + } + } + } else { + let masked_args = &masked[open + 1..close]; + let mut joined = String::new(); + let mut depth = 0usize; + let mut arg_start = 0usize; + for (index, byte) in masked_args.bytes().chain(std::iter::once(b',')).enumerate() { + match byte { + b'(' | b'[' | b'{' => depth += 1, + b')' | b']' | b'}' => depth = depth.saturating_sub(1), + b',' if depth == 0 => { + let arg = &args[arg_start..index.min(args.len())]; + arg_start = index + 1; + if arg.trim().is_empty() { + continue; + } + match sole_string_literal(arg) + .and_then(|span| plain_string_body(&span, arg)) + { + Some(body) => joined.push_str(&body), + None => joined.push_str("{opaque}"), + } + } + _ => {} + } + } + format!("\"{joined}\"") + }; + text.replace_range(at..=close, &replacement); + } + } + // sqlx's `query_file*!` macros load SQL from a file at compile time; + // nothing uses them, so ban them rather than resolve them. + let masked = mask_comments_and_literals(&text); + for (at, _) in masked.match_indices("query_file") { + let starts_token = !masked[..at] + .chars() + .next_back() + .is_some_and(|ch| ch.is_ascii_alphanumeric() || ch == '_'); + let name_len = masked[at..] + .find(|ch: char| !(ch.is_ascii_alphanumeric() || ch == '_')) + .unwrap_or(masked.len() - at); + if starts_token && masked[at + name_len..].trim_start().starts_with('!') { + violations.push(format!( + "{}! loads SQL from a file", + &masked[at..at + name_len] + )); + } + } + (text, violations) +} + +/// Length of the function item that starts `masked_function` (comments and +/// literals blanked): through the `}` closing its body, or through the `;` of +/// a bodiless declaration. Unlike [`cfg_test_item_len`], a `;` inside the +/// signature's parentheses or brackets (`[u8; 4]`) does not end it. Without a +/// body the whole input counts, so an unrecognized shape never hides SQL. +fn function_item_len(masked_function: &str) -> usize { + let mut nesting = 0usize; + for (index, byte) in masked_function.bytes().enumerate() { + match byte { + b'(' | b'[' => nesting += 1, + b')' | b']' => nesting = nesting.saturating_sub(1), + b';' if nesting == 0 => return index + 1, + b'{' if nesting == 0 => { + return matching_close(masked_function, index) + .map_or(masked_function.len(), |close| close + 1); + } + _ => {} + } + } + masked_function.len() +} + +/// Each function slice cut to the function's own item, so a const or static +/// after its body is not attributed to it (see [`item_level_literals`]). +fn function_items(production: &str) -> Vec<&str> { + function_slices(production) + .into_iter() + .map(|function| &function[..function_item_len(&mask_comments_and_literals(function))]) + .collect() +} + +/// String literals outside every function body (consts, statics and macro +/// bodies), as plain literals one per line. A function's floor guard cannot +/// be attributed to SQL there. +fn item_level_literals(production: &str) -> String { + let base = production.as_ptr() as usize; + let functions: Vec<(usize, usize)> = function_items(production) + .into_iter() + .map(|function| { + let start = function.as_ptr() as usize - base; + (start, start + function.len()) + }) + .collect(); + let mut literals = String::new(); + for span in source_spans(production) { + if functions + .iter() + .any(|(start, end)| *start <= span.start && span.end <= *end) + { + continue; + } + if let Some(body) = plain_string_body(&span, production) { + literals.push_str(&format!("\"{body}\"\n")); + } + } + literals +} + +/// Every floor-policy violation in one production file, after +/// [`sql_visible_source`] has exposed included and concatenated SQL. +fn file_event_floor_policy_violations(path: &std::path::Path, source: &str) -> Vec { + let (production, mut violations) = sql_visible_source(path, &strip_cfg_test_items(source)); + // The fence cannot bound a prepared transaction's rows without the + // retired trigger (see `replica_fence::sample_writer`), so no supported + // writer may prepare one. + for span in source_spans(&production) { + if let Some(body) = span.string_body(&production) { + for violation in literal_sql_violations(&normalized_sql_source(body)) { + if !violations.iter().any(|seen| seen == violation) { + violations.push(violation.to_owned()); + } + } + } + } + let item_level = item_level_literals(&production); + if inserts_events_row(&item_level) || updates_event_floor_columns(&item_level) { + violations.push("event-write SQL outside any function body".to_owned()); + } + violations.extend(event_floor_policy_violations(&production)); + violations +} + +fn event_floor_policy_violations(production: &str) -> Vec { + function_items(production) + .into_iter() + .filter(|function| { + let header = function_header(function); + if EVENT_FLOOR_PROBE_EXCEPTIONS + .iter() + .any(|exception| header.starts_with(exception)) + { + return false; + } + updates_event_floor_columns(function) + || (inserts_events_row(function) + && !is_channel_null_event_writer(function) + && !mask_comments_and_literals(function) + .contains("replica_fence::guard_event_created_at_floor(")) + }) + .map(|function| function_header(function).to_owned()) + .collect() +} + +#[test] +fn event_floor_policy_rejects_unguarded_inserts_and_floor_column_updates() { + let source = r#" +async fn missing_guard(tx: &mut AdmittedTx) { + sqlx::query("INSERT INTO events (id) VALUES ($1)"); + // replica_fence::guard_event_created_at_floor(tx, created_at, channel).await?; +} +async fn guarded(tx: &mut AdmittedTx) { + crate::replica_fence::guard_event_created_at_floor(tx, created_at, channel).await?; + sqlx::query("INSERT INTO public.events (id) VALUES ($1)"); +} +async fn rewrite_time(tx: &mut AdmittedTx) { + sqlx::query("UPDATE events SET tags = $1, created_at = $2 WHERE id = $3"); +} +async fn attach_channel(tx: &mut AdmittedTx) { + sqlx::query("uPdAtE public.events\nSET channel_id=$1 WHERE id=$2"); +} +async fn alias_time(tx: &mut AdmittedTx) { + sqlx::query("UPDATE events e SET created_at = $1 WHERE e.id = $2"); +} +async fn as_alias_time(tx: &mut AdmittedTx) { + sqlx::query("UPDATE events AS e SET created_at = $1 WHERE e.id = $2"); +} +async fn only_channel(tx: &mut AdmittedTx) { + sqlx::query("UPDATE ONLY events SET channel_id = $1 WHERE id = $2"); +} +async fn only_public_alias(tx: &mut AdmittedTx) { + sqlx::query("UPDATE ONLY public.events AS e SET channel_id = $1 WHERE e.id = $2"); +} +async fn row_time(tx: &mut AdmittedTx) { + sqlx::query("UPDATE events SET (created_at, kind) = ($1, 1) WHERE id = $2"); +} +async fn row_channel(tx: &mut AdmittedTx) { + sqlx::query("UPDATE events SET (kind, channel_id) = (1, $1) WHERE id = $2"); +} +async fn quoted_insert(tx: &mut AdmittedTx) { + sqlx::query("INSERT INTO \"events\" (id) VALUES ($1)"); +} +async fn quoted_schema_insert(tx: &mut AdmittedTx) { + sqlx::query("INSERT INTO \"public\".\"events\" (id) VALUES ($1)"); +} +async fn quoted_update(tx: &mut AdmittedTx) { + sqlx::query("UPDATE \"events\" SET created_at = $1 WHERE id = $2"); +} +async fn quoted_schema_update(tx: &mut AdmittedTx) { + sqlx::query("UPDATE \"public\".\"events\" SET channel_id = $1 WHERE id = $2"); +} +async fn delete_only(tx: &mut AdmittedTx) { + sqlx::query("UPDATE events SET deleted_at=now() WHERE channel_id=$1 AND created_at<$2"); +} +async fn row_time_no_space(tx: &mut AdmittedTx) { + sqlx::query("UPDATE events SET(created_at,kind)=($1,1) WHERE id=$2"); +} +async fn guarded_upsert_rehome(tx: &mut AdmittedTx) { + crate::replica_fence::guard_event_created_at_floor(tx, created_at, channel).await?; + sqlx::query("INSERT INTO events (id, channel_id) VALUES ($1, $2) ON CONFLICT (id) DO UPDATE SET channel_id = EXCLUDED.channel_id"); +} +async fn guarded_upsert_row_time(tx: &mut AdmittedTx) { + crate::replica_fence::guard_event_created_at_floor(tx, created_at, channel).await?; + sqlx::query("INSERT INTO events (id) VALUES ($1) ON CONFLICT (id) DO UPDATE SET(kind, created_at) = (1, $2)"); +} +async fn guarded_upsert_other_column(tx: &mut AdmittedTx) { + crate::replica_fence::guard_event_created_at_floor(tx, created_at, channel).await?; + sqlx::query("INSERT INTO events (id) VALUES ($1) ON CONFLICT (id) DO UPDATE SET tags = EXCLUDED.tags"); +} +async fn guarded_upsert_do_nothing(tx: &mut AdmittedTx) { + crate::replica_fence::guard_event_created_at_floor(tx, created_at, channel).await?; + sqlx::query("INSERT INTO events (id) VALUES ($1) ON CONFLICT (id) DO NOTHING"); +} +async fn guarded_insert_then_other_table_upsert(tx: &mut AdmittedTx) { + crate::replica_fence::guard_event_created_at_floor(tx, created_at, channel).await?; + sqlx::query("INSERT INTO events (id) VALUES ($1) ON CONFLICT DO NOTHING"); + sqlx::query("INSERT INTO parameterized_event_watermarks (d_tag, created_at) VALUES ($1, $2) ON CONFLICT (d_tag) DO UPDATE SET created_at = EXCLUDED.created_at"); +} +"#; + assert_eq!( + event_floor_policy_violations(source), + [ + "async fn missing_guard(tx: &mut AdmittedTx) {", + "async fn rewrite_time(tx: &mut AdmittedTx) {", + "async fn attach_channel(tx: &mut AdmittedTx) {", + "async fn alias_time(tx: &mut AdmittedTx) {", + "async fn as_alias_time(tx: &mut AdmittedTx) {", + "async fn only_channel(tx: &mut AdmittedTx) {", + "async fn only_public_alias(tx: &mut AdmittedTx) {", + "async fn row_time(tx: &mut AdmittedTx) {", + "async fn row_channel(tx: &mut AdmittedTx) {", + "async fn quoted_insert(tx: &mut AdmittedTx) {", + "async fn quoted_schema_insert(tx: &mut AdmittedTx) {", + "async fn quoted_update(tx: &mut AdmittedTx) {", + "async fn quoted_schema_update(tx: &mut AdmittedTx) {", + "async fn row_time_no_space(tx: &mut AdmittedTx) {", + "async fn guarded_upsert_rehome(tx: &mut AdmittedTx) {", + "async fn guarded_upsert_row_time(tx: &mut AdmittedTx) {", + ] + ); +} + +#[test] +fn event_floor_policy_requires_nip43_channel_null_at_its_insert_position() { + let literal_null = r#" +pub async fn publish_nip43_membership_locked(tx: &mut AdmittedTx) { + sqlx::query("INSERT INTO events (community_id,id,pubkey,kind,content,tags,sig,created_at,received_at,channel_id,d_tag) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,NULL,$10)"); +} +"#; + assert!(event_floor_policy_violations(literal_null).is_empty()); + let unrelated_null_bind = literal_null.replace("$9,NULL,$10", "$9,$10,$11").replace( + "sqlx::query(", + "let unrelated = query.bind::>(None); sqlx::query(", + ); + assert_eq!( + event_floor_policy_violations(&unrelated_null_bind).len(), + 1, + "an unrelated NULL bind must not exempt a channel-bearing INSERT" + ); +} + +#[test] +fn event_floor_policy_fails_closed_on_sql_it_cannot_attribute() { + let dir = std::env::temp_dir().join(format!("buzz-floor-policy-{}", uuid::Uuid::new_v4())); + std::fs::create_dir_all(&dir).expect("fixture dir"); + std::fs::write( + dir.join("insert_event.sql"), + "INSERT INTO \"events\" (id) VALUES ($1)", + ) + .expect("sql fixture"); + std::fs::write(dir.join("notes.md"), "Prose that quotes nothing.").expect("md fixture"); + let path = dir.join("writer.rs"); + let source = r##" +const LEADING_INSERT: &str = "INSERT INTO events (id) VALUES ($1)"; +const USER_AGENT: &str = concat!("buzz-agent/", env!("CARGO_PKG_VERSION")); +static NOTES: &str = include_str!("notes.md"); +async fn joined_insert(tx: &mut AdmittedTx) { + sqlx::query(concat!("INSERT INTO ", "events (id) VALUES ($1)")); +} +async fn guarded_joined_insert(tx: &mut AdmittedTx) { + crate::replica_fence::guard_event_created_at_floor(tx, created_at, channel).await?; + sqlx::query(concat!("INSERT INTO ", r#"events"#, " (id) VALUES ($1)")); +} +const TRAILING_REWRITE: &str = "UPDATE events SET created_at = $1 WHERE id = $2"; +async fn joined_rewrite(tx: &mut AdmittedTx) { + crate::replica_fence::guard_event_created_at_floor(tx, created_at, channel).await?; + sqlx::query(concat!("UPDATE events ", "SET created_at = $1 WHERE id = $2")); +} +async fn included_insert(tx: &mut AdmittedTx) { + sqlx::query(include_str!("insert_event.sql")); +} +async fn guarded_included_insert(tx: &mut AdmittedTx) { + crate::replica_fence::guard_event_created_at_floor(tx, created_at, channel).await?; + sqlx::query(include_str!("insert_event.sql")); +} +async fn opaque_include(tx: &mut AdmittedTx) { + sqlx::query(include_str!(concat!(env!("OUT_DIR"), "/insert.sql"))); +} +async fn opaque_concat(tx: &mut AdmittedTx) { + sqlx::query(concat!("INSERT INTO ", env!("EVENTS_TABLE"))); +} +async fn two_phase(tx: &mut AdmittedTx) { + sqlx::query("PREPARE TRANSACTION 'fence'"); +} +"##; + let violations = file_event_floor_policy_violations(&path, source); + + // One writer per shape: the SQL is split, assembled, loaded or spelled so + // the per-literal checks cannot read it whole. + const WRITER: &str = "async fn writer(tx: &mut AdmittedTx) {"; + const OPAQUE_INSERT: &str = "INSERT target is not a literal table"; + let shapes: [(&str, &[&str]); 23] = [ + // Controls: an opaque WHERE predicate (the retention deletes), a + // placeholder table with no floor column (tenant severing), and a + // builder whose upsert continues in another literal. + ( + r#"sqlx::query(concat!("UPDATE events SET deleted_at = NOW() WHERE NOT ", pred!(), ""))"#, + &[], + ), + ( + r#"format!("UPDATE {table} SET community_id = NULL WHERE community_id = $1")"#, + &[], + ), + ( + r#"QueryBuilder::new("INSERT INTO listeners (a) ").push(" ON CONFLICT (a) DO UPDATE SET created_at = EXCLUDED.created_at")"#, + &[], + ), + // A non-literal `concat!` argument as the target or a SET column. + ( + r#"concat!("UPDATE ", table!(), " SET created_at = $1")"#, + &[WRITER], + ), + ( + r#"concat!("UPDATE events SET ", cols!(), " = $1")"#, + &[WRITER], + ), + // Every macro delimiter. + ( + r#"concat!["INSERT INTO ", "events (id) VALUES ($1)"]"#, + &[WRITER], + ), + ( + r#"::core::concat!{"INSERT INTO ", "events (id) VALUES ($1)"}"#, + &[WRITER], + ), + (r#"include_str!["insert_event.sql"]"#, &[WRITER]), + (r#"include_str! {"insert_event.sql"}"#, &[WRITER]), + // Compile-time file SQL is banned outright. + ( + r#"sqlx::query_file!("insert_event.sql")"#, + &["query_file! loads SQL from a file"], + ), + ( + r#"sqlx::query_file_as!(Row, "insert_event.sql")"#, + &["query_file_as! loads SQL from a file"], + ), + // SQL assembled at runtime. + ( + r#"format!("{}{}", "INSERT INTO ", "events (id) VALUES ($1)")"#, + &[OPAQUE_INSERT], + ), + ( + r#"format!("INSERT INTO {t} (id) VALUES ($1)")"#, + &[OPAQUE_INSERT], + ), + ( + r#"["INSERT INTO ", "events (id) VALUES ($1)"].concat()"#, + &[OPAQUE_INSERT], + ), + ( + r#"format!("INSERT INTO events (id) VALUES ($1) {suffix}")"#, + &["events INSERT with a non-literal part", WRITER], + ), + ( + r#"format!("{}{}", "UPDATE ", "events SET created_at = $1")"#, + &[WRITER], + ), + ( + r#"format!("UPDATE public.events SET {column} = $1")"#, + &[WRITER], + ), + // A SET list continued in another literal or argument. + ( + r#"format!("UPDATE events SET {}", "created_at = $1")"#, + &[WRITER], + ), + ( + r#"["UPDATE events SET ", "channel_id = $1"].concat()"#, + &[WRITER], + ), + ( + r#"QueryBuilder::new("UPDATE events SET ").push("created_at = ")"#, + &[WRITER], + ), + ( + r#"QueryBuilder::new("INSERT INTO events (id) ").push(" ON CONFLICT (id) DO UPDATE SET ").push("channel_id = EXCLUDED.channel_id")"#, + &[WRITER], + ), + ( + r#"sqlx::query("MERGE INTO events e USING src ON e.id = src.id WHEN MATCHED THEN DO NOTHING")"#, + &["events written by a statement other than INSERT"], + ), + ( + r#"format!("{} TRANSACTION 'x'", "PREPARE")"#, + &["PREPARE TRANSACTION in a production writer"], + ), + ]; + let shape_violations = shapes.map(|(shape, _)| { + file_event_floor_policy_violations(&path, &format!("{WRITER}\n {shape};\n}}\n")) + }); + + // Each item-level form is caught on its own, wherever it sits. + std::fs::write( + dir.join("insert_event.sql"), + "insert into events (id) values ($1)", + ) + .expect("sql fixture"); + let items = [ + "const LEADING: &str = \"INSERT INTO events (id) VALUES ($1)\";\nfn f() {}\n", + "fn f() {}\nconst TRAILING: &str = \"UPDATE events SET channel_id = $1\";\nfn g() {}\n", + "fn f() {}\nstatic INCLUDED: &str = include_str!(\"insert_event.sql\");\n", + ]; + let item_violations = items.map(|item| file_event_floor_policy_violations(&path, item)); + std::fs::remove_dir_all(&dir).expect("remove fixture dir"); + + assert_eq!( + violations, + [ + r#"include_str!(concat!(env!("OUT_DIR"), "/insert.sql")) is opaque"#, + OPAQUE_INSERT, + "PREPARE TRANSACTION in a production writer", + "event-write SQL outside any function body", + "async fn joined_insert(tx: &mut AdmittedTx) {", + "async fn joined_rewrite(tx: &mut AdmittedTx) {", + "async fn included_insert(tx: &mut AdmittedTx) {", + ] + ); + for ((shape, expected), found) in shapes.iter().zip(&shape_violations) { + assert_eq!(found, expected, "shape `{shape}`"); + } + for (item, found) in items.iter().zip(&item_violations) { + assert_eq!( + found, + &["event-write SQL outside any function body"], + "item-level SQL in `{item}`" + ); + } +} + +#[test] +fn every_production_channel_event_insert_checks_the_floor_and_never_rewrites_it() { + let files = collect_production_rs_files(); + let crates_root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .expect("crates root"); + assert!( + files.contains(&crates_root.join("buzz-relay/src/state.rs")), + "floor policy must scan the workspace, including relay writers" + ); + let mut guarded_writers = 0; + let mut violations = Vec::new(); + for path in files { + let source = std::fs::read_to_string(&path).expect("source"); + let (production, _) = sql_visible_source(&path, &strip_cfg_test_items(&source)); + guarded_writers += function_items(&production) + .into_iter() + .filter(|function| { + inserts_events_row(function) + && mask_comments_and_literals(function) + .contains("replica_fence::guard_event_created_at_floor(") + }) + .count(); + violations.extend( + file_event_floor_policy_violations(&path, &source) + .into_iter() + .map(|header| format!("{}: {header}", path.display())), + ); + } + assert!( + violations.is_empty(), + "event floor policy violations: {violations:?}" + ); + assert!( + guarded_writers >= 5, + "expected five channel-bearing event writers, found {guarded_writers}" + ); +} + +#[test] +fn event_floor_guard_uses_the_database_statement_clock_and_shared_floor() { + let source = strip_cfg_test_items(include_str!("../src/runtime/replica_fence.rs")); + let guard = function_slices(&source) + .into_iter() + .find(|function| function_header(function).contains("fn guard_event_created_at_floor(")) + .expect("app floor guard"); + assert!( + guard.contains("SELECT $1::timestamptz < clock_timestamp()"), + "the guard must sample the database statement clock, after BEGIN" + ); + assert!( + guard.contains(".bind(CREATED_AT_FLOOR_SECS as f64)"), + "app check and fence wall must share the same floor constant" + ); +} + +#[cfg(unix)] +#[test] +fn admin_seed_refuses_remote_database_before_invoking_psql() { + use std::os::unix::fs::PermissionsExt; + let root = std::env::temp_dir().join(format!("buzz-seed-policy-{}", uuid::Uuid::new_v4())); + let scripts = root.join("scripts"); + std::fs::create_dir_all(&scripts).expect("scratch scripts directory"); + let script = scripts.join("seed-admin-dashboard.sh"); + std::fs::write( + &script, + include_str!("../../../scripts/seed-admin-dashboard.sh"), + ) + .expect("copy script without developer .env"); + let psql = root.join("psql"); + std::fs::write( + &psql, + "#!/bin/sh\nprintf '%s\\n' \"$@\" > \"$SEED_PSQL_WITNESS\"\nexit 1\n", + ) + .expect("stub psql"); + std::fs::set_permissions(&psql, std::fs::Permissions::from_mode(0o755)) + .expect("executable stub"); + let witness = root.join("psql-was-invoked"); + let run = |host: &str, address: &str, service: &str, database: &str| { + let output = std::process::Command::new("/bin/bash") + .arg(&script) + .env("PGHOST", host) + .env("PGHOSTADDR", address) + .env("PGSERVICE", service) + .env("PGDATABASE", database) + .env("PATH", format!("{}:/usr/bin:/bin", root.display())) + .env("SEED_PSQL_WITNESS", &witness) + .output() + .expect("run local seed guard"); + let args = std::fs::read_to_string(&witness).ok(); + if args.is_some() { + std::fs::remove_file(&witness).expect("clear witness"); + } + (output, args) + }; + // Control: a local host with a plain database name reaches psql, so the + // refusals below come from the guard rather than an unrunnable fixture. + // It must also pass -X so a psqlrc cannot \connect after the guard. + let (_, control_args) = run("localhost", "", "", "buzz"); + let cases = [ + ("remote host", "shared-db.example.invalid", "", "", ""), + ("address override", "localhost", "192.0.2.1", "", ""), + ("service override", "localhost", "", "production", ""), + ( + "socket-prefixed host list", + "/tmp,shared-db.example.invalid", + "", + "", + "", + ), + ( + "host list", + "localhost,shared-db.example.invalid", + "", + "", + "", + ), + ( + "database URI", + "localhost", + "", + "", + "postgresql://buzz@shared-db.example.invalid/buzz", + ), + ( + "short database URI", + "localhost", + "", + "", + "postgres://shared-db.example.invalid/buzz", + ), + ( + "database key/value host", + "localhost", + "", + "", + "host=shared-db.example.invalid dbname=buzz", + ), + ( + "database key/value hostaddr", + "localhost", + "", + "", + "hostaddr=192.0.2.1 dbname=buzz", + ), + ( + "database key/value service", + "localhost", + "", + "", + "service=production", + ), + ]; + let results = cases.map(|(name, host, address, service, database)| { + let (output, args) = run(host, address, service, database); + (name, output, args.is_some()) + }); + std::fs::remove_dir_all(root).expect("remove scratch fixture"); + let control_args = control_args.expect("plain local target did not reach psql"); + assert!( + control_args.lines().any(|arg| arg == "-X"), + "psql invoked without -X: {control_args:?}" + ); + for (name, output, invoked) in results { + assert!(!invoked, "{name} reached psql"); + assert!(!output.status.success(), "{name} accepted"); + assert!( + String::from_utf8_lossy(&output.stderr).contains("requires a local PostgreSQL target"), + "{name}: missing refusal" + ); + } +} diff --git a/crates/buzz-relay/src/api/admin/mod.rs b/crates/buzz-relay/src/api/admin/mod.rs index b5cfa1ef490..c9e071b6b94 100644 --- a/crates/buzz-relay/src/api/admin/mod.rs +++ b/crates/buzz-relay/src/api/admin/mod.rs @@ -9029,18 +9029,6 @@ mod postgres_tests { // `fail_outbox_row` (not `mark_outbox_delivered`) when the insert itself // fails — so nothing is durably persisted, and the row is NOT marked delivered. // - // The failure is induced AFTER tenant resolution, inside `emit_system_message`'s - // `insert_event` call, by: - // 1. Building a dedicated test pool whose `after_connect` sets the - // `buzz.created_at_floor` GUC session-locally (not database-globally). - // Every connection from that pool inherits the floor; no other pool or - // test is affected, and there is no cleanup race on panic. - // 2. Backdating the outbox row's `created_at` beyond that floor. - // `emit_system_message` derives the Nostr event's `created_at` from - // `row.created_at` (the idempotency timestamp). With the floor active, the - // deferrable trigger fires on INSERT and raises a check_violation, which - // `insert_event` propagates as `Err`. The `?` in `emit_system_message` then - // propagates it up through `deliver_tombstone → deliver_one → fail_outbox_row`. let pool = e2e_pool().await; let (community_id, _host) = e2e_community(&pool, "race-c3").await; let target = vec![24u8; 32]; @@ -9144,43 +9132,22 @@ mod postgres_tests { .await .expect("insert fail-outbox row"); - // Backdate the outbox row's created_at so emit_system_message uses an old - // idempotency_ts. The events_created_at_floor trigger will reject the INSERT - // once we arm the GUC below. - sqlx::query( - "UPDATE relay_admin_outbox SET created_at = now() - interval '10 seconds' WHERE id = $1", + // Inject a durable insert failure only in this per-test database. This + // exercises error propagation and retry state independently of the floor. + sqlx::raw_sql( + "CREATE FUNCTION reject_c3_system_message() RETURNS trigger LANGUAGE plpgsql AS $$ \ + BEGIN IF NEW.kind = 40099 THEN \ + RAISE EXCEPTION 'injected system-message insert failure' USING ERRCODE = '23514'; \ + END IF; RETURN NEW; END $$; \ + CREATE TRIGGER reject_c3_system_message BEFORE INSERT ON events \ + FOR EACH ROW EXECUTE FUNCTION reject_c3_system_message();", ) - .bind(fail_outbox_id) .execute(&pool) .await - .expect("backdate outbox created_at"); - - // Build a dedicated pool whose after_connect sets buzz.created_at_floor = 5 - // session-locally on each connection (set_config 3rd arg false = session scope). - // A floor of 5 s means any event with created_at > 5 s ago is rejected. - // Our outbox row's created_at is ~10 s ago → trigger fires on insert_event. - // This pool is fully isolated: no other pool or test is affected, and there - // is no cleanup dependence (dropping the pool closes all its connections). - let db_url = database_url(); - let floor_pool = sqlx::postgres::PgPoolOptions::new() - .max_connections(4) - .after_connect(|conn, _meta| { - Box::pin(async move { - sqlx::query("SELECT set_config('buzz.created_at_floor', '5', false)") - .execute(conn) - .await?; - Ok(()) - }) - }) - .connect(&db_url) - .await - .expect("connect floor pool"); - - // Build an AppState around the floor pool so deliver_one's insert_event call - // runs on a connection where the deferrable trigger is active. - let fresh_state = state_from_pool(floor_pool.clone()).await; + .expect("install isolated insert failure"); + let fresh_state = state_from_pool(pool.clone()).await; - // Claim the row via the floor pool so deliver_one has a real claim token. + // Claim the row so deliver_one has a real claim token. let lease_until = chrono::Utc::now() + chrono::Duration::seconds(30); let mut batch = fresh_state .db @@ -9193,14 +9160,18 @@ mod postgres_tests { .expect("fail_outbox_id must be in batch"); let row = batch.remove(row_idx); - // deliver_one fails inside emit_system_message at insert_event (deferrable - // floor-guard trigger → check_violation) and must call fail_outbox_row — + // deliver_one fails inside emit_system_message at insert_event and + // must call fail_outbox_row — // NOT mark_outbox_delivered. crate::handlers::admin_outbox_worker::deliver_one(&fresh_state, &row).await; - // Drop the floor pool — all its connections close, GUC vanishes with them. - // No ALTER DATABASE, no global state, no reset required. - drop(floor_pool); + sqlx::raw_sql( + "DROP TRIGGER reject_c3_system_message ON events; \ + DROP FUNCTION reject_c3_system_message();", + ) + .execute(&pool) + .await + .expect("remove isolated insert failure"); // No tombstone event was persisted — the failure was inside insert_event. let post_event_count: i64 = sqlx::query_scalar( diff --git a/crates/buzz-relay/src/api/bridge/thread_window/postgres_tests.rs b/crates/buzz-relay/src/api/bridge/thread_window/postgres_tests.rs index 02ef106cdae..3f39f2e4101 100644 --- a/crates/buzz-relay/src/api/bridge/thread_window/postgres_tests.rs +++ b/crates/buzz-relay/src/api/bridge/thread_window/postgres_tests.rs @@ -30,7 +30,7 @@ impl Fixture { .await .unwrap(); let mut state = (*state).clone(); - // Use production after_connect policy (floor guard, isolation and + // Use production after_connect policy (isolation and // session timeouts), not raw SQLx pools that mask deployed failures. state.db = production_db(buzz_db::DbConfig::default()).await; Arc::make_mut(&mut state.config).require_auth_token = true; diff --git a/crates/buzz-relay/src/main.rs b/crates/buzz-relay/src/main.rs index 4ed127dfffa..031ef8e8de0 100644 --- a/crates/buzz-relay/src/main.rs +++ b/crates/buzz-relay/src/main.rs @@ -551,26 +551,11 @@ async fn run_relay_main(boot: BootTracker) -> anyhow::Result<()> { info!("Community deletion serving fences verified"); step.finish(); - // Freshness fence probe: cursor pages route to the replica only for - // history the probe has verified as fully replayed. Deliberately AFTER - // the migration decision: spawn_fence_probe first verifies the - // commit-time floor guard (catalog shape + observed behavior through the - // armed pool) against the live schema, so a relay running with - // BUZZ_AUTO_MIGRATE off and migration 0021 unapplied can never open the - // fence over an unenforced floor. Verification failure is loud but - // non-fatal: the fence stays closed and every cursor page routes to the - // writer. - let mut step = StepTimer::start(StartupStep::ReplicaFenceProbe); - match db.spawn_fence_probe().await { - Ok(true) => info!("Replica fence probe started (floor guard verified)"), - Ok(false) => {} - Err(e) => { - step.degrade(); - error!( - "Replica fence disabled — floor guard verification failed: {e}. \ - All cursor reads stay on the writer." - ); - } + // The ordered heartbeat probe opens replica routing only after a + // successful sample. Errors keep cursor reads on the writer. + let step = StepTimer::start(StartupStep::ReplicaFenceProbe); + if db.spawn_fence_probe() { + info!("Replica fence probe started"); } step.finish(); diff --git a/crates/buzz-relay/src/startup_steps.rs b/crates/buzz-relay/src/startup_steps.rs index 3980b6b86af..ee248e6395a 100644 --- a/crates/buzz-relay/src/startup_steps.rs +++ b/crates/buzz-relay/src/startup_steps.rs @@ -47,7 +47,7 @@ pub enum StartupStep { PartitionEnsure, /// Verify the community-deletion serving fences. DeletionFenceVerify, - /// Verify the replica floor guard and start the fence probe. + /// Start the replica freshness fence probe. ReplicaFenceProbe, /// Ensure the deployment community, backfill the allowlist, and bootstrap the owner. MembershipBootstrap, diff --git a/deploy/charts/buzz/README.md b/deploy/charts/buzz/README.md index 5704a7622b8..4cc1901ea4a 100644 --- a/deploy/charts/buzz/README.md +++ b/deploy/charts/buzz/README.md @@ -354,7 +354,7 @@ can give it to a caller. | `buzz_db_connection_step_duration_seconds` | histogram | `pool_role`, `step` | | `buzz_db_connection_step_attempts_total` | counter | `pool_role`, `step`, `outcome` | -The fixed steps are `writer_pool`, `physical_connect`, `created_at_floor`, +The fixed steps are `writer_pool`, `physical_connect`, `session_timeouts`, `isolation`, and `ready`. Measurable phases emit start, duration, and terminal evidence. `physical_connect` and `ready` are success milestones: SQLx 0.9 exposes `after_connect` only after DNS, network, TLS, and @@ -368,8 +368,8 @@ start counter to derive the number of attempts currently in progress. The session phases run in order, so a later phase start also proves the earlier phases succeeded for that attempt. -Four start counters, four 13-series histograms, and fifteen terminal counters -create a hard ceiling of 71 raw Prometheus series per pod. Connection ordinals, +Three start counters, three 13-series histograms, and twelve terminal counters +create a hard ceiling of 54 raw Prometheus series per pod. Connection ordinals, database URLs, hosts, usernames, SQL, and raw errors are forbidden as metric labels. diff --git a/migrations/0062_retire_created_at_floor.sql b/migrations/0062_retire_created_at_floor.sql new file mode 100644 index 00000000000..057171e9752 --- /dev/null +++ b/migrations/0062_retire_created_at_floor.sql @@ -0,0 +1,29 @@ +SET LOCAL lock_timeout = '5s'; + +-- DESTRUCTIVE CONTRACT: deploy phase 1's application floor check to every +-- writer pod/job first, with no rollback expected. Stop out-of-band writers +-- and hold the replica breaker closed through this drop until replay catches +-- up. Old pods that already started do not recheck the guard on each probe. +-- Keep the 960-second application floor and the ordered heartbeat handshake. +-- DROP TRIGGER takes ACCESS EXCLUSIVE on events and every partition, so it +-- can stall events reads/writes for up to lock_timeout. Run off-peak; a lock +-- timeout aborts the whole migration and is safe to retry. +-- No other trigger, function, FK, or tenant fence is retired here. +DROP TRIGGER IF EXISTS events_created_at_floor ON events; + +-- Dropping the parent removes its inherited clones (including partitions +-- created since 0021). Also remove any independently attached child copy. +DO $$ +DECLARE + partition_table regclass; +BEGIN + FOR partition_table IN + SELECT relid FROM pg_partition_tree('events'::regclass) + WHERE relid <> 'events'::regclass + LOOP + EXECUTE format('DROP TRIGGER IF EXISTS events_created_at_floor ON %s', partition_table); + END LOOP; +END +$$; + +DROP FUNCTION IF EXISTS events_created_at_floor_guard(); diff --git a/schema/functions/events_created_at_floor_guard.sql b/schema/functions/events_created_at_floor_guard.sql deleted file mode 100644 index 263886f83f9..00000000000 --- a/schema/functions/events_created_at_floor_guard.sql +++ /dev/null @@ -1,29 +0,0 @@ --- Replica-fence floor guard (keep in sync with migrations/0021). A deferred --- constraint trigger re-checks, inside COMMIT processing, that channel-bearing --- event rows are no older than `buzz.created_at_floor` seconds before commit --- time (clock_timestamp(), NOT the transaction-frozen now()). This turns the --- relay's ingest-time created_at envelope into a commit-time storage --- invariant, which is what lets keyset-cursor pages below the replica fence --- be served by a read replica without holes. Enforcement is armed per session --- via the GUC (set by the relay's writer pool on connect); sessions without --- the GUC (pg_restore, manual backfills) bypass it and must hold the replica --- fence closed for their duration. The only structural exemption is --- channel_id IS NULL: those rows never appear in keyset-paged windows. -CREATE FUNCTION events_created_at_floor_guard() RETURNS trigger -LANGUAGE plpgsql AS $$ -DECLARE - floor_secs numeric := nullif(current_setting('buzz.created_at_floor', true), '')::numeric; -BEGIN - IF floor_secs IS NOT NULL - AND floor_secs > 0 - AND NEW.channel_id IS NOT NULL - AND NEW.created_at < clock_timestamp() - make_interval(secs => floor_secs) - THEN - RAISE EXCEPTION - 'events.created_at % is more than % s before commit time %; below the replica-fence floor', - NEW.created_at, floor_secs, clock_timestamp() - USING ERRCODE = 'check_violation'; - END IF; - RETURN NULL; -END -$$; diff --git a/schema/schema.sql b/schema/schema.sql index 5ec8bd5829a..30670abb820 100644 --- a/schema/schema.sql +++ b/schema/schema.sql @@ -67,7 +67,6 @@ CREATE EXTENSION IF NOT EXISTS pgcrypto; \i functions/enqueue_push_match_job.sql \i functions/refresh_channel_ttl_after_event_insert.sql \i functions/guard_channel_roster_snapshot.sql -\i functions/events_created_at_floor_guard.sql \i functions/prevent_community_deletion_request_retargeting.sql \i functions/prevent_community_deletion_approval_removal.sql \i functions/protect_community_deletion_manifest_keys.sql diff --git a/schema/triggers/events.sql b/schema/triggers/events.sql index 5491d1631d6..aea4facc0e9 100644 --- a/schema/triggers/events.sql +++ b/schema/triggers/events.sql @@ -10,15 +10,4 @@ CREATE TRIGGER trg_events_guard_channel_roster_snapshot BEFORE INSERT ON events FOR EACH ROW EXECUTE FUNCTION guard_channel_roster_snapshot(); --- INSERT OR UPDATE OF: an UPDATE can move a previously exempt row into the --- guarded set (channel_id NULL -> NOT NULL) or move a channel row's --- created_at below the fence, so both mutation paths re-run the guard on the --- NEW row. A created_at rewrite that crosses partition bounds runs as --- DELETE + INSERT and hits the cloned AFTER INSERT guard on the destination --- partition; an in-partition rewrite fires the UPDATE OF arm. -CREATE CONSTRAINT TRIGGER events_created_at_floor - AFTER INSERT OR UPDATE OF created_at, channel_id ON events - DEFERRABLE INITIALLY DEFERRED - FOR EACH ROW - EXECUTE FUNCTION events_created_at_floor_guard(); SELECT attach_community_write_fence('events'); diff --git a/scripts/reconcile-schema-after-pgschema.sql b/scripts/reconcile-schema-after-pgschema.sql index 7eef8c4bbe5..131c1085bd3 100644 --- a/scripts/reconcile-schema-after-pgschema.sql +++ b/scripts/reconcile-schema-after-pgschema.sql @@ -18,7 +18,6 @@ BEGIN -- triggers while attaching and rejects same-named child triggers. DROP TRIGGER IF EXISTS events_enqueue_push_match ON events_p_past; DROP TRIGGER IF EXISTS events_refresh_channel_ttl ON events_p_past; - DROP TRIGGER IF EXISTS events_created_at_floor ON events_p_past; DROP TRIGGER IF EXISTS community_write_fence_events ON events_p_past; DROP TRIGGER IF EXISTS trg_events_guard_channel_roster_snapshot ON events_p_past; ALTER TABLE events ATTACH PARTITION events_p_past @@ -32,7 +31,6 @@ BEGIN ) THEN DROP TRIGGER IF EXISTS events_enqueue_push_match ON events_p2026_01; DROP TRIGGER IF EXISTS events_refresh_channel_ttl ON events_p2026_01; - DROP TRIGGER IF EXISTS events_created_at_floor ON events_p2026_01; DROP TRIGGER IF EXISTS community_write_fence_events ON events_p2026_01; DROP TRIGGER IF EXISTS trg_events_guard_channel_roster_snapshot ON events_p2026_01; ALTER TABLE events ATTACH PARTITION events_p2026_01 @@ -46,7 +44,6 @@ BEGIN ) THEN DROP TRIGGER IF EXISTS events_enqueue_push_match ON events_p2026_02; DROP TRIGGER IF EXISTS events_refresh_channel_ttl ON events_p2026_02; - DROP TRIGGER IF EXISTS events_created_at_floor ON events_p2026_02; DROP TRIGGER IF EXISTS community_write_fence_events ON events_p2026_02; DROP TRIGGER IF EXISTS trg_events_guard_channel_roster_snapshot ON events_p2026_02; ALTER TABLE events ATTACH PARTITION events_p2026_02 @@ -60,7 +57,6 @@ BEGIN ) THEN DROP TRIGGER IF EXISTS events_enqueue_push_match ON events_p2026_03; DROP TRIGGER IF EXISTS events_refresh_channel_ttl ON events_p2026_03; - DROP TRIGGER IF EXISTS events_created_at_floor ON events_p2026_03; DROP TRIGGER IF EXISTS community_write_fence_events ON events_p2026_03; DROP TRIGGER IF EXISTS trg_events_guard_channel_roster_snapshot ON events_p2026_03; ALTER TABLE events ATTACH PARTITION events_p2026_03 @@ -74,7 +70,6 @@ BEGIN ) THEN DROP TRIGGER IF EXISTS events_enqueue_push_match ON events_p2026_04; DROP TRIGGER IF EXISTS events_refresh_channel_ttl ON events_p2026_04; - DROP TRIGGER IF EXISTS events_created_at_floor ON events_p2026_04; DROP TRIGGER IF EXISTS community_write_fence_events ON events_p2026_04; DROP TRIGGER IF EXISTS trg_events_guard_channel_roster_snapshot ON events_p2026_04; ALTER TABLE events ATTACH PARTITION events_p2026_04 @@ -88,7 +83,6 @@ BEGIN ) THEN DROP TRIGGER IF EXISTS events_enqueue_push_match ON events_p2026_05; DROP TRIGGER IF EXISTS events_refresh_channel_ttl ON events_p2026_05; - DROP TRIGGER IF EXISTS events_created_at_floor ON events_p2026_05; DROP TRIGGER IF EXISTS community_write_fence_events ON events_p2026_05; DROP TRIGGER IF EXISTS trg_events_guard_channel_roster_snapshot ON events_p2026_05; ALTER TABLE events ATTACH PARTITION events_p2026_05 @@ -102,7 +96,6 @@ BEGIN ) THEN DROP TRIGGER IF EXISTS events_enqueue_push_match ON events_p2026_06; DROP TRIGGER IF EXISTS events_refresh_channel_ttl ON events_p2026_06; - DROP TRIGGER IF EXISTS events_created_at_floor ON events_p2026_06; DROP TRIGGER IF EXISTS community_write_fence_events ON events_p2026_06; DROP TRIGGER IF EXISTS trg_events_guard_channel_roster_snapshot ON events_p2026_06; ALTER TABLE events ATTACH PARTITION events_p2026_06 @@ -116,7 +109,6 @@ BEGIN ) THEN DROP TRIGGER IF EXISTS events_enqueue_push_match ON events_p_future; DROP TRIGGER IF EXISTS events_refresh_channel_ttl ON events_p_future; - DROP TRIGGER IF EXISTS events_created_at_floor ON events_p_future; DROP TRIGGER IF EXISTS community_write_fence_events ON events_p_future; DROP TRIGGER IF EXISTS trg_events_guard_channel_roster_snapshot ON events_p_future; ALTER TABLE events ATTACH PARTITION events_p_future diff --git a/scripts/seed-admin-dashboard.sh b/scripts/seed-admin-dashboard.sh index 32320718fad..96fd2fb0915 100755 --- a/scripts/seed-admin-dashboard.sh +++ b/scripts/seed-admin-dashboard.sh @@ -19,15 +19,35 @@ export PGUSER="${PGUSER:-buzz}" export PGPASSWORD="${PGPASSWORD:-buzz_dev}" export PGDATABASE="${PGDATABASE:-buzz}" +# This raw fixture writer is unarmed and must never target a live replica fence. +if [[ -n "${PGHOSTADDR:-}" || -n "${PGSERVICE:-}" || "${PGHOST}" == *,* ]]; then + echo "error: admin seed requires a local PostgreSQL target without PGHOSTADDR, PGSERVICE or multiple hosts" >&2 + exit 1 +fi +case "${PGHOST}" in + localhost|127.0.0.1|::1|/*) ;; + *) + echo "error: admin seed requires a local PostgreSQL target (PGHOST=${PGHOST})" >&2 + exit 1 + ;; +esac +# psql -d treats URI and key=value values as full connection strings that +# override -h, so accept only a plain database name. +if [[ ! "${PGDATABASE}" =~ ^[A-Za-z0-9_-]+$ ]]; then + echo "error: admin seed requires a local PostgreSQL target with a plain PGDATABASE name" >&2 + exit 1 +fi + +# -X skips psqlrc, which runs after this guard and could \connect elsewhere. if command -v psql >/dev/null 2>&1; then run_psql() { - PGPASSWORD="${PGPASSWORD}" psql -h "${PGHOST}" -p "${PGPORT}" \ + PGPASSWORD="${PGPASSWORD}" psql -X -h "${PGHOST}" -p "${PGPORT}" \ -U "${PGUSER}" -d "${PGDATABASE}" "$@" } elif docker exec buzz-postgres psql --version >/dev/null 2>&1; then run_psql() { docker exec -i -e PGPASSWORD="${PGPASSWORD}" buzz-postgres \ - psql -U "${PGUSER}" -d "${PGDATABASE}" "$@" + psql -X -U "${PGUSER}" -d "${PGDATABASE}" "$@" } else echo "error: neither psql nor buzz-postgres docker psql is available" >&2