diff --git a/crates/buzz-db/src/runtime/migration.rs b/crates/buzz-db/src/runtime/migration.rs index b6ead54e06f..79df1aa9549 100644 --- a/crates/buzz-db/src/runtime/migration.rs +++ b/crates/buzz-db/src/runtime/migration.rs @@ -705,7 +705,7 @@ mod postgres_tests { let mut migrations: Vec<_> = MIGRATOR.iter().collect(); migrations.sort_by_key(|migration| migration.version); - assert_eq!(migrations.len(), 60); + assert_eq!(migrations.len(), 61); assert_eq!(migrations[58].version, 59); assert!(migrations[58].sql.as_str().contains("buzz.push_enabled")); assert_eq!(migrations[59].version, 60); @@ -713,6 +713,11 @@ mod postgres_tests { .sql .as_str() .contains("ALTER TABLE push_leases DROP COLUMN app_profile")); + assert_eq!(migrations[60].version, 61); + assert!(migrations[60] + .sql + .as_str() + .contains("idx_events_read_timeline")); assert_eq!(migrations[55].version, 56); assert!(migrations[55] .sql @@ -2253,6 +2258,25 @@ mod postgres_tests { .sql .as_str(); assert!(through_message.contains("DROP COLUMN threads_through_timestamp")); + // 0061 adds the sticky thread unfollow and follow liveness. + let unfollowed_column = "unfollowed boolean not null default false \ + check (not unfollowed or root_id <> ''::bytea), \ + explicit boolean not null default false, \ + active_at timestamptz not null default now(), "; + let access_paths = MIGRATOR + .iter() + .find(|m| m.version == 61) + .expect("read-state access path migration") + .sql + .as_str(); + assert!(access_paths.contains( + "$d$BOOLEAN NOT NULL DEFAULT false CHECK (NOT unfollowed OR root_id <> ''::bytea)$d$" + )); + assert!(access_paths + .contains("('personal_read_frontiers', 'explicit', 'BOOLEAN NOT NULL DEFAULT false')")); + assert!(access_paths.contains( + "('personal_read_frontiers', 'active_at', 'TIMESTAMPTZ NOT NULL DEFAULT now()')" + )); for (table, definition) in personal.tables { let in_schema = schema.tables.get(&table).map(|schema_definition| { if table == "personal_read_accounts" { @@ -2262,7 +2286,11 @@ mod postgres_tests { } }); let definition = if table == "personal_read_frontiers" { - definition.replacen(threads_through_column, through_message_column, 1) + definition.replacen( + threads_through_column, + &format!("{through_message_column}{unfollowed_column}"), + 1, + ) } else { definition }; diff --git a/crates/buzz-db/src/runtime/tests/thread_window_postgres_tests.rs b/crates/buzz-db/src/runtime/tests/thread_window_postgres_tests.rs index beda4b199fc..eca8ea37652 100644 --- a/crates/buzz-db/src/runtime/tests/thread_window_postgres_tests.rs +++ b/crates/buzz-db/src/runtime/tests/thread_window_postgres_tests.rs @@ -270,9 +270,11 @@ async fn migration_schema_thread_window_prebuild_does_not_queue_behind_writer() .await .unwrap(); - // Also bind the public migrator: the advisory schema/destruction lock, - // ledger write and post-migration catalog checks must work with ingestion. - let production_result = migration::run_migrations(&pool).await; + // Also bind the migrator: the advisory schema/destruction lock, ledger + // write and post-migration catalog checks must work with ingestion. 0061 + // adds a column to thread_metadata, which takes a brief exclusive lock + // bounded by its lock_timeout, so it runs after the writer below. + let production_result = migration::run_migrations_through(&pool, 60).await; sqlx::query(insert) .bind(community) .bind(channel) @@ -280,10 +282,6 @@ async fn migration_schema_thread_window_prebuild_does_not_queue_behind_writer() .execute(&mut *witness) .await .unwrap(); - let version: i64 = sqlx::query_scalar("SELECT max(version) FROM _sqlx_migrations") - .fetch_one(&mut *witness) - .await - .unwrap(); let final_oid: i64 = sqlx::query_scalar("SELECT 'idx_thread_metadata_window'::regclass::oid::bigint") .fetch_one(&mut *witness) @@ -297,6 +295,11 @@ async fn migration_schema_thread_window_prebuild_does_not_queue_behind_writer() .unwrap(); drop(witness); writer.rollback().await.unwrap(); + migration::run_migrations(&pool).await.unwrap(); + let version: i64 = sqlx::query_scalar("SELECT max(version) FROM _sqlx_migrations") + .fetch_one(&pool) + .await + .unwrap(); drop_scratch_db(&admin, pool, &name).await; assert!( diff --git a/crates/buzz-db/src/store/event.rs b/crates/buzz-db/src/store/event.rs index 183585e2785..a217251d4ed 100644 --- a/crates/buzz-db/src/store/event.rs +++ b/crates/buzz-db/src/store/event.rs @@ -447,6 +447,10 @@ pub async fn insert_event_in_transaction( ) -> Result<(StoredEvent, bool)> { let result = insert_event_on(tx, event, channel_id).await?; if result.1 { + if let Some((channel, place)) = crate::personal_read::place(event, channel_id, None) { + crate::personal_read::record_message(tx, event, channel, place, result.0.received_at) + .await?; + } crate::operator_listener::enqueue_mentions_in_transaction(tx, event).await?; } Ok(result) @@ -482,8 +486,8 @@ async fn insert_event_on( let not_before = extract_not_before(event); let result = sqlx::query( r#" - INSERT INTO events (community_id, id, pubkey, created_at, kind, tags, content, sig, received_at, channel_id, d_tag, not_before) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) + INSERT INTO events (community_id, id, pubkey, created_at, kind, tags, content, sig, received_at, channel_id, d_tag, not_before, read_timeline) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13) ON CONFLICT DO NOTHING "#, ) @@ -499,6 +503,7 @@ async fn insert_event_on( .bind(channel_id) .bind(d_tag.as_deref()) .bind(not_before) + .bind(crate::personal_read::place(event, channel_id, None).is_some_and(|(_, p)| p.on_timeline())) .execute(tx.conn()) .await?; @@ -1730,11 +1735,22 @@ pub(crate) async fn insert_event_with_thread_metadata_tx( let received_at = Utc::now(); let d_tag = extract_d_tag(event); let not_before = extract_not_before(event); + let place = match &thread_meta { + Some(meta) => crate::personal_read::place( + event, + Some(meta.channel_id), + meta.parent_event_id.map(|parent| match meta.root_event_id { + Some(root) => (root, meta.root_event_created_at, meta.broadcast), + None => (parent, meta.parent_event_created_at, meta.broadcast), + }), + ), + None => crate::personal_read::place(event, channel_id, None), + }; let result = sqlx::query( r#" - INSERT INTO events (community_id, id, pubkey, created_at, kind, tags, content, sig, received_at, channel_id, d_tag, not_before) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) + INSERT INTO events (community_id, id, pubkey, created_at, kind, tags, content, sig, received_at, channel_id, d_tag, not_before, read_timeline) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13) ON CONFLICT DO NOTHING "#, ) @@ -1750,6 +1766,7 @@ pub(crate) async fn insert_event_with_thread_metadata_tx( .bind(channel_id) .bind(d_tag.as_deref()) .bind(not_before) + .bind(place.is_some_and(|(_, p)| p.on_timeline())) .execute(tx.conn()) .await?; @@ -1764,6 +1781,12 @@ pub(crate) async fn insert_event_with_thread_metadata_tx( ) .await?; + // Before the thread counters below: their root row is shared by every + // reply, so its lock must not be held across these writes. + if let Some((channel, place)) = place { + crate::personal_read::record_message(tx, event, channel, place, received_at).await?; + } + if let Some(ref meta) = thread_meta { let broadcast_val: bool = meta.broadcast; @@ -1773,8 +1796,8 @@ pub(crate) async fn insert_event_with_thread_metadata_tx( (community_id, event_created_at, event_id, channel_id, parent_event_id, parent_event_created_at, root_event_id, root_event_created_at, - depth, broadcast) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) + depth, broadcast, received_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) ON CONFLICT DO NOTHING "#, ) @@ -1788,6 +1811,8 @@ pub(crate) async fn insert_event_with_thread_metadata_tx( .bind(meta.root_event_created_at) .bind(meta.depth) .bind(broadcast_val) + // An eligible reply's arrival, for the read-state thread index. + .bind(place.and(meta.parent_event_id).map(|_| received_at)) .execute(tx.conn()) .await?; diff --git a/crates/buzz-db/src/store/personal_read/arrival_postgres_tests.rs b/crates/buzz-db/src/store/personal_read/arrival_postgres_tests.rs index a36c2d3c9c4..d0bb2ca9006 100644 --- a/crates/buzz-db/src/store/personal_read/arrival_postgres_tests.rs +++ b/crates/buzz-db/src/store/personal_read/arrival_postgres_tests.rs @@ -1,6 +1,9 @@ //! Read progress follows relay arrival (`received_at`), never author time. //! Each case sets every arrival explicitly: back-to-back inserts share a clock. -use super::{postgres_tests::fixture, *}; +use super::{ + postgres_tests::{fixture, insert_reply, sync_reply_arrivals}, + *, +}; use crate::Db; use buzz_core::CommunityId; use nostr::{EventBuilder, Keys, Kind, Tag}; @@ -20,6 +23,7 @@ async fn arrive(pool: &PgPool, community: CommunityId, event: &nostr::Event, arr .unwrap() .rows_affected(); assert_eq!(updated, 1, "arrival must land on exactly one stored event"); + sync_reply_arrivals(pool).await; } /// A message authored at `authored` that arrives at `arrived`. It mentions the @@ -45,7 +49,8 @@ async fn post( event } -/// A reply to `root` that mentions the actor, so it counts without membership. +/// A reply to `root` that mentions the actor, so the actor follows the thread +/// from just before the earliest such reply to arrive. #[allow(clippy::too_many_arguments)] async fn reply( db: &Db, @@ -57,12 +62,29 @@ async fn reply( authored: u64, arrived: u64, ) -> nostr::Event { - let event = post(db, pool, community, channel, actor, authored, arrived).await; - sqlx::query("INSERT INTO thread_metadata (community_id,event_id,event_created_at,channel_id,root_event_id,parent_event_id,depth) - VALUES ($1,$2,to_timestamp($3),$4,$5,$5,1)") - .bind(community.as_uuid()).bind(event.id.as_bytes().as_slice()) - .bind(event.created_at.as_secs() as f64) - .bind(channel).bind(root.id.as_bytes().as_slice()).execute(pool).await.unwrap(); + let event = EventBuilder::new(Kind::Custom(9), format!("authored {authored}")) + .tags([Tag::public_key(actor.public_key())]) + .custom_created_at(nostr::Timestamp::from(authored)) + .sign_with_keys(&Keys::generate()) + .unwrap(); + insert_reply(db, community, channel, root, &event).await; + arrive(pool, community, &event, arrived).await; + // Ingest started the follow just before the real arrival; move it to + // just before the rewritten one. + let moved = sqlx::query( + "UPDATE personal_read_frontiers + SET through_timestamp=LEAST(through_timestamp, to_timestamp($4)-interval '1 microsecond') + WHERE community_id=$1 AND actor=$2 AND root_id=$3", + ) + .bind(community.as_uuid()) + .bind(actor.public_key().to_bytes().as_slice()) + .bind(root.id.as_bytes().as_slice()) + .bind(arrived as f64) + .execute(pool) + .await + .unwrap() + .rows_affected(); + assert_eq!(moved, 1, "the mention follows the thread"); event } @@ -233,11 +255,38 @@ async fn mark_thread_read_with_the_sidebar_anchor_clears_a_late_reply() { assert!(row.threads.is_empty()); } -/// The shallow latest probe reads the newest `MAX_CHANNEL_SCAN + 1` events by -/// author time, of any kind. A late arrival with an older author time than -/// that many events is counted unread, so the unread scan must supply the -/// anchor or Mark as read with the sidebar anchor leaves it. -async fn mark_as_read_behind_fillers(fillers: u64) -> (bool, u32) { +/// A late arrival with an older author time than the uncounted events after +/// it must still be the sidebar anchor, or Mark as read with that anchor +/// leaves it unread. +async fn reactions( + pool: &PgPool, + db: &Db, + community: CommunityId, + channel: Uuid, + count: u64, + now: u64, +) { + for i in 0..count { + // Reactions are not counted, but the probe's LIMIT sees them. + let filler = EventBuilder::new(Kind::Custom(7), "+") + .custom_created_at(nostr::Timestamp::from(now - 500 + i % 400)) + .sign_with_keys(&Keys::generate()) + .unwrap(); + db.insert_event(community, &filler, Some(channel)) + .await + .unwrap(); + } + sqlx::query("UPDATE events SET received_at=to_timestamp($2) WHERE community_id=$1 AND kind=7") + .bind(community.as_uuid()) + .bind((now - 50) as f64) + .execute(pool) + .await + .unwrap(); +} + +#[tokio::test] +#[ignore = "requires Postgres"] +async fn mark_as_read_clears_a_late_arrival_behind_reactions() { let (db, pool, community, channel, actor, first) = fixture().await; let now = first.created_at.as_secs(); arrive(&pool, community, &first, now - 60).await; @@ -248,17 +297,8 @@ async fn mark_as_read_behind_fillers(fillers: u64) -> (bool, u32) { mark_through(channel, None, &first.id.to_hex()), ) .await; - for i in 0..fillers { - // Reactions are not counted, but the probe's LIMIT sees them. - let filler = EventBuilder::new(Kind::Custom(7), "+") - .custom_created_at(nostr::Timestamp::from(now - 500 + i)) - .sign_with_keys(&Keys::generate()) - .unwrap(); - db.insert_event(community, &filler, Some(channel)) - .await - .unwrap(); - arrive(&pool, community, &filler, now - 50).await; - } + // More than main's old 256-event probe. + reactions(&pool, &db, community, channel, 300, now).await; post(&db, &pool, community, channel, &actor, now - 600, now - 10).await; assert_eq!(counts(&db, community, &actor).await, (true, 1)); @@ -267,54 +307,7 @@ async fn mark_as_read_behind_fillers(fillers: u64) -> (bool, u32) { .latest_id .expect("a channel with messages has a latest message"); apply(&db, community, &actor, mark_through(channel, None, &anchor)).await; - counts(&db, community, &actor).await -} - -/// Control: with one slot to spare the late arrival is inside the probe. -#[tokio::test] -#[ignore = "requires Postgres"] -async fn mark_as_read_clears_a_late_arrival_inside_the_latest_probe() { - let fillers = MAX_CHANNEL_SCAN as u64 - 1; - assert_eq!(mark_as_read_behind_fillers(fillers).await, (false, 0)); -} - -#[tokio::test] -#[ignore = "requires Postgres"] -async fn mark_as_read_clears_a_late_arrival_outside_the_latest_probe() { - let fillers = MAX_CHANNEL_SCAN as u64; - assert_eq!(mark_as_read_behind_fillers(fillers).await, (false, 0)); -} - -/// When the newest 257 events are all uncounted, the only message is still in -/// the unread scan: the summary reports it as latest and unread. -#[tokio::test] -#[ignore = "requires Postgres"] -async fn latest_message_behind_a_full_probe_of_reactions_is_found() { - let (db, pool, community, channel, actor, first) = fixture().await; - let now = first.created_at.as_secs(); - let demoted = sqlx::query("UPDATE events SET kind=7 WHERE community_id=$1 AND id=$2") - .bind(community.as_uuid()) - .bind(first.id.as_bytes().as_slice()) - .execute(&pool) - .await - .unwrap() - .rows_affected(); - assert_eq!(demoted, 1); - let only = post(&db, &pool, community, channel, &actor, now - 600, now - 10).await; - for i in 0..MAX_CHANNEL_SCAN as u64 { - let filler = EventBuilder::new(Kind::Custom(7), "+") - .custom_created_at(nostr::Timestamp::from(now - 500 + i)) - .sign_with_keys(&Keys::generate()) - .unwrap(); - db.insert_event(community, &filler, Some(channel)) - .await - .unwrap(); - arrive(&pool, community, &filler, now - 50).await; - } - - let row = sidebar(&db, community, &actor).await; - assert_eq!(row.latest_id, Some(only.id.to_hex())); - assert_eq!((row.unread, row.mentions), (true, 1)); + assert_eq!(counts(&db, community, &actor).await, (false, 0)); } /// Store `event` as having arrived at exactly `seconds` plus `micros`. Built @@ -336,6 +329,7 @@ async fn arrive_exact( .unwrap() .rows_affected(); assert_eq!(updated, 1, "arrival must land on exactly one stored event"); + sync_reply_arrivals(pool).await; let stored: chrono::DateTime = sqlx::query_scalar("SELECT received_at FROM events WHERE community_id=$1 AND id=$2") .bind(community.as_uuid()) diff --git a/crates/buzz-db/src/store/personal_read/follows.rs b/crates/buzz-db/src/store/personal_read/follows.rs new file mode 100644 index 00000000000..2ed532a3290 --- /dev/null +++ b/crates/buzz-db/src/store/personal_read/follows.rs @@ -0,0 +1,224 @@ +//! Read state that ingest writes. A thread frontier row exists for a thread +//! its actor follows or has unfollowed; ingest and follow intents create one. +use super::model::{DEFAULT_RETENTION_SECONDS, ELIGIBLE_KINDS}; +use crate::{AdmittedTx, Result}; +use chrono::{DateTime, Utc}; +use nostr::Event; +use std::sync::atomic::{AtomicU32, Ordering}; +use uuid::Uuid; + +/// The relay's follow dormancy window, in seconds (`BUZZ_V1_RETENTION_SECONDS`). +static FOLLOW_WINDOW_SECONDS: AtomicU32 = AtomicU32::new(DEFAULT_RETENTION_SECONDS); + +/// Set the follow dormancy window ingest refreshes against: the same window +/// the read projections are given. Process-wide; the relay sets it once at +/// startup. +pub fn set_follow_window_seconds(seconds: u32) { + FOLLOW_WINDOW_SECONDS.store(seconds, Ordering::Relaxed); +} + +/// How stale a live follow's involvement may grow before an involvement that +/// does not otherwise write its row refreshes it: a day, or a quarter of the +/// window when that is shorter. Never longer than the window, so such an +/// involvement always revives a dormant follow. +fn refresh_after_seconds() -> f64 { + f64::from((FOLLOW_WINDOW_SECONDS.load(Ordering::Relaxed) / 4).min(24 * 60 * 60)) +} + +/// Where a newly stored channel message sits. +#[derive(Clone, Copy)] +pub(crate) enum Place<'a> { + /// A top-level timeline message. + TopLevel, + /// A reply in the thread rooted at `root`, authored at `root_created_at`; + /// a broadcast reply is also on the channel timeline. + Reply { + root: &'a [u8], + root_created_at: DateTime, + broadcast: bool, + }, +} + +impl Place<'_> { + /// Whether the message counts on its channel timeline (`read_timeline`). + pub(crate) fn on_timeline(&self) -> bool { + matches!( + self, + Place::TopLevel + | Place::Reply { + broadcast: true, + .. + } + ) + } +} + +/// A reply's recorded ancestry: its root, the root's author time, and whether +/// it is a broadcast. +pub(crate) type Ancestry<'a> = (&'a [u8], Option>, bool); + +/// Where an eligible channel message sits, from its recorded ancestry. None +/// for other events, and for a reply whose ancestry was never (fully) +/// recorded: it is on neither read path. +pub(crate) fn place<'a>( + event: &Event, + channel: Option, + reply: Option>, +) -> Option<(Uuid, Place<'a>)> { + let channel = channel?; + if !ELIGIBLE_KINDS.contains(&i32::from(event.kind.as_u16())) { + return None; + } + match reply { + Some((root, root_created_at, broadcast)) => Some(( + channel, + Place::Reply { + root, + root_created_at: root_created_at?, + broadcast, + }, + )), + None if buzz_core::nip10::parse_thread_markers(&event.tags) + .resolve() + .is_some() => + { + None + } + None => Some((channel, Place::TopLevel)), + } +} + +/// Record what a newly stored eligible message changes, in its transaction: +/// - its author has read through it: the timeline for a top-level message, +/// otherwise the thread, which the author then follows (and the timeline +/// too for a broadcast reply); +/// - each channel member it mentions follows its thread (for a top-level +/// message, the thread rooted at it); +/// - a reply makes the root's author, and in a DM every member, follow too. +/// +/// Everyone but the author starts just before the message, so it is unread +/// to them. A follower keeps an existing position. An unfollow sticks until +/// its actor posts in the thread or is mentioned there again. +/// +/// Each of these involves its actor in the thread (`active_at`), which keeps +/// an implicit follow live. Agent accounts keep no read state. +pub(crate) async fn record_message( + tx: &mut AdmittedTx, + event: &Event, + channel: Uuid, + place: Place<'_>, + received_at: DateTime, +) -> Result<()> { + if !ELIGIBLE_KINDS.contains(&i32::from(event.kind.as_u16())) { + return Ok(()); + } + let community = tx.community(); + let id = event.id.as_bytes().as_slice(); + let mentioned: Vec> = event + .tags + .public_keys() + .map(|key| key.to_bytes().to_vec()) + .collect(); + let (read, thread, reply, root_created_at, broadcast) = match place { + Place::TopLevel => (&[][..], id, false, None, false), + Place::Reply { + root, + root_created_at, + broadcast, + } => (root, root, true, Some(root_created_at), broadcast), + }; + // Accounts, then frontiers, each in key order, so concurrent messages that + // involve the same people cannot deadlock: the frontier insert joins the + // finished account insert. Foreign-key checks run at the end of the + // statement, after the account rows exist. + sqlx::query( + "WITH involved (actor, root_id, through, message, refollow) AS ( + SELECT $4::bytea, $5::bytea, $6::timestamptz, $3::bytea, true + UNION ALL + SELECT $4, ''::bytea, $6, $3, true WHERE $11 + UNION ALL + SELECT cm.pubkey, $7, $6-interval '1 microsecond', NULL, true FROM channel_members cm + WHERE cm.community_id=$1 AND cm.channel_id=$2 AND cm.removed_at IS NULL + AND cm.pubkey=ANY($8) AND cm.pubkey<>$4 + UNION ALL + SELECT e.pubkey, $7, $6-interval '1 microsecond', NULL, false FROM events e + WHERE $9 AND e.community_id=$1 AND e.created_at=$10 AND e.id=$7 + AND e.channel_id=$2 AND e.pubkey<>$4 + UNION ALL + SELECT cm.pubkey, $7, $6-interval '1 microsecond', NULL, false + FROM channel_members cm JOIN channels c + ON c.community_id=cm.community_id AND c.id=cm.channel_id + WHERE $9 AND cm.community_id=$1 AND cm.channel_id=$2 AND cm.removed_at IS NULL + AND c.channel_type='dm' AND cm.pubkey<>$4 + ), rows AS ( + -- An existing follower keeps their position, so is only touched + -- to keep the follow live, at most once per refresh interval ($12) + -- rather than per message; + -- an unfollow is not touched unless this message re-follows (the + -- author posting, or a mention). Existence probes are LIMIT 1 + -- laterals: always one index lookup. + SELECT DISTINCT ON (i.actor, i.root_id) i.* FROM involved i + LEFT JOIN LATERAL ( + SELECT f.unfollowed, f.active_at FROM personal_read_frontiers f + WHERE f.community_id=$1 AND f.actor=i.actor AND f.channel_id=$2 + AND f.root_id=i.root_id LIMIT 1 + ) f ON true + WHERE NOT EXISTS (SELECT 1 FROM users u WHERE u.community_id=$1 + AND u.pubkey=i.actor AND u.agent_owner_pubkey IS NOT NULL) + AND (i.message IS NOT NULL OR f.unfollowed IS NULL + OR (i.refollow AND f.unfollowed) + OR (i.root_id<>''::bytea AND NOT f.unfollowed + AND f.active_at < now()-make_interval(secs=>$12))) + ORDER BY i.actor, i.root_id, i.message IS NULL, NOT i.refollow + ), accounts AS ( + INSERT INTO personal_read_accounts (community_id, actor) + SELECT $1, r.actor FROM (SELECT DISTINCT actor FROM rows) r + LEFT JOIN LATERAL ( + SELECT true AS found FROM personal_read_accounts a + WHERE a.community_id=$1 AND a.actor=r.actor LIMIT 1 + ) a ON true + WHERE a.found IS NULL ORDER BY r.actor + ON CONFLICT DO NOTHING RETURNING 1 + ) + -- A re-follow starts just before this message, like a new follow, + -- unless the unfollow is later. A row that a concurrent message + -- created is only advanced when it carries an anchor; one kept live + -- keeps its position. + INSERT INTO personal_read_frontiers + (community_id, actor, channel_id, root_id, through_timestamp, through_message_id) + SELECT $1, actor, $2, root_id, through, message + FROM rows CROSS JOIN (SELECT count(*) FROM accounts) written + ORDER BY actor, root_id + ON CONFLICT (community_id, actor, channel_id, root_id) DO UPDATE + SET through_timestamp=CASE + WHEN excluded.through_message_id IS NOT NULL OR personal_read_frontiers.unfollowed + THEN GREATEST(personal_read_frontiers.through_timestamp, excluded.through_timestamp) + ELSE personal_read_frontiers.through_timestamp END, + through_message_id=CASE + WHEN (excluded.through_message_id IS NOT NULL OR personal_read_frontiers.unfollowed) + AND excluded.through_timestamp > personal_read_frontiers.through_timestamp + THEN excluded.through_message_id ELSE personal_read_frontiers.through_message_id END, + unfollowed=false, + active_at=CASE WHEN excluded.root_id<>''::bytea THEN now() + ELSE personal_read_frontiers.active_at END + WHERE excluded.through_message_id IS NOT NULL OR (personal_read_frontiers.unfollowed + AND excluded.through_timestamp > personal_read_frontiers.through_timestamp) + OR (excluded.root_id<>''::bytea AND NOT personal_read_frontiers.unfollowed + AND personal_read_frontiers.active_at < now()-make_interval(secs=>$12))", + ) + .bind(community.as_uuid()) + .bind(channel) + .bind(id) + .bind(event.pubkey.to_bytes().as_slice()) + .bind(read) + .bind(received_at) + .bind(thread) + .bind(&mentioned) + .bind(reply) + .bind(root_created_at) + .bind(broadcast) + .bind(refresh_after_seconds()) + .execute(tx.conn()) + .await?; + Ok(()) +} diff --git a/crates/buzz-db/src/store/personal_read/follows_postgres_tests.rs b/crates/buzz-db/src/store/personal_read/follows_postgres_tests.rs new file mode 100644 index 00000000000..2f89a9ea72b --- /dev/null +++ b/crates/buzz-db/src/store/personal_read/follows_postgres_tests.rs @@ -0,0 +1,940 @@ +//! Who follows a thread, as recorded by ingest, and what following shows. +use super::{ + postgres_tests::{fixture, insert_reply, start_before_everything}, + *, +}; +use crate::Db; +use buzz_core::{channel::MemberRole, CommunityId}; +use nostr::{EventBuilder, Keys, Kind, Tag}; +use sqlx::PgPool; +use uuid::Uuid; + +fn message(author: &Keys, tags: Vec) -> nostr::Event { + EventBuilder::new(Kind::Custom(9), Uuid::new_v4().to_string()) + .tags(tags) + .sign_with_keys(author) + .unwrap() +} + +async fn post( + db: &Db, + community: CommunityId, + channel: Uuid, + author: &Keys, + tags: Vec, +) -> nostr::Event { + let event = message(author, tags); + db.insert_event(community, &event, Some(channel)) + .await + .unwrap(); + event +} + +async fn reply( + db: &Db, + community: CommunityId, + channel: Uuid, + root: &nostr::Event, + author: &Keys, + tags: Vec, +) -> nostr::Event { + let event = message(author, tags); + insert_reply(db, community, channel, root, &event).await; + event +} + +fn mention(who: &Keys) -> Vec { + vec![Tag::public_key(who.public_key())] +} + +async fn member(db: &Db, pool: &PgPool, community: CommunityId, channel: Uuid) -> Keys { + let keys = Keys::generate(); + db.add_member( + community, + channel, + &keys.public_key().to_bytes(), + MemberRole::Member, + None, + ) + .await + .unwrap(); + start_before_everything(pool, community, &keys.public_key()).await; + keys +} + +async fn row(db: &Db, community: CommunityId, channel: Uuid, actor: &Keys) -> ChannelReadSummary { + db.personal_read_sidebar_channels( + community, + &actor.public_key(), + DEFAULT_RETENTION_SECONDS, + &[channel], + ) + .await + .unwrap() + .channels + .remove(0) +} + +/// The actor's frontier message for a thread, if the actor follows it. A +/// follower who has read nothing yet has no message. +async fn follow( + pool: &PgPool, + community: CommunityId, + actor: &Keys, + root: &nostr::Event, +) -> Option> { + sqlx::query_scalar( + "SELECT encode(through_message_id,'hex') FROM personal_read_frontiers + WHERE community_id=$1 AND actor=$2 AND root_id=$3", + ) + .bind(community.as_uuid()) + .bind(actor.public_key().to_bytes().as_slice()) + .bind(root.id.as_bytes().as_slice()) + .fetch_optional(pool) + .await + .unwrap() +} + +/// (root, mentions) for each listed thread. +fn threads(row: &ChannelReadSummary) -> Vec<(String, u32)> { + row.threads + .iter() + .map(|t| (t.root_id.clone(), t.mentions)) + .collect() +} + +fn mark_thread(channel: Uuid, root: &nostr::Event, message: &str) -> ReadIntent { + ReadIntent::MarkThrough { + target: ReadTarget { + channel_id: channel, + root_id: Some(root.id.to_hex()), + }, + message_id: message.to_owned(), + } +} + +#[tokio::test] +#[ignore = "requires Postgres"] +async fn replying_follows_the_thread_and_reads_through_the_reply() { + let (db, pool, community, channel, actor, root) = fixture().await; + let own = reply(&db, community, channel, &root, &actor, vec![]).await; + assert_eq!( + follow(&pool, community, &actor, &root).await, + Some(Some(own.id.to_hex())) + ); + assert!(row(&db, community, channel, &actor) + .await + .threads + .is_empty()); + + let other = reply(&db, community, channel, &root, &Keys::generate(), vec![]).await; + let summary = row(&db, community, channel, &actor).await; + // Unread, but not directed at the actor. + assert_eq!(threads(&summary), [(root.id.to_hex(), 0)]); + assert_eq!(summary.threads[0].latest_id, other.id.to_hex()); + assert_eq!(summary.threads[0].read_through_id, Some(own.id.to_hex())); +} + +#[tokio::test] +#[ignore = "requires Postgres"] +async fn a_mention_follows_a_member_from_just_before_it() { + let (db, pool, community, channel, _, root) = fixture().await; + let bee = member(&db, &pool, community, channel).await; + let outsider = Keys::generate(); + let earlier = reply(&db, community, channel, &root, &Keys::generate(), vec![]).await; + let mut tags = mention(&bee); + tags.extend(mention(&outsider)); + let mentioning = reply(&db, community, channel, &root, &Keys::generate(), tags).await; + reply(&db, community, channel, &root, &Keys::generate(), vec![]).await; + + assert_eq!(follow(&pool, community, &bee, &root).await, Some(None)); + assert_eq!(follow(&pool, community, &outsider, &root).await, None); + // The mention and what came after it are unread; not the reply before it. + let summary = row(&db, community, channel, &bee).await; + assert_eq!(threads(&summary), [(root.id.to_hex(), 1)]); + assert_ne!(summary.threads[0].latest_id, earlier.id.to_hex()); + assert_ne!(summary.threads[0].latest_id, mentioning.id.to_hex()); +} + +#[tokio::test] +#[ignore = "requires Postgres"] +async fn a_later_mention_keeps_the_followers_position() { + let (db, pool, community, channel, actor, root) = fixture().await; + let own = reply(&db, community, channel, &root, &actor, vec![]).await; + reply(&db, community, channel, &root, &Keys::generate(), vec![]).await; + reply( + &db, + community, + channel, + &root, + &Keys::generate(), + mention(&actor), + ) + .await; + assert_eq!( + follow(&pool, community, &actor, &root).await, + Some(Some(own.id.to_hex())) + ); + let summary = row(&db, community, channel, &actor).await; + assert_eq!(threads(&summary), [(root.id.to_hex(), 1)]); +} + +#[tokio::test] +#[ignore = "requires Postgres"] +async fn the_root_author_follows_on_the_first_reply() { + let (db, pool, community, channel, actor, _) = fixture().await; + let root = post(&db, community, channel, &actor, vec![]).await; + assert_eq!(follow(&pool, community, &actor, &root).await, None); + + reply(&db, community, channel, &root, &Keys::generate(), vec![]).await; + assert_eq!(follow(&pool, community, &actor, &root).await, Some(None)); + let summary = row(&db, community, channel, &actor).await; + assert_eq!(threads(&summary), [(root.id.to_hex(), 0)]); +} + +#[tokio::test] +#[ignore = "requires Postgres"] +async fn a_top_level_mention_follows_the_thread_rooted_at_it() { + let (db, pool, community, channel, actor, _) = fixture().await; + let root = post(&db, community, channel, &Keys::generate(), mention(&actor)).await; + assert_eq!(follow(&pool, community, &actor, &root).await, Some(None)); + reply(&db, community, channel, &root, &Keys::generate(), vec![]).await; + + let summary = row(&db, community, channel, &actor).await; + // The fixture message and the mention are unread; one is a mention. + assert_eq!((summary.unread, summary.mentions), (true, 1)); + // The reply is unread but does not mention the actor. + assert_eq!(threads(&summary), [(root.id.to_hex(), 0)]); +} + +#[tokio::test] +#[ignore = "requires Postgres"] +async fn dm_members_follow_every_thread() { + let (db, pool, community, _, actor, _) = fixture().await; + let (bee, wasp) = (Keys::generate(), Keys::generate()); + let keys = [&actor, &bee, &wasp].map(|k| k.public_key().to_bytes()); + let participants: Vec<&[u8]> = keys.iter().map(|k| k.as_slice()).collect(); + let dm = db + .create_dm(community, &participants, &keys[1]) + .await + .unwrap() + .id; + let root = post(&db, community, dm, &bee, vec![]).await; + reply(&db, community, dm, &root, &wasp, vec![]).await; + + assert_eq!(follow(&pool, community, &actor, &root).await, Some(None)); + let summary = row(&db, community, dm, &actor).await; + // Every DM message counts as a mention. + assert_eq!((summary.unread, summary.mentions), (true, 1)); + assert_eq!(threads(&summary), [(root.id.to_hex(), 1)]); +} + +#[tokio::test] +#[ignore = "requires Postgres"] +async fn an_unfollowed_thread_never_counts_and_marking_it_follows_nothing() { + let (db, pool, community, channel, actor, root) = fixture().await; + reply(&db, community, channel, &root, &Keys::generate(), vec![]).await; + let broadcast = vec![Tag::parse(["broadcast", "1"]).unwrap()]; + let last = reply(&db, community, channel, &root, &Keys::generate(), broadcast).await; + let summary = row(&db, community, channel, &actor).await; + assert!(summary.threads.is_empty()); + // Only the broadcast reply is also on the timeline. + assert_eq!((summary.unread, summary.mentions), (true, 0)); + assert_eq!(summary.latest_id, Some(last.id.to_hex())); + + let outcome = db + .apply_personal_read_intent( + community, + &actor.public_key(), + &mark_thread(channel, &root, &last.id.to_hex()), + ) + .await + .unwrap(); + assert_eq!(outcome, IntentOutcome::Applied); + assert_eq!(follow(&pool, community, &actor, &root).await, None); +} + +#[tokio::test] +#[ignore = "requires Postgres"] +async fn posting_reads_the_timeline_through_the_post() { + let (db, _pool, community, channel, actor, _) = fixture().await; + assert!(row(&db, community, channel, &actor).await.unread); + let own = post(&db, community, channel, &actor, vec![]).await; + let summary = row(&db, community, channel, &actor).await; + assert!(!summary.unread); + assert_eq!(summary.read_through_id, Some(own.id.to_hex())); + assert_eq!(summary.latest_id, Some(own.id.to_hex())); +} + +#[tokio::test] +#[ignore = "requires Postgres"] +async fn thread_counts_stop_at_the_cap() { + let (db, _pool, community, channel, actor, root) = fixture().await; + reply(&db, community, channel, &root, &actor, vec![]).await; + let other = Keys::generate(); + for _ in 0..=MAX_UNREAD_COUNT { + reply(&db, community, channel, &root, &other, mention(&actor)).await; + } + let summary = row(&db, community, channel, &actor).await; + assert_eq!( + threads(&summary), + [(root.id.to_hex(), MAX_UNREAD_COUNT as u32)] + ); +} + +#[tokio::test] +#[ignore = "requires Postgres"] +async fn ingest_does_not_start_an_account() { + let (db, pool, community, channel, _, root) = fixture().await; + let bee = Keys::generate(); + db.add_member( + community, + channel, + &bee.public_key().to_bytes(), + MemberRole::Member, + None, + ) + .await + .unwrap(); + reply( + &db, + community, + channel, + &root, + &Keys::generate(), + mention(&bee), + ) + .await; + reply(&db, community, channel, &root, &bee, vec![]).await; + + let started: Option> = sqlx::query_scalar( + "SELECT started_at FROM personal_read_accounts WHERE community_id=$1 AND actor=$2", + ) + .bind(community.as_uuid()) + .bind(bee.public_key().to_bytes().as_slice()) + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!(started, None); + // Before its first intent, nothing counts. + let summary = row(&db, community, channel, &bee).await; + assert_eq!((summary.unread, summary.mentions), (false, 0)); + assert!(summary.threads.is_empty()); +} + +#[tokio::test] +#[ignore = "requires Postgres"] +async fn a_nested_reply_follows_the_root_thread() { + let (db, pool, community, channel, actor, _) = fixture().await; + let root = post(&db, community, channel, &actor, vec![]).await; + let parent = reply(&db, community, channel, &root, &Keys::generate(), vec![]).await; + let bee = Keys::generate(); + let at = |e: &nostr::Event| { + chrono::DateTime::from_timestamp(e.created_at.as_secs() as i64, 0).unwrap() + }; + let nested = message(&bee, vec![]); + db.insert_event_with_thread_metadata( + community, + &nested, + Some(channel), + Some(crate::event::ThreadMetadataParams { + event_id: nested.id.as_bytes(), + event_created_at: at(&nested), + channel_id: channel, + parent_event_id: Some(parent.id.as_bytes()), + parent_event_created_at: Some(at(&parent)), + root_event_id: Some(root.id.as_bytes()), + root_event_created_at: Some(at(&root)), + depth: 2, + broadcast: false, + }), + ) + .await + .unwrap(); + + assert_eq!( + follow(&pool, community, &bee, &root).await, + Some(Some(nested.id.to_hex())) + ); + assert_eq!(follow(&pool, community, &bee, &parent).await, None); + let summary = row(&db, community, channel, &actor).await; + assert_eq!(threads(&summary), [(root.id.to_hex(), 0)]); + assert_eq!(summary.threads[0].latest_id, nested.id.to_hex()); +} + +#[tokio::test] +#[ignore = "requires Postgres"] +async fn a_message_stored_without_thread_metadata_reads_the_timeline() { + let (db, _pool, community, channel, actor, _) = fixture().await; + let own = message(&actor, vec![]); + db.insert_event_with_thread_metadata(community, &own, Some(channel), None) + .await + .unwrap(); + let summary = row(&db, community, channel, &actor).await; + assert!(!summary.unread); + assert_eq!(summary.read_through_id, Some(own.id.to_hex())); +} + +fn follow_intent(channel: Uuid, root: Option<&nostr::Event>, follow: bool) -> ReadIntent { + let target = ReadTarget { + channel_id: channel, + root_id: root.map(|r| r.id.to_hex()), + }; + if follow { + ReadIntent::Follow { target } + } else { + ReadIntent::Unfollow { target } + } +} + +async fn intent( + db: &Db, + community: CommunityId, + actor: &Keys, + intent: ReadIntent, +) -> IntentOutcome { + db.apply_personal_read_intent(community, &actor.public_key(), &intent) + .await + .unwrap() +} + +#[tokio::test] +#[ignore = "requires Postgres"] +async fn following_starts_now_and_needs_a_thread() { + let (db, pool, community, channel, actor, root) = fixture().await; + reply(&db, community, channel, &root, &Keys::generate(), vec![]).await; + assert_eq!( + intent(&db, community, &actor, follow_intent(channel, None, true)).await, + IntentOutcome::Invalid + ); + let missing = message(&Keys::generate(), vec![]); + assert_eq!( + intent( + &db, + community, + &actor, + follow_intent(channel, Some(&missing), true) + ) + .await, + IntentOutcome::Blocked + ); + assert_eq!( + intent( + &db, + community, + &actor, + follow_intent(channel, Some(&root), true) + ) + .await, + IntentOutcome::Applied + ); + assert_eq!(follow(&pool, community, &actor, &root).await, Some(None)); + // The reply before the follow is read; the next one is not. + assert!(row(&db, community, channel, &actor) + .await + .threads + .is_empty()); + let next = reply(&db, community, channel, &root, &Keys::generate(), vec![]).await; + let summary = row(&db, community, channel, &actor).await; + assert_eq!(threads(&summary), [(root.id.to_hex(), 0)]); + assert_eq!(summary.threads[0].latest_id, next.id.to_hex()); + // Following again keeps the position. + intent( + &db, + community, + &actor, + follow_intent(channel, Some(&root), true), + ) + .await; + assert_eq!( + threads(&row(&db, community, channel, &actor).await).len(), + 1 + ); +} + +#[tokio::test] +#[ignore = "requires Postgres"] +async fn an_unfollow_sticks_until_a_mention_or_own_post() { + let (db, pool, community, channel, actor, _) = fixture().await; + // The actor's own root: its author follows on every reply. + let root = post(&db, community, channel, &actor, vec![]).await; + reply(&db, community, channel, &root, &Keys::generate(), vec![]).await; + assert_eq!( + threads(&row(&db, community, channel, &actor).await).len(), + 1 + ); + assert_eq!( + intent( + &db, + community, + &actor, + follow_intent(channel, Some(&root), false) + ) + .await, + IntentOutcome::Applied + ); + reply(&db, community, channel, &root, &Keys::generate(), vec![]).await; + assert!(row(&db, community, channel, &actor) + .await + .threads + .is_empty()); + + // A mention follows again, from just before it. + let mentioning = reply( + &db, + community, + channel, + &root, + &Keys::generate(), + mention(&actor), + ) + .await; + let summary = row(&db, community, channel, &actor).await; + assert_eq!(threads(&summary), [(root.id.to_hex(), 1)]); + assert_eq!(summary.threads[0].latest_id, mentioning.id.to_hex()); + assert_eq!(summary.threads[0].read_through_id, None); + + // So does posting, which reads through the post. + intent( + &db, + community, + &actor, + follow_intent(channel, Some(&root), false), + ) + .await; + let own = reply(&db, community, channel, &root, &actor, vec![]).await; + assert_eq!( + follow(&pool, community, &actor, &root).await, + Some(Some(own.id.to_hex())) + ); + reply(&db, community, channel, &root, &Keys::generate(), vec![]).await; + assert_eq!( + threads(&row(&db, community, channel, &actor).await).len(), + 1 + ); +} + +#[tokio::test] +#[ignore = "requires Postgres"] +async fn a_broadcast_reply_is_a_timeline_message_and_anchor() { + let (db, _pool, community, channel, actor, root) = fixture().await; + apply_mark(&db, community, channel, &actor, &root).await; + let broadcast = vec![ + Tag::parse(["broadcast", "1"]).unwrap(), + Tag::public_key(actor.public_key()), + ]; + let last = reply(&db, community, channel, &root, &Keys::generate(), broadcast).await; + let summary = row(&db, community, channel, &actor).await; + assert_eq!((summary.unread, summary.mentions), (true, 1)); + assert_eq!(summary.latest_id, Some(last.id.to_hex())); + assert_eq!( + apply_mark(&db, community, channel, &actor, &last).await, + IntentOutcome::Applied + ); + assert!(!row(&db, community, channel, &actor).await.unread); + + // A broadcast by the reader reads the timeline through it, as a post does. + post(&db, community, channel, &Keys::generate(), vec![]).await; + let broadcast = vec![Tag::parse(["broadcast", "1"]).unwrap()]; + let own = reply(&db, community, channel, &root, &actor, broadcast).await; + let summary = row(&db, community, channel, &actor).await; + assert!(!summary.unread); + assert_eq!(summary.read_through_id, Some(own.id.to_hex())); +} + +async fn apply_mark( + db: &Db, + community: CommunityId, + channel: Uuid, + actor: &Keys, + message: &nostr::Event, +) -> IntentOutcome { + let mark = ReadIntent::MarkThrough { + target: ReadTarget { + channel_id: channel, + root_id: None, + }, + message_id: message.id.to_hex(), + }; + intent(db, community, actor, mark).await +} + +async fn threads_page( + db: &Db, + community: CommunityId, + actor: &Keys, + limit: usize, + after: Option<&str>, +) -> ThreadsPage { + let cursor = after.map(|c| ThreadsCursor::parse(c).unwrap()); + db.personal_read_threads( + community, + &actor.public_key(), + DEFAULT_RETENTION_SECONDS, + limit, + cursor.as_ref(), + ) + .await + .unwrap() +} + +#[tokio::test] +#[ignore = "requires Postgres"] +async fn the_threads_view_lists_follows_newest_arrival_first() { + let (db, pool, community, channel, actor, _) = fixture().await; + let mut roots = Vec::new(); + for _ in 0..3 { + let root = post(&db, community, channel, &Keys::generate(), vec![]).await; + reply(&db, community, channel, &root, &actor, vec![]).await; + roots.push(root); + } + // The first thread has news, so it comes first; the rest are read. + reply( + &db, + community, + channel, + &roots[0], + &Keys::generate(), + mention(&actor), + ) + .await; + // Neither an unfollowed thread nor one with no reply is listed. + let quiet = post(&db, community, channel, &Keys::generate(), mention(&actor)).await; + let left = post(&db, community, channel, &Keys::generate(), vec![]).await; + reply(&db, community, channel, &left, &actor, vec![]).await; + intent( + &db, + community, + &actor, + follow_intent(channel, Some(&left), false), + ) + .await; + assert_eq!(follow(&pool, community, &actor, &quiet).await, Some(None)); + + let first = threads_page(&db, community, &actor, 2, None).await; + let rows: Vec<_> = first + .threads + .iter() + .map(|t| (t.thread.root_id.clone(), t.thread.unread, t.thread.mentions)) + .collect(); + assert_eq!( + rows, + [ + (roots[0].id.to_hex(), true, 1), + (roots[2].id.to_hex(), false, 0) + ] + ); + assert!(first.threads.iter().all(|t| t.channel_id == channel)); + let rest = threads_page(&db, community, &actor, 2, first.next_cursor.as_deref()).await; + let rest: Vec<_> = rest + .threads + .iter() + .map(|t| t.thread.root_id.clone()) + .collect(); + assert_eq!(rest, [roots[1].id.to_hex()]); + + // Leaving the channel hides its threads. + sqlx::query("UPDATE channel_members SET removed_at=now() WHERE community_id=$1 AND pubkey=$2") + .bind(community.as_uuid()) + .bind(actor.public_key().to_bytes().as_slice()) + .execute(&pool) + .await + .unwrap(); + assert!(threads_page(&db, community, &actor, 2, None) + .await + .threads + .is_empty()); +} + +/// Put every one of the actor's follows past the default window. +async fn go_dormant(pool: &PgPool, community: CommunityId, actor: &Keys) { + sqlx::query( + "UPDATE personal_read_frontiers SET active_at=now()-interval '31 days' + WHERE community_id=$1 AND actor=$2", + ) + .bind(community.as_uuid()) + .bind(actor.public_key().to_bytes().as_slice()) + .execute(pool) + .await + .unwrap(); +} + +/// Roots of the threads listed for the actor, by the sidebar and by the +/// threads view. +async fn listed(db: &Db, community: CommunityId, channel: Uuid, actor: &Keys) -> [Vec; 2] { + let sidebar = row(db, community, channel, actor) + .await + .threads + .into_iter() + .map(|t| t.root_id) + .collect(); + let view = threads_page(db, community, actor, MAX_THREADS, None) + .await + .threads + .into_iter() + .map(|t| t.thread.root_id) + .collect(); + [sidebar, view] +} + +#[tokio::test] +#[ignore = "requires Postgres"] +async fn a_dormant_follow_is_hidden_until_a_mention_or_post_revives_it() { + let (db, pool, community, channel, actor, root) = fixture().await; + let id = root.id.to_hex(); + let own = reply(&db, community, channel, &root, &actor, vec![]).await; + reply(&db, community, channel, &root, &Keys::generate(), vec![]).await; + assert_eq!( + listed(&db, community, channel, &actor).await, + [vec![id.clone()], vec![id.clone()]] + ); + + go_dormant(&pool, community, &actor).await; + // A reply that does not involve the actor leaves it dormant; the row and + // its position stay. + reply(&db, community, channel, &root, &Keys::generate(), vec![]).await; + assert_eq!( + listed(&db, community, channel, &actor).await, + [Vec::::new(), vec![]] + ); + assert_eq!( + follow(&pool, community, &actor, &root).await, + Some(Some(own.id.to_hex())) + ); + + // A mention revives it at its old position. + reply( + &db, + community, + channel, + &root, + &Keys::generate(), + mention(&actor), + ) + .await; + let summary = row(&db, community, channel, &actor).await; + assert_eq!(threads(&summary), [(id.clone(), 1)]); + assert_eq!(summary.threads[0].read_through_id, Some(own.id.to_hex())); + + // So does posting, which reads through the post. + go_dormant(&pool, community, &actor).await; + assert!(row(&db, community, channel, &actor) + .await + .threads + .is_empty()); + let again = reply(&db, community, channel, &root, &actor, vec![]).await; + reply(&db, community, channel, &root, &Keys::generate(), vec![]).await; + let summary = row(&db, community, channel, &actor).await; + assert_eq!(threads(&summary), [(id.clone(), 0)]); + assert_eq!(summary.threads[0].read_through_id, Some(again.id.to_hex())); + + // And reading it. + go_dormant(&pool, community, &actor).await; + let last = reply(&db, community, channel, &root, &Keys::generate(), vec![]).await; + assert!(row(&db, community, channel, &actor) + .await + .threads + .is_empty()); + assert_eq!( + intent( + &db, + community, + &actor, + mark_thread(channel, &root, &last.id.to_hex()) + ) + .await, + IntentOutcome::Applied + ); + reply(&db, community, channel, &root, &Keys::generate(), vec![]).await; + assert_eq!( + listed(&db, community, channel, &actor).await, + [vec![id.clone()], vec![id]] + ); +} + +#[tokio::test] +#[ignore = "requires Postgres"] +async fn an_explicit_follow_never_goes_dormant() { + let (db, pool, community, channel, actor, _) = fixture().await; + // One thread followed only by intent, one followed implicitly first. + let chosen = post(&db, community, channel, &Keys::generate(), vec![]).await; + let joined = post(&db, community, channel, &Keys::generate(), vec![]).await; + reply(&db, community, channel, &joined, &actor, vec![]).await; + for root in [&chosen, &joined] { + assert_eq!( + intent( + &db, + community, + &actor, + follow_intent(channel, Some(root), true) + ) + .await, + IntentOutcome::Applied + ); + } + let implicit = post(&db, community, channel, &Keys::generate(), vec![]).await; + reply(&db, community, channel, &implicit, &actor, vec![]).await; + for root in [&chosen, &joined, &implicit] { + reply(&db, community, channel, root, &Keys::generate(), vec![]).await; + } + go_dormant(&pool, community, &actor).await; + let mut explicit = vec![chosen.id.to_hex(), joined.id.to_hex()]; + explicit.sort(); + let [mut sidebar, mut view] = listed(&db, community, channel, &actor).await; + sidebar.sort(); + view.sort(); + assert_eq!([sidebar, view], [explicit.clone(), explicit]); + + // Unfollowing ends it; a later mention re-follows it implicitly, so it + // can go dormant again. + intent( + &db, + community, + &actor, + follow_intent(channel, Some(&chosen), false), + ) + .await; + reply( + &db, + community, + channel, + &chosen, + &Keys::generate(), + mention(&actor), + ) + .await; + assert_eq!( + threads(&row(&db, community, channel, &actor).await).len(), + 2 + ); + go_dormant(&pool, community, &actor).await; + assert_eq!( + listed(&db, community, channel, &actor).await, + [vec![joined.id.to_hex()], vec![joined.id.to_hex()]] + ); +} + +#[tokio::test] +#[ignore = "requires Postgres"] +async fn an_agent_keeps_no_read_state() { + let (db, pool, community, channel, actor, _) = fixture().await; + let agent = member(&db, &pool, community, channel).await; + sqlx::query("DELETE FROM personal_read_accounts WHERE community_id=$1 AND actor=$2") + .bind(community.as_uuid()) + .bind(agent.public_key().to_bytes().as_slice()) + .execute(&pool) + .await + .unwrap(); + for (who, owner) in [(&actor, None), (&agent, Some(&actor))] { + sqlx::query( + "INSERT INTO users (community_id, pubkey, agent_owner_pubkey) VALUES ($1,$2,$3)", + ) + .bind(community.as_uuid()) + .bind(who.public_key().to_bytes().as_slice()) + .bind(owner.map(|o| o.public_key().to_bytes().to_vec())) + .execute(&pool) + .await + .unwrap(); + } + // The agent posts, replies to its owner's root, is mentioned, and has a + // root replied to: every way a person would gain a position or a follow. + let own_root = post(&db, community, channel, &agent, mention(&actor)).await; + reply(&db, community, channel, &own_root, &actor, mention(&agent)).await; + let owner_root = post(&db, community, channel, &actor, mention(&agent)).await; + reply( + &db, + community, + channel, + &owner_root, + &agent, + mention(&actor), + ) + .await; + let rows: i64 = sqlx::query_scalar( + "SELECT (SELECT count(*) FROM personal_read_frontiers WHERE community_id=$1 AND actor=$2) + + (SELECT count(*) FROM personal_read_accounts WHERE community_id=$1 AND actor=$2)", + ) + .bind(community.as_uuid()) + .bind(agent.public_key().to_bytes().as_slice()) + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!(rows, 0, "an agent has no positions, follows or account"); + // The agent's messages still involve its owner. + assert!(follow(&pool, community, &actor, &own_root).await.is_some()); + assert!(follow(&pool, community, &actor, &owner_root) + .await + .is_some()); +} + +/// How many threads the sidebar and the threads view list under `window`. +async fn listed_within( + db: &Db, + community: CommunityId, + channel: Uuid, + actor: &Keys, + window: u32, +) -> (usize, usize) { + let sidebar = db + .personal_read_sidebar_channels(community, &actor.public_key(), window, &[channel]) + .await + .unwrap() + .channels + .remove(0) + .threads + .len(); + let view = db + .personal_read_threads(community, &actor.public_key(), window, MAX_THREADS, None) + .await + .unwrap() + .threads + .len(); + (sidebar, view) +} + +/// A window shorter than a day still revives on a mention: ingest refreshes +/// against the same window the projections read. +#[tokio::test] +#[ignore = "requires Postgres"] +async fn a_mention_revives_a_follow_under_a_sub_day_window() { + const HOUR: u32 = 60 * 60; + set_follow_window_seconds(HOUR); + let (db, pool, community, channel, actor, root) = fixture().await; + reply( + &db, + community, + channel, + &root, + &Keys::generate(), + mention(&actor), + ) + .await; + sqlx::query( + "UPDATE personal_read_frontiers SET active_at=now()-interval '2 hours' + WHERE community_id=$1 AND actor=$2", + ) + .bind(community.as_uuid()) + .bind(actor.public_key().to_bytes().as_slice()) + .execute(&pool) + .await + .unwrap(); + assert_eq!( + listed_within(&db, community, channel, &actor, HOUR).await, + (0, 0), + "dormant under the window" + ); + reply( + &db, + community, + channel, + &root, + &Keys::generate(), + mention(&actor), + ) + .await; + assert_eq!( + listed_within(&db, community, channel, &actor, HOUR).await, + (1, 1), + "a fresh mention revives it" + ); +} diff --git a/crates/buzz-db/src/store/personal_read/mod.rs b/crates/buzz-db/src/store/personal_read/mod.rs index bd01af0cdeb..323894adfd2 100644 --- a/crates/buzz-db/src/store/personal_read/mod.rs +++ b/crates/buzz-db/src/store/personal_read/mod.rs @@ -1,21 +1,26 @@ //! Private, signer-owned accessory read progress, separate from NIP-RS events. //! //! A frontier is the relay arrival time of the message a context was read -//! through; the unread horizon alone uses author time. Only fixed context -//! intents advance frontiers, never a query scan cap. +//! through. Read intents and the author's own posts advance frontiers. A +//! thread frontier exists only for a followed thread, and the sidebar counts +//! forward from each frontier. An implicit follow goes dormant after the +//! relay's window without the actor's involvement; an explicit one never does. +//! Agent accounts keep no read state. +mod follows; mod model; -mod participation; mod projection; mod writes; +pub use follows::set_follow_window_seconds; +pub(crate) use follows::{place, record_message}; pub use model::*; #[cfg(test)] mod postgres_tests; #[cfg(test)] -mod participation_postgres_tests; +mod follows_postgres_tests; #[cfg(test)] mod projection_postgres_tests; diff --git a/crates/buzz-db/src/store/personal_read/model.rs b/crates/buzz-db/src/store/personal_read/model.rs index 448d2491cb3..72cd27955e5 100644 --- a/crates/buzz-db/src/store/personal_read/model.rs +++ b/crates/buzz-db/src/store/personal_read/model.rs @@ -4,7 +4,8 @@ use uuid::Uuid; /// Maximum independent operations in one HTTP request. pub const MAX_INTENTS: usize = 100; -/// Default unread-tracking duration, not event or encrypted NIP-RS retention. +/// Default time an implicit thread follow stays live without the actor's +/// involvement; not event or encrypted NIP-RS retention. pub const DEFAULT_RETENTION_SECONDS: u32 = 30 * 24 * 60 * 60; /// A channel or canonical thread; absence of a root denotes only the channel timeline. @@ -28,6 +29,29 @@ pub enum ReadIntent { /// Fixed anchor; retry must not substitute the latest message. message_id: String, }, + /// Follow a thread. A new follow, or one after an unfollow, starts now; + /// an existing follow keeps its position. + Follow { + /// The thread; a channel-only target is invalid. + target: ReadTarget, + }, + /// Stop following a thread until the actor posts in it or is mentioned + /// there again. + Unfollow { + /// The thread; a channel-only target is invalid. + target: ReadTarget, + }, +} + +impl ReadIntent { + /// The intent's target. + pub fn target(&self) -> &ReadTarget { + match self { + ReadIntent::MarkThrough { target, .. } + | ReadIntent::Follow { target } + | ReadIntent::Unfollow { target } => target, + } + } } /// Outcome for one independent transaction, never acknowledged before commit. @@ -45,7 +69,8 @@ pub enum IntentOutcome { /// The tracking boundary for the authenticated account. Not on the wire. #[derive(Clone, Debug)] pub(super) struct ReadAccount { - /// Read-time author-time cutoff (Unix milliseconds), not a discard boundary. + /// Implicit follows not involved since this instant (Unix milliseconds) are + /// dormant; nothing is discarded. pub(super) cutoff_ms: i64, /// Arrival floor of every frontier: the account's first applied read /// intent. `None` until then, and nothing counts as unread. @@ -57,10 +82,12 @@ pub const MAX_CHANNELS: usize = 20; /// Maximum thread rows per channel row. A POST can return rows for /// `MAX_INTENTS` channels, and they must fit the 1 MiB response limit. pub const MAX_THREAD_SUMMARIES: usize = 25; -/// Latest-probe event budget per channel, before eligibility filtering. -pub const MAX_CHANNEL_SCAN: usize = 256; -/// Unread-window work budget per channel, before eligibility/ancestry joins. -pub const MAX_UNREAD_SCAN: usize = 4096; +/// Counts stop here: a count of 100 means at least 100. +pub const MAX_UNREAD_COUNT: usize = 100; +/// The relay accepts author times up to this far from arrival either way. +/// Read positions are arrival times and mentions are indexed by author time, +/// so a mention walk starts this much before the position. +pub const MAX_ARRIVAL_SKEW_SECONDS: u32 = 15 * 60; /// Conversation kinds eligible for ordinary unread state (not edits/reactions). pub const ELIGIBLE_KINDS: [i32; 4] = [9, 40002, 45001, 45003]; @@ -74,32 +101,32 @@ pub struct ChannelReadSummary { /// only on their thread row. pub unread: bool, /// Unread top-level messages directed at the actor: every one in a DM, - /// otherwise those that tag the actor with `p` or carry `broadcast=1`. + /// otherwise those that tag the actor with `p`. pub mentions: u32, - /// The anchor of the timeline's frontier: the marked message that arrived - /// last. None until the timeline is marked. + /// The anchor of the timeline's frontier: the marked or own top-level + /// message that arrived last. None until either exists. pub read_through_id: Option, /// Last eligible top-level message to arrive, whatever its author or read /// progress: marking the timeline through it reads the timeline. pub latest_id: Option, - /// Threads with unread replies, newest unread reply first. + /// Followed threads with unread replies, newest arrival first. pub threads: Vec, } /// One thread's read state within a channel row. No conversation bytes. -#[derive(Debug, Serialize)] +#[derive(Debug, Deserialize, Serialize)] pub struct ThreadReadSummary { /// Canonical thread-root event ID. pub root_id: String, - /// Whether an unread reply that counts was found. + /// Whether the last reply to arrive is unread. pub unread: bool, - /// Unread replies that count: directed at the actor, or in one of the - /// actor's conversations. + /// Unread replies directed at the actor: every one in a DM, otherwise + /// those that tag the actor with `p`. pub mentions: u32, - /// The anchor of the thread's frontier: the marked message that arrived - /// last. None until the thread is marked. + /// The anchor of the thread's frontier: the marked or own reply that + /// arrived last. None for a follower who has read nothing yet. pub read_through_id: Option, - /// Last counted reply to arrive: marking through it reads the thread. + /// Last reply to arrive: marking through it reads the thread. pub latest_id: String, } @@ -111,3 +138,43 @@ pub struct SidebarPage { /// Exclusive UUID roster cursor. None means this roster scan exhausted. pub next_cursor: Option, } + +/// Maximum rows in one threads page. +pub const MAX_THREADS: usize = 50; + +/// One followed thread in the threads view. +#[derive(Debug, Serialize)] +pub struct FollowedThread { + /// Joined channel UUID. + pub channel_id: Uuid, + /// The thread's read state; `unread` may be false here. + #[serde(flatten)] + pub thread: ThreadReadSummary, +} + +/// A page of followed threads, newest arrival first. +#[derive(Debug, Serialize)] +pub struct ThreadsPage { + /// Followed threads in joined channels. + pub threads: Vec, + /// Opaque exclusive cursor. None means this scan exhausted. + pub next_cursor: Option, +} + +/// Where a threads page resumes: after this arrival and root. +#[derive(Clone, Debug)] +pub struct ThreadsCursor { + pub(super) latest_at: DateTime, + pub(super) root_id: Vec, +} + +impl ThreadsCursor { + /// Parse a `next_cursor` value; None if it is not one. + pub fn parse(value: &str) -> Option { + let (micros, root) = value.split_once('.')?; + Some(Self { + latest_at: DateTime::from_timestamp_micros(micros.parse().ok()?)?, + root_id: super::writes::event_id(root)?, + }) + } +} diff --git a/crates/buzz-db/src/store/personal_read/participation.rs b/crates/buzz-db/src/store/personal_read/participation.rs deleted file mode 100644 index 682bca30489..00000000000 --- a/crates/buzz-db/src/store/personal_read/participation.rs +++ /dev/null @@ -1,127 +0,0 @@ -//! Conversation membership from the existing thread store, bounded by time. -use super::model::ELIGIBLE_KINDS; -use crate::Result; -use buzz_core::CommunityId; -use sqlx::{Acquire, PgConnection, Row}; -use std::collections::HashMap; -use uuid::Uuid; - -// Bound multiplicative work independently of the unread evidence window. -const MAX_PARENTS: usize = 1024; - -/// Whether each `(channel, parent)` is one of the actor's conversations: the -/// actor wrote the parent or has a reply to it. Only live eligible messages -/// qualify. An absent key is undecided: past the target cap, or the statement -/// deadline expired. Membership is independent of unread retention and read -/// frontiers. -pub(super) async fn resolve( - conn: &mut PgConnection, - community: CommunityId, - actor: &[u8], - targets: &[(Uuid, Vec)], -) -> Result), bool>> { - if targets.is_empty() { - return Ok(HashMap::new()); - } - let targets = select_targets(targets); - let channels: Vec<_> = targets.iter().map(|(channel, _)| *channel).collect(); - let parents: Vec<_> = targets.iter().map(|(_, parent)| parent.clone()).collect(); - // Optional inference must not abort authoritative unread/frontier reads. - // A nested transaction is a savepoint; rollback also restores the caller's - // statement timeout. No state is written in this read-only inference. - let mut budget = conn.begin().await?; - sqlx::query("SET LOCAL statement_timeout = '500ms'") - .execute(&mut *budget) - .await?; - sqlx::query("SET LOCAL jit = off") - .execute(&mut *budget) - .await?; - // Exact: a row cap would lose a real member behind a busy parent's other - // replies. LATERAL ... LIMIT 1 keeps the work on the replies under the - // requested parents; a plain EXISTS may be hashed over the whole tenant. - let result = sqlx::query( - "SELECT t.channel_id,t.parent_id,(own.hit OR replied.hit) IS TRUE AS member - FROM unnest($3::uuid[],$4::bytea[]) t(channel_id,parent_id) - LEFT JOIN LATERAL ( - SELECT e.pubkey=$2 AND e.deleted_at IS NULL AND e.kind=ANY($5) AS hit - FROM events e WHERE e.community_id=$1 AND e.channel_id=t.channel_id AND e.id=t.parent_id - ORDER BY e.created_at DESC LIMIT 1 - ) own ON true - LEFT JOIN LATERAL ( - SELECT true AS hit FROM thread_metadata tm JOIN events e ON e.community_id=$1 - AND e.channel_id=t.channel_id AND e.created_at=tm.event_created_at AND e.id=tm.event_id - WHERE tm.community_id=$1 AND tm.channel_id=t.channel_id AND tm.parent_event_id=t.parent_id - AND e.pubkey=$2 AND e.deleted_at IS NULL AND e.kind=ANY($5) - AND own.hit IS NOT TRUE - LIMIT 1 - ) replied ON true", - ) - .bind(community.as_uuid()) - .bind(actor) - .bind(channels) - .bind(parents) - .bind(ELIGIBLE_KINDS.as_slice()) - .fetch_all(&mut *budget) - .await; - budget.rollback().await?; - let rows = match result { - Ok(rows) => rows, - Err(sqlx::Error::Database(error)) - if matches!(error.code().as_deref(), Some("57014" | "55P03")) => - { - return Ok(HashMap::new()); - } - Err(error) => return Err(error.into()), - }; - rows.into_iter() - .map(|row| { - Ok(( - (row.try_get("channel_id")?, row.try_get("parent_id")?), - row.try_get("member")?, - )) - }) - .collect() -} - -// Keep selection independent of the optional SQL deadline: a timeout must not -// hide a regression in the cardinality bound. -fn select_targets(targets: &[(Uuid, Vec)]) -> Vec<(Uuid, Vec)> { - let mut targets = targets.to_vec(); - targets.sort_unstable(); - targets.dedup(); - targets.truncate(MAX_PARENTS); - targets -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn target_selection_caps_unique_parents_independently_of_sql_timeout() { - // Literal contract boundaries deliberately do not derive from MAX_PARENTS. - for count in [0_u32, 1, 1023, 1024, 1025] { - let unique: Vec<_> = (0..count) - .map(|i| (Uuid::nil(), i.to_be_bytes().to_vec())) - .collect(); - let input: Vec<_> = unique - .iter() - .rev() - .chain(unique.iter().rev()) - .cloned() - .collect(); - let selected = select_targets(&input); - assert_eq!(selected, unique[..unique.len().min(1024)], "count {count}"); - } - } - - #[test] - fn target_selection_keeps_channel_identity() { - let first = (Uuid::from_u128(1), vec![7; 32]); - let second = (Uuid::from_u128(2), vec![7; 32]); - assert_eq!( - select_targets(&[second.clone(), first.clone(), second.clone()]), - vec![first, second] - ); - } -} diff --git a/crates/buzz-db/src/store/personal_read/participation_postgres_tests.rs b/crates/buzz-db/src/store/personal_read/participation_postgres_tests.rs deleted file mode 100644 index 5a3d3a0acad..00000000000 --- a/crates/buzz-db/src/store/personal_read/participation_postgres_tests.rs +++ /dev/null @@ -1,537 +0,0 @@ -//! Which replies count: the direct-parent conversation rule, its deletion and -//! channel edges, and the bounds on the membership lookup. -use super::*; -use crate::{ - channel::{ChannelType, ChannelVisibility}, - Db, -}; -use buzz_core::CommunityId; -use nostr::{EventBuilder, Keys, Kind, Tag, Timestamp}; -use serde_json::{json, Value}; -use sqlx::PgPool; -use uuid::Uuid; - -const DAY: u64 = 86_400; - -/// One community with the reading actor and a peer. -struct World { - db: Db, - pool: PgPool, - community: CommunityId, - actor: Keys, - peer: Keys, - now: u64, -} - -impl World { - async fn new() -> Self { - let pool = PgPool::connect(&crate::test_support::database_url()) - .await - .unwrap(); - let db = Db::from_pool(pool.clone()); - let community = db - .ensure_configured_community(&format!("conversation-{}.local", Uuid::new_v4())) - .await - .unwrap() - .id; - let actor = Keys::generate(); - super::postgres_tests::start_before_everything(&pool, community, &actor.public_key()).await; - Self { - db, - pool, - community, - actor, - peer: Keys::generate(), - now: Timestamp::now().as_secs(), - } - } - - /// A channel the actor has joined. - async fn channel(&self) -> Uuid { - self.db - .create_channel( - self.community, - &Uuid::new_v4().to_string(), - ChannelType::Stream, - ChannelVisibility::Open, - None, - &self.actor.public_key().to_bytes(), - None, - ) - .await - .unwrap() - .id - } - - /// A top-level message. - async fn post(&self, channel: Uuid, author: &Keys, at: u64, tags: Vec) -> nostr::Event { - let event = EventBuilder::new(Kind::Custom(9), Uuid::new_v4().to_string()) - .tags(tags) - .custom_created_at(Timestamp::from(at)) - .sign_with_keys(author) - .unwrap(); - self.db - .insert_event(self.community, &event, Some(channel)) - .await - .unwrap(); - event - } - - /// A canonical reply to `parent` in `root`'s thread. `parent` need not be - /// stored in `channel`: thread metadata records what the reply claims. - async fn reply( - &self, - channel: Uuid, - author: &Keys, - root: &nostr::Event, - parent: Option<&nostr::Event>, - at: u64, - tags: Vec, - ) -> nostr::Event { - let event = self.post(channel, author, at, tags).await; - sqlx::query("INSERT INTO thread_metadata (community_id,event_id,event_created_at,channel_id,root_event_id,parent_event_id,depth) - VALUES ($1,$2,to_timestamp($3),$4,$5,$6,1)") - .bind(self.community.as_uuid()).bind(event.id.as_bytes().as_slice()).bind(at as f64) - .bind(channel).bind(root.id.as_bytes().as_slice()) - .bind(parent.map(|parent| parent.id.as_bytes().as_slice())) - .execute(&self.pool).await.unwrap(); - event - } - - /// The storage write that author and staff deletions share. - async fn delete(&self, event: &nostr::Event) { - assert!(self - .db - .soft_delete_event_and_update_thread(self.community, event.id.as_bytes(), None, None) - .await - .unwrap()); - } - - async fn row(&self, channel: Uuid) -> ChannelReadSummary { - self.db - .personal_read_sidebar_channels( - self.community, - &self.actor.public_key(), - DEFAULT_RETENTION_SECONDS, - &[channel], - ) - .await - .unwrap() - .channels - .remove(0) - } - - /// The row's counts and thread rows: what the sidebar says counts. - async fn counts(&self, channel: Uuid) -> Value { - let row = self.row(channel).await; - json!({"unread":row.unread,"mentions":row.mentions,"threads":wire(&row.threads)}) - } - - fn mention(&self) -> Tag { - Tag::parse(["p", &self.actor.public_key().to_hex()]).unwrap() - } -} - -fn wire(value: &T) -> Value { - serde_json::to_value(value).unwrap() -} - -/// Expected `counts`: whether a top-level message is unread, how many of those -/// are directed, and the listed thread rows. -fn counts(unread: bool, mentions: u32, threads: Vec) -> Value { - json!({"unread":unread,"mentions":mentions,"threads":threads}) -} - -fn broadcast() -> Tag { - Tag::parse(["broadcast", "1"]).unwrap() -} - -/// A listed, never-marked thread with `mentions` counted unread replies. -fn item(root: &nostr::Event, mentions: u32, latest: &nostr::Event) -> Value { - json!({"root_id":root.id.to_hex(),"unread":true,"mentions":mentions, - "read_through_id":null,"latest_id":latest.id.to_hex()}) -} - -#[tokio::test] -#[ignore = "requires Postgres"] -async fn a_reply_counts_only_in_a_conversation_the_actor_wrote_or_replied_to() { - let w = World::new().await; - // Every witness predates the unread horizon: membership does not expire. - let old = w.now - 40 * DAY; - - // The actor wrote the root. A peer answers it; another peer reply continues - // under that answer, where the actor has written nothing. - let c = w.channel().await; - let root = w.post(c, &w.actor, old, vec![]).await; - let answer = w - .reply(c, &w.peer, &root, Some(&root), w.now + 1, vec![]) - .await; - let nested = w - .reply(c, &w.peer, &root, Some(&answer), w.now + 2, vec![]) - .await; - // Only `answer` counts. The newer reply that does not count is not the - // thread's latest. - assert_eq!( - w.counts(c).await, - counts(false, 0, vec![item(&root, 1, &answer)]) - ); - - // Joining the nested conversation makes its earlier reply count. The - // actor's own reply adds nothing. - w.reply(c, &w.actor, &root, Some(&answer), w.now + 3, vec![]) - .await; - assert_eq!( - w.counts(c).await, - counts(false, 0, vec![item(&root, 2, &nested)]) - ); - - // A peer's thread. The actor replied to one of two parents, long ago. - let c = w.channel().await; - let root = w.post(c, &w.peer, old, vec![]).await; - let joined = w.reply(c, &w.peer, &root, Some(&root), old, vec![]).await; - let other = w.reply(c, &w.peer, &root, Some(&root), old, vec![]).await; - w.reply(c, &w.actor, &root, Some(&joined), old, vec![]) - .await; - let sibling = w - .reply(c, &w.peer, &root, Some(&joined), w.now + 1, vec![]) - .await; - w.reply(c, &w.peer, &root, Some(&other), w.now + 2, vec![]) - .await; - assert_eq!( - w.counts(c).await, - counts(false, 0, vec![item(&root, 1, &sibling)]) - ); - - // A conversation the actor never joined: one unread top-level message, and - // a reply that is not unread even in its own thread. - let c = w.channel().await; - let root = w.post(c, &w.peer, w.now, vec![]).await; - w.reply(c, &w.peer, &root, Some(&root), w.now + 1, vec![]) - .await; - assert_eq!(w.counts(c).await, counts(true, 0, vec![])); - - // A reply whose parent was never recorded is undecided: left out. - w.reply(c, &w.peer, &root, None, w.now + 2, vec![]).await; - assert_eq!(w.counts(c).await, counts(true, 0, vec![])); -} - -#[tokio::test] -#[ignore = "requires Postgres"] -async fn a_directed_reply_counts_outside_the_actors_conversations() { - let w = World::new().await; - let c = w.channel().await; - let root = w.post(c, &w.peer, w.now, vec![]).await; - let parent = w.reply(c, &w.peer, &root, Some(&root), w.now, vec![]).await; - w.reply(c, &w.actor, &root, Some(&root), w.now + 1, vec![]) - .await; - for (at, tags) in [(2, vec![w.mention()]), (3, vec![broadcast()])] { - w.reply(c, &w.peer, &root, Some(&parent), w.now + at, tags) - .await; - } - let both = w - .reply( - c, - &w.peer, - &root, - Some(&parent), - w.now + 4, - vec![broadcast(), w.mention()], - ) - .await; - // The root is an undirected top-level message. The thread counts the - // actor's sibling `parent` and the three directed replies, not the - // actor's own. - let expected = counts(true, 0, vec![item(&root, 4, &both)]); - assert_eq!(w.counts(c).await, expected); - - // Joining the directed replies' conversation changes no count. - w.reply(c, &w.actor, &root, Some(&parent), w.now + 5, vec![]) - .await; - assert_eq!(w.counts(c).await, expected); - - // In a DM every peer message is direct, tagged or not, joined or not. - let dm = w.channel().await; - sqlx::query("UPDATE channels SET channel_type='dm' WHERE community_id=$1 AND id=$2") - .bind(w.community.as_uuid()) - .bind(dm) - .execute(&w.pool) - .await - .unwrap(); - let root = w.post(dm, &w.peer, w.now, vec![w.mention()]).await; - let parent = w - .reply(dm, &w.peer, &root, Some(&root), w.now, vec![]) - .await; - let reply = w - .reply(dm, &w.peer, &root, Some(&parent), w.now + 1, vec![]) - .await; - assert_eq!( - w.counts(dm).await, - counts(true, 1, vec![item(&root, 2, &reply)]) - ); -} - -#[tokio::test] -#[ignore = "requires Postgres"] -async fn eligibility_frontier_and_membership_each_leave_replies_out() { - let w = World::new().await; - let c = w.channel().await; - let root = w.post(c, &w.peer, w.now, vec![]).await; - let parent = w.reply(c, &w.peer, &root, Some(&root), w.now, vec![]).await; - let reply = |author, at, tags| w.reply(c, author, &root, Some(&parent), w.now + at, tags); - // Peer replies to a parent the actor neither wrote nor replied to. - reply(&w.peer, 1, vec![]).await; - let deleted = reply(&w.peer, 2, vec![w.mention()]).await; - w.delete(&deleted).await; - w.reply(c, &w.actor, &root, Some(&root), w.now + 3, vec![]) - .await; - let anchor = reply(&w.peer, 4, vec![w.mention()]).await; - // Undirected, outside the actor's conversations: never counts. - reply(&w.peer, 5, vec![]).await; - // `parent` (a reply to the root, which the actor answered) and `anchor` - // count: not the deleted, own or undirected replies. - assert_eq!( - w.counts(c).await, - counts(true, 0, vec![item(&root, 2, &anchor)]) - ); - - assert_eq!( - w.db.apply_personal_read_intent( - w.community, - &w.actor.public_key(), - &ReadIntent::MarkThrough { - target: ReadTarget { - channel_id: c, - root_id: Some(root.id.to_hex()), - }, - message_id: anchor.id.to_hex(), - }, - ) - .await - .unwrap(), - IntentOutcome::Applied - ); - assert_eq!(w.counts(c).await, counts(true, 0, vec![])); - // A later directed reply is unread past the thread's anchor; the channel - // timeline was never marked. - let after = reply(&w.peer, 6, vec![w.mention()]).await; - let mut thread = item(&root, 1, &after); - thread["read_through_id"] = json!(anchor.id.to_hex()); - assert_eq!(w.counts(c).await, counts(true, 0, vec![thread])); - assert_eq!(w.row(c).await.read_through_id, None); -} - -#[tokio::test] -#[ignore = "requires Postgres"] -async fn a_deleted_message_is_no_witness_and_a_surviving_reply_still_is() { - let w = World::new().await; - let c = w.channel().await; - let root = w.post(c, &w.peer, w.now - 40 * DAY, vec![]).await; - let reply = |author, parent, at| w.reply(c, author, &root, Some(parent), w.now + at, vec![]); - - // The actor wrote the parent and never replied under it. - let wrote = reply(&w.actor, &root, 0).await; - reply(&w.peer, &wrote, 10).await; - // The actor's only reply to a peer's parent. - let once = reply(&w.peer, &root, 0).await; - let only = reply(&w.actor, &once, 1).await; - reply(&w.peer, &once, 11).await; - // Two replies by the actor to a peer's parent. - let twice = reply(&w.peer, &root, 0).await; - let first = reply(&w.actor, &twice, 1).await; - reply(&w.actor, &twice, 2).await; - let to_twice = reply(&w.peer, &twice, 12).await; - // The actor wrote the parent and also replied under it. - let both = reply(&w.actor, &root, 0).await; - let under = reply(&w.actor, &both, 1).await; - let to_both = reply(&w.peer, &both, 13).await; - - // How many replies the thread counts, the latest of them `to_both` until - // it stops counting. - let w = &w; - let thread = |mentions: u32| async move { - let row = w.row(c).await; - assert!(!row.unread, "the root predates the horizon"); - assert_eq!(row.threads.len(), 1); - assert_eq!(row.threads[0].mentions, mentions); - row.threads[0].latest_id.clone() - }; - // The four peer replies to the actor's conversations count. So do `once` - // and `twice`: peer replies to the root, which the actor has replied to - // (`wrote`, `both`). - assert_eq!(thread(6).await, to_both.id.to_hex()); - - // Each deletion changes only its own parent's conversation. - w.delete(&wrote).await; - assert_eq!(thread(5).await, to_both.id.to_hex()); - w.delete(&only).await; - assert_eq!(thread(4).await, to_both.id.to_hex()); - w.delete(&first).await; - assert_eq!(thread(4).await, to_both.id.to_hex()); - // With `wrote` and `both` gone the actor has no reply to the root either. - w.delete(&both).await; - assert_eq!(thread(2).await, to_both.id.to_hex()); - w.delete(&under).await; - assert_eq!(thread(1).await, to_twice.id.to_hex()); -} - -#[tokio::test] -#[ignore = "requires Postgres"] -async fn membership_is_scoped_to_the_replys_own_channel() { - let w = World::new().await; - let (a, b) = (w.channel().await, w.channel().await); - let old = w.now - 40 * DAY; - - // The actor replied to a peer's parent, but that reply is stored in B. - let theirs = w.post(a, &w.peer, old, vec![]).await; - w.reply(b, &w.actor, &theirs, Some(&theirs), old, vec![]) - .await; - let in_a = w - .reply(a, &w.peer, &theirs, Some(&theirs), w.now, vec![]) - .await; - assert_eq!(w.counts(a).await, counts(false, 0, vec![])); - - // The actor wrote a parent in A. A peer reply in B claims it as its parent. - let mine = w.post(a, &w.actor, old, vec![]).await; - w.reply(b, &w.peer, &mine, Some(&mine), w.now, vec![]).await; - assert_eq!(w.counts(b).await, counts(false, 0, vec![])); - - // The same parents count once the actor is a member in the reply's channel. - w.reply(a, &w.actor, &theirs, Some(&theirs), old, vec![]) - .await; - let to_mine = w.reply(a, &w.peer, &mine, Some(&mine), w.now, vec![]).await; - // Equal reply times: root ID order. - let mut threads = [item(&theirs, 1, &in_a), item(&mine, 1, &to_mine)]; - threads.sort_by(|x, y| x["root_id"].as_str().cmp(&y["root_id"].as_str())); - assert_eq!(w.counts(a).await, counts(false, 0, threads.to_vec())); -} - -#[tokio::test] -#[ignore = "requires Postgres"] -async fn membership_is_exact_behind_a_busy_parent() { - let w = World::new().await; - let c = w.channel().await; - let old = w.now - 40 * DAY; - let root = w.post(c, &w.peer, old, vec![]).await; - // The actor's reply is the oldest of 259 under one parent: a 256-row - // window over the parent's replies would never reach it. - w.reply(c, &w.actor, &root, Some(&root), old, vec![]).await; - let mut last = root.clone(); - for i in 0..258 { - last = w - .reply(c, &w.peer, &root, Some(&root), w.now + i, vec![]) - .await; - } - assert_eq!( - w.counts(c).await, - counts(false, 0, vec![item(&root, 258, &last)]) - ); -} - -#[tokio::test] -#[ignore = "requires Postgres"] -async fn threads_are_capped_after_replies_that_do_not_count_are_removed() { - let w = World::new().await; - let c = w.channel().await; - let old = w.now - 40 * DAY; - // The actor's thread has the oldest unread reply. A cap's worth of - // unjoined threads are newer and would fill the list if the cap came - // first. - let mine = w.post(c, &w.actor, old, vec![]).await; - let answer = w.reply(c, &w.peer, &mine, Some(&mine), w.now, vec![]).await; - for i in 1..=MAX_THREAD_SUMMARIES as u64 { - let root = w.post(c, &w.peer, old, vec![]).await; - w.reply(c, &w.peer, &root, Some(&root), w.now + i, vec![]) - .await; - } - assert_eq!( - w.counts(c).await, - counts(false, 0, vec![item(&mine, 1, &answer)]) - ); -} - -#[tokio::test] -#[ignore = "requires Postgres"] -async fn the_parent_budget_leaves_undecided_replies_out() { - let w = World::new().await; - let c = w.channel().await; - // The actor is in the first conversation. Whether it is inside the budget - // depends on ID order, so only the bounds are asserted. - let mut mine = None; - for i in 0..1025 { - let author = if i == 0 { &w.actor } else { &w.peer }; - let root = w.post(c, author, w.now, vec![]).await; - let reply = w - .reply(c, &w.peer, &root, Some(&root), w.now + 1, vec![]) - .await; - if i == 0 { - mine = Some(item(&root, 1, &reply)); - } - // 1024 parents fit: 1023 peer roots, and the reply to the actor's. - // The independent SQL deadline may still withhold the answer. Past - // 1024, one reply is undecided. An undecided reply is left out. - if i >= 1023 { - let counted = w.counts(c).await; - assert!( - [vec![], vec![mine.clone().unwrap()]] - .map(|threads| counts(true, 0, threads)) - .contains(&counted), - "{i}: {counted}" - ); - } - } -} - -#[tokio::test] -#[ignore = "requires Postgres"] -async fn a_lookup_timeout_decides_nothing_and_preserves_the_callers_transaction() { - let w = World::new().await; - let c = w.channel().await; - let root = w.post(c, &w.peer, w.now, vec![]).await; - w.reply(c, &w.peer, &root, Some(&root), w.now + 1, vec![]) - .await; - let mut held = w.pool.begin().await.unwrap(); - // Force the real resolver to time out, then check that its outer snapshot - // and original statement budget remain usable. - sqlx::query("LOCK TABLE thread_metadata IN ACCESS EXCLUSIVE MODE") - .execute(&mut *held) - .await - .unwrap(); - let mut reader = w.pool.begin().await.unwrap(); - sqlx::query("SET LOCAL statement_timeout='2000ms'") - .execute(&mut *reader) - .await - .unwrap(); - for lock_timeout in ["0", "10ms"] { - sqlx::query("SELECT set_config('lock_timeout',$1,true)") - .bind(lock_timeout) - .execute(&mut *reader) - .await - .unwrap(); - sqlx::query("SET LOCAL jit=on") - .execute(&mut *reader) - .await - .unwrap(); - let result = participation::resolve( - &mut reader, - w.community, - &w.actor.public_key().to_bytes(), - &[(c, root.id.as_bytes().to_vec())], - ) - .await - .unwrap(); - assert!(result.is_empty(), "timeout provides no negative evidence"); - let setting: String = sqlx::query_scalar("SHOW statement_timeout") - .fetch_one(&mut *reader) - .await - .unwrap(); - assert_eq!(setting, "2s"); - let jit: String = sqlx::query_scalar("SHOW jit") - .fetch_one(&mut *reader) - .await - .unwrap(); - assert_eq!(jit, "on", "optional inference restores caller settings"); - } - reader.rollback().await.unwrap(); - held.rollback().await.unwrap(); - assert_eq!(w.counts(c).await, counts(true, 0, vec![])); -} diff --git a/crates/buzz-db/src/store/personal_read/postgres_tests.rs b/crates/buzz-db/src/store/personal_read/postgres_tests.rs index 136eb89148d..d8e2c5f3e6e 100644 --- a/crates/buzz-db/src/store/personal_read/postgres_tests.rs +++ b/crates/buzz-db/src/store/personal_read/postgres_tests.rs @@ -51,7 +51,8 @@ pub(super) async fn start_before_everything( ) { sqlx::query( "INSERT INTO personal_read_accounts (community_id,actor,started_at) - VALUES ($1,$2,'1900-01-01T00:00:00Z')", + VALUES ($1,$2,'1900-01-01T00:00:00Z') + ON CONFLICT (community_id,actor) DO UPDATE SET started_at=excluded.started_at", ) .bind(community.as_uuid()) .bind(actor.to_bytes().as_slice()) @@ -60,14 +61,69 @@ pub(super) async fn start_before_everything( .unwrap(); } -/// Move every community message past the default horizon by author time, the -/// only clock the unread window reads. -async fn expire(pool: &PgPool, community: CommunityId) { - sqlx::query("UPDATE events SET created_at=created_at-interval '31 days' WHERE community_id=$1") +/// Store `event` as a canonical reply in `root`'s thread, through ingest. +/// Copy rewritten event arrivals onto their replies' thread rows, as ingest +/// stores them. +pub(super) async fn sync_reply_arrivals(pool: &PgPool) { + sqlx::query( + "UPDATE thread_metadata tm SET received_at=e.received_at FROM events e + WHERE e.community_id=tm.community_id AND e.created_at=tm.event_created_at + AND e.id=tm.event_id AND tm.received_at IS NOT NULL + AND tm.received_at<>e.received_at", + ) + .execute(pool) + .await + .unwrap(); +} + +pub(super) async fn insert_reply( + db: &Db, + community: CommunityId, + channel: Uuid, + root: &nostr::Event, + event: &nostr::Event, +) { + let at = |e: &nostr::Event| { + chrono::DateTime::from_timestamp(e.created_at.as_secs() as i64, 0).unwrap() + }; + db.insert_event_with_thread_metadata( + community, + event, + Some(channel), + Some(crate::event::ThreadMetadataParams { + event_id: event.id.as_bytes(), + event_created_at: at(event), + channel_id: channel, + parent_event_id: Some(root.id.as_bytes()), + parent_event_created_at: Some(at(root)), + root_event_id: Some(root.id.as_bytes()), + root_event_created_at: Some(at(root)), + depth: 1, + broadcast: event.tags.iter().any(|t| { + let t = t.as_slice(); + t.len() >= 2 && t[0] == "broadcast" && t[1] == "1" + }), + }), + ) + .await + .unwrap(); +} + +/// Move every community message's author time back past the default window. +/// Unread reads arrival, so nothing it counts may change. +async fn age_author_time(pool: &PgPool, community: CommunityId) { + for table in [ + "events SET created_at=created_at", + "event_mentions SET event_created_at=event_created_at", + ] { + sqlx::query(sqlx::AssertSqlSafe(format!( + "UPDATE {table}-interval '31 days' WHERE community_id=$1" + ))) .bind(community.as_uuid()) .execute(pool) .await .unwrap(); + } } async fn sidebar(db: &Db, community: CommunityId, actor: &Keys) -> SidebarPage { @@ -105,12 +161,17 @@ fn mark_thread(channel: Uuid, root: &nostr::Event, message: &nostr::Event) -> Re /// Address every stored community message to `actor`, so the channel's /// mention count is exactly its unread top-level count. async fn mention_everywhere(pool: &PgPool, community: CommunityId, actor: &Keys) { - sqlx::query("UPDATE events SET tags=$2 WHERE community_id=$1") - .bind(community.as_uuid()) - .bind(serde_json::json!([["p", actor.public_key().to_hex()]])) - .execute(pool) - .await - .unwrap(); + sqlx::query( + "INSERT INTO event_mentions + (community_id,pubkey_hex,event_id,event_created_at,channel_id,event_kind) + SELECT community_id,$2,id,created_at,channel_id,kind FROM events WHERE community_id=$1 + ON CONFLICT DO NOTHING", + ) + .bind(community.as_uuid()) + .bind(actor.public_key().to_hex()) + .execute(pool) + .await + .unwrap(); } async fn add_history(db: &Db, community: CommunityId, channel: Uuid, count: usize) -> nostr::Event { @@ -132,9 +193,9 @@ async fn add_history(db: &Db, community: CommunityId, channel: Uuid, count: usiz #[tokio::test] #[ignore = "requires Postgres"] -async fn personal_read_sidebar_marked_history_past_the_scan_cap_is_read_and_keeps_latest() { +async fn personal_read_sidebar_marked_history_is_read_and_keeps_latest() { let (db, _pool, community, channel, actor, _) = fixture().await; - let last = add_history(&db, community, channel, MAX_UNREAD_SCAN + 44).await; + let last = add_history(&db, community, channel, 300).await; let outcome = db .apply_personal_read_intent( community, @@ -144,7 +205,6 @@ async fn personal_read_sidebar_marked_history_past_the_scan_cap_is_read_and_keep .await .unwrap(); assert_eq!(outcome, IntentOutcome::Applied); - // The scan never reaches the oldest 45 messages; nothing is unread. let row = &sidebar(&db, community, &actor).await.channels[0]; assert!(!row.unread); assert_eq!(row.mentions, 0); @@ -154,20 +214,16 @@ async fn personal_read_sidebar_marked_history_past_the_scan_cap_is_read_and_keep #[tokio::test] #[ignore = "requires Postgres"] -async fn personal_read_sidebar_author_window_excludes_expired_and_late_old_messages() { +async fn personal_read_author_time_never_hides_unread() { let (db, pool, community, channel, actor, _) = fixture().await; - let latest = add_history(&db, community, channel, MAX_UNREAD_SCAN + 44).await; - expire(&pool, community).await; + let latest = add_history(&db, community, channel, 300).await; + age_author_time(&pool, community).await; let row = &sidebar(&db, community, &actor).await.channels[0]; - assert!( - !row.unread, - "an empty author-time window has nothing unread" - ); + assert!(row.unread, "old author time is still an unread arrival"); assert_eq!(row.latest_id.as_deref(), Some(latest.id.to_hex().as_str())); - // Acceptance time is irrelevant: a message accepted now with an author - // time beyond the horizon stays out; one authored now is counted. + // A message accepted now counts whatever its author time. let now = nostr::Timestamp::now().as_secs(); - for (age, unread, mentions) in [(40 * 86400, false, 0), (0, true, 1)] { + for (age, mentions) in [(40 * 86400, 1), (0, 2)] { let event = EventBuilder::new(Kind::Custom(9), "accepted now") .tags([Tag::public_key(actor.public_key())]) .custom_created_at(nostr::Timestamp::from(now - age)) @@ -177,40 +233,34 @@ async fn personal_read_sidebar_author_window_excludes_expired_and_late_old_messa .await .unwrap(); let row = &sidebar(&db, community, &actor).await.channels[0]; - assert_eq!((row.unread, row.mentions), (unread, mentions), "age={age}"); + assert_eq!((row.unread, row.mentions), (true, mentions), "age={age}"); } } #[tokio::test] #[ignore = "requires Postgres"] -async fn personal_read_sidebar_window_budget_counts_boundary_and_ineligible_tail() { +async fn personal_read_counts_stop_at_the_cap_after_eligibility() { let (db, pool, community, channel, actor, _) = fixture().await; - // The fixture contributes one event, so this is exactly the evidence budget. - add_history(&db, community, channel, MAX_UNREAD_SCAN - 1).await; + // The fixture contributes one message, so the count reaches the cap. + add_history(&db, community, channel, MAX_UNREAD_COUNT - 1).await; mention_everywhere(&pool, community, &actor).await; let mentions = |page: SidebarPage| page.channels[0].mentions; assert_eq!( mentions(sidebar(&db, community, &actor).await), - MAX_UNREAD_SCAN as u32 + MAX_UNREAD_COUNT as u32 ); - let overflow = EventBuilder::new(Kind::Custom(9), "one beyond the budget") - .sign_with_keys(&Keys::generate()) - .unwrap(); - db.insert_event(community, &overflow, Some(channel)) - .await - .unwrap(); + add_history(&db, community, channel, 50).await; mention_everywhere(&pool, community, &actor).await; - // The budget bounds the count; the unexamined message is left out. assert_eq!( mentions(sidebar(&db, community, &actor).await), - MAX_UNREAD_SCAN as u32 + MAX_UNREAD_COUNT as u32, + "a count of the cap means at least that many" ); - // Expire all but 601 messages. Of these, 300 are own and 300 deleted. - // Eligibility is downstream of the bounded unread window, not the old 256 cap. - sqlx::query("WITH ranked AS (SELECT created_at,id,row_number() OVER (ORDER BY created_at DESC,id) AS n FROM events WHERE community_id=$1 AND channel_id=$2) - UPDATE events e SET created_at=CASE WHEN r.n>601 THEN e.created_at-interval '31 days' ELSE e.created_at END, - pubkey=CASE WHEN r.n<=300 THEN $3 ELSE e.pubkey END, - deleted_at=CASE WHEN r.n>300 AND r.n<=600 THEN now() ELSE NULL END + // The last to arrive stays; of the others, 50 become own and the rest + // are deleted. The cap applies after eligibility, so one remains. + sqlx::query("WITH ranked AS (SELECT created_at,id,row_number() OVER (ORDER BY received_at DESC,id) AS n FROM events WHERE community_id=$1 AND channel_id=$2) + UPDATE events e SET pubkey=CASE WHEN r.n BETWEEN 2 AND 51 THEN $3 ELSE e.pubkey END, + deleted_at=CASE WHEN r.n>51 THEN now() ELSE NULL END FROM ranked r WHERE e.community_id=$1 AND e.created_at=r.created_at AND e.id=r.id") .bind(community.as_uuid()).bind(channel).bind(actor.public_key().to_bytes().as_slice()).execute(&pool).await.unwrap(); let page = sidebar(&db, community, &actor).await; @@ -368,7 +418,7 @@ async fn personal_read_latest_includes_own_and_excludes_deleted_auxiliary() { latest(sidebar(&db, community, &actor).await), Some(own.id.to_hex()) ); - expire(&pool, community).await; + age_author_time(&pool, community).await; assert_eq!( latest(sidebar(&db, community, &actor).await), Some(own.id.to_hex()) @@ -394,8 +444,8 @@ async fn personal_read_latest_skips_a_reply_without_ancestry() { latest(sidebar(&db, community, &actor).await), Some(first.id.to_hex()) ); - // The shallow probe applies the same rule once the horizon is empty. - expire(&pool, community).await; + // Author time plays no part in the rule. + age_author_time(&pool, community).await; assert_eq!( latest(sidebar(&db, community, &actor).await), Some(first.id.to_hex()) @@ -415,7 +465,7 @@ async fn personal_read_latest_skips_a_reply_without_ancestry() { #[ignore = "requires Postgres"] async fn personal_read_latest_old_message_is_not_an_empty_channel() { let (db, pool, community, channel, actor, event) = fixture().await; - expire(&pool, community).await; + age_author_time(&pool, community).await; let empty = db .create_channel( community, @@ -436,7 +486,7 @@ async fn personal_read_latest_old_message_is_not_an_empty_channel() { .find(|c| c.channel_id == channel) .unwrap(); assert_eq!(old.latest_id.as_deref(), Some(event.id.to_hex().as_str())); - assert!(!old.unread); + assert!(old.unread); let empty = page .channels .iter() @@ -691,13 +741,13 @@ async fn personal_read_counts_only_arrivals_after_the_account_starts() { #[tokio::test] #[ignore = "requires Postgres"] -async fn personal_read_unstarted_account_is_caught_up_past_the_scan_cap() { +async fn personal_read_unstarted_account_is_caught_up() { let (db, pool, community, channel, actor, _) = fixture().await; unstart(&pool, community).await; - let last = add_history(&db, community, channel, MAX_UNREAD_SCAN + 44).await; + let last = add_history(&db, community, channel, 300).await; let page = sidebar(&db, community, &actor).await; let row = &page.channels[0]; - // The start floor covers every arrival, examined by the scan or not. + // The start floor covers every arrival. assert!(!row.unread); assert_eq!(row.mentions, 0); assert!(row.threads.is_empty()); @@ -710,24 +760,18 @@ async fn personal_read_unstarted_account_is_caught_up_past_the_scan_cap() { async fn personal_read_channel_and_thread_never_inherit_each_other() { let (db, pool, community, channel, actor, root) = fixture().await; let base = root.created_at.as_secs(); - // Directed, so they count outside the actor's conversations. + // Mentioning the actor, who therefore follows the thread. let reply_at = |at: u64| { - let (db, pool) = (db.clone(), pool.clone()); + let db = db.clone(); let mention = Tag::public_key(actor.public_key()); - let root = root.id; + let root = root.clone(); async move { let reply = EventBuilder::new(Kind::Custom(9), "unseen thread reply") .tags([mention]) .custom_created_at(nostr::Timestamp::from(at)) .sign_with_keys(&Keys::generate()) .unwrap(); - db.insert_event(community, &reply, Some(channel)) - .await - .unwrap(); - sqlx::query("INSERT INTO thread_metadata (community_id,event_id,event_created_at,channel_id,root_event_id,parent_event_id,depth) - VALUES ($1,$2,to_timestamp($3),$4,$5,$5,1)") - .bind(community.as_uuid()).bind(reply.id.as_bytes().as_slice()).bind(at as f64) - .bind(channel).bind(root.as_bytes().as_slice()).execute(&pool).await.unwrap(); + insert_reply(&db, community, channel, &root, &reply).await; reply } }; @@ -812,19 +856,18 @@ async fn personal_read_channel_and_thread_never_inherit_each_other() { .fetch_all(&pool) .await .unwrap(); - assert_eq!( - roots, - vec![root.id.as_bytes().to_vec()], - "thread reading never creates a channel frontier" + assert!( + roots.is_empty(), + "reading a thread never follows it or creates a channel frontier" ); } #[tokio::test] #[ignore = "requires Postgres"] -async fn personal_read_author_horizon_unresolved_ancestry_and_deletion() { +async fn personal_read_unresolved_ancestry_and_deletion_not_author_age() { let (db, pool, community, channel, actor, root) = fixture().await; let mention = Tag::public_key(actor.public_key()); - let old = EventBuilder::new(Kind::Custom(9), "directed, about to expire") + let old = EventBuilder::new(Kind::Custom(9), "directed, about to age") .tags([mention.clone()]) .sign_with_keys(&Keys::generate()) .unwrap(); @@ -845,20 +888,32 @@ async fn personal_read_author_horizon_unresolved_ancestry_and_deletion() { let row = &sidebar(&db, community, &actor).await.channels[0]; assert_eq!((row.unread, row.mentions), (true, 1)); assert!(row.threads.is_empty()); - sqlx::query( - "UPDATE events SET created_at=created_at-interval '31 days' WHERE community_id=$1 AND id=$2", - ) - .bind(community.as_uuid()) - .bind(old.id.as_bytes().as_slice()) - .execute(&pool) - .await - .unwrap(); - sqlx::query("UPDATE events SET deleted_at=now() WHERE community_id=$1 AND id=$2") - .bind(community.as_uuid()) - .bind(root.id.as_bytes().as_slice()) - .execute(&pool) - .await - .unwrap(); + for table in [ + "events SET created_at=created_at-interval '31 days' WHERE community_id=$1 AND id=$2", + "event_mentions SET event_created_at=event_created_at-interval '31 days' + WHERE community_id=$1 AND event_id=$2", + ] { + sqlx::query(sqlx::AssertSqlSafe(format!("UPDATE {table}"))) + .bind(community.as_uuid()) + .bind(old.id.as_bytes().as_slice()) + .execute(&pool) + .await + .unwrap(); + } + let row = &sidebar(&db, community, &actor).await.channels[0]; + assert_eq!( + (row.unread, row.mentions), + (true, 1), + "author age hides nothing" + ); + for gone in [&root, &old] { + sqlx::query("UPDATE events SET deleted_at=now() WHERE community_id=$1 AND id=$2") + .bind(community.as_uuid()) + .bind(gone.id.as_bytes().as_slice()) + .execute(&pool) + .await + .unwrap(); + } let row = &sidebar(&db, community, &actor).await.channels[0]; assert_eq!((row.unread, row.mentions), (false, 0)); } diff --git a/crates/buzz-db/src/store/personal_read/projection.rs b/crates/buzz-db/src/store/personal_read/projection.rs index 38e6e026009..43dc0426a8a 100644 --- a/crates/buzz-db/src/store/personal_read/projection.rs +++ b/crates/buzz-db/src/store/personal_read/projection.rs @@ -1,20 +1,55 @@ -//! Bounded evidence projection. The cap limits evidence, not the definition of -//! unread: an unexamined tail goes uncounted, so counts may undercount. +//! Sidebar and threads read state, counted forward from each read position. +//! Each channel timeline and each followed thread is one index probe for its +//! last arrival; counts walk only what arrived after the position and stop at +//! `MAX_UNREAD_COUNT`. -use super::{model::*, participation, writes}; +use super::{model::*, writes}; use buzz_core::CommunityId; use chrono::{DateTime, Utc}; -use serde_json::Value; use sqlx::{Acquire, PgConnection, Row}; -use std::cmp::Reverse; -use std::collections::{hash_map::Entry, HashMap}; use uuid::Uuid; use crate::{observability, Db, DbError, Result}; +/// The last eligible reply `tl` to arrive in thread `tf` (idx_thread_metadata_read_arrival). +/// Deletion is a per-row filter, not a join, so the walk stops at the first +/// living reply instead of sorting them all. +const LATEST_REPLY: &str = "SELECT tm.event_id AS id, tm.received_at, e.pubkey, e.created_at + FROM (SELECT tm.event_id, tm.event_created_at, tm.received_at FROM thread_metadata tm + WHERE tm.community_id=$1 AND tm.root_event_id=tf.root_id AND tm.received_at IS NOT NULL + AND tm.channel_id=tf.channel_id + AND (SELECT e.deleted_at IS NULL FROM events e WHERE e.community_id=$1 + AND e.created_at=tm.event_created_at AND e.id=tm.event_id) + ORDER BY tm.received_at DESC, tm.event_id LIMIT 1) tm + JOIN events e ON e.community_id=$1 AND e.created_at=tm.event_created_at AND e.id=tm.event_id"; + +/// Unread replies in thread `t` at position `t.position` that are directed at +/// the actor: every one in a DM (`t.dm`), otherwise those that tag the actor. +/// Nothing is walked unless `t.unread`. Root first: the thread's unread +/// replies (idx_thread_metadata_read_arrival), each probed against the +/// mention key, never the actor's mentions across the channel. +pub(super) const THREAD_MENTIONS: &str = "SELECT count(*) AS n FROM ( + SELECT 1 FROM thread_metadata tm JOIN events e ON e.community_id=$1 + AND e.created_at=tm.event_created_at AND e.id=tm.event_id + WHERE t.unread AND t.dm AND tm.community_id=$1 AND tm.root_event_id=t.root_id + AND tm.received_at > t.position AND tm.channel_id=t.channel_id + AND e.deleted_at IS NULL AND e.pubkey<>$2 + UNION ALL + SELECT 1 FROM ( + SELECT tm.event_id, tm.event_created_at FROM thread_metadata tm + WHERE t.unread AND NOT t.dm AND tm.community_id=$1 AND tm.root_event_id=t.root_id + AND tm.channel_id=t.channel_id AND tm.received_at > t.position + OFFSET 0 + ) tm JOIN event_mentions em ON em.community_id=$1 AND em.pubkey_hex=$5 + AND em.event_created_at=tm.event_created_at AND em.event_id=tm.event_id + JOIN events e ON e.community_id=$1 AND e.created_at=tm.event_created_at AND e.id=tm.event_id + WHERE em.channel_id=t.channel_id AND em.event_created_at >= t.position-$6 + AND e.deleted_at IS NULL AND e.pubkey<>$2 + LIMIT $7 + ) x"; + impl Db { - /// Read a bounded joined roster from the writer. One SQL statement projects - /// channels, event evidence and read authority at a compatible MVCC cut. + /// Read a bounded joined roster from the writer in one read-only snapshot. /// Callers must recheck admission/resource access before releasing this data. pub async fn personal_read_sidebar( &self, @@ -70,7 +105,7 @@ impl Db { ) .await?; let mut tx = conn.begin().await?; - // The horizon and frontier evidence share one read-only cut. + // The dormancy cutoff and every read position share one read-only cut. sqlx::query("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ READ ONLY") .execute(&mut *tx) .await?; @@ -78,291 +113,235 @@ impl Db { sqlx::query("SET LOCAL jit = off").execute(&mut *tx).await?; let actor_bytes = actor.to_bytes(); let account = read_account(&mut tx, community, &actor_bytes, retention_seconds).await?; - // The inner event LIMIT is deliberately before eligibility filtering. - // This bounds rows/joins even with long deleted or self-authored runs. - // Aggregate equivalent eligible evidence before transfer. Multiplicity - // preserves counts. - // Validate relevant tag parts before compacting directed/ancestry facts. - // PostgreSQL scalar "p" ->> 0 is "p": the type check must reject it too. - // Canonical timeline roots share an empty (present) root sentinel. - // Tags are bounded before transfer; oversized/corrupt evidence is - // skipped, never falsely top-level/unmentioned/read. - // Latest comes from the unread scan, so its ID arrives no earlier than - // anything counted; the shallow probe answers only when the horizon - // holds no top-level message. Thread replies are never the timeline's - // latest: a timeline mark cannot use one as its anchor. - let rows = sqlx::query( + // A position is its frontier floored at the account's start (infinity + // before the actor's first intent, so nothing counts). + // + // Timeline, per channel: + // - latest_id: the last timeline message to arrive (idx_events_read_timeline); + // - unread: latest is someone else's and arrived after the position + // (an own post moves the position past it); + // - mentions: unread timeline messages that tag the actor, found by + // author time (event_mentions) from the skew before the position, + // or every one in a DM. + // Threads: each live follow (explicit, or involved since the cutoff) + // whose last reply is unread the same way, then its mentions for the + // listed ones only. A dormant follow is filtered before its probe. + let sql = format!( r#"WITH roster AS MATERIALIZED ( - SELECT c.id,c.channel_type::text AS channel_type + SELECT c.id, c.channel_type='dm' AS dm, + GREATEST(cf.through_timestamp, COALESCE($3::timestamptz, 'infinity')) AS position, + encode(cf.through_message_id,'hex') AS read_through_id FROM channel_members cm JOIN channels c ON c.community_id=cm.community_id AND c.id=cm.channel_id + LEFT JOIN personal_read_frontiers cf ON cf.community_id=$1 AND cf.actor=$2 + AND cf.channel_id=c.id AND cf.root_id=''::bytea WHERE cm.community_id=$1 AND cm.pubkey=$2 AND cm.removed_at IS NULL - AND c.deleted_at IS NULL AND ($3::uuid IS NULL OR c.id>$3) + AND c.deleted_at IS NULL AND ($8::uuid IS NULL OR c.id>$8) AND ($9::uuid[] IS NULL OR c.id=ANY($9)) - ORDER BY c.id LIMIT $4 + ORDER BY c.id LIMIT $10 ) - SELECT r.*, COALESCE(e.latest_id, latest.latest_id) AS latest_id, - encode(cf.through_message_id,'hex') AS read_through_id, - COALESCE(e.evidence,'[]'::jsonb) AS evidence FROM roster r + SELECT r.id, r.read_through_id, encode(l.id,'hex') AS latest_id, u.unread, + COALESCE(m.mentions, 0) AS mentions, COALESCE(th.items, '[]') AS threads + FROM roster r LEFT JOIN LATERAL ( - WITH candidates AS MATERIALIZED ( - SELECT id,created_at,received_at,kind,deleted_at,tags FROM events - WHERE community_id=$1 AND channel_id=r.id - ORDER BY created_at DESC,id LIMIT $5 - ) - SELECT (array_agg(encode(c.id,'hex') ORDER BY c.received_at DESC,c.id) - FILTER (WHERE c.kind=ANY($6) AND c.deleted_at IS NULL - AND NOT COALESCE(tm.root_event_id<>c.id,false) - -- Same unresolved-reply test as `threaded` below. - AND NOT (tm.root_event_id IS NULL AND jsonb_typeof(c.tags)='array' - AND EXISTS (SELECT 1 FROM jsonb_array_elements(c.tags) t(tag) - WHERE jsonb_typeof(tag)='array' AND tag->>0='e' - AND tag->>3='reply' - AND (tag->>1) COLLATE "C" ~ '^[0123456789abcdefABCDEF]{64}$')) - ))[1] AS latest_id - FROM candidates c LEFT JOIN thread_metadata tm ON tm.community_id=$1 - AND tm.channel_id=r.id AND tm.event_created_at=c.created_at AND tm.event_id=c.id - ) latest ON true - LEFT JOIN personal_read_frontiers cf ON cf.community_id=$1 AND cf.actor=$2 - AND cf.channel_id=r.id AND cf.root_id=''::bytea + SELECT e.id, e.pubkey, e.created_at, e.received_at FROM events e + WHERE e.community_id=$1 AND e.channel_id=r.id AND e.read_timeline + AND e.deleted_at IS NULL + ORDER BY e.received_at DESC, e.id LIMIT 1 + ) l ON true + CROSS JOIN LATERAL ( + SELECT COALESCE(l.received_at > r.position AND l.pubkey<>$2, false) AS unread + ) u LEFT JOIN LATERAL ( - WITH candidates AS MATERIALIZED ( - SELECT id,pubkey,created_at,received_at,deleted_at,kind,tags - FROM events WHERE community_id=$1 AND channel_id=r.id AND created_at >= $7 - ORDER BY created_at DESC,id LIMIT $8 - ), threaded AS MATERIALIZED ( - SELECT e.*, tm.root_event_id AS root, tm.parent_event_id AS parent, - COALESCE(tm.root_event_id<>e.id,false) AS is_reply, - -- A NIP-10 reply marker without canonical ancestry: neither - -- counted nor a timeline anchor, since a mark cannot validate - -- it. Parity-tested against the shared NIP-10 parser. - tm.root_event_id IS NULL AND jsonb_typeof(e.tags)='array' - AND EXISTS (SELECT 1 FROM jsonb_array_elements(e.tags) t(tag) - WHERE jsonb_typeof(tag)='array' AND tag->>0='e' - AND tag->>3='reply' - AND (tag->>1) COLLATE "C" ~ '^[0123456789abcdefABCDEF]{64}$') - AS unresolved - FROM candidates e - LEFT JOIN thread_metadata tm ON tm.community_id=$1 AND tm.channel_id=r.id - AND tm.event_created_at=e.created_at AND tm.event_id=e.id - WHERE e.kind=ANY($6) AND e.deleted_at IS NULL - ), classified AS ( - SELECT e.*, - -- Every frontier is floored at the account's start. - COALESCE(e.received_at <= GREATEST(COALESCE($10::timestamptz,'infinity'), - CASE WHEN e.is_reply THEN tf.through_timestamp - ELSE cf.through_timestamp END),false) AS covered, - encode(tf.through_message_id,'hex') AS read_through_id - FROM threaded e - LEFT JOIN personal_read_frontiers tf ON tf.community_id=$1 AND tf.actor=$2 - AND tf.channel_id=r.id AND tf.root_id=e.root AND e.is_reply - WHERE e.pubkey<>$2 AND NOT e.unresolved - ), grouped AS ( - SELECT CASE WHEN root IS NULL THEN NULL - WHEN is_reply THEN encode(root,'hex') ELSE '' END AS root, - CASE WHEN is_reply THEN encode(parent,'hex') END AS parent, - is_reply, covered, read_through_id, - -- ->>0 also selects scalar "p"/"e": reject nonarrays first. - -- C collation matches Rust's ASCII case rules. - CASE WHEN octet_length(tags::text)<=8192 - AND jsonb_typeof(tags)='array' THEN - (SELECT CASE WHEN bool_or(jsonb_typeof(tag)<>'array' - OR jsonb_path_exists(tag,'strict $[*] ? (@.type() != "string")')) - THEN NULL ELSE jsonb_build_object( - 'directed',COALESCE(bool_or( - (tag->>0='p' AND lower((tag->>1) COLLATE "C")=encode($2,'hex')) - OR (tag->>0='broadcast' AND tag->>1='1')),false)) END - FROM jsonb_array_elements(tags) t(tag) - WHERE tag->>0 IN ('p','broadcast','e')) ELSE NULL END AS facts, - count(*) AS n, - (extract(epoch FROM max(received_at))*1000000)::bigint AS newest_arrival, - (array_agg(encode(id,'hex') ORDER BY received_at DESC,id))[1] AS newest_id, - (array_agg(extract(epoch FROM created_at)::bigint - ORDER BY received_at DESC,id))[1] AS newest_at - FROM classified - WHERE NOT covered OR root IS NULL - GROUP BY 1,2,3,4,5,6 - ) - SELECT (array_agg(encode(id,'hex') ORDER BY received_at DESC,id) - FILTER (WHERE NOT is_reply AND NOT unresolved))[1] AS latest_id, - (SELECT jsonb_agg(to_jsonb(grouped)) FROM grouped) AS evidence - FROM threaded - ) e ON true ORDER BY r.id"#, - ).bind(community.as_uuid()).bind(actor_bytes.as_slice()).bind(after) - .bind((limit+1) as i64).bind(MAX_CHANNEL_SCAN as i64) - .bind(ELIGIBLE_KINDS.as_slice()) - .bind(DateTime::from_timestamp_millis(account.cutoff_ms) - .ok_or_else(|| DbError::InvalidData("invalid unread cutoff".into()))?) - .bind(MAX_UNREAD_SCAN as i64) - .bind(only) + SELECT count(*) AS mentions FROM ( + SELECT 1 FROM events e WHERE r.dm AND e.community_id=$1 + AND e.channel_id=r.id AND e.read_timeline AND e.deleted_at IS NULL + AND e.received_at > r.position AND e.pubkey<>$2 + UNION ALL + SELECT 1 FROM event_mentions em JOIN events e ON e.community_id=$1 + AND e.created_at=em.event_created_at AND e.id=em.event_id + WHERE NOT r.dm AND em.community_id=$1 AND em.pubkey_hex=$5 + AND em.channel_id=r.id AND em.event_created_at >= r.position-$6 + AND e.read_timeline AND e.deleted_at IS NULL + AND e.received_at > r.position AND e.pubkey<>$2 + LIMIT $7 + ) x WHERE u.unread + ) m ON true + LEFT JOIN LATERAL ( + SELECT jsonb_agg(jsonb_build_object('root_id',encode(t.root_id,'hex'), + 'unread',true,'mentions',tc.n,'read_through_id',t.read_through_id, + 'latest_id',encode(t.latest_id,'hex')) + ORDER BY t.latest_at DESC, t.root_id) AS items + FROM ( + SELECT tf.root_id, tf.channel_id, r.dm, p.position, true AS unread, + encode(tf.through_message_id,'hex') AS read_through_id, + tl.id AS latest_id, tl.received_at AS latest_at + FROM personal_read_frontiers tf + CROSS JOIN LATERAL (SELECT GREATEST(tf.through_timestamp, + COALESCE($3, 'infinity')) AS position) p + CROSS JOIN LATERAL ({LATEST_REPLY}) tl + WHERE tf.community_id=$1 AND tf.actor=$2 AND tf.channel_id=r.id + AND tf.root_id<>''::bytea AND NOT tf.unfollowed + AND (tf.explicit OR tf.active_at >= $4) + AND tl.received_at > p.position AND tl.pubkey<>$2 + ORDER BY tl.received_at DESC, tf.root_id LIMIT $11 + ) t CROSS JOIN LATERAL ({THREAD_MENTIONS}) tc + ) th ON true + ORDER BY r.id"# + ); + let cutoff = DateTime::from_timestamp_millis(account.cutoff_ms) + .ok_or_else(|| DbError::InvalidData("invalid dormancy cutoff".into()))?; + // Only compile-time constants are interpolated. + let rows = sqlx::query(sqlx::AssertSqlSafe(sql)) + .bind(community.as_uuid()) + .bind(actor_bytes.as_slice()) .bind(account.started_at) - .fetch_all(&mut *tx).await?; + .bind(cutoff) + .bind(actor.to_hex()) + .bind(skew()) + .bind(MAX_UNREAD_COUNT as i64) + .bind(after) + .bind(only) + .bind((limit + 1) as i64) + .bind(MAX_THREAD_SUMMARIES as i64) + .fetch_all(&mut *tx) + .await?; + tx.commit().await?; let has_more = rows.len() > limit; - // An unstarted account has read every arrival, so nothing is unread. - let started = account.started_at.is_some(); - let mut channels = Vec::new(); - let mut pending = Vec::new(); + let mut channels = Vec::with_capacity(limit); for row in rows.into_iter().take(limit) { - let evidence: Value = row.try_get("evidence")?; - let evidence = evidence - .as_array() - .ok_or_else(|| DbError::InvalidData("invalid sidebar evidence".into()))?; - let evidence = if started { evidence.as_slice() } else { &[] }; - let channel_type: String = row.try_get("channel_type")?; - let mut unread = false; - let mut mentions = 0; - let mut threads: HashMap, Replies> = HashMap::new(); - let mut undirected = Vec::new(); - for e in evidence { - let n = e["n"] - .as_u64() - .filter(|n| *n <= MAX_UNREAD_SCAN as u64) - .ok_or_else(|| DbError::InvalidData("invalid evidence multiplicity".into()))? - as u32; - // Unusable tags prove nothing: the message is left out. - let Some(facts) = e["facts"].as_object() else { - continue; - }; - if e["covered"] == true { - continue; - } - let directed = - channel_type == "dm" || facts.get("directed") == Some(&Value::Bool(true)); - // Roots are timeline messages; descendants belong exclusively - // to their canonical thread. Never inherit the channel prefix. - if e["is_reply"] != true { - unread = true; - if directed { - mentions += n; - } - continue; - } - let Some(root) = e["root"].as_str().and_then(writes::event_id) else { - continue; - }; - let replies = Replies { - n, - newest: ( - e["newest_arrival"].as_i64().ok_or_else(invalid_newest)?, - e["newest_id"] - .as_str() - .ok_or_else(invalid_newest)? - .to_owned(), - ), - newest_at: e["newest_at"].as_i64().ok_or_else(invalid_newest)?, - read_through_id: e["read_through_id"].as_str().map(str::to_owned), - }; - // A directed reply counts whatever its conversation. Any other - // reply counts only in one of the actor's conversations. - if directed { - count(&mut threads, root, replies); - } else if let Some(parent) = e["parent"].as_str().and_then(writes::event_id) { - undirected.push((root, parent, replies)); - } - } - pending.push((threads, undirected)); + let threads: Vec = + serde_json::from_value(row.try_get("threads")?) + .map_err(|_| DbError::InvalidData("invalid thread summaries".into()))?; + let mentions: i64 = row.try_get("mentions")?; channels.push(ChannelReadSummary { channel_id: row.try_get("id")?, - unread, - mentions, + unread: row.try_get("unread")?, + mentions: mentions as u32, read_through_id: row.try_get("read_through_id")?, latest_id: row.try_get("latest_id")?, - // Threads wait for conversation membership below. - threads: Vec::new(), + threads, }); } - let targets: Vec<_> = channels - .iter() - .zip(&pending) - .flat_map(|(channel, (_, undirected))| { - undirected - .iter() - .map(|(_, parent, _)| (channel.channel_id, parent.clone())) - }) - .collect(); - let members = participation::resolve(&mut tx, community, &actor_bytes, &targets).await?; - for (channel, (mut threads, undirected)) in channels.iter_mut().zip(pending) { - // Relevance is decided before the thread cap. An undecided - // membership leaves the reply out. - for (root, parent, replies) in undirected { - if members.get(&(channel.channel_id, parent)) == Some(&true) { - count(&mut threads, root, replies); - } - } - channel.threads = summarize(threads); - } let next_cursor = if has_more { channels.last().map(|c| c.channel_id) } else { None }; - tx.commit().await?; Ok(SidebarPage { channels, next_cursor, }) } -} -/// Uncovered replies in one canonical thread: one evidence group, or the -/// thread's counted total. -struct Replies { - n: u32, - /// Last of them to arrive: (arrival microseconds, lowercase hex ID). - newest: (i64, String), - /// Its author seconds. - newest_at: i64, - /// The thread's frontier anchor, shared by every group of the thread. - read_through_id: Option, -} - -/// Add replies that count to their thread. -fn count(threads: &mut HashMap, Replies>, root: Vec, replies: Replies) { - match threads.entry(root) { - Entry::Vacant(slot) => { - slot.insert(replies); + /// Followed threads in joined channels, newest arrival first, read or + /// not, from the writer in one read-only snapshot. A thread with no reply + /// yet is omitted. Callers must recheck admission/resource access before + /// releasing this data. + pub async fn personal_read_threads( + &self, + community: CommunityId, + actor: &nostr::PublicKey, + retention_seconds: u32, + limit: usize, + after: Option<&ThreadsCursor>, + ) -> Result { + if !(1..=MAX_THREADS).contains(&limit) { + return Err(DbError::InvalidData("invalid threads limit".into())); } - Entry::Occupied(mut slot) => { - let thread = slot.get_mut(); - thread.n += replies.n; - // Latest arrival first; equal arrivals break toward the smaller ID. - if (replies.newest.0, Reverse(&replies.newest.1)) - > (thread.newest.0, Reverse(&thread.newest.1)) - { - thread.newest = replies.newest; - thread.newest_at = replies.newest_at; - } + let mut conn = observability::acquire_writer( + &self.pool, + observability::WriterOperation::SubscriptionHistory, + ) + .await?; + let mut tx = conn.begin().await?; + sqlx::query("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ READ ONLY") + .execute(&mut *tx) + .await?; + writes::deadlines(&mut tx).await?; + sqlx::query("SET LOCAL jit = off").execute(&mut *tx).await?; + let actor_bytes = actor.to_bytes(); + let account = read_account(&mut tx, community, &actor_bytes, retention_seconds).await?; + // Every live follow is probed for its last arrival and sorted: the + // cost grows with the actor's recent follows, not with community size. + let sql = format!( + r#"SELECT t.channel_id, encode(t.root_id,'hex') AS root_id, t.unread, + tc.n AS mentions, + encode(t.through_message_id,'hex') AS read_through_id, + encode(t.latest_id,'hex') AS latest_id, t.latest_at + FROM ( + SELECT tf.root_id, tf.channel_id, c.channel_type='dm' AS dm, p.position, + tf.through_message_id, tl.id AS latest_id, tl.received_at AS latest_at, + (tl.received_at > p.position AND tl.pubkey<>$2) AS unread + FROM personal_read_frontiers tf + JOIN channel_members cm ON cm.community_id=$1 AND cm.channel_id=tf.channel_id + AND cm.pubkey=$2 AND cm.removed_at IS NULL + JOIN channels c ON c.community_id=$1 AND c.id=tf.channel_id AND c.deleted_at IS NULL + CROSS JOIN LATERAL (SELECT GREATEST(tf.through_timestamp, + COALESCE($3::timestamptz, 'infinity')) AS position) p + CROSS JOIN LATERAL ({LATEST_REPLY}) tl + WHERE tf.community_id=$1 AND tf.actor=$2 AND tf.root_id<>''::bytea + AND NOT tf.unfollowed AND (tf.explicit OR tf.active_at >= $4) + AND ($8::timestamptz IS NULL OR tl.received_at < $8 + OR (tl.received_at=$8 AND tf.root_id>$9)) + ORDER BY tl.received_at DESC, tf.root_id LIMIT $10 + ) t CROSS JOIN LATERAL ({THREAD_MENTIONS}) tc + ORDER BY t.latest_at DESC, t.root_id"# + ); + let cutoff = DateTime::from_timestamp_millis(account.cutoff_ms) + .ok_or_else(|| DbError::InvalidData("invalid dormancy cutoff".into()))?; + // Only compile-time constants are interpolated. + let rows = sqlx::query(sqlx::AssertSqlSafe(sql)) + .bind(community.as_uuid()) + .bind(actor_bytes.as_slice()) + .bind(account.started_at) + .bind(cutoff) + .bind(actor.to_hex()) + .bind(skew()) + .bind(MAX_UNREAD_COUNT as i64) + .bind(after.map(|c| c.latest_at)) + .bind(after.map(|c| c.root_id.as_slice())) + .bind((limit + 1) as i64) + .fetch_all(&mut *tx) + .await?; + tx.commit().await?; + let has_more = rows.len() > limit; + let mut threads = Vec::with_capacity(limit); + let mut last = None; + for row in rows.into_iter().take(limit) { + let mentions: i64 = row.try_get("mentions")?; + let root_id: String = row.try_get("root_id")?; + last = Some(( + row.try_get::, _>("latest_at")?, + root_id.clone(), + )); + threads.push(FollowedThread { + channel_id: row.try_get("channel_id")?, + thread: ThreadReadSummary { + root_id, + unread: row.try_get("unread")?, + mentions: mentions as u32, + read_through_id: row.try_get("read_through_id")?, + latest_id: row.try_get("latest_id")?, + }, + }); } + let next_cursor = last + .filter(|_| has_more) + .map(|(at, root)| format!("{}.{root}", at.timestamp_micros())); + Ok(ThreadsPage { + threads, + next_cursor, + }) } } -fn invalid_newest() -> DbError { - DbError::InvalidData("invalid thread evidence".into()) -} - -/// Order newest unread reply first (root ID breaks ties) and cap the list. -fn summarize(threads: HashMap, Replies>) -> Vec { - let mut threads: Vec<_> = threads.into_iter().collect(); - threads.sort_unstable_by(|(a_root, a), (b_root, b)| { - b.newest_at - .cmp(&a.newest_at) - .then_with(|| a_root.cmp(b_root)) - }); - threads - .into_iter() - .take(MAX_THREAD_SUMMARIES) - .map(|(root, thread)| ThreadReadSummary { - root_id: hex::encode(root), - unread: true, - mentions: thread.n, - read_through_id: thread.read_through_id, - latest_id: thread.newest.1, - }) - .collect() +fn skew() -> chrono::Duration { + chrono::Duration::seconds(i64::from(MAX_ARRIVAL_SKEW_SECONDS)) } -/// Read-time horizon and the account's start, in the caller's snapshot. -/// Frontier state is not discarded on expiry. +/// The follow dormancy cutoff and the account's start, in the caller's +/// snapshot. A dormant follow keeps its row and position. pub(super) async fn read_account( conn: &mut PgConnection, community: CommunityId, @@ -384,37 +363,3 @@ pub(super) async fn read_account( started_at: row.try_get("started_at")?, }) } - -#[cfg(test)] -mod tests { - use super::*; - - fn replies(at: i64) -> Replies { - Replies { - n: 1, - newest: (at, String::new()), - newest_at: at, - read_through_id: None, - } - } - - #[test] - fn thread_rows_order_newest_first_break_ties_by_root_and_cap_at_twenty_five() { - // Literal contract boundaries deliberately do not derive from the constant. - for count in [0_u8, 1, 24, 25, 26, 27] { - // Pairwise-equal times exercise the tie-break. - let threads = (0..count) - .map(|i| (vec![i; 32], replies(i64::from(i / 2)))) - .collect(); - let roots: Vec<_> = summarize(threads).into_iter().map(|t| t.root_id).collect(); - let mut expected: Vec<_> = (0..count).map(|i| (i64::from(i / 2), i)).collect(); - expected.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(&b.1))); - let expected: Vec<_> = expected - .into_iter() - .take(25) - .map(|(_, i)| hex::encode([i; 32])) - .collect(); - assert_eq!(roots, expected, "count {count}"); - } - } -} diff --git a/crates/buzz-db/src/store/personal_read/projection_postgres_tests.rs b/crates/buzz-db/src/store/personal_read/projection_postgres_tests.rs index 3dc5c131b36..5af5abe53bf 100644 --- a/crates/buzz-db/src/store/personal_read/projection_postgres_tests.rs +++ b/crates/buzz-db/src/store/personal_read/projection_postgres_tests.rs @@ -1,14 +1,20 @@ -use super::{postgres_tests::fixture, *}; -use serde_json::json; +use super::{ + postgres_tests::{fixture, insert_reply}, + *, +}; +use crate::Db; +use nostr::{EventBuilder, Keys, Kind, Tag}; +use uuid::Uuid; +/// Eligibility is classified at ingest (kind) and at read (author, deletion); +/// author time plays no part. Each case is stored through ingest, then +/// rewritten in place for the read-time facts. #[tokio::test] #[ignore = "requires Postgres"] -async fn sidebar_sql_eligibility_follows_kind_author_deletion_and_horizon() { - let (db, pool, community, _, actor, event) = fixture().await; +async fn sidebar_sql_eligibility_follows_kind_author_and_deletion() { + let (db, pool, community, channel, actor, _) = fixture().await; let now = chrono::Utc::now().timestamp_millis(); let horizon = i64::from(DEFAULT_RETENTION_SECONDS) * 1000; - // Addressed to the actor, so a counted message is also exactly one mention. - let tags = json!([["p", actor.public_key().to_hex()]]); for (kind, eligible_kind) in [ (9, true), (40002, true), @@ -16,22 +22,34 @@ async fn sidebar_sql_eligibility_follows_kind_author_deletion_and_horizon() { (45003, true), (1, false), (7, false), - (39002, false), (40008, false), ] { + sqlx::query("UPDATE events SET deleted_at=now() WHERE community_id=$1") + .bind(community.as_uuid()) + .execute(&pool) + .await + .unwrap(); + // Addressed to the actor, so a counted message is also exactly one mention. + let author = Keys::generate(); + let event = EventBuilder::new(Kind::Custom(kind), "case") + .tags([Tag::public_key(actor.public_key())]) + .sign_with_keys(&author) + .unwrap(); + db.insert_event(community, &event, Some(channel)) + .await + .unwrap(); for own in [false, true] { for deleted in [false, true] { - // Now, then one minute inside and one minute outside the horizon. - for (age, inside) in [ - (0, true), - (horizon - 60_000, true), - (horizon + 60_000, false), - ] { + // Now, and one minute past the default follow window. + for age in [0, horizon + 60_000] { let created = now - age; - sqlx::query("UPDATE events SET kind=$2,pubkey=$3,deleted_at=CASE WHEN $4 THEN now() ELSE NULL END,created_at=to_timestamp($5::double precision/1000),tags=$6 WHERE community_id=$1") - .bind(community.as_uuid()).bind(kind) - .bind(if own { actor.public_key().to_bytes() } else { event.pubkey.to_bytes() }.as_slice()) - .bind(deleted).bind(created as f64).bind(&tags).execute(&pool).await.unwrap(); + sqlx::query("UPDATE events SET pubkey=$3,deleted_at=CASE WHEN $4 THEN now() ELSE NULL END,created_at=to_timestamp($5::double precision/1000) WHERE community_id=$1 AND id=$2") + .bind(community.as_uuid()).bind(event.id.as_bytes().as_slice()) + .bind(if own { actor.public_key() } else { author.public_key() }.to_bytes().as_slice()) + .bind(deleted).bind(created as f64).execute(&pool).await.unwrap(); + sqlx::query("UPDATE event_mentions SET event_created_at=to_timestamp($3::double precision/1000) WHERE community_id=$1 AND event_id=$2") + .bind(community.as_uuid()).bind(event.id.as_bytes().as_slice()) + .bind(created as f64).execute(&pool).await.unwrap(); let page = db .personal_read_sidebar( community, @@ -42,7 +60,7 @@ async fn sidebar_sql_eligibility_follows_kind_author_deletion_and_horizon() { ) .await .unwrap(); - let expected = eligible_kind && !own && !deleted && inside; + let expected = eligible_kind && !own && !deleted; let row = &page.channels[0]; assert_eq!( (row.unread, row.mentions), @@ -55,60 +73,12 @@ async fn sidebar_sql_eligibility_follows_kind_author_deletion_and_horizon() { } } -#[tokio::test] -#[ignore = "requires Postgres"] -async fn sidebar_compacted_tags_preserve_directed_and_corruption_rules() { - let (db, pool, community, _, actor, _) = fixture().await; - let actor_hex = actor.public_key().to_hex().to_uppercase(); - // Unusable tags prove nothing: that message is left out, neither unread - // nor a mention. - for (tags, unread, mentions) in [ - (json!([]), true, 0), - (json!(["p"]), false, 0), - (json!(["e"]), false, 0), - (json!(["broadcast"]), false, 0), - (json!([["p", actor_hex, "relay", "petname"]]), true, 1), - (json!([["p", "00".repeat(32)]]), true, 0), - (json!([["broadcast", "1", "extra"]]), true, 1), - (json!([["broadcast", "0"]]), true, 0), - (json!([["p", "00".repeat(32), 42]]), false, 0), - (json!([["broadcast", "0", 42]]), false, 0), - (json!([["e", "00".repeat(32), "", "root", 42]]), false, 0), - (json!([["p", actor_hex], ["p", "other", 42]]), false, 0), - (json!([["e", "00".repeat(32), "", "reply"]]), false, 0), - (json!([["x", 42]]), true, 0), - (json!({"p":actor_hex}), false, 0), - (json!([["p", "x".repeat(8193)]]), false, 0), - ] { - sqlx::query("UPDATE events SET tags=$2 WHERE community_id=$1") - .bind(community.as_uuid()) - .bind(&tags) - .execute(&pool) - .await - .unwrap(); - let page = db - .personal_read_sidebar( - community, - &actor.public_key(), - DEFAULT_RETENTION_SECONDS, - 20, - None, - ) - .await - .unwrap(); - let row = &page.channels[0]; - assert_eq!( - (row.unread, row.mentions), - (unread, mentions), - "tags={tags}" - ); - } -} - +/// A message with a NIP-10 reply marker but no recorded ancestry is on +/// neither read path, by the shared parser's definition of a reply. #[tokio::test] #[ignore = "requires Postgres"] async fn sidebar_ancestry_fact_matches_shared_nip10_parser() { - let (db, pool, community, _, actor, _) = fixture().await; + let (db, pool, community, channel, actor, _) = fixture().await; let ids = [ "a".repeat(64), "A".repeat(64), @@ -140,12 +110,22 @@ async fn sidebar_ancestry_fact_matches_shared_nip10_parser() { buzz_core::nip10::parse_thread_markers_from_parts(tags.iter().map(Vec::as_slice)) .resolve() .is_some(); - sqlx::query("UPDATE events SET tags=$2 WHERE community_id=$1") + sqlx::query("UPDATE events SET deleted_at=now() WHERE community_id=$1") .bind(community.as_uuid()) - .bind(json!(tags)) .execute(&pool) .await .unwrap(); + let parsed: Vec = tags + .iter() + .map(|tag| Tag::parse(tag.iter().map(String::as_str)).unwrap()) + .collect(); + let event = EventBuilder::new(Kind::Custom(9), "case") + .tags(parsed) + .sign_with_keys(&Keys::generate()) + .unwrap(); + db.insert_event(community, &event, Some(channel)) + .await + .unwrap(); let page = db .personal_read_sidebar( community, @@ -169,7 +149,7 @@ async fn sidebar_ancestry_fact_matches_shared_nip10_parser() { #[tokio::test] #[ignore = "requires Postgres"] -async fn sidebar_directed_fact_follows_dm_mention_and_broadcast_rules() { +async fn sidebar_mentions_are_p_tags_in_a_stream_and_everything_in_a_dm() { let (db, pool, community, channel, actor, _) = fixture().await; let actor_hex = actor.public_key().to_hex(); let fullwidth: String = actor_hex @@ -177,54 +157,224 @@ async fn sidebar_directed_fact_follows_dm_mention_and_broadcast_rules() { .map(|c| if c.is_ascii_alphabetic() { 'Ａ' } else { c }) .collect(); assert_ne!(fullwidth, actor_hex); - // Whether each tag set is directed in a stream; in a DM every one is. + let tag = |parts: &[&str]| Tag::parse(parts.iter().copied()).unwrap(); + // Whether each message mentions the actor in a stream. A top-level + // broadcast is not a mention. let cases = [ - (json!([]), false), - (json!([["p", actor_hex]]), true), - (json!([["p", actor_hex.to_uppercase()]]), true), - (json!([["p", fullwidth]]), false), - (json!([["p"]]), false), - (json!([["broadcast", "1"]]), true), - (json!([["broadcast", "true"]]), false), - (json!([["p", "00".repeat(32)]]), false), + (vec![], false), + (vec![tag(&["p", &actor_hex])], true), + (vec![tag(&["p", &actor_hex.to_uppercase()])], true), + (vec![tag(&["p", &fullwidth])], false), + (vec![tag(&["broadcast", "1"])], false), + (vec![tag(&["p", &"00".repeat(32)])], false), ( - json!([["broadcast", "0"], ["p", actor_hex.to_uppercase()]]), + vec![tag(&["broadcast", "0"]), tag(&["p", &actor_hex])], true, ), ]; - for channel_type in ["stream", "dm"] { - sqlx::query( - "UPDATE channels SET channel_type=$3::channel_type WHERE community_id=$1 AND id=$2", - ) + let mut expected = 0; + for (tags, mentioned) in &cases { + let event = EventBuilder::new(Kind::Custom(9), "case") + .tags(tags.clone()) + .sign_with_keys(&Keys::generate()) + .unwrap(); + db.insert_event(community, &event, Some(channel)) + .await + .unwrap(); + expected += u32::from(*mentioned); + assert_eq!( + mentions(&db, community, &actor).await, + expected, + "tags={tags:?}" + ); + } + sqlx::query("UPDATE channels SET channel_type='dm' WHERE community_id=$1 AND id=$2") .bind(community.as_uuid()) .bind(channel) - .bind(channel_type) .execute(&pool) .await .unwrap(); - for (tags, in_stream) in &cases { - let expected = u32::from(channel_type == "dm" || *in_stream); - sqlx::query("UPDATE events SET tags=$2 WHERE community_id=$1") - .bind(community.as_uuid()) - .bind(tags) - .execute(&pool) - .await - .unwrap(); - let page = db - .personal_read_sidebar( - community, - &actor.public_key(), - DEFAULT_RETENTION_SECONDS, - 20, - None, - ) + // Every message by someone else, the fixture's included. + assert_eq!( + mentions(&db, community, &actor).await, + cases.len() as u32 + 1 + ); +} + +async fn mentions(db: &Db, community: buzz_core::CommunityId, actor: &Keys) -> u32 { + db.personal_read_sidebar( + community, + &actor.public_key(), + DEFAULT_RETENTION_SECONDS, + 20, + None, + ) + .await + .unwrap() + .channels[0] + .mentions +} + +/// THREAD_MENTIONS before the root-first rewrite, kept as the oracle: it +/// walked the actor's mentions in the channel and filtered them by root. `$4` +/// was the author-time horizon. +const CHANNEL_FIRST_THREAD_MENTIONS: &str = "SELECT count(*) AS n FROM ( + SELECT 1 FROM thread_metadata tm JOIN events e ON e.community_id=$1 + AND e.created_at=tm.event_created_at AND e.id=tm.event_id + WHERE t.unread AND t.dm AND tm.community_id=$1 AND tm.root_event_id=t.root_id + AND tm.received_at > t.position AND tm.channel_id=t.channel_id + AND e.deleted_at IS NULL AND e.pubkey<>$2 AND e.created_at >= $4 + UNION ALL + SELECT 1 FROM event_mentions em JOIN thread_metadata tm ON tm.community_id=$1 + AND tm.event_created_at=em.event_created_at AND tm.event_id=em.event_id + JOIN events e ON e.community_id=$1 AND e.created_at=em.event_created_at AND e.id=em.event_id + WHERE t.unread AND NOT t.dm AND em.community_id=$1 AND em.pubkey_hex=$5 AND em.channel_id=t.channel_id + AND em.event_created_at >= GREATEST(t.position-$6, $4) + AND tm.root_event_id=t.root_id AND tm.received_at > t.position + AND e.deleted_at IS NULL AND e.pubkey<>$2 + LIMIT $7 + ) x"; + +/// Each of the actor's thread frontiers and its mention count by `query`. +async fn thread_mentions( + pool: &sqlx::PgPool, + community: buzz_core::CommunityId, + actor: &Keys, + query: &str, +) -> Vec<(String, i64)> { + let sql = format!( + "SELECT encode(t.root_id,'hex') AS root, tc.n FROM ( + SELECT tf.root_id, tf.channel_id, c.channel_type='dm' AS dm, true AS unread, + GREATEST(tf.through_timestamp, COALESCE($3::timestamptz, 'infinity')) AS position + FROM personal_read_frontiers tf JOIN channels c + ON c.community_id=tf.community_id AND c.id=tf.channel_id + WHERE tf.community_id=$1 AND tf.actor=$2 AND tf.root_id<>''::bytea + AND $4::timestamptz IS NOT NULL + ) t CROSS JOIN LATERAL ({query}) tc ORDER BY 1" + ); + sqlx::query_as(sqlx::AssertSqlSafe(sql)) + .bind(community.as_uuid()) + .bind(actor.public_key().to_bytes().as_slice()) + .bind(chrono::DateTime::from_timestamp(0, 0).unwrap()) + .bind(chrono::DateTime::from_timestamp(0, 0).unwrap()) + .bind(actor.public_key().to_hex()) + .bind(chrono::Duration::seconds(i64::from( + MAX_ARRIVAL_SKEW_SECONDS, + ))) + .bind(MAX_UNREAD_COUNT as i64) + .fetch_all(pool) + .await + .unwrap() +} + +/// The root-first thread mention count matches the channel-first query it +/// replaced, across positions, own and deleted replies, mentions in sibling +/// threads and channels, and DMs. +#[tokio::test] +#[ignore = "requires Postgres"] +async fn root_first_thread_mentions_match_the_channel_first_query() { + let (db, pool, community, channel, actor, _) = fixture().await; + let tag = || vec![Tag::public_key(actor.public_key())]; + let (bee, wasp) = (Keys::generate(), Keys::generate()); + let keys = [&actor, &bee, &wasp].map(|k| k.public_key().to_bytes()); + let participants: Vec<&[u8]> = keys.iter().map(|k| k.as_slice()).collect(); + let dm = db + .create_dm(community, &participants, &keys[1]) + .await + .unwrap() + .id; + let other = db + .create_channel( + community, + "sibling channel", + crate::channel::ChannelType::Stream, + crate::channel::ChannelVisibility::Open, + None, + &actor.public_key().to_bytes(), + None, + ) + .await + .unwrap() + .id; + let mut expected = Vec::new(); + for (place, mentions) in [ + (channel, 0), + (channel, 1), + (channel, 3), + (other, 2), + (dm, 2), + ] { + let author = if place == dm { + bee.clone() + } else { + Keys::generate() + }; + let root = send(&db, community, place, None, &author, vec![]).await; + // A mention before the actor's own reply is read. + send(&db, community, place, Some(&root), &Keys::generate(), tag()).await; + db.apply_personal_read_intent( + community, + &actor.public_key(), + &ReadIntent::Follow { + target: ReadTarget { + channel_id: place, + root_id: Some(root.id.to_hex()), + }, + }, + ) + .await + .unwrap(); + // Posting reads through the post; own and deleted mentions never count. + send(&db, community, place, Some(&root), &actor, tag()).await; + for _ in 0..mentions { + send(&db, community, place, Some(&root), &wasp, tag()).await; + } + send( + &db, + community, + place, + Some(&root), + &Keys::generate(), + vec![], + ) + .await; + let gone = send(&db, community, place, Some(&root), &Keys::generate(), tag()).await; + sqlx::query("UPDATE events SET deleted_at=now() WHERE community_id=$1 AND id=$2") + .bind(community.as_uuid()) + .bind(gone.id.as_bytes().as_slice()) + .execute(&pool) + .await + .unwrap(); + // In a DM every reply is directed at the actor. + let n = if place == dm { mentions + 1 } else { mentions }; + expected.push((root.id.to_hex(), n)); + } + expected.sort(); + let before = thread_mentions(&pool, community, &actor, CHANNEL_FIRST_THREAD_MENTIONS).await; + let after = thread_mentions(&pool, community, &actor, super::projection::THREAD_MENTIONS).await; + assert_eq!(before, expected); + assert_eq!(after, expected); +} + +async fn send( + db: &Db, + community: buzz_core::CommunityId, + channel: Uuid, + root: Option<&nostr::Event>, + author: &Keys, + tags: Vec, +) -> nostr::Event { + let event = EventBuilder::new(Kind::Custom(9), Uuid::new_v4().to_string()) + .tags(tags) + .sign_with_keys(author) + .unwrap(); + match root { + Some(root) => insert_reply(db, community, channel, root, &event).await, + None => { + db.insert_event(community, &event, Some(channel)) .await .unwrap(); - assert!(page.channels[0].unread); - assert_eq!( - page.channels[0].mentions, expected, - "channel_type={channel_type} tags={tags}" - ); } } + event } diff --git a/crates/buzz-db/src/store/personal_read/threads_postgres_tests.rs b/crates/buzz-db/src/store/personal_read/threads_postgres_tests.rs index 2bb86aab8da..2259649b6bc 100644 --- a/crates/buzz-db/src/store/personal_read/threads_postgres_tests.rs +++ b/crates/buzz-db/src/store/personal_read/threads_postgres_tests.rs @@ -1,12 +1,14 @@ //! Thread summaries, anchor validation and targeted refresh. -use super::{postgres_tests::fixture, *}; +use super::{ + postgres_tests::{fixture, insert_reply, sync_reply_arrivals}, + *, +}; use crate::{ channel::{ChannelType, ChannelVisibility}, Db, }; use buzz_core::CommunityId; use nostr::{EventBuilder, Keys, Kind, Tag}; -use sqlx::PgPool; use uuid::Uuid; async fn post( @@ -38,48 +40,41 @@ async fn post_as( event } -/// A canonical reply to the root: stored event plus its thread metadata. +/// A canonical reply to the root, through ingest. async fn reply( db: &Db, - pool: &PgPool, community: CommunityId, channel: Uuid, root: &nostr::Event, at: u64, tags: Vec, ) -> nostr::Event { - let event = post(db, community, channel, at, tags).await; - link(pool, community, channel, root, &event).await; - event + reply_as(db, community, channel, root, &Keys::generate(), at, tags).await } -async fn link( - pool: &PgPool, +async fn reply_as( + db: &Db, community: CommunityId, channel: Uuid, root: &nostr::Event, - event: &nostr::Event, -) { - sqlx::query("INSERT INTO thread_metadata (community_id,event_id,event_created_at,channel_id,root_event_id,parent_event_id,depth) - VALUES ($1,$2,to_timestamp($3),$4,$5,$5,1)") - .bind(community.as_uuid()).bind(event.id.as_bytes().as_slice()) - .bind(event.created_at.as_secs() as f64) - .bind(channel).bind(root.id.as_bytes().as_slice()).execute(pool).await.unwrap(); + author: &Keys, + at: u64, + tags: Vec, +) -> nostr::Event { + let event = EventBuilder::new(Kind::Custom(9), format!("reply at {at}")) + .tags(tags) + .custom_created_at(nostr::Timestamp::from(at)) + .sign_with_keys(author) + .unwrap(); + insert_reply(db, community, channel, root, &event).await; + event } -/// Put the actor in the root's conversation, so that plain replies to it -/// count. The actor's own reply is never unread. -async fn join( - db: &Db, - pool: &PgPool, - community: CommunityId, - channel: Uuid, - actor: &Keys, - root: &nostr::Event, -) { +/// The actor replies to the root, and so follows its thread. The actor's own +/// reply is never unread. +async fn join(db: &Db, community: CommunityId, channel: Uuid, actor: &Keys, root: &nostr::Event) { let at = root.created_at.as_secs(); - let own = post_as(db, community, channel, actor, at, vec![]).await; - link(pool, community, channel, root, &own).await; + reply_as(db, community, channel, root, actor, at, vec![]).await; } async fn sidebar(db: &Db, community: CommunityId, actor: &Keys) -> ChannelReadSummary { @@ -162,35 +157,6 @@ async fn mark_through_rejects_unresolved_missing_auxiliary_and_malformed_anchors assert_eq!(row.read_through_id, None, "nothing was marked"); } -#[tokio::test] -#[ignore = "requires Postgres"] -async fn channel_frontier_never_covers_an_epoch_zero_reply() { - let (db, pool, community, channel, actor, root) = fixture().await; - join(&db, &pool, community, channel, &actor, &root).await; - let ancient = reply(&db, &pool, community, channel, &root, 0, vec![]).await; - // A channel frontier exists, with no thread frontier: absent must not - // read as epoch zero or as the channel's. - apply( - &db, - community, - &actor, - channel_mark(channel, root.id.to_hex()), - ) - .await; - // Widen the horizon to reach the epoch, so that only a misread absent - // frontier could hide this reply. - let row = db - .personal_read_sidebar(community, &actor.public_key(), u32::MAX, 20, None) - .await - .unwrap() - .channels - .remove(0); - assert!(!row.unread); - assert_eq!(row.threads.len(), 1, "epoch-zero reply stays unread"); - assert_eq!(row.threads[0].mentions, 1); - assert_eq!(row.threads[0].latest_id, ancient.id.to_hex()); -} - #[tokio::test] #[ignore = "requires Postgres"] async fn thread_summaries_order_cap_and_anchor() { @@ -202,7 +168,7 @@ async fn thread_summaries_order_cap_and_anchor() { let mut roots = Vec::new(); for i in 0..n { let root = post(&db, community, channel, base + i, vec![]).await; - join(&db, &pool, community, channel, &actor, &root).await; + join(&db, community, channel, &actor, &root).await; roots.push(root); } let newest_root = roots.last().unwrap().clone(); @@ -213,18 +179,17 @@ async fn thread_summaries_order_cap_and_anchor() { channel_mark(channel, newest_root.id.to_hex()), ) .await; - // Threads 0 and 1 tie on newest reply time and the rest follow, newest + // Threads 0 and 1 tie on last arrival and the rest follow, newest // first. Thread 2 has one directed reply and two that arrive last, // together. let offset = |i: u64| if i < 2 { n * 10 } else { (n - i) * 10 }; let mut anchors = Vec::new(); for (i, root) in (0..).zip(&roots) { let at = base + 100 + offset(i); - anchors.push(reply(&db, &pool, community, channel, root, at, vec![]).await); + anchors.push(reply(&db, community, channel, root, at, vec![]).await); } reply( &db, - &pool, community, channel, &roots[2], @@ -234,7 +199,6 @@ async fn thread_summaries_order_cap_and_anchor() { .await; let twin = reply( &db, - &pool, community, channel, &roots[2], @@ -242,14 +206,18 @@ async fn thread_summaries_order_cap_and_anchor() { vec![], ) .await; + // Replies arrive in author order, so the twin and thread 2's anchor + // arrive together. sqlx::query( - "UPDATE events SET received_at=(SELECT received_at FROM events WHERE id=$1) WHERE id=$2", + "UPDATE events e SET received_at=e.created_at FROM thread_metadata tm + WHERE tm.community_id=e.community_id AND tm.event_id=e.id + AND tm.received_at IS NOT NULL AND e.community_id=$1", ) - .bind(twin.id.as_bytes().as_slice()) - .bind(anchors[2].id.as_bytes().as_slice()) + .bind(community.as_uuid()) .execute(&pool) .await .unwrap(); + sync_reply_arrivals(&pool).await; let row = sidebar(&db, community, &actor).await; // Every root is read; the eight unread replies count only on threads. @@ -266,7 +234,7 @@ async fn thread_summaries_order_cap_and_anchor() { // The oldest unread thread is past the cap. assert_eq!(order(&row), [tied.to_vec(), ids(&roots[2..last])].concat()); let third = &row.threads[2]; - assert_eq!(third.mentions, 3); + assert_eq!(third.mentions, 1, "only the directed reply is a mention"); assert_eq!( third.latest_id, std::cmp::min(anchors[2].id.to_hex(), twin.id.to_hex()), @@ -285,8 +253,8 @@ async fn thread_summaries_order_cap_and_anchor() { [vec![tied[1].clone()], ids(&roots[2..])].concat() ); let mentions: Vec<_> = row.threads.iter().map(|t| t.mentions).collect(); - let mut expected = vec![1; MAX_THREAD_SUMMARIES]; - expected[1] = 3; + let mut expected = vec![0; MAX_THREAD_SUMMARIES]; + expected[1] = 1; assert_eq!(mentions, expected); } @@ -305,7 +273,7 @@ impl ThreadReadSummary { #[tokio::test] #[ignore = "requires Postgres"] async fn thread_on_a_never_unread_root_is_selectable_and_readable_by_itself() { - let (db, pool, community, channel, actor, root) = fixture().await; + let (db, _pool, community, channel, actor, root) = fixture().await; let base = root.created_at.as_secs(); let mention = || vec![Tag::parse(["p", &actor.public_key().to_hex()]).unwrap()]; // A diff is never unread, not even one addressed to the actor. @@ -317,8 +285,8 @@ async fn thread_on_a_never_unread_root_is_selectable_and_readable_by_itself() { db.insert_event(community, &diff, Some(channel)) .await .unwrap(); - let on_diff = reply(&db, &pool, community, channel, &diff, base + 10, mention()).await; - let elsewhere = reply(&db, &pool, community, channel, &root, base + 20, mention()).await; + let on_diff = reply(&db, community, channel, &diff, base + 10, mention()).await; + let elsewhere = reply(&db, community, channel, &root, base + 20, mention()).await; let row = sidebar(&db, community, &actor).await; // The undirected fixture root is unread; the directed diff is not counted. diff --git a/crates/buzz-db/src/store/personal_read/writes.rs b/crates/buzz-db/src/store/personal_read/writes.rs index 9d4c4a6586c..5e9b7be4351 100644 --- a/crates/buzz-db/src/store/personal_read/writes.rs +++ b/crates/buzz-db/src/store/personal_read/writes.rs @@ -24,7 +24,8 @@ pub(super) async fn deadlines(conn: &mut PgConnection) -> Result<()> { } /// Start the account if this is the actor's first read intent, then serialize -/// private frontier writes, never shared conversation rows. +/// private frontier writes, never shared conversation rows. NO KEY UPDATE +/// leaves ingest's foreign-key checks (which create thread rows) unblocked. pub(super) async fn lock_account( conn: &mut PgConnection, community: CommunityId, @@ -40,7 +41,8 @@ pub(super) async fn lock_account( .execute(&mut *conn) .await?; sqlx::query( - "SELECT actor FROM personal_read_accounts WHERE community_id=$1 AND actor=$2 FOR UPDATE", + "SELECT actor FROM personal_read_accounts WHERE community_id=$1 AND actor=$2 + FOR NO KEY UPDATE", ) .bind(community.as_uuid()) .bind(actor) @@ -86,6 +88,8 @@ struct Message { id: Vec, received_at: DateTime, root: Option>, + /// A reply that is also on the channel timeline. + broadcast: bool, } /// A channel event and its canonical ancestry. `kinds` bounds the lookup: an @@ -98,7 +102,7 @@ async fn message( kinds: Option<&[i32]>, ) -> Result> { let row = sqlx::query( - "SELECT e.id, e.received_at, e.tags, tm.root_event_id + "SELECT e.id, e.received_at, e.tags, tm.root_event_id, COALESCE(tm.broadcast, false) AS broadcast FROM events e LEFT JOIN thread_metadata tm ON tm.community_id=e.community_id AND tm.event_created_at=e.created_at AND tm.event_id=e.id AND tm.channel_id=e.channel_id WHERE e.community_id=$1 AND e.channel_id=$2 AND e.id=$3 @@ -123,6 +127,7 @@ async fn message( id, received_at, root, + broadcast: row.try_get("broadcast")?, }) }) .transpose() @@ -159,7 +164,9 @@ pub(super) async fn valid_target( } /// Advance a monotone frontier. The anchor ID follows the greatest arrival; -/// an equal arrival keeps the existing anchor. +/// an equal arrival keeps the existing anchor. Reading a thread never follows +/// it: only a followed thread has a row to advance, and reading involves the +/// actor, which keeps the follow live. async fn frontier( conn: &mut PgConnection, community: CommunityId, @@ -168,7 +175,7 @@ async fn frontier( root: &[u8], msg: &Message, ) -> Result<()> { - sqlx::query( + let sql = if root.is_empty() { "INSERT INTO personal_read_frontiers (community_id, actor, channel_id, root_id, through_timestamp, through_message_id) VALUES ($1,$2,$3,$4,$5,$6) @@ -176,10 +183,61 @@ async fn frontier( SET through_timestamp=GREATEST(personal_read_frontiers.through_timestamp, EXCLUDED.through_timestamp), through_message_id=CASE WHEN EXCLUDED.through_timestamp > personal_read_frontiers.through_timestamp - THEN EXCLUDED.through_message_id ELSE personal_read_frontiers.through_message_id END", - ).bind(community.as_uuid()).bind(actor).bind(target.channel_id).bind(root) - .bind(msg.received_at).bind(&msg.id) - .execute(&mut *conn).await?; + THEN EXCLUDED.through_message_id ELSE personal_read_frontiers.through_message_id END" + } else { + "UPDATE personal_read_frontiers + SET through_timestamp=GREATEST(through_timestamp, $5), + through_message_id=CASE WHEN $5 > through_timestamp + THEN $6 ELSE through_message_id END, + active_at=now() + WHERE community_id=$1 AND actor=$2 AND channel_id=$3 AND root_id=$4" + }; + sqlx::query(sql) + .bind(community.as_uuid()) + .bind(actor) + .bind(target.channel_id) + .bind(root) + .bind(msg.received_at) + .bind(&msg.id) + .execute(&mut *conn) + .await?; + Ok(()) +} + +/// Follow or unfollow a thread. A follow that changes state starts now, so +/// earlier replies are read; an unfollowed row keeps ingest from following +/// again, except as `record_message` allows. An explicit follow, including of +/// a thread already followed implicitly, never goes dormant. +async fn follow( + conn: &mut PgConnection, + community: CommunityId, + actor: &[u8], + channel: Uuid, + root: &[u8], + unfollow: bool, +) -> Result<()> { + sqlx::query( + "INSERT INTO personal_read_frontiers + (community_id, actor, channel_id, root_id, through_timestamp, unfollowed, explicit) + VALUES ($1,$2,$3,$4,now(),$5,NOT $5) + ON CONFLICT (community_id, actor, channel_id, root_id) DO UPDATE + SET unfollowed=$5, explicit=NOT $5, active_at=now(), + through_timestamp=CASE WHEN personal_read_frontiers.unfollowed <> $5 + THEN GREATEST(personal_read_frontiers.through_timestamp, now()) + ELSE personal_read_frontiers.through_timestamp END, + through_message_id=CASE WHEN personal_read_frontiers.unfollowed <> $5 + AND now() > personal_read_frontiers.through_timestamp + THEN NULL ELSE personal_read_frontiers.through_message_id END + WHERE personal_read_frontiers.unfollowed <> $5 + OR personal_read_frontiers.explicit = $5", + ) + .bind(community.as_uuid()) + .bind(actor) + .bind(channel) + .bind(root) + .bind(unfollow) + .execute(conn) + .await?; Ok(()) } @@ -209,13 +267,30 @@ pub(super) async fn apply( return Ok(IntentOutcome::Blocked); }; let is_reply = msg.root.as_ref().is_some_and(|r| r != &msg.id); - if (root.is_empty() && is_reply) + if (root.is_empty() && is_reply && !msg.broadcast) || (!root.is_empty() && msg.id != root && msg.root.as_ref() != Some(&root)) { return Ok(IntentOutcome::Blocked); } frontier(conn, community, actor, target, &root, &msg).await?; } + ReadIntent::Follow { target } | ReadIntent::Unfollow { target } => { + if target.root_id.is_none() { + return Ok(IntentOutcome::Invalid); + } + let Some(root) = valid_target(conn, community, actor, target).await? else { + return Ok(IntentOutcome::Blocked); + }; + follow( + conn, + community, + actor, + target.channel_id, + &root, + matches!(intent, ReadIntent::Unfollow { .. }), + ) + .await?; + } } Ok(IntentOutcome::Applied) } diff --git a/crates/buzz-relay/src/api/buzz_v1/auth.rs b/crates/buzz-relay/src/api/buzz_v1/auth.rs index dd65d3dad66..ed7ed647f4f 100644 --- a/crates/buzz-relay/src/api/buzz_v1/auth.rs +++ b/crates/buzz-relay/src/api/buzz_v1/auth.rs @@ -132,6 +132,16 @@ pub(super) async fn authorize( signed_at, }; recheck(state, headers, &principal).await?; + // Read state is for people: an agent account has none to read or write. + let agent = state + .db + .get_agent_channel_policy(principal.tenant.community(), &actor.to_bytes()) + .await + .map_err(|_| Error::unavailable())? + .is_some_and(|(_, owner)| owner.is_some()); + if agent { + return Err(Error::new(StatusCode::FORBIDDEN, "forbidden")); + } Ok(principal) } diff --git a/crates/buzz-relay/src/api/buzz_v1/handlers.rs b/crates/buzz-relay/src/api/buzz_v1/handlers.rs index f3dbf095654..8f7d1eb17c1 100644 --- a/crates/buzz-relay/src/api/buzz_v1/handlers.rs +++ b/crates/buzz-relay/src/api/buzz_v1/handlers.rs @@ -7,7 +7,8 @@ use axum::{ response::Response, }; use buzz_db::personal_read::{ - ChannelReadSummary, ReadIntent, SidebarPage, MAX_CHANNELS, MAX_INTENTS, + ChannelReadSummary, ReadIntent, ThreadsCursor, ThreadsPage, MAX_CHANNELS, MAX_INTENTS, + MAX_THREADS, }; use serde::Deserialize; use serde_json::{json, Value}; @@ -74,27 +75,72 @@ pub(super) async fn sidebar( } } .map_err(|_| Error::unavailable())?; - auth::response(release(&state, &headers, &principal, page).await?) + let channels: Vec<_> = page.channels.iter().map(|c| c.channel_id).collect(); + release(&state, &headers, &principal, &channels).await?; + auth::response(page) + }) + .await + .map_err(|_| Error::unavailable())? +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct ThreadsQuery { + limit: Option, + cursor: Option, +} + +pub(super) async fn threads( + State(state): State>, + headers: HeaderMap, + OriginalUri(uri): OriginalUri, + query: Result, axum::extract::rejection::QueryRejection>, +) -> Result { + tokio::time::timeout(Duration::from_secs(8), async { + let principal = auth::authorize(&state, &headers, &uri, "GET", None).await?; + let Query(query) = query.map_err(|_| Error::invalid())?; + let limit = query.limit.unwrap_or(MAX_THREADS); + let cursor = query + .cursor + .map(|c| ThreadsCursor::parse(&c).ok_or_else(Error::invalid)) + .transpose()?; + if !(1..=MAX_THREADS).contains(&limit) { + return Err(Error::invalid()); + } + let page: ThreadsPage = state + .db + .personal_read_threads( + principal.tenant.community(), + &principal.actor, + state.config.buzz_v1_retention_seconds, + limit, + cursor.as_ref(), + ) + .await + .map_err(|_| Error::unavailable())?; + let channels: BTreeSet<_> = page.threads.iter().map(|t| t.channel_id).collect(); + let channels: Vec<_> = channels.into_iter().collect(); + release(&state, &headers, &principal, &channels).await?; + auth::response(page) }) .await .map_err(|_| Error::unavailable())? } /// Release projected rows only after admission and the actor's membership of -/// every row are rechecked outside the projection snapshot. +/// every row's channel are rechecked outside the projection snapshot. async fn release( state: &AppState, headers: &HeaderMap, principal: &auth::Principal, - page: SidebarPage, -) -> Result { + channels: &[Uuid], +) -> Result<(), Error> { auth::recheck(state, headers, principal).await?; - let channels: Vec<_> = page.channels.iter().map(|c| c.channel_id).collect(); let memberships = state .db .membership_pairs( principal.tenant.community(), - &channels, + channels, &[principal.actor.to_bytes().to_vec()], ) .await @@ -102,7 +148,7 @@ async fn release( if memberships.len() != channels.len() { return Err(Error::unavailable()); } - Ok(page) + Ok(()) } /// Updated rows for the channels a batch applied to. A channel the actor has @@ -126,7 +172,9 @@ async fn refreshed( ) .await .map_err(|_| Error::unavailable())?; - rows.extend(release(state, headers, principal, page).await?.channels); + let ids: Vec<_> = page.channels.iter().map(|c| c.channel_id).collect(); + release(state, headers, principal, &ids).await?; + rows.extend(page.channels); } Ok(rows) } @@ -168,8 +216,7 @@ pub(super) async fn write( .await .unwrap_or_else(|_| json!({"status":"unknown","retryable":true})); if outcome["status"] == "applied" { - let ReadIntent::MarkThrough { target, .. } = &intent; - applied.insert(target.channel_id); + applied.insert(intent.target().channel_id); } outcomes.push(outcome); } diff --git a/crates/buzz-relay/src/api/buzz_v1/mod.rs b/crates/buzz-relay/src/api/buzz_v1/mod.rs index d462ac15aae..0236dd7761e 100644 --- a/crates/buzz-relay/src/api/buzz_v1/mod.rs +++ b/crates/buzz-relay/src/api/buzz_v1/mod.rs @@ -17,6 +17,7 @@ pub const BASE_PATH: &str = "/buzz/v1"; pub fn router(state: Arc) -> Router { Router::new() .route("/me/sidebar", get(handlers::sidebar)) + .route("/me/threads", get(handlers::threads)) .route("/me/read-state", post(handlers::write)) .fallback(|| async { auth::Error::new(axum::http::StatusCode::NOT_FOUND, "not_found") }) .with_state(state) diff --git a/crates/buzz-relay/src/api/buzz_v1/postgres_tests.rs b/crates/buzz-relay/src/api/buzz_v1/postgres_tests.rs index 63942726793..2e913d2478c 100644 --- a/crates/buzz-relay/src/api/buzz_v1/postgres_tests.rs +++ b/crates/buzz-relay/src/api/buzz_v1/postgres_tests.rs @@ -361,6 +361,195 @@ async fn accessory_batch_outcomes_are_independent_and_return_applied_rows() { assert_ne!(response.status(), StatusCode::OK); } +#[tokio::test] +#[ignore = "requires Postgres"] +async fn accessory_follow_intents_and_threads_view() { + let fixture = crate::api::bridge::postgres_tests::bridge_handler_test_state() + .await + .unwrap(); + let mut state = (*fixture).clone(); + Arc::make_mut(&mut state.config).buzz_v1_enabled = true; + let state = Arc::new(state); + let host = format!("bff-threads-{}.local", uuid::Uuid::new_v4()); + let community = state + .db + .ensure_configured_community(&host) + .await + .unwrap() + .id; + let actor = Keys::generate(); + let channel = state + .db + .create_channel( + community, + "threads", + buzz_db::channel::ChannelType::Stream, + buzz_db::channel::ChannelVisibility::Open, + None, + &actor.public_key().to_bytes(), + None, + ) + .await + .unwrap() + .id; + start_before_everything(&state, community, &actor).await; + let mut roots = Vec::new(); + for _ in 0..2 { + let root = EventBuilder::new(Kind::Custom(9), "root") + .sign_with_keys(&Keys::generate()) + .unwrap(); + state + .db + .insert_event(community, &root, Some(channel)) + .await + .unwrap(); + roots.push(root); + } + let write_path = "/buzz/v1/me/read-state"; + let body = serde_json::to_vec(&json!({"intents":[ + {"type":"follow","target":{"channel_id":channel}}, + {"type":"follow","target":{"channel_id":channel,"root_id":roots[0].id.to_hex()}}, + {"type":"follow","target":{"channel_id":channel,"root_id":roots[1].id.to_hex()}}, + {"type":"unfollow","target":{"channel_id":channel,"root_id":roots[1].id.to_hex()}} + ]})) + .unwrap(); + let auth = proof(&actor, &host, write_path, "POST", Some(&body)); + let result = request(state.clone(), &host, write_path, "POST", Some(&auth), &body).await; + assert_eq!(result.0, StatusCode::OK, "{}", result.1); + assert_eq!( + result.1["outcomes"], + json!([{"status":"invalid"},{"status":"applied"},{"status":"applied"},{"status":"applied"}]) + ); + assert_eq!(result.1["channels"][0]["channel_id"], json!(channel)); + + let mut replies = Vec::new(); + for root in &roots { + let reply = EventBuilder::new(Kind::Custom(9), "reply") + .tags([Tag::parse(["e", &root.id.to_hex(), "", "root"]).unwrap()]) + .sign_with_keys(&Keys::generate()) + .unwrap(); + let at = |e: &nostr::Event| { + chrono::DateTime::from_timestamp(e.created_at.as_secs() as i64, 0).unwrap() + }; + state + .db + .insert_event_with_thread_metadata( + community, + &reply, + Some(channel), + Some(buzz_db::event::ThreadMetadataParams { + event_id: reply.id.as_bytes(), + event_created_at: at(&reply), + channel_id: channel, + parent_event_id: Some(root.id.as_bytes()), + parent_event_created_at: Some(at(root)), + root_event_id: Some(root.id.as_bytes()), + root_event_created_at: Some(at(root)), + depth: 1, + broadcast: false, + }), + ) + .await + .unwrap(); + replies.push(reply); + } + let path = "/buzz/v1/me/threads?limit=1"; + let auth = proof(&actor, &host, path, "GET", None); + let result = request(state.clone(), &host, path, "GET", Some(&auth), b"").await; + assert_eq!(result.0, StatusCode::OK, "{}", result.1); + // Only the followed thread, unread since the follow. + assert_eq!( + result.1, + json!({"threads":[{"channel_id":channel,"root_id":roots[0].id.to_hex(), + "unread":true,"mentions":0,"read_through_id":null, + "latest_id":replies[0].id.to_hex()}],"next_cursor":null}) + ); + let path = "/buzz/v1/me/threads?cursor=not-a-cursor"; + let auth = proof(&actor, &host, path, "GET", None); + let result = request(state.clone(), &host, path, "GET", Some(&auth), b"").await; + assert_eq!(result.0, StatusCode::BAD_REQUEST, "{}", result.1); +} + +/// Read state is for people: an agent account is refused every endpoint, +/// while its owner's sidebar is served. +#[tokio::test] +#[ignore = "requires Postgres"] +async fn accessory_refuses_agent_accounts() { + let fixture = crate::api::bridge::postgres_tests::bridge_handler_test_state() + .await + .unwrap(); + let mut state = (*fixture).clone(); + Arc::make_mut(&mut state.config).buzz_v1_enabled = true; + let state = Arc::new(state); + let host = format!("bff-agents-{}.local", uuid::Uuid::new_v4()); + let community = state + .db + .ensure_configured_community(&host) + .await + .unwrap() + .id; + let (owner, agent) = (Keys::generate(), Keys::generate()); + let channel = state + .db + .create_channel( + community, + "agents", + buzz_db::channel::ChannelType::Stream, + buzz_db::channel::ChannelVisibility::Open, + None, + &owner.public_key().to_bytes(), + None, + ) + .await + .unwrap() + .id; + for (who, by) in [(&owner, None), (&agent, Some(&owner))] { + sqlx::query( + "INSERT INTO users (community_id, pubkey, agent_owner_pubkey) VALUES ($1,$2,$3)", + ) + .bind(community.as_uuid()) + .bind(who.public_key().to_bytes().as_slice()) + .bind(by.map(|k| k.public_key().to_bytes().to_vec())) + .execute(state.db.pool()) + .await + .unwrap(); + } + let write = serde_json::to_vec(&json!({"intents":[ + {"type":"mark_through","target":{"channel_id":channel},"message_id":"ab".repeat(32)} + ]})) + .unwrap(); + for (path, method, body) in [ + ("/buzz/v1/me/sidebar", "GET", &b""[..]), + ("/buzz/v1/me/threads", "GET", &b""[..]), + ("/buzz/v1/me/read-state", "POST", write.as_slice()), + ] { + let auth = proof( + &agent, + &host, + path, + method, + (method == "POST").then_some(body), + ); + let (status, json) = request(state.clone(), &host, path, method, Some(&auth), body).await; + assert_eq!(status, StatusCode::FORBIDDEN, "{method} {path}"); + assert_eq!(json["error"]["code"], "forbidden"); + } + let path = "/buzz/v1/me/sidebar"; + let auth = proof(&owner, &host, path, "GET", None); + let (status, json) = request(state.clone(), &host, path, "GET", Some(&auth), b"").await; + assert_eq!(status, StatusCode::OK); + assert_eq!(json["channels"][0]["channel_id"], channel.to_string()); + let accounts: i64 = sqlx::query_scalar( + "SELECT count(*) FROM personal_read_accounts WHERE community_id=$1 AND actor=$2", + ) + .bind(community.as_uuid()) + .bind(agent.public_key().to_bytes().as_slice()) + .fetch_one(state.db.pool()) + .await + .unwrap(); + assert_eq!(accounts, 0); +} + #[tokio::test] #[ignore = "requires Postgres"] async fn accessory_write_revocation_is_terminal_before_persistence() { @@ -437,12 +626,15 @@ async fn accessory_write_revocation_is_terminal_before_persistence() { // Exercise the same per-item function the batch handler uses after admission. let result = super::handlers::write_intent(&state, &headers, &principal, &intent).await; assert_eq!(result, json!({"status":"blocked"})); - let persisted: i64 = - sqlx::query_scalar("SELECT count(*) FROM personal_read_accounts WHERE community_id=$1") - .bind(community.as_uuid()) - .fetch_one(state.db.pool()) - .await - .unwrap(); + // Ingest gave the message's author an account; the actor has none. + let persisted: i64 = sqlx::query_scalar( + "SELECT count(*) FROM personal_read_accounts WHERE community_id=$1 AND actor=$2", + ) + .bind(community.as_uuid()) + .bind(actor.public_key().to_bytes().as_slice()) + .fetch_one(state.db.pool()) + .await + .unwrap(); assert_eq!(persisted, 0, "denied intent must not persist"); } @@ -492,7 +684,8 @@ async fn accessory_discovery_is_host_bound_and_opt_in() { doc["buzz_v1"], json!({"version":1,"base_path":"/buzz/v1","retention_seconds":1234, "max_channels":20,"max_intents":100, - "eligible_kinds":[9, 40002, 45001, 45003]}) + "eligible_kinds":[9, 40002, 45001, 45003], + "capabilities":["follow","threads"],"max_threads":50}) ); } } diff --git a/crates/buzz-relay/src/config.rs b/crates/buzz-relay/src/config.rs index 44662bb9e10..f19a3327ec1 100644 --- a/crates/buzz-relay/src/config.rs +++ b/crates/buzz-relay/src/config.rs @@ -187,7 +187,8 @@ pub struct Config { pub require_auth_token: bool, /// Opt-in private accessory API; disabled until explicitly deployed. pub buzz_v1_enabled: bool, - /// Author-time unread tracking duration, independent of NIP-RS retention. + /// How long an automatic thread follow stays live without the actor's + /// involvement; independent of NIP-RS retention. pub buzz_v1_retention_seconds: u32, /// Comma-separated list of allowed CORS origins. /// If empty, permissive CORS is used (dev mode). diff --git a/crates/buzz-relay/src/main.rs b/crates/buzz-relay/src/main.rs index e42c6a49756..8ec1a618bc9 100644 --- a/crates/buzz-relay/src/main.rs +++ b/crates/buzz-relay/src/main.rs @@ -388,6 +388,7 @@ async fn run_relay_main(boot: BootTracker) -> anyhow::Result<()> { anyhow::anyhow!("Configuration error: {error}") })?; boot = next_boot; + buzz_db::personal_read::set_follow_window_seconds(config.buzz_v1_retention_seconds); let key_failure = if config.relay_private_key.is_some() { LifecycleReason::RequiredInvalid diff --git a/crates/buzz-relay/src/nip11.rs b/crates/buzz-relay/src/nip11.rs index d60771b2bc4..2a885f14b23 100644 --- a/crates/buzz-relay/src/nip11.rs +++ b/crates/buzz-relay/src/nip11.rs @@ -97,6 +97,11 @@ pub struct BuzzV1Descriptor { /// Message kinds that count as unread and as latest activity. Clients /// classify live arrivals with this set instead of keeping a copy. pub eligible_kinds: [i32; 4], + /// Additions beyond the first v1 contract: `follow` (follow and unfollow + /// intents) and `threads` (`GET /me/threads`). Send one only when listed. + pub capabilities: Vec, + /// Maximum followed threads per threads page. + pub max_threads: usize, } /// Public capability descriptor for relay-proxied GIF search. @@ -382,7 +387,7 @@ pub(crate) async fn nip11_document(state: &crate::state::AppState, raw_host: &st ); if let Ok(tenant) = crate::tenant::bind_community(&state.db, raw_host).await { if state.config.buzz_v1_enabled { - use buzz_db::personal_read::{ELIGIBLE_KINDS, MAX_CHANNELS, MAX_INTENTS}; + use buzz_db::personal_read::{ELIGIBLE_KINDS, MAX_CHANNELS, MAX_INTENTS, MAX_THREADS}; info.buzz_v1 = Some(BuzzV1Descriptor { version: 1, base_path: crate::api::buzz_v1::BASE_PATH.to_owned(), @@ -390,6 +395,8 @@ pub(crate) async fn nip11_document(state: &crate::state::AppState, raw_host: &st max_channels: MAX_CHANNELS, max_intents: MAX_INTENTS, eligible_kinds: ELIGIBLE_KINDS, + capabilities: vec!["follow".to_owned(), "threads".to_owned()], + max_threads: MAX_THREADS, }); } info.read_state_snapshot = Some(serde_json::json!({ diff --git a/docs/buzz-v1-read-state.md b/docs/buzz-v1-read-state.md index f89e2e07740..4a96c894cfe 100644 --- a/docs/buzz-v1-read-state.md +++ b/docs/buzz-v1-read-state.md @@ -12,9 +12,15 @@ application/nostr+json`, or `GET /info`) includes `buzz_v1` only when enabled: ```json {"buzz_v1":{"version":1,"base_path":"/buzz/v1","retention_seconds":2592000, -"max_channels":20,"max_intents":100,"eligible_kinds":[9,40002,45001,45003]}} +"max_channels":20,"max_intents":100,"eligible_kinds":[9,40002,45001,45003], +"capabilities":["follow","threads"],"max_threads":50}} ``` +`capabilities` lists additions to the first v1 contract: `follow` for the +`follow` and `unfollow` intents, `threads` for `GET /me/threads`. Use one only +when it is listed. `retention_seconds` is how long an automatic thread follow +stays live without your involvement (see [Following](#following)). + Use the requesting origin plus this relative prefix. Discovery is a configured capability, not a promise that the next request cannot fail. Unknown hosts and disabled deployments omit it, and a disabled deployment does not mount @@ -31,6 +37,8 @@ NIP-FI admission is enforced and its asserted key must match the request signer. The host chooses the community; the signer chooses `/me`. NIP-OA admission does not grant access to the owner's personal state. Relay membership, bans and resource access are enforced; moderation timeouts do not prohibit reading. +Read state is for people: an agent account (one with an owner) gets 403 from +every endpoint, and the relay keeps no positions or follows for it. Responses produced by the v1 handlers are `Cache-Control: private, no-store`. Application errors (including NIP-98 failures with NIP-FI Off or Shadow) use @@ -74,61 +82,97 @@ marking one never moves another. | Field | Channel row | Thread row | |---|---|---| -| `unread` | an unread top-level message counts | an unread reply counts | -| `mentions` | unread top-level messages directed at you | unread replies that count | +| `unread` | an unread timeline message counts | an unread reply counts | +| `mentions` | unread timeline messages that tag you with `p`, or every unread one in a DM (at most 100) | unread replies that tag you with `p`, or every unread one in a DM (at most 100) | | `read_through_id` | the anchor of the timeline's frontier, or null | the anchor of the thread's frontier, or null | -| `latest_id` | the last eligible top-level message to arrive, or null | the last counted unread reply to arrive | - -A thread reply never makes the channel row unread and never becomes its -`latest_id`, including a reply carrying `broadcast=1`: it shows on its thread -row. `latest_id` is the anchor that reads its scope: "mark channel read" sends a -channel `mark_through` at the channel's `latest_id`, plus one thread -`mark_through` at each listed thread's `latest_id` to clear those too. The -channel `latest_id` counts any author and ignores read progress. Null means the -bounded scans found no top-level message, not that the channel is empty. -`read_through_id` may name a message that has since been deleted. - -`threads` lists threads with unread replies that count, newest unread reply -first by author time (then `root_id`), at most 25. Threads past the 25th are -omitted with their mentions, and nothing signals the omission: after marking -the listed threads read, a refresh can list the next ones. A thread row's -`unread` is always true in this version. No message bytes are included. +| `latest_id` | the last eligible timeline message to arrive, or null | the last reply to arrive | + +The timeline holds top-level messages and replies carrying `broadcast=1`; a +broadcast reply also counts on its thread row. No other reply makes the channel +row unread or becomes its `latest_id`. `latest_id` is the anchor that reads its +scope: "mark channel read" sends a channel `mark_through` at the channel's +`latest_id`, plus one thread `mark_through` at each listed thread's `latest_id` +to clear those too. The channel `latest_id` counts any author and ignores read +progress. Null means no timeline message was recorded since 0061, not that the +channel is empty. `read_through_id` may name a message that has since been +deleted. + +`threads` lists the threads you follow that have unread replies, newest +arrival first (then `root_id`), at most 25. Threads past +the 25th are omitted with their mentions, and nothing signals the omission: +after marking the listed threads read, a refresh can list the next ones. A +thread row's `unread` is always true in this version. No message bytes are +included. A message is eligible when it is non-own, nondeleted, of the advertised -`eligible_kinds` and inside the horizon. The same kinds alone define latest +`eligible_kinds`; its author time plays no part. The same kinds alone define latest activity, so an edit, reaction or diff (40008) neither makes a channel unread nor moves it. Classify live arrivals with the advertised set, not a client copy. -A top-level message is directed at you when its channel is a DM, it tags you -with `p`, or it carries `broadcast=1`. A reply counts only when it is directed -in the same way, or it is in one of your conversations: you wrote its direct -parent or have a reply to that same parent, in that channel. Conversation -membership uses only your live eligible messages (a deleted parent proves -nothing; a surviving reply to it still does), looks at the direct parent only -(owning the root or replying elsewhere in the thread proves nothing), and is -independent of read progress and retention. This is not Desktop notification -policy: follows and mutes do not affect these counts. - -Counts are what the bounded scans could establish, and may undercount. A message -whose tags are unusable, whose ancestry is unresolved, or whose conversation -membership is undecided is left out, as are messages beyond the scan bounds -(see [Bounds](#bounds-and-deployment)). - -The unread horizon defaults to 30 days (`BUZZ_V1_RETENTION_SECONDS`) and is -measured in author time (`created_at`). It filters unread and mentions, not -latest activity, event storage or frontier state. Frontiers use a different -clock, relay arrival (see [Order](#order)). Three consequences: - -- A message accepted late with an author time beyond the horizon (an import, a - backfill, a long-offline sender) is excluded under the current horizon, - however recently the relay accepted it. It can still be latest when the - horizon holds no message. -- Unread expires at author time plus the horizon, so a future-dated author - time extends how long a message counts. -- A context's unread set is two tests, not one range: arrived after the - frontier, and author time at or after the cutoff. - -A later configuration expansion can change counts without having lost progress. +### Following + +Every top-level message counts on its channel row. A reply counts only in a +thread you follow. The relay records follows when it stores an eligible message, +in the same transaction: + +- Its author has read through it: the timeline for a top-level message, + otherwise the thread, which the author now follows. A broadcast reply reads + only its thread. +- Each channel member it tags with `p` follows its thread (for a top-level + message, the thread rooted at it). +- A reply also makes the root's author follow, and in a DM every member. + +None of this applies to an agent account: it neither reads through its posts +nor follows, though its messages still make people follow. + +A new follower starts just before the message that made them follow, so it is +unread; one who already follows keeps their position. Reading never follows: +a thread `mark_through` moves an existing follow and otherwise changes nothing. +`broadcast=1` does not make anyone follow. Follows start when the relay that +records them is deployed; there is no backfill. Desktop notification follows +and mutes do not affect these counts. + +The `follow` intent follows a thread explicitly, from now: replies that arrived +before it are read. Following a thread you already follow keeps its position +and only makes the follow explicit. +`unfollow` stops counting it, and the stored message rules above no longer +follow you again, with two exceptions: posting in the thread follows you through +your post, and a mention follows you from just before it. + +Counts are what the bounded walks could establish, and may undercount. A +message whose ancestry is unresolved is left out, as are messages beyond the +bounds (see [Bounds](#bounds-and-deployment)). + +An automatic follow goes dormant after 30 days (`BUZZ_V1_RETENTION_SECONDS`) +without your involvement in the thread: posting there, being mentioned there, +a reply to your root or in your DM, marking it read, or following it. A dormant +thread is neither counted nor listed, but keeps its row and position, and any +of those involvements makes it live again at that position. Replies that do not +involve you never revive it. A follow made by the `follow` intent never goes +dormant; an unfollow ends that. Being mentioned, a reply to your root and DM +replies refresh a live follow at most once a day, or once a quarter of the +window when that is shorter, so dormancy is measured to that interval. + +## Threads view + +`GET /buzz/v1/me/threads?limit=50&cursor=` lists the threads you follow +in joined, nondeleted channels, newest last reply first by arrival (then +`root_id`), read or not, at most 50 per page. Omit the cursor on the first +request and pass `next_cursor` back unchanged; null means the scan is done. A +thread with no reply yet is not listed. + +```json +{"threads":[{"channel_id":"","root_id":"<64-hex>","unread":false, + "mentions":0,"read_through_id":"<64-hex>","latest_id":"<64-hex>"}], + "next_cursor":""} +``` + +Each row has the sidebar thread row's fields and meanings plus its +`channel_id`, except that `unread` may be false and `mentions` is then 0. Pages +are separate snapshots: a thread that gets a new reply while you page moves +ahead of the cursor, so refresh from the first page after a change. Every +page probes each of your live follows, so its cost grows with how many threads +you are involved in, not with community size or your whole history. ## Fixed-operand writes @@ -137,7 +181,9 @@ A later configuration expansion can change counts without having lost progress. ```json {"intents":[ {"type":"mark_through","target":{"channel_id":""},"message_id":"<64-hex-event>"}, - {"type":"mark_through","target":{"channel_id":"","root_id":"<64-hex-root>"},"message_id":"<64-hex-event>"} + {"type":"mark_through","target":{"channel_id":"","root_id":"<64-hex-root>"},"message_id":"<64-hex-event>"}, + {"type":"follow","target":{"channel_id":"","root_id":"<64-hex-root>"}}, + {"type":"unfollow","target":{"channel_id":"","root_id":"<64-hex-root>"}} ]} ``` @@ -155,8 +201,9 @@ until its outcome is resolved. A mark-through validates a fixed message and advances its context's monotone frontier to that message's relay arrival (see [Order](#order)). A channel -anchor must be a top-level message; a thread anchor must be the thread's root -or one of its replies. The frontier records the anchor as `read_through_id` +anchor must be a top-level message or a broadcast reply; a thread anchor must +be the thread's root or one of its replies. `follow` and `unfollow` need a +thread target; a channel-only one is `invalid`. The frontier records the anchor as `read_through_id` when it advances; an anchor that arrived at or before the current frontier changes nothing. Opening a view is not itself a reading action; client dwell/focus policy determines when to send an actual observed anchor. Old or @@ -185,12 +232,22 @@ microsecond resolution. Three limits follow, none of which strands a badge: - Relay processes with different clocks can stamp out of true order by their skew. - Messages with the identical stamp are read together. - -A channel's `latest_id` is the last top-level message to arrive among those the -unread count examined: the 4,096 most recent events by author time inside the -horizon. So marking through it reads every top-level message counted. When -that scan holds no top-level message, it is the last top-level message to -arrive among the channel's 256 most recent events. +- A `follow` and the account's start use the database clock, so they can be + off from relay arrivals by the skew between the two. + +Counting walks forward from a position over indexes in arrival order: the +timeline's messages per channel and each thread's replies. A channel's +`latest_id` is exactly its last timeline arrival and a thread's its last reply +arrival, so marking through it reads everything counted. Mentions are indexed +by author time, which the relay accepts within 15 minutes of arrival, so a +mention walk starts 15 minutes of author time before the position: a mention +backdated further than that is unread but not counted as a mention. + +A channel row is unread exactly when its `latest_id` is someone else's and +arrived after the timeline's position. Nothing earlier needs +checking: posting reads the timeline through the post, so an own `latest_id` +means everything before it is read, and an unread one is the anchor that clears +the row. A row is never unread without an anchor to mark. There is no import of earlier client read state. The relay records the account's `started_at` at its first applied read intent and never moves it. @@ -204,27 +261,71 @@ device-local. - 20 sidebar rows and 100 intents per request; 25 thread rows per channel, so a POST's rows for 100 channels fit the response limit. - 64 KiB write body; 1 MiB serialized API response. -- 4096 raw events per channel inside the horizon, before eligibility. -- The latest fallback probes 256 events; a long ineligible or reply-only tail - can leave `latest_id` null. -- Tag documents over 8192 bytes or malformed relevant tags leave the message - out. Compact boolean facts cross the database boundary, never raw tag - payloads. -- Conversation membership: at most 1024 unique parents per request, with a - 500 ms savepoint budget. The lookup is exact, so its work grows with the - replies under each parent. Past either bound a reply is undecided and left - out. +- Counts stop at 100 per channel and per thread. +- A caught-up reader costs one index probe per channel and per followed + thread; a reader who is behind walks only what arrived after their position. +- 50 rows per threads page; each page probes every live follow of the reader. - DB statement/lock deadlines and HTTP read deadlines bound work; a write's intents and its row refresh share one eight-second deadline after admission. Limits are containment, not a production capacity claim. -Apply migrations 0056 through 0058 (or the equivalent desired schema). 0056 -creates two empty private tables and no index on `events`, 0057 adds the -nullable `personal_read_accounts.started_at` column, and 0058 adds the nullable -`personal_read_frontiers.through_message_id` column and drops -`threads_through_timestamp`. Both sidebar scans are -served by the existing `idx_events_community_channel_created`. No new -per-message ingest write path or stored unread counters are introduced. +Apply migrations 0056 through 0058 and 0061 (or the equivalent desired +schema). 0056 creates two empty private tables and no index on `events`, 0057 +adds the nullable `personal_read_accounts.started_at` column, and 0058 adds the +nullable `personal_read_frontiers.through_message_id` column and drops +`threads_through_timestamp`. 0061 adds `events.read_timeline` (set at insert +for timeline messages), `thread_metadata.received_at` (set at insert for +eligible replies) and `personal_read_frontiers.unfollowed`, `explicit` and +`active_at` (existing rows start at migration time), plus four indexes: +`idx_events_read_timeline` for timeline walks, `idx_thread_metadata_read_arrival` +for thread walks, `idx_event_mentions_pubkey_channel_created` for mention +counts and `idx_personal_read_frontiers_active` for an actor's threads by +involvement. There is no backfill: messages stored before the new relay serves, +including by older relay processes still running after 0061, are on neither +arrival index, so they never count as unread or become `latest_id`. Ingest +writes a few frontier rows per eligible channel message (the author, and anyone +it makes follow), whether or not the API is enabled, so follows exist once it +is; there are no stored unread counters. + +The column additions are catalog-only but briefly take an exclusive lock on +`events`, `thread_metadata` and `personal_read_frontiers`, bounded by the +migration's one-second `lock_timeout`; a busy writer fails the deployment, +which retries, rather than stalling ingestion. Building the indexes inside the +migration is bounded the same way, so on a database with data, add the columns +and prebuild the indexes before deploying, as for +[0049](thread-window-deployment.md). 0061 then skips both and only validates +the indexes. Run each statement on its own, retrying any that times out; +`CREATE INDEX CONCURRENTLY` cannot run in a transaction block: + +```sql +SET lock_timeout = '1s'; +ALTER TABLE events ADD COLUMN IF NOT EXISTS read_timeline BOOLEAN NOT NULL DEFAULT false; +ALTER TABLE thread_metadata ADD COLUMN IF NOT EXISTS received_at TIMESTAMPTZ; +ALTER TABLE personal_read_frontiers ADD COLUMN IF NOT EXISTS unfollowed BOOLEAN NOT NULL + DEFAULT false CHECK (NOT unfollowed OR root_id <> ''::bytea); +ALTER TABLE personal_read_frontiers ADD COLUMN IF NOT EXISTS explicit BOOLEAN NOT NULL + DEFAULT false; +ALTER TABLE personal_read_frontiers ADD COLUMN IF NOT EXISTS active_at TIMESTAMPTZ NOT NULL + DEFAULT now(); +CREATE INDEX CONCURRENTLY idx_personal_read_frontiers_active ON public.personal_read_frontiers + (community_id, actor, active_at DESC) WHERE root_id <> ''::bytea; +CREATE INDEX CONCURRENTLY idx_thread_metadata_read_arrival ON public.thread_metadata + (community_id, root_event_id, received_at DESC, event_id) WHERE received_at IS NOT NULL; +CREATE INDEX CONCURRENTLY idx_event_mentions_pubkey_channel_created ON public.event_mentions + (community_id, pubkey_hex, channel_id, event_created_at DESC); +-- events is partitioned: an invalid parent, then each partition concurrently. +CREATE INDEX idx_events_read_timeline ON ONLY public.events + (community_id, channel_id, received_at DESC, id) WHERE read_timeline AND deleted_at IS NULL; +-- For each partition p in +-- SELECT inhrelid::regclass FROM pg_inherits WHERE inhparent = 'public.events'::regclass: +CREATE INDEX CONCURRENTLY p_read_timeline ON p + (community_id, channel_id, received_at DESC, id) WHERE read_timeline AND deleted_at IS NULL; +ALTER INDEX idx_events_read_timeline ATTACH PARTITION p_read_timeline; +``` + +The parent becomes valid once every partition's index is attached. Verify the +four like 0049's, against the definitions the migration expects; recover a +failed build the same way. Use existing HTTP route/status/latency metrics for `/buzz/v1/me/sidebar` and `/buzz/v1/me/read-state`, plus database pool/statement metrics. No payload, @@ -236,7 +337,7 @@ local seed is not a DAU/concurrency or p95/p99 production acceptance result. Within `/buzz/v1`, clients must ignore unknown response object fields. The shape is stable: existing fields, their types, status variants and what a `mark_through` does remain. What the counts include is not yet: which messages -are unread or directed, which threads are listed, and the bounds may change +are unread or mentions, which threads are listed, and the bounds may change while v1 is pre-release. A client therefore replaces its rows with each response rather than reconciling them with its own counts. Breaking shape changes require an explicitly negotiated contract or a new API version. @@ -253,8 +354,8 @@ additional capabilities. These typed relational frontiers are signer-private application state, **not self-encrypted**. Database operators can see reading progress; ordinary Nostr queries, search and moderator interfaces do not expose it. No public receipts -are emitted. Storage grows by touched contexts, not observed messages, and has -no fixed context-count ceiling. +are emitted. Storage grows by followed threads and posted-in channels, not +observed messages, and has no fixed context-count ceiling. Leaving/rejoining does not erase progress; revoked access hides it. Soft-deleted channels are inaccessible, while hard channel deletion cascades their frontiers. @@ -262,7 +363,7 @@ Deleting an account row cascades that actor's frontiers in the same community; community erasure inventories both tables under the existing write fence. There is no new public account export/reset endpoint. Operator-assisted erasure/export must use the established authenticated operational process and explicitly scope -both community and actor; never equate the read-time horizon with data erasure. +both community and actor; never equate follow dormancy with data erasure. There is no down migration, and disabling the API is not a rollback. A relay built before 0056 that restarts with `BUZZ_AUTO_MIGRATE=true` (the Helm default) diff --git a/migrations/0061_read_state_access_paths.sql b/migrations/0061_read_state_access_paths.sql new file mode 100644 index 00000000000..7ab79b05a47 --- /dev/null +++ b/migrations/0061_read_state_access_paths.sql @@ -0,0 +1,76 @@ +-- Read-state access paths: one index seek answers "newest countable message +-- here" and "what arrived after this position" for a channel timeline or a +-- thread, and "my mentions in this channel since then". Arrival order +-- (received_at), not author time. No backfill: rows written before this +-- migration are not on the new paths. +-- +-- events.read_timeline: an eligible message that counts on its channel +-- timeline (top-level, or a broadcast reply), set at insert. +-- thread_metadata.received_at: a reply's arrival, set at insert; NULL on root +-- stubs and earlier rows. +-- personal_read_frontiers.unfollowed: a sticky unfollow of a thread. +-- personal_read_frontiers.explicit: a follow intent, never dormant. +-- personal_read_frontiers.active_at: the actor's last involvement in a +-- thread; an implicit follow is dormant once it is older than the relay's +-- window. Earlier rows start at migration time. +-- +-- Brownfield operators MUST add the five columns and prebuild the four +-- indexes concurrently as documented in docs/buzz-v1-read-state.md. Startup is +-- bounded like 0049: a large table fails deployment rather than blocking +-- ingestion. +SET LOCAL lock_timeout = '1s'; +SET LOCAL statement_timeout = '5s'; + +DO $$ +DECLARE + spec RECORD; +BEGIN + -- Even ADD COLUMN IF NOT EXISTS requests a writer-conflicting lock: skip + -- a column an operator added beforehand. + FOR spec IN SELECT * FROM (VALUES + ('events', 'read_timeline', 'BOOLEAN NOT NULL DEFAULT false'), + ('thread_metadata', 'received_at', 'TIMESTAMPTZ'), + ('personal_read_frontiers', 'unfollowed', + $d$BOOLEAN NOT NULL DEFAULT false CHECK (NOT unfollowed OR root_id <> ''::bytea)$d$), + ('personal_read_frontiers', 'explicit', 'BOOLEAN NOT NULL DEFAULT false'), + ('personal_read_frontiers', 'active_at', 'TIMESTAMPTZ NOT NULL DEFAULT now()') + ) AS s(tbl, col, definition) LOOP + IF NOT EXISTS ( + SELECT 1 FROM pg_attribute + WHERE attrelid = ('public.' || spec.tbl)::regclass + AND attname = spec.col AND NOT attisdropped + ) THEN + EXECUTE format('ALTER TABLE public.%I ADD COLUMN %I %s', + spec.tbl, spec.col, spec.definition); + END IF; + END LOOP; + + -- A partitioned parent's definition reads ON ONLY; CREATE recurses to + -- every partition regardless. + FOR spec IN SELECT * FROM (VALUES + ('idx_events_read_timeline', + 'CREATE INDEX idx_events_read_timeline ON ONLY public.events USING btree (community_id, channel_id, received_at DESC, id) WHERE (read_timeline AND (deleted_at IS NULL))'), + ('idx_thread_metadata_read_arrival', + 'CREATE INDEX idx_thread_metadata_read_arrival ON public.thread_metadata USING btree (community_id, root_event_id, received_at DESC, event_id) WHERE (received_at IS NOT NULL)'), + ('idx_event_mentions_pubkey_channel_created', + 'CREATE INDEX idx_event_mentions_pubkey_channel_created ON public.event_mentions USING btree (community_id, pubkey_hex, channel_id, event_created_at DESC)'), + ('idx_personal_read_frontiers_active', + $d$CREATE INDEX idx_personal_read_frontiers_active ON public.personal_read_frontiers USING btree (community_id, actor, active_at DESC) WHERE (root_id <> '\x'::bytea)$d$) + ) AS s(name, definition) LOOP + -- IF NOT EXISTS still requests a writer-conflicting lock: skip CREATE + -- for a prebuilt index, then validate it. + IF to_regclass('public.' || spec.name) IS NULL THEN + EXECUTE replace(spec.definition, ' ON ONLY ', ' ON '); + END IF; + IF NOT EXISTS ( + SELECT 1 FROM pg_index i + JOIN pg_class c ON c.oid = i.indexrelid + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = 'public' AND c.relname = spec.name + AND i.indisvalid AND i.indisready AND i.indislive + AND pg_get_indexdef(i.indexrelid) = spec.definition + ) THEN + RAISE EXCEPTION '% invalid or wrong definition; see docs/buzz-v1-read-state.md', spec.name; + END IF; + END LOOP; +END $$; diff --git a/schema/tables/public/event_mentions.sql b/schema/tables/public/event_mentions.sql index b0eb9dcb4a7..df0ee76e534 100644 --- a/schema/tables/public/event_mentions.sql +++ b/schema/tables/public/event_mentions.sql @@ -18,5 +18,7 @@ CREATE INDEX idx_event_mentions_pubkey_created ON event_mentions (community_id, pubkey_hex, event_created_at DESC); CREATE INDEX idx_event_mentions_pubkey_kind_created ON event_mentions (community_id, pubkey_hex, event_kind, event_created_at DESC); +CREATE INDEX idx_event_mentions_pubkey_channel_created + ON event_mentions (community_id, pubkey_hex, channel_id, event_created_at DESC); CREATE INDEX idx_event_mentions_community_event ON event_mentions (community_id, event_id); diff --git a/schema/tables/public/events.sql b/schema/tables/public/events.sql index 10c1adc19e9..936d7d737d5 100644 --- a/schema/tables/public/events.sql +++ b/schema/tables/public/events.sql @@ -37,6 +37,8 @@ CREATE TABLE events ( d_tag TEXT, not_before BIGINT, delivered_at BIGINT, + -- An eligible message that counts on its channel's read-state timeline. + read_timeline BOOLEAN NOT NULL DEFAULT false, PRIMARY KEY (community_id, created_at, id) ) PARTITION BY RANGE (created_at); @@ -52,6 +54,10 @@ CREATE INDEX idx_events_community_pubkey_kind_created CREATE INDEX idx_events_community_kind_created ON events (community_id, kind, created_at DESC, id); CREATE INDEX idx_events_community_deleted ON events (community_id, deleted_at); +-- Read state: newest timeline message, and what arrived after a position. +CREATE INDEX idx_events_read_timeline + ON events (community_id, channel_id, received_at DESC, id) + WHERE read_timeline AND deleted_at IS NULL; -- Addressable (replaceable) and NIP-33 parameterized lookups. CREATE INDEX idx_events_addressable ON events (community_id, kind, pubkey, channel_id, deleted_at); diff --git a/schema/tables/public/personal_read_frontiers.sql b/schema/tables/public/personal_read_frontiers.sql index e62eb9c8b94..cc2f05e1f92 100644 --- a/schema/tables/public/personal_read_frontiers.sql +++ b/schema/tables/public/personal_read_frontiers.sql @@ -6,9 +6,21 @@ CREATE TABLE personal_read_frontiers ( through_timestamp TIMESTAMPTZ NOT NULL, -- The message whose arrival is through_timestamp; NULL before 0058. through_message_id BYTEA CHECK (octet_length(through_message_id) = 32), + -- A sticky thread unfollow; thread rows only. + unfollowed BOOLEAN NOT NULL DEFAULT false CHECK (NOT unfollowed OR root_id <> ''::bytea), + -- A follow intent: never dormant. Thread rows only. + explicit BOOLEAN NOT NULL DEFAULT false, + -- The actor's last involvement in the thread: posting, being mentioned, + -- reading, following. An implicit follow older than the window is dormant. + active_at TIMESTAMPTZ NOT NULL DEFAULT now(), PRIMARY KEY (community_id, actor, channel_id, root_id), FOREIGN KEY (community_id, actor) REFERENCES personal_read_accounts (community_id, actor) ON DELETE CASCADE, FOREIGN KEY (community_id, channel_id) REFERENCES channels (community_id, id) ON DELETE CASCADE ); + +-- Live follows, and every thread the actor was involved in, newest first. +CREATE INDEX idx_personal_read_frontiers_active + ON personal_read_frontiers (community_id, actor, active_at DESC) + WHERE root_id <> ''::bytea; diff --git a/schema/tables/public/thread_metadata.sql b/schema/tables/public/thread_metadata.sql index 2c9ed9800ac..94a162316e0 100644 --- a/schema/tables/public/thread_metadata.sql +++ b/schema/tables/public/thread_metadata.sql @@ -15,6 +15,8 @@ CREATE TABLE thread_metadata ( descendant_count INT NOT NULL DEFAULT 0, last_reply_at TIMESTAMPTZ, broadcast BOOLEAN NOT NULL DEFAULT FALSE, + -- A reply's relay arrival; NULL on root stubs and rows before 0061. + received_at TIMESTAMPTZ, PRIMARY KEY (community_id, event_created_at, event_id), FOREIGN KEY (community_id, channel_id) REFERENCES channels (community_id, id) ); @@ -26,3 +28,6 @@ CREATE INDEX idx_thread_metadata_window CREATE INDEX idx_thread_metadata_channel_depth ON thread_metadata (community_id, channel_id, depth, event_created_at); CREATE INDEX idx_thread_metadata_event_id ON thread_metadata (community_id, event_id); +CREATE INDEX idx_thread_metadata_read_arrival + ON thread_metadata (community_id, root_event_id, received_at DESC, event_id) + WHERE received_at IS NOT NULL;