From 419a53029a45530d262c8b79468a4c64ff0a80d7 Mon Sep 17 00:00:00 2001 From: Honey <8e307ae0076a4dab6b94b036ea3edc7e08f823a625269c1e6919e881a048b4d2@buzz.block.builderlab.xyz> Date: Fri, 9 Oct 2026 19:17:22 -0700 Subject: [PATCH 1/4] feat(read-state): count thread replies only in followed threads Ingest records follows in the message's transaction: the author reads through their own message, and mentioned channel members, the root's author and DM members follow the thread just before it. The sidebar then counts forward from each read position instead of resolving participation and walking 4096 events back per channel, and counts stop at 100. A thread mark_through moves an existing follow and never creates one. participation.rs is removed. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Honey <8e307ae0076a4dab6b94b036ea3edc7e08f823a625269c1e6919e881a048b4d2@buzz.block.builderlab.xyz> --- crates/buzz-db/src/store/event.rs | 26 + .../personal_read/arrival_postgres_tests.rs | 136 +++-- .../src/store/personal_read/follows.rs | 113 ++++ .../personal_read/follows_postgres_tests.rs | 373 ++++++++++++ crates/buzz-db/src/store/personal_read/mod.rs | 10 +- .../buzz-db/src/store/personal_read/model.rs | 34 +- .../src/store/personal_read/participation.rs | 127 ----- .../participation_postgres_tests.rs | 537 ------------------ .../src/store/personal_read/postgres_tests.rs | 122 ++-- .../src/store/personal_read/projection.rs | 476 ++++++---------- .../projection_postgres_tests.rs | 152 ++--- .../personal_read/threads_postgres_tests.rs | 92 +-- .../buzz-db/src/store/personal_read/writes.rs | 32 +- .../src/api/buzz_v1/postgres_tests.rs | 15 +- docs/buzz-v1-read-state.md | 106 ++-- 15 files changed, 1065 insertions(+), 1286 deletions(-) create mode 100644 crates/buzz-db/src/store/personal_read/follows.rs create mode 100644 crates/buzz-db/src/store/personal_read/follows_postgres_tests.rs delete mode 100644 crates/buzz-db/src/store/personal_read/participation.rs delete mode 100644 crates/buzz-db/src/store/personal_read/participation_postgres_tests.rs diff --git a/crates/buzz-db/src/store/event.rs b/crates/buzz-db/src/store/event.rs index 183585e2785..9eec332b44c 100644 --- a/crates/buzz-db/src/store/event.rs +++ b/crates/buzz-db/src/store/event.rs @@ -447,6 +447,16 @@ pub async fn insert_event_in_transaction( ) -> Result<(StoredEvent, bool)> { let result = insert_event_on(tx, event, channel_id).await?; if result.1 { + if let Some(channel) = channel_id { + crate::personal_read::record_message( + tx, + event, + channel, + crate::personal_read::Place::TopLevel, + result.0.received_at, + ) + .await?; + } crate::operator_listener::enqueue_mentions_in_transaction(tx, event).await?; } Ok(result) @@ -1872,6 +1882,22 @@ pub(crate) async fn insert_event_with_thread_metadata_tx( } } + let place = match &thread_meta { + Some(meta) => Some(( + meta.channel_id, + match meta.parent_event_id { + Some(parent) => crate::personal_read::Place::Reply { + root: meta.root_event_id.unwrap_or(parent), + }, + // Workflow messages store a depth-0 metadata row. + None => crate::personal_read::Place::TopLevel, + }, + )), + None => channel_id.map(|channel| (channel, crate::personal_read::Place::TopLevel)), + }; + if let Some((channel, place)) = place { + crate::personal_read::record_message(tx, event, channel, place, received_at).await?; + } crate::operator_listener::enqueue_mentions_in_transaction(tx, event).await?; } diff --git a/crates/buzz-db/src/store/personal_read/arrival_postgres_tests.rs b/crates/buzz-db/src/store/personal_read/arrival_postgres_tests.rs index a36c2d3c9c4..f8add017bf1 100644 --- a/crates/buzz-db/src/store/personal_read/arrival_postgres_tests.rs +++ b/crates/buzz-db/src/store/personal_read/arrival_postgres_tests.rs @@ -1,6 +1,9 @@ //! Read progress follows relay arrival (`received_at`), never author time. //! Each case sets every arrival explicitly: back-to-back inserts share a clock. -use super::{postgres_tests::fixture, *}; +use super::{ + postgres_tests::{fixture, insert_reply}, + *, +}; use crate::Db; use buzz_core::CommunityId; use nostr::{EventBuilder, Keys, Kind, Tag}; @@ -45,7 +48,8 @@ async fn post( event } -/// A reply to `root` that mentions the actor, so it counts without membership. +/// A reply to `root` that mentions the actor, so the actor follows the thread +/// from just before the earliest such reply to arrive. #[allow(clippy::too_many_arguments)] async fn reply( db: &Db, @@ -57,12 +61,29 @@ async fn reply( authored: u64, arrived: u64, ) -> nostr::Event { - let event = post(db, pool, community, channel, actor, authored, arrived).await; - sqlx::query("INSERT INTO thread_metadata (community_id,event_id,event_created_at,channel_id,root_event_id,parent_event_id,depth) - VALUES ($1,$2,to_timestamp($3),$4,$5,$5,1)") - .bind(community.as_uuid()).bind(event.id.as_bytes().as_slice()) - .bind(event.created_at.as_secs() as f64) - .bind(channel).bind(root.id.as_bytes().as_slice()).execute(pool).await.unwrap(); + let event = EventBuilder::new(Kind::Custom(9), format!("authored {authored}")) + .tags([Tag::public_key(actor.public_key())]) + .custom_created_at(nostr::Timestamp::from(authored)) + .sign_with_keys(&Keys::generate()) + .unwrap(); + insert_reply(db, community, channel, root, &event).await; + arrive(pool, community, &event, arrived).await; + // Ingest started the follow just before the real arrival; move it to + // just before the rewritten one. + let moved = sqlx::query( + "UPDATE personal_read_frontiers + SET through_timestamp=LEAST(through_timestamp, to_timestamp($4)-interval '1 microsecond') + WHERE community_id=$1 AND actor=$2 AND root_id=$3", + ) + .bind(community.as_uuid()) + .bind(actor.public_key().to_bytes().as_slice()) + .bind(root.id.as_bytes().as_slice()) + .bind(arrived as f64) + .execute(pool) + .await + .unwrap() + .rows_affected(); + assert_eq!(moved, 1, "the mention follows the thread"); event } @@ -233,11 +254,39 @@ async fn mark_thread_read_with_the_sidebar_anchor_clears_a_late_reply() { assert!(row.threads.is_empty()); } -/// The shallow latest probe reads the newest `MAX_CHANNEL_SCAN + 1` events by -/// author time, of any kind. A late arrival with an older author time than -/// that many events is counted unread, so the unread scan must supply the -/// anchor or Mark as read with the sidebar anchor leaves it. -async fn mark_as_read_behind_fillers(fillers: u64) -> (bool, u32) { +/// The latest probe reads the newest `MAX_TIMELINE_SCAN` events by author +/// time, of any kind. A late arrival with an older author time than the +/// uncounted events after it must still be the sidebar anchor, or Mark as read +/// with that anchor leaves it unread. +async fn reactions( + pool: &PgPool, + db: &Db, + community: CommunityId, + channel: Uuid, + count: u64, + now: u64, +) { + for i in 0..count { + // Reactions are not counted, but the probe's LIMIT sees them. + let filler = EventBuilder::new(Kind::Custom(7), "+") + .custom_created_at(nostr::Timestamp::from(now - 500 + i % 400)) + .sign_with_keys(&Keys::generate()) + .unwrap(); + db.insert_event(community, &filler, Some(channel)) + .await + .unwrap(); + } + sqlx::query("UPDATE events SET received_at=to_timestamp($2) WHERE community_id=$1 AND kind=7") + .bind(community.as_uuid()) + .bind((now - 50) as f64) + .execute(pool) + .await + .unwrap(); +} + +#[tokio::test] +#[ignore = "requires Postgres"] +async fn mark_as_read_clears_a_late_arrival_behind_reactions() { let (db, pool, community, channel, actor, first) = fixture().await; let now = first.created_at.as_secs(); arrive(&pool, community, &first, now - 60).await; @@ -248,17 +297,8 @@ async fn mark_as_read_behind_fillers(fillers: u64) -> (bool, u32) { mark_through(channel, None, &first.id.to_hex()), ) .await; - for i in 0..fillers { - // Reactions are not counted, but the probe's LIMIT sees them. - let filler = EventBuilder::new(Kind::Custom(7), "+") - .custom_created_at(nostr::Timestamp::from(now - 500 + i)) - .sign_with_keys(&Keys::generate()) - .unwrap(); - db.insert_event(community, &filler, Some(channel)) - .await - .unwrap(); - arrive(&pool, community, &filler, now - 50).await; - } + // More than main's old 256-event probe. + reactions(&pool, &db, community, channel, 300, now).await; post(&db, &pool, community, channel, &actor, now - 600, now - 10).await; assert_eq!(counts(&db, community, &actor).await, (true, 1)); @@ -267,29 +307,14 @@ async fn mark_as_read_behind_fillers(fillers: u64) -> (bool, u32) { .latest_id .expect("a channel with messages has a latest message"); apply(&db, community, &actor, mark_through(channel, None, &anchor)).await; - counts(&db, community, &actor).await -} - -/// Control: with one slot to spare the late arrival is inside the probe. -#[tokio::test] -#[ignore = "requires Postgres"] -async fn mark_as_read_clears_a_late_arrival_inside_the_latest_probe() { - let fillers = MAX_CHANNEL_SCAN as u64 - 1; - assert_eq!(mark_as_read_behind_fillers(fillers).await, (false, 0)); -} - -#[tokio::test] -#[ignore = "requires Postgres"] -async fn mark_as_read_clears_a_late_arrival_outside_the_latest_probe() { - let fillers = MAX_CHANNEL_SCAN as u64; - assert_eq!(mark_as_read_behind_fillers(fillers).await, (false, 0)); + assert_eq!(counts(&db, community, &actor).await, (false, 0)); } -/// When the newest 257 events are all uncounted, the only message is still in -/// the unread scan: the summary reports it as latest and unread. +/// Past the budget the probe finds no anchor, and nothing counts: a badge +/// with no latest to mark could never be cleared. #[tokio::test] #[ignore = "requires Postgres"] -async fn latest_message_behind_a_full_probe_of_reactions_is_found() { +async fn a_message_behind_a_full_budget_of_reactions_is_not_a_badge_without_an_anchor() { let (db, pool, community, channel, actor, first) = fixture().await; let now = first.created_at.as_secs(); let demoted = sqlx::query("UPDATE events SET kind=7 WHERE community_id=$1 AND id=$2") @@ -300,21 +325,20 @@ async fn latest_message_behind_a_full_probe_of_reactions_is_found() { .unwrap() .rows_affected(); assert_eq!(demoted, 1); - let only = post(&db, &pool, community, channel, &actor, now - 600, now - 10).await; - for i in 0..MAX_CHANNEL_SCAN as u64 { - let filler = EventBuilder::new(Kind::Custom(7), "+") - .custom_created_at(nostr::Timestamp::from(now - 500 + i)) - .sign_with_keys(&Keys::generate()) - .unwrap(); - db.insert_event(community, &filler, Some(channel)) - .await - .unwrap(); - arrive(&pool, community, &filler, now - 50).await; - } + post(&db, &pool, community, channel, &actor, now - 600, now - 10).await; + reactions( + &pool, + &db, + community, + channel, + MAX_TIMELINE_SCAN as u64, + now, + ) + .await; let row = sidebar(&db, community, &actor).await; - assert_eq!(row.latest_id, Some(only.id.to_hex())); - assert_eq!((row.unread, row.mentions), (true, 1)); + assert_eq!(row.latest_id, None); + assert_eq!((row.unread, row.mentions), (false, 0)); } /// Store `event` as having arrived at exactly `seconds` plus `micros`. Built diff --git a/crates/buzz-db/src/store/personal_read/follows.rs b/crates/buzz-db/src/store/personal_read/follows.rs new file mode 100644 index 00000000000..680ccfb9dcc --- /dev/null +++ b/crates/buzz-db/src/store/personal_read/follows.rs @@ -0,0 +1,113 @@ +//! Read state that ingest writes. A thread frontier row exists only for a +//! thread its actor follows; only this module creates one. +use super::model::ELIGIBLE_KINDS; +use crate::{AdmittedTx, Result}; +use chrono::{DateTime, Utc}; +use nostr::Event; +use uuid::Uuid; + +/// Where a newly stored channel message sits. +#[derive(Clone, Copy)] +pub(crate) enum Place<'a> { + /// A top-level timeline message. + TopLevel, + /// A reply in the thread rooted at `root`. + Reply { root: &'a [u8] }, +} + +/// Record what a newly stored eligible message changes, in its transaction: +/// - its author has read through it: the timeline for a top-level message, +/// otherwise the thread, which the author then follows; +/// - each channel member it mentions follows its thread (for a top-level +/// message, the thread rooted at it); +/// - a reply makes the root's author, and in a DM every member, follow too. +/// +/// Everyone but the author starts just before the message, so it is unread +/// to them. A follower keeps an existing position. +pub(crate) async fn record_message( + tx: &mut AdmittedTx, + event: &Event, + channel: Uuid, + place: Place<'_>, + received_at: DateTime, +) -> Result<()> { + if !ELIGIBLE_KINDS.contains(&i32::from(event.kind.as_u16())) { + return Ok(()); + } + let community = tx.community(); + let id = event.id.as_bytes().as_slice(); + let mentioned: Vec> = event + .tags + .public_keys() + .map(|key| key.to_bytes().to_vec()) + .collect(); + let (read, thread, reply) = match place { + Place::TopLevel => (&[][..], id, false), + Place::Reply { root } => (root, root, true), + }; + // Accounts, then frontiers, each in key order, so concurrent messages that + // involve the same people cannot deadlock: the frontier insert joins the + // finished account insert. Foreign-key checks run at the end of the + // statement, after the account rows exist. + sqlx::query( + "WITH involved (actor, root_id, through, message) AS ( + SELECT $4::bytea, $5::bytea, $6::timestamptz, $3::bytea + UNION ALL + SELECT cm.pubkey, $7, $6-interval '1 microsecond', NULL FROM channel_members cm + WHERE cm.community_id=$1 AND cm.channel_id=$2 AND cm.removed_at IS NULL + AND cm.pubkey=ANY($8) AND cm.pubkey<>$4 + UNION ALL + SELECT e.pubkey, $7, $6-interval '1 microsecond', NULL FROM events e + WHERE $9 AND e.community_id=$1 AND e.channel_id=$2 AND e.id=$7 AND e.pubkey<>$4 + UNION ALL + SELECT cm.pubkey, $7, $6-interval '1 microsecond', NULL + FROM channel_members cm JOIN channels c + ON c.community_id=cm.community_id AND c.id=cm.channel_id + WHERE $9 AND cm.community_id=$1 AND cm.channel_id=$2 AND cm.removed_at IS NULL + AND c.channel_type='dm' AND cm.pubkey<>$4 + ), rows AS ( + -- An existing follower keeps their position, so is not touched. + -- Existence probes are LIMIT 1 laterals: always one index lookup. + SELECT DISTINCT ON (i.actor, i.root_id) i.* FROM involved i + LEFT JOIN LATERAL ( + SELECT true AS found FROM personal_read_frontiers f WHERE f.community_id=$1 + AND f.actor=i.actor AND f.channel_id=$2 AND f.root_id=i.root_id LIMIT 1 + ) f ON true + WHERE i.message IS NOT NULL OR f.found IS NULL + ORDER BY i.actor, i.root_id + ), accounts AS ( + INSERT INTO personal_read_accounts (community_id, actor) + SELECT $1, r.actor FROM (SELECT DISTINCT actor FROM rows) r + LEFT JOIN LATERAL ( + SELECT true AS found FROM personal_read_accounts a + WHERE a.community_id=$1 AND a.actor=r.actor LIMIT 1 + ) a ON true + WHERE a.found IS NULL ORDER BY r.actor + ON CONFLICT DO NOTHING RETURNING 1 + ) + INSERT INTO personal_read_frontiers + (community_id, actor, channel_id, root_id, through_timestamp, through_message_id) + SELECT $1, actor, $2, root_id, through, message + FROM rows CROSS JOIN (SELECT count(*) FROM accounts) written + ORDER BY actor, root_id + ON CONFLICT (community_id, actor, channel_id, root_id) DO UPDATE + SET through_timestamp=GREATEST(personal_read_frontiers.through_timestamp, + excluded.through_timestamp), + through_message_id=CASE + WHEN excluded.through_timestamp > personal_read_frontiers.through_timestamp + THEN excluded.through_message_id ELSE personal_read_frontiers.through_message_id END + WHERE excluded.through_message_id IS NOT NULL", + ) + .bind(community.as_uuid()) + .bind(channel) + .bind(id) + .bind(event.pubkey.to_bytes().as_slice()) + .bind(read) + .bind(received_at) + .bind(thread) + .bind(&mentioned) + .bind(reply) + .execute(tx.conn()) + .await?; + Ok(()) +} diff --git a/crates/buzz-db/src/store/personal_read/follows_postgres_tests.rs b/crates/buzz-db/src/store/personal_read/follows_postgres_tests.rs new file mode 100644 index 00000000000..33a2e75929e --- /dev/null +++ b/crates/buzz-db/src/store/personal_read/follows_postgres_tests.rs @@ -0,0 +1,373 @@ +//! Who follows a thread, as recorded by ingest, and what following shows. +use super::{ + postgres_tests::{fixture, insert_reply, start_before_everything}, + *, +}; +use crate::Db; +use buzz_core::{channel::MemberRole, CommunityId}; +use nostr::{EventBuilder, Keys, Kind, Tag}; +use sqlx::PgPool; +use uuid::Uuid; + +fn message(author: &Keys, tags: Vec) -> nostr::Event { + EventBuilder::new(Kind::Custom(9), Uuid::new_v4().to_string()) + .tags(tags) + .sign_with_keys(author) + .unwrap() +} + +async fn post( + db: &Db, + community: CommunityId, + channel: Uuid, + author: &Keys, + tags: Vec, +) -> nostr::Event { + let event = message(author, tags); + db.insert_event(community, &event, Some(channel)) + .await + .unwrap(); + event +} + +async fn reply( + db: &Db, + community: CommunityId, + channel: Uuid, + root: &nostr::Event, + author: &Keys, + tags: Vec, +) -> nostr::Event { + let event = message(author, tags); + insert_reply(db, community, channel, root, &event).await; + event +} + +fn mention(who: &Keys) -> Vec { + vec![Tag::public_key(who.public_key())] +} + +async fn member(db: &Db, pool: &PgPool, community: CommunityId, channel: Uuid) -> Keys { + let keys = Keys::generate(); + db.add_member( + community, + channel, + &keys.public_key().to_bytes(), + MemberRole::Member, + None, + ) + .await + .unwrap(); + start_before_everything(pool, community, &keys.public_key()).await; + keys +} + +async fn row(db: &Db, community: CommunityId, channel: Uuid, actor: &Keys) -> ChannelReadSummary { + db.personal_read_sidebar_channels( + community, + &actor.public_key(), + DEFAULT_RETENTION_SECONDS, + &[channel], + ) + .await + .unwrap() + .channels + .remove(0) +} + +/// The actor's frontier message for a thread, if the actor follows it. A +/// follower who has read nothing yet has no message. +async fn follow( + pool: &PgPool, + community: CommunityId, + actor: &Keys, + root: &nostr::Event, +) -> Option> { + sqlx::query_scalar( + "SELECT encode(through_message_id,'hex') FROM personal_read_frontiers + WHERE community_id=$1 AND actor=$2 AND root_id=$3", + ) + .bind(community.as_uuid()) + .bind(actor.public_key().to_bytes().as_slice()) + .bind(root.id.as_bytes().as_slice()) + .fetch_optional(pool) + .await + .unwrap() +} + +/// (root, unread replies) for each listed thread. +fn threads(row: &ChannelReadSummary) -> Vec<(String, u32)> { + row.threads + .iter() + .map(|t| (t.root_id.clone(), t.mentions)) + .collect() +} + +fn mark_thread(channel: Uuid, root: &nostr::Event, message: &str) -> ReadIntent { + ReadIntent::MarkThrough { + target: ReadTarget { + channel_id: channel, + root_id: Some(root.id.to_hex()), + }, + message_id: message.to_owned(), + } +} + +#[tokio::test] +#[ignore = "requires Postgres"] +async fn replying_follows_the_thread_and_reads_through_the_reply() { + let (db, pool, community, channel, actor, root) = fixture().await; + let own = reply(&db, community, channel, &root, &actor, vec![]).await; + assert_eq!( + follow(&pool, community, &actor, &root).await, + Some(Some(own.id.to_hex())) + ); + assert!(row(&db, community, channel, &actor) + .await + .threads + .is_empty()); + + let other = reply(&db, community, channel, &root, &Keys::generate(), vec![]).await; + let summary = row(&db, community, channel, &actor).await; + assert_eq!(threads(&summary), [(root.id.to_hex(), 1)]); + assert_eq!(summary.threads[0].latest_id, other.id.to_hex()); + assert_eq!(summary.threads[0].read_through_id, Some(own.id.to_hex())); +} + +#[tokio::test] +#[ignore = "requires Postgres"] +async fn a_mention_follows_a_member_from_just_before_it() { + let (db, pool, community, channel, _, root) = fixture().await; + let bee = member(&db, &pool, community, channel).await; + let outsider = Keys::generate(); + let earlier = reply(&db, community, channel, &root, &Keys::generate(), vec![]).await; + let mut tags = mention(&bee); + tags.extend(mention(&outsider)); + let mentioning = reply(&db, community, channel, &root, &Keys::generate(), tags).await; + reply(&db, community, channel, &root, &Keys::generate(), vec![]).await; + + assert_eq!(follow(&pool, community, &bee, &root).await, Some(None)); + assert_eq!(follow(&pool, community, &outsider, &root).await, None); + // The mention and what came after it; not the reply before it. + let summary = row(&db, community, channel, &bee).await; + assert_eq!(threads(&summary), [(root.id.to_hex(), 2)]); + assert_ne!(summary.threads[0].latest_id, earlier.id.to_hex()); + assert_ne!(summary.threads[0].latest_id, mentioning.id.to_hex()); +} + +#[tokio::test] +#[ignore = "requires Postgres"] +async fn a_later_mention_keeps_the_followers_position() { + let (db, pool, community, channel, actor, root) = fixture().await; + let own = reply(&db, community, channel, &root, &actor, vec![]).await; + reply(&db, community, channel, &root, &Keys::generate(), vec![]).await; + reply( + &db, + community, + channel, + &root, + &Keys::generate(), + mention(&actor), + ) + .await; + assert_eq!( + follow(&pool, community, &actor, &root).await, + Some(Some(own.id.to_hex())) + ); + let summary = row(&db, community, channel, &actor).await; + assert_eq!(threads(&summary), [(root.id.to_hex(), 2)]); +} + +#[tokio::test] +#[ignore = "requires Postgres"] +async fn the_root_author_follows_on_the_first_reply() { + let (db, pool, community, channel, actor, _) = fixture().await; + let root = post(&db, community, channel, &actor, vec![]).await; + assert_eq!(follow(&pool, community, &actor, &root).await, None); + + reply(&db, community, channel, &root, &Keys::generate(), vec![]).await; + assert_eq!(follow(&pool, community, &actor, &root).await, Some(None)); + let summary = row(&db, community, channel, &actor).await; + assert_eq!(threads(&summary), [(root.id.to_hex(), 1)]); +} + +#[tokio::test] +#[ignore = "requires Postgres"] +async fn a_top_level_mention_follows_the_thread_rooted_at_it() { + let (db, pool, community, channel, actor, _) = fixture().await; + let root = post(&db, community, channel, &Keys::generate(), mention(&actor)).await; + assert_eq!(follow(&pool, community, &actor, &root).await, Some(None)); + reply(&db, community, channel, &root, &Keys::generate(), vec![]).await; + + let summary = row(&db, community, channel, &actor).await; + // The fixture message and the mention are unread; one is a mention. + assert_eq!((summary.unread, summary.mentions), (true, 1)); + assert_eq!(threads(&summary), [(root.id.to_hex(), 1)]); +} + +#[tokio::test] +#[ignore = "requires Postgres"] +async fn dm_members_follow_every_thread() { + let (db, pool, community, _, actor, _) = fixture().await; + let (bee, wasp) = (Keys::generate(), Keys::generate()); + let keys = [&actor, &bee, &wasp].map(|k| k.public_key().to_bytes()); + let participants: Vec<&[u8]> = keys.iter().map(|k| k.as_slice()).collect(); + let dm = db + .create_dm(community, &participants, &keys[1]) + .await + .unwrap() + .id; + let root = post(&db, community, dm, &bee, vec![]).await; + reply(&db, community, dm, &root, &wasp, vec![]).await; + + assert_eq!(follow(&pool, community, &actor, &root).await, Some(None)); + let summary = row(&db, community, dm, &actor).await; + // Every DM message counts as a mention. + assert_eq!((summary.unread, summary.mentions), (true, 1)); + assert_eq!(threads(&summary), [(root.id.to_hex(), 1)]); +} + +#[tokio::test] +#[ignore = "requires Postgres"] +async fn an_unfollowed_thread_never_counts_and_marking_it_follows_nothing() { + let (db, pool, community, channel, actor, root) = fixture().await; + reply(&db, community, channel, &root, &Keys::generate(), vec![]).await; + let broadcast = vec![Tag::parse(["broadcast", "1"]).unwrap()]; + let last = reply(&db, community, channel, &root, &Keys::generate(), broadcast).await; + let summary = row(&db, community, channel, &actor).await; + assert!(summary.threads.is_empty()); + // Replies are not on the timeline; only the fixture message is unread. + assert_eq!((summary.unread, summary.mentions), (true, 0)); + + let outcome = db + .apply_personal_read_intent( + community, + &actor.public_key(), + &mark_thread(channel, &root, &last.id.to_hex()), + ) + .await + .unwrap(); + assert_eq!(outcome, IntentOutcome::Applied); + assert_eq!(follow(&pool, community, &actor, &root).await, None); +} + +#[tokio::test] +#[ignore = "requires Postgres"] +async fn posting_reads_the_timeline_through_the_post() { + let (db, _pool, community, channel, actor, _) = fixture().await; + assert!(row(&db, community, channel, &actor).await.unread); + let own = post(&db, community, channel, &actor, vec![]).await; + let summary = row(&db, community, channel, &actor).await; + assert!(!summary.unread); + assert_eq!(summary.read_through_id, Some(own.id.to_hex())); + assert_eq!(summary.latest_id, Some(own.id.to_hex())); +} + +#[tokio::test] +#[ignore = "requires Postgres"] +async fn thread_counts_stop_at_the_cap() { + let (db, _pool, community, channel, actor, root) = fixture().await; + reply(&db, community, channel, &root, &actor, vec![]).await; + let other = Keys::generate(); + for _ in 0..=MAX_UNREAD_COUNT { + reply(&db, community, channel, &root, &other, vec![]).await; + } + let summary = row(&db, community, channel, &actor).await; + assert_eq!( + threads(&summary), + [(root.id.to_hex(), MAX_UNREAD_COUNT as u32)] + ); +} + +#[tokio::test] +#[ignore = "requires Postgres"] +async fn ingest_does_not_start_an_account() { + let (db, pool, community, channel, _, root) = fixture().await; + let bee = Keys::generate(); + db.add_member( + community, + channel, + &bee.public_key().to_bytes(), + MemberRole::Member, + None, + ) + .await + .unwrap(); + reply( + &db, + community, + channel, + &root, + &Keys::generate(), + mention(&bee), + ) + .await; + reply(&db, community, channel, &root, &bee, vec![]).await; + + let started: Option> = sqlx::query_scalar( + "SELECT started_at FROM personal_read_accounts WHERE community_id=$1 AND actor=$2", + ) + .bind(community.as_uuid()) + .bind(bee.public_key().to_bytes().as_slice()) + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!(started, None); + // Before its first intent, nothing counts. + let summary = row(&db, community, channel, &bee).await; + assert_eq!((summary.unread, summary.mentions), (false, 0)); + assert!(summary.threads.is_empty()); +} + +#[tokio::test] +#[ignore = "requires Postgres"] +async fn a_nested_reply_follows_the_root_thread() { + let (db, pool, community, channel, actor, _) = fixture().await; + let root = post(&db, community, channel, &actor, vec![]).await; + let parent = reply(&db, community, channel, &root, &Keys::generate(), vec![]).await; + let bee = Keys::generate(); + let at = |e: &nostr::Event| { + chrono::DateTime::from_timestamp(e.created_at.as_secs() as i64, 0).unwrap() + }; + let nested = message(&bee, vec![]); + db.insert_event_with_thread_metadata( + community, + &nested, + Some(channel), + Some(crate::event::ThreadMetadataParams { + event_id: nested.id.as_bytes(), + event_created_at: at(&nested), + channel_id: channel, + parent_event_id: Some(parent.id.as_bytes()), + parent_event_created_at: Some(at(&parent)), + root_event_id: Some(root.id.as_bytes()), + root_event_created_at: Some(at(&root)), + depth: 2, + broadcast: false, + }), + ) + .await + .unwrap(); + + assert_eq!( + follow(&pool, community, &bee, &root).await, + Some(Some(nested.id.to_hex())) + ); + assert_eq!(follow(&pool, community, &bee, &parent).await, None); + let summary = row(&db, community, channel, &actor).await; + assert_eq!(threads(&summary), [(root.id.to_hex(), 2)]); + assert_eq!(summary.threads[0].latest_id, nested.id.to_hex()); +} + +#[tokio::test] +#[ignore = "requires Postgres"] +async fn a_message_stored_without_thread_metadata_reads_the_timeline() { + let (db, _pool, community, channel, actor, _) = fixture().await; + let own = message(&actor, vec![]); + db.insert_event_with_thread_metadata(community, &own, Some(channel), None) + .await + .unwrap(); + let summary = row(&db, community, channel, &actor).await; + assert!(!summary.unread); + assert_eq!(summary.read_through_id, Some(own.id.to_hex())); +} diff --git a/crates/buzz-db/src/store/personal_read/mod.rs b/crates/buzz-db/src/store/personal_read/mod.rs index bd01af0cdeb..130b55d81c4 100644 --- a/crates/buzz-db/src/store/personal_read/mod.rs +++ b/crates/buzz-db/src/store/personal_read/mod.rs @@ -1,21 +1,23 @@ //! Private, signer-owned accessory read progress, separate from NIP-RS events. //! //! A frontier is the relay arrival time of the message a context was read -//! through; the unread horizon alone uses author time. Only fixed context -//! intents advance frontiers, never a query scan cap. +//! through; the unread horizon alone uses author time. Read intents and the +//! author's own posts advance frontiers. A thread frontier exists only for a +//! followed thread, and the sidebar counts forward from each frontier. +mod follows; mod model; -mod participation; mod projection; mod writes; +pub(crate) use follows::{record_message, Place}; pub use model::*; #[cfg(test)] mod postgres_tests; #[cfg(test)] -mod participation_postgres_tests; +mod follows_postgres_tests; #[cfg(test)] mod projection_postgres_tests; diff --git a/crates/buzz-db/src/store/personal_read/model.rs b/crates/buzz-db/src/store/personal_read/model.rs index 448d2491cb3..1b276918f6c 100644 --- a/crates/buzz-db/src/store/personal_read/model.rs +++ b/crates/buzz-db/src/store/personal_read/model.rs @@ -57,10 +57,17 @@ pub const MAX_CHANNELS: usize = 20; /// Maximum thread rows per channel row. A POST can return rows for /// `MAX_INTENTS` channels, and they must fit the 1 MiB response limit. pub const MAX_THREAD_SUMMARIES: usize = 25; -/// Latest-probe event budget per channel, before eligibility filtering. -pub const MAX_CHANNEL_SCAN: usize = 256; -/// Unread-window work budget per channel, before eligibility/ancestry joins. -pub const MAX_UNREAD_SCAN: usize = 4096; +/// Events each walk examines before eligibility: newest-first for the +/// newest timeline message, then for the last arrival among those authored up +/// to twice the skew before it (in a thread too), and forward from the read +/// position for DM counts. What lies past the budget goes unseen. +pub const MAX_TIMELINE_SCAN: usize = 4096; +/// Counts stop here: a count of 100 means at least 100. +pub const MAX_UNREAD_COUNT: usize = 100; +/// The relay accepts author times up to this far from arrival either way. +/// Read positions are arrival times and the indexes are by author time, so +/// every forward walk starts this much before the position. +pub const MAX_ARRIVAL_SKEW_SECONDS: u32 = 15 * 60; /// Conversation kinds eligible for ordinary unread state (not edits/reactions). pub const ELIGIBLE_KINDS: [i32; 4] = [9, 40002, 45001, 45003]; @@ -74,32 +81,31 @@ pub struct ChannelReadSummary { /// only on their thread row. pub unread: bool, /// Unread top-level messages directed at the actor: every one in a DM, - /// otherwise those that tag the actor with `p` or carry `broadcast=1`. + /// otherwise those that tag the actor with `p`. pub mentions: u32, - /// The anchor of the timeline's frontier: the marked message that arrived - /// last. None until the timeline is marked. + /// The anchor of the timeline's frontier: the marked or own top-level + /// message that arrived last. None until either exists. pub read_through_id: Option, /// Last eligible top-level message to arrive, whatever its author or read /// progress: marking the timeline through it reads the timeline. pub latest_id: Option, - /// Threads with unread replies, newest unread reply first. + /// Followed threads with unread replies, newest unread reply first. pub threads: Vec, } /// One thread's read state within a channel row. No conversation bytes. -#[derive(Debug, Serialize)] +#[derive(Debug, Deserialize, Serialize)] pub struct ThreadReadSummary { /// Canonical thread-root event ID. pub root_id: String, /// Whether an unread reply that counts was found. pub unread: bool, - /// Unread replies that count: directed at the actor, or in one of the - /// actor's conversations. + /// Unread replies by others in this followed thread. pub mentions: u32, - /// The anchor of the thread's frontier: the marked message that arrived - /// last. None until the thread is marked. + /// The anchor of the thread's frontier: the marked or own reply that + /// arrived last. None for a follower who has read nothing yet. pub read_through_id: Option, - /// Last counted reply to arrive: marking through it reads the thread. + /// Last reply to arrive: marking through it reads the thread. pub latest_id: String, } diff --git a/crates/buzz-db/src/store/personal_read/participation.rs b/crates/buzz-db/src/store/personal_read/participation.rs deleted file mode 100644 index 682bca30489..00000000000 --- a/crates/buzz-db/src/store/personal_read/participation.rs +++ /dev/null @@ -1,127 +0,0 @@ -//! Conversation membership from the existing thread store, bounded by time. -use super::model::ELIGIBLE_KINDS; -use crate::Result; -use buzz_core::CommunityId; -use sqlx::{Acquire, PgConnection, Row}; -use std::collections::HashMap; -use uuid::Uuid; - -// Bound multiplicative work independently of the unread evidence window. -const MAX_PARENTS: usize = 1024; - -/// Whether each `(channel, parent)` is one of the actor's conversations: the -/// actor wrote the parent or has a reply to it. Only live eligible messages -/// qualify. An absent key is undecided: past the target cap, or the statement -/// deadline expired. Membership is independent of unread retention and read -/// frontiers. -pub(super) async fn resolve( - conn: &mut PgConnection, - community: CommunityId, - actor: &[u8], - targets: &[(Uuid, Vec)], -) -> Result), bool>> { - if targets.is_empty() { - return Ok(HashMap::new()); - } - let targets = select_targets(targets); - let channels: Vec<_> = targets.iter().map(|(channel, _)| *channel).collect(); - let parents: Vec<_> = targets.iter().map(|(_, parent)| parent.clone()).collect(); - // Optional inference must not abort authoritative unread/frontier reads. - // A nested transaction is a savepoint; rollback also restores the caller's - // statement timeout. No state is written in this read-only inference. - let mut budget = conn.begin().await?; - sqlx::query("SET LOCAL statement_timeout = '500ms'") - .execute(&mut *budget) - .await?; - sqlx::query("SET LOCAL jit = off") - .execute(&mut *budget) - .await?; - // Exact: a row cap would lose a real member behind a busy parent's other - // replies. LATERAL ... LIMIT 1 keeps the work on the replies under the - // requested parents; a plain EXISTS may be hashed over the whole tenant. - let result = sqlx::query( - "SELECT t.channel_id,t.parent_id,(own.hit OR replied.hit) IS TRUE AS member - FROM unnest($3::uuid[],$4::bytea[]) t(channel_id,parent_id) - LEFT JOIN LATERAL ( - SELECT e.pubkey=$2 AND e.deleted_at IS NULL AND e.kind=ANY($5) AS hit - FROM events e WHERE e.community_id=$1 AND e.channel_id=t.channel_id AND e.id=t.parent_id - ORDER BY e.created_at DESC LIMIT 1 - ) own ON true - LEFT JOIN LATERAL ( - SELECT true AS hit FROM thread_metadata tm JOIN events e ON e.community_id=$1 - AND e.channel_id=t.channel_id AND e.created_at=tm.event_created_at AND e.id=tm.event_id - WHERE tm.community_id=$1 AND tm.channel_id=t.channel_id AND tm.parent_event_id=t.parent_id - AND e.pubkey=$2 AND e.deleted_at IS NULL AND e.kind=ANY($5) - AND own.hit IS NOT TRUE - LIMIT 1 - ) replied ON true", - ) - .bind(community.as_uuid()) - .bind(actor) - .bind(channels) - .bind(parents) - .bind(ELIGIBLE_KINDS.as_slice()) - .fetch_all(&mut *budget) - .await; - budget.rollback().await?; - let rows = match result { - Ok(rows) => rows, - Err(sqlx::Error::Database(error)) - if matches!(error.code().as_deref(), Some("57014" | "55P03")) => - { - return Ok(HashMap::new()); - } - Err(error) => return Err(error.into()), - }; - rows.into_iter() - .map(|row| { - Ok(( - (row.try_get("channel_id")?, row.try_get("parent_id")?), - row.try_get("member")?, - )) - }) - .collect() -} - -// Keep selection independent of the optional SQL deadline: a timeout must not -// hide a regression in the cardinality bound. -fn select_targets(targets: &[(Uuid, Vec)]) -> Vec<(Uuid, Vec)> { - let mut targets = targets.to_vec(); - targets.sort_unstable(); - targets.dedup(); - targets.truncate(MAX_PARENTS); - targets -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn target_selection_caps_unique_parents_independently_of_sql_timeout() { - // Literal contract boundaries deliberately do not derive from MAX_PARENTS. - for count in [0_u32, 1, 1023, 1024, 1025] { - let unique: Vec<_> = (0..count) - .map(|i| (Uuid::nil(), i.to_be_bytes().to_vec())) - .collect(); - let input: Vec<_> = unique - .iter() - .rev() - .chain(unique.iter().rev()) - .cloned() - .collect(); - let selected = select_targets(&input); - assert_eq!(selected, unique[..unique.len().min(1024)], "count {count}"); - } - } - - #[test] - fn target_selection_keeps_channel_identity() { - let first = (Uuid::from_u128(1), vec![7; 32]); - let second = (Uuid::from_u128(2), vec![7; 32]); - assert_eq!( - select_targets(&[second.clone(), first.clone(), second.clone()]), - vec![first, second] - ); - } -} diff --git a/crates/buzz-db/src/store/personal_read/participation_postgres_tests.rs b/crates/buzz-db/src/store/personal_read/participation_postgres_tests.rs deleted file mode 100644 index 5a3d3a0acad..00000000000 --- a/crates/buzz-db/src/store/personal_read/participation_postgres_tests.rs +++ /dev/null @@ -1,537 +0,0 @@ -//! Which replies count: the direct-parent conversation rule, its deletion and -//! channel edges, and the bounds on the membership lookup. -use super::*; -use crate::{ - channel::{ChannelType, ChannelVisibility}, - Db, -}; -use buzz_core::CommunityId; -use nostr::{EventBuilder, Keys, Kind, Tag, Timestamp}; -use serde_json::{json, Value}; -use sqlx::PgPool; -use uuid::Uuid; - -const DAY: u64 = 86_400; - -/// One community with the reading actor and a peer. -struct World { - db: Db, - pool: PgPool, - community: CommunityId, - actor: Keys, - peer: Keys, - now: u64, -} - -impl World { - async fn new() -> Self { - let pool = PgPool::connect(&crate::test_support::database_url()) - .await - .unwrap(); - let db = Db::from_pool(pool.clone()); - let community = db - .ensure_configured_community(&format!("conversation-{}.local", Uuid::new_v4())) - .await - .unwrap() - .id; - let actor = Keys::generate(); - super::postgres_tests::start_before_everything(&pool, community, &actor.public_key()).await; - Self { - db, - pool, - community, - actor, - peer: Keys::generate(), - now: Timestamp::now().as_secs(), - } - } - - /// A channel the actor has joined. - async fn channel(&self) -> Uuid { - self.db - .create_channel( - self.community, - &Uuid::new_v4().to_string(), - ChannelType::Stream, - ChannelVisibility::Open, - None, - &self.actor.public_key().to_bytes(), - None, - ) - .await - .unwrap() - .id - } - - /// A top-level message. - async fn post(&self, channel: Uuid, author: &Keys, at: u64, tags: Vec) -> nostr::Event { - let event = EventBuilder::new(Kind::Custom(9), Uuid::new_v4().to_string()) - .tags(tags) - .custom_created_at(Timestamp::from(at)) - .sign_with_keys(author) - .unwrap(); - self.db - .insert_event(self.community, &event, Some(channel)) - .await - .unwrap(); - event - } - - /// A canonical reply to `parent` in `root`'s thread. `parent` need not be - /// stored in `channel`: thread metadata records what the reply claims. - async fn reply( - &self, - channel: Uuid, - author: &Keys, - root: &nostr::Event, - parent: Option<&nostr::Event>, - at: u64, - tags: Vec, - ) -> nostr::Event { - let event = self.post(channel, author, at, tags).await; - sqlx::query("INSERT INTO thread_metadata (community_id,event_id,event_created_at,channel_id,root_event_id,parent_event_id,depth) - VALUES ($1,$2,to_timestamp($3),$4,$5,$6,1)") - .bind(self.community.as_uuid()).bind(event.id.as_bytes().as_slice()).bind(at as f64) - .bind(channel).bind(root.id.as_bytes().as_slice()) - .bind(parent.map(|parent| parent.id.as_bytes().as_slice())) - .execute(&self.pool).await.unwrap(); - event - } - - /// The storage write that author and staff deletions share. - async fn delete(&self, event: &nostr::Event) { - assert!(self - .db - .soft_delete_event_and_update_thread(self.community, event.id.as_bytes(), None, None) - .await - .unwrap()); - } - - async fn row(&self, channel: Uuid) -> ChannelReadSummary { - self.db - .personal_read_sidebar_channels( - self.community, - &self.actor.public_key(), - DEFAULT_RETENTION_SECONDS, - &[channel], - ) - .await - .unwrap() - .channels - .remove(0) - } - - /// The row's counts and thread rows: what the sidebar says counts. - async fn counts(&self, channel: Uuid) -> Value { - let row = self.row(channel).await; - json!({"unread":row.unread,"mentions":row.mentions,"threads":wire(&row.threads)}) - } - - fn mention(&self) -> Tag { - Tag::parse(["p", &self.actor.public_key().to_hex()]).unwrap() - } -} - -fn wire(value: &T) -> Value { - serde_json::to_value(value).unwrap() -} - -/// Expected `counts`: whether a top-level message is unread, how many of those -/// are directed, and the listed thread rows. -fn counts(unread: bool, mentions: u32, threads: Vec) -> Value { - json!({"unread":unread,"mentions":mentions,"threads":threads}) -} - -fn broadcast() -> Tag { - Tag::parse(["broadcast", "1"]).unwrap() -} - -/// A listed, never-marked thread with `mentions` counted unread replies. -fn item(root: &nostr::Event, mentions: u32, latest: &nostr::Event) -> Value { - json!({"root_id":root.id.to_hex(),"unread":true,"mentions":mentions, - "read_through_id":null,"latest_id":latest.id.to_hex()}) -} - -#[tokio::test] -#[ignore = "requires Postgres"] -async fn a_reply_counts_only_in_a_conversation_the_actor_wrote_or_replied_to() { - let w = World::new().await; - // Every witness predates the unread horizon: membership does not expire. - let old = w.now - 40 * DAY; - - // The actor wrote the root. A peer answers it; another peer reply continues - // under that answer, where the actor has written nothing. - let c = w.channel().await; - let root = w.post(c, &w.actor, old, vec![]).await; - let answer = w - .reply(c, &w.peer, &root, Some(&root), w.now + 1, vec![]) - .await; - let nested = w - .reply(c, &w.peer, &root, Some(&answer), w.now + 2, vec![]) - .await; - // Only `answer` counts. The newer reply that does not count is not the - // thread's latest. - assert_eq!( - w.counts(c).await, - counts(false, 0, vec![item(&root, 1, &answer)]) - ); - - // Joining the nested conversation makes its earlier reply count. The - // actor's own reply adds nothing. - w.reply(c, &w.actor, &root, Some(&answer), w.now + 3, vec![]) - .await; - assert_eq!( - w.counts(c).await, - counts(false, 0, vec![item(&root, 2, &nested)]) - ); - - // A peer's thread. The actor replied to one of two parents, long ago. - let c = w.channel().await; - let root = w.post(c, &w.peer, old, vec![]).await; - let joined = w.reply(c, &w.peer, &root, Some(&root), old, vec![]).await; - let other = w.reply(c, &w.peer, &root, Some(&root), old, vec![]).await; - w.reply(c, &w.actor, &root, Some(&joined), old, vec![]) - .await; - let sibling = w - .reply(c, &w.peer, &root, Some(&joined), w.now + 1, vec![]) - .await; - w.reply(c, &w.peer, &root, Some(&other), w.now + 2, vec![]) - .await; - assert_eq!( - w.counts(c).await, - counts(false, 0, vec![item(&root, 1, &sibling)]) - ); - - // A conversation the actor never joined: one unread top-level message, and - // a reply that is not unread even in its own thread. - let c = w.channel().await; - let root = w.post(c, &w.peer, w.now, vec![]).await; - w.reply(c, &w.peer, &root, Some(&root), w.now + 1, vec![]) - .await; - assert_eq!(w.counts(c).await, counts(true, 0, vec![])); - - // A reply whose parent was never recorded is undecided: left out. - w.reply(c, &w.peer, &root, None, w.now + 2, vec![]).await; - assert_eq!(w.counts(c).await, counts(true, 0, vec![])); -} - -#[tokio::test] -#[ignore = "requires Postgres"] -async fn a_directed_reply_counts_outside_the_actors_conversations() { - let w = World::new().await; - let c = w.channel().await; - let root = w.post(c, &w.peer, w.now, vec![]).await; - let parent = w.reply(c, &w.peer, &root, Some(&root), w.now, vec![]).await; - w.reply(c, &w.actor, &root, Some(&root), w.now + 1, vec![]) - .await; - for (at, tags) in [(2, vec![w.mention()]), (3, vec![broadcast()])] { - w.reply(c, &w.peer, &root, Some(&parent), w.now + at, tags) - .await; - } - let both = w - .reply( - c, - &w.peer, - &root, - Some(&parent), - w.now + 4, - vec![broadcast(), w.mention()], - ) - .await; - // The root is an undirected top-level message. The thread counts the - // actor's sibling `parent` and the three directed replies, not the - // actor's own. - let expected = counts(true, 0, vec![item(&root, 4, &both)]); - assert_eq!(w.counts(c).await, expected); - - // Joining the directed replies' conversation changes no count. - w.reply(c, &w.actor, &root, Some(&parent), w.now + 5, vec![]) - .await; - assert_eq!(w.counts(c).await, expected); - - // In a DM every peer message is direct, tagged or not, joined or not. - let dm = w.channel().await; - sqlx::query("UPDATE channels SET channel_type='dm' WHERE community_id=$1 AND id=$2") - .bind(w.community.as_uuid()) - .bind(dm) - .execute(&w.pool) - .await - .unwrap(); - let root = w.post(dm, &w.peer, w.now, vec![w.mention()]).await; - let parent = w - .reply(dm, &w.peer, &root, Some(&root), w.now, vec![]) - .await; - let reply = w - .reply(dm, &w.peer, &root, Some(&parent), w.now + 1, vec![]) - .await; - assert_eq!( - w.counts(dm).await, - counts(true, 1, vec![item(&root, 2, &reply)]) - ); -} - -#[tokio::test] -#[ignore = "requires Postgres"] -async fn eligibility_frontier_and_membership_each_leave_replies_out() { - let w = World::new().await; - let c = w.channel().await; - let root = w.post(c, &w.peer, w.now, vec![]).await; - let parent = w.reply(c, &w.peer, &root, Some(&root), w.now, vec![]).await; - let reply = |author, at, tags| w.reply(c, author, &root, Some(&parent), w.now + at, tags); - // Peer replies to a parent the actor neither wrote nor replied to. - reply(&w.peer, 1, vec![]).await; - let deleted = reply(&w.peer, 2, vec![w.mention()]).await; - w.delete(&deleted).await; - w.reply(c, &w.actor, &root, Some(&root), w.now + 3, vec![]) - .await; - let anchor = reply(&w.peer, 4, vec![w.mention()]).await; - // Undirected, outside the actor's conversations: never counts. - reply(&w.peer, 5, vec![]).await; - // `parent` (a reply to the root, which the actor answered) and `anchor` - // count: not the deleted, own or undirected replies. - assert_eq!( - w.counts(c).await, - counts(true, 0, vec![item(&root, 2, &anchor)]) - ); - - assert_eq!( - w.db.apply_personal_read_intent( - w.community, - &w.actor.public_key(), - &ReadIntent::MarkThrough { - target: ReadTarget { - channel_id: c, - root_id: Some(root.id.to_hex()), - }, - message_id: anchor.id.to_hex(), - }, - ) - .await - .unwrap(), - IntentOutcome::Applied - ); - assert_eq!(w.counts(c).await, counts(true, 0, vec![])); - // A later directed reply is unread past the thread's anchor; the channel - // timeline was never marked. - let after = reply(&w.peer, 6, vec![w.mention()]).await; - let mut thread = item(&root, 1, &after); - thread["read_through_id"] = json!(anchor.id.to_hex()); - assert_eq!(w.counts(c).await, counts(true, 0, vec![thread])); - assert_eq!(w.row(c).await.read_through_id, None); -} - -#[tokio::test] -#[ignore = "requires Postgres"] -async fn a_deleted_message_is_no_witness_and_a_surviving_reply_still_is() { - let w = World::new().await; - let c = w.channel().await; - let root = w.post(c, &w.peer, w.now - 40 * DAY, vec![]).await; - let reply = |author, parent, at| w.reply(c, author, &root, Some(parent), w.now + at, vec![]); - - // The actor wrote the parent and never replied under it. - let wrote = reply(&w.actor, &root, 0).await; - reply(&w.peer, &wrote, 10).await; - // The actor's only reply to a peer's parent. - let once = reply(&w.peer, &root, 0).await; - let only = reply(&w.actor, &once, 1).await; - reply(&w.peer, &once, 11).await; - // Two replies by the actor to a peer's parent. - let twice = reply(&w.peer, &root, 0).await; - let first = reply(&w.actor, &twice, 1).await; - reply(&w.actor, &twice, 2).await; - let to_twice = reply(&w.peer, &twice, 12).await; - // The actor wrote the parent and also replied under it. - let both = reply(&w.actor, &root, 0).await; - let under = reply(&w.actor, &both, 1).await; - let to_both = reply(&w.peer, &both, 13).await; - - // How many replies the thread counts, the latest of them `to_both` until - // it stops counting. - let w = &w; - let thread = |mentions: u32| async move { - let row = w.row(c).await; - assert!(!row.unread, "the root predates the horizon"); - assert_eq!(row.threads.len(), 1); - assert_eq!(row.threads[0].mentions, mentions); - row.threads[0].latest_id.clone() - }; - // The four peer replies to the actor's conversations count. So do `once` - // and `twice`: peer replies to the root, which the actor has replied to - // (`wrote`, `both`). - assert_eq!(thread(6).await, to_both.id.to_hex()); - - // Each deletion changes only its own parent's conversation. - w.delete(&wrote).await; - assert_eq!(thread(5).await, to_both.id.to_hex()); - w.delete(&only).await; - assert_eq!(thread(4).await, to_both.id.to_hex()); - w.delete(&first).await; - assert_eq!(thread(4).await, to_both.id.to_hex()); - // With `wrote` and `both` gone the actor has no reply to the root either. - w.delete(&both).await; - assert_eq!(thread(2).await, to_both.id.to_hex()); - w.delete(&under).await; - assert_eq!(thread(1).await, to_twice.id.to_hex()); -} - -#[tokio::test] -#[ignore = "requires Postgres"] -async fn membership_is_scoped_to_the_replys_own_channel() { - let w = World::new().await; - let (a, b) = (w.channel().await, w.channel().await); - let old = w.now - 40 * DAY; - - // The actor replied to a peer's parent, but that reply is stored in B. - let theirs = w.post(a, &w.peer, old, vec![]).await; - w.reply(b, &w.actor, &theirs, Some(&theirs), old, vec![]) - .await; - let in_a = w - .reply(a, &w.peer, &theirs, Some(&theirs), w.now, vec![]) - .await; - assert_eq!(w.counts(a).await, counts(false, 0, vec![])); - - // The actor wrote a parent in A. A peer reply in B claims it as its parent. - let mine = w.post(a, &w.actor, old, vec![]).await; - w.reply(b, &w.peer, &mine, Some(&mine), w.now, vec![]).await; - assert_eq!(w.counts(b).await, counts(false, 0, vec![])); - - // The same parents count once the actor is a member in the reply's channel. - w.reply(a, &w.actor, &theirs, Some(&theirs), old, vec![]) - .await; - let to_mine = w.reply(a, &w.peer, &mine, Some(&mine), w.now, vec![]).await; - // Equal reply times: root ID order. - let mut threads = [item(&theirs, 1, &in_a), item(&mine, 1, &to_mine)]; - threads.sort_by(|x, y| x["root_id"].as_str().cmp(&y["root_id"].as_str())); - assert_eq!(w.counts(a).await, counts(false, 0, threads.to_vec())); -} - -#[tokio::test] -#[ignore = "requires Postgres"] -async fn membership_is_exact_behind_a_busy_parent() { - let w = World::new().await; - let c = w.channel().await; - let old = w.now - 40 * DAY; - let root = w.post(c, &w.peer, old, vec![]).await; - // The actor's reply is the oldest of 259 under one parent: a 256-row - // window over the parent's replies would never reach it. - w.reply(c, &w.actor, &root, Some(&root), old, vec![]).await; - let mut last = root.clone(); - for i in 0..258 { - last = w - .reply(c, &w.peer, &root, Some(&root), w.now + i, vec![]) - .await; - } - assert_eq!( - w.counts(c).await, - counts(false, 0, vec![item(&root, 258, &last)]) - ); -} - -#[tokio::test] -#[ignore = "requires Postgres"] -async fn threads_are_capped_after_replies_that_do_not_count_are_removed() { - let w = World::new().await; - let c = w.channel().await; - let old = w.now - 40 * DAY; - // The actor's thread has the oldest unread reply. A cap's worth of - // unjoined threads are newer and would fill the list if the cap came - // first. - let mine = w.post(c, &w.actor, old, vec![]).await; - let answer = w.reply(c, &w.peer, &mine, Some(&mine), w.now, vec![]).await; - for i in 1..=MAX_THREAD_SUMMARIES as u64 { - let root = w.post(c, &w.peer, old, vec![]).await; - w.reply(c, &w.peer, &root, Some(&root), w.now + i, vec![]) - .await; - } - assert_eq!( - w.counts(c).await, - counts(false, 0, vec![item(&mine, 1, &answer)]) - ); -} - -#[tokio::test] -#[ignore = "requires Postgres"] -async fn the_parent_budget_leaves_undecided_replies_out() { - let w = World::new().await; - let c = w.channel().await; - // The actor is in the first conversation. Whether it is inside the budget - // depends on ID order, so only the bounds are asserted. - let mut mine = None; - for i in 0..1025 { - let author = if i == 0 { &w.actor } else { &w.peer }; - let root = w.post(c, author, w.now, vec![]).await; - let reply = w - .reply(c, &w.peer, &root, Some(&root), w.now + 1, vec![]) - .await; - if i == 0 { - mine = Some(item(&root, 1, &reply)); - } - // 1024 parents fit: 1023 peer roots, and the reply to the actor's. - // The independent SQL deadline may still withhold the answer. Past - // 1024, one reply is undecided. An undecided reply is left out. - if i >= 1023 { - let counted = w.counts(c).await; - assert!( - [vec![], vec![mine.clone().unwrap()]] - .map(|threads| counts(true, 0, threads)) - .contains(&counted), - "{i}: {counted}" - ); - } - } -} - -#[tokio::test] -#[ignore = "requires Postgres"] -async fn a_lookup_timeout_decides_nothing_and_preserves_the_callers_transaction() { - let w = World::new().await; - let c = w.channel().await; - let root = w.post(c, &w.peer, w.now, vec![]).await; - w.reply(c, &w.peer, &root, Some(&root), w.now + 1, vec![]) - .await; - let mut held = w.pool.begin().await.unwrap(); - // Force the real resolver to time out, then check that its outer snapshot - // and original statement budget remain usable. - sqlx::query("LOCK TABLE thread_metadata IN ACCESS EXCLUSIVE MODE") - .execute(&mut *held) - .await - .unwrap(); - let mut reader = w.pool.begin().await.unwrap(); - sqlx::query("SET LOCAL statement_timeout='2000ms'") - .execute(&mut *reader) - .await - .unwrap(); - for lock_timeout in ["0", "10ms"] { - sqlx::query("SELECT set_config('lock_timeout',$1,true)") - .bind(lock_timeout) - .execute(&mut *reader) - .await - .unwrap(); - sqlx::query("SET LOCAL jit=on") - .execute(&mut *reader) - .await - .unwrap(); - let result = participation::resolve( - &mut reader, - w.community, - &w.actor.public_key().to_bytes(), - &[(c, root.id.as_bytes().to_vec())], - ) - .await - .unwrap(); - assert!(result.is_empty(), "timeout provides no negative evidence"); - let setting: String = sqlx::query_scalar("SHOW statement_timeout") - .fetch_one(&mut *reader) - .await - .unwrap(); - assert_eq!(setting, "2s"); - let jit: String = sqlx::query_scalar("SHOW jit") - .fetch_one(&mut *reader) - .await - .unwrap(); - assert_eq!(jit, "on", "optional inference restores caller settings"); - } - reader.rollback().await.unwrap(); - held.rollback().await.unwrap(); - assert_eq!(w.counts(c).await, counts(true, 0, vec![])); -} diff --git a/crates/buzz-db/src/store/personal_read/postgres_tests.rs b/crates/buzz-db/src/store/personal_read/postgres_tests.rs index 136eb89148d..ad850d219c6 100644 --- a/crates/buzz-db/src/store/personal_read/postgres_tests.rs +++ b/crates/buzz-db/src/store/personal_read/postgres_tests.rs @@ -51,7 +51,8 @@ pub(super) async fn start_before_everything( ) { sqlx::query( "INSERT INTO personal_read_accounts (community_id,actor,started_at) - VALUES ($1,$2,'1900-01-01T00:00:00Z')", + VALUES ($1,$2,'1900-01-01T00:00:00Z') + ON CONFLICT (community_id,actor) DO UPDATE SET started_at=excluded.started_at", ) .bind(community.as_uuid()) .bind(actor.to_bytes().as_slice()) @@ -60,14 +61,55 @@ pub(super) async fn start_before_everything( .unwrap(); } +/// Store `event` as a canonical reply in `root`'s thread, through ingest. +pub(super) async fn insert_reply( + db: &Db, + community: CommunityId, + channel: Uuid, + root: &nostr::Event, + event: &nostr::Event, +) { + let at = |e: &nostr::Event| { + chrono::DateTime::from_timestamp(e.created_at.as_secs() as i64, 0).unwrap() + }; + db.insert_event_with_thread_metadata( + community, + event, + Some(channel), + Some(crate::event::ThreadMetadataParams { + event_id: event.id.as_bytes(), + event_created_at: at(event), + channel_id: channel, + parent_event_id: Some(root.id.as_bytes()), + parent_event_created_at: Some(at(root)), + root_event_id: Some(root.id.as_bytes()), + root_event_created_at: Some(at(root)), + depth: 1, + broadcast: event.tags.iter().any(|t| { + let t = t.as_slice(); + t.len() >= 2 && t[0] == "broadcast" && t[1] == "1" + }), + }), + ) + .await + .unwrap(); +} + /// Move every community message past the default horizon by author time, the /// only clock the unread window reads. async fn expire(pool: &PgPool, community: CommunityId) { - sqlx::query("UPDATE events SET created_at=created_at-interval '31 days' WHERE community_id=$1") + for table in [ + "events SET created_at=created_at", + "event_mentions SET event_created_at=event_created_at", + ] { + sqlx::query(sqlx::AssertSqlSafe(format!( + "UPDATE {table}-interval '31 days' WHERE community_id=$1" + ))) .bind(community.as_uuid()) .execute(pool) .await .unwrap(); + } } async fn sidebar(db: &Db, community: CommunityId, actor: &Keys) -> SidebarPage { @@ -105,12 +147,17 @@ fn mark_thread(channel: Uuid, root: &nostr::Event, message: &nostr::Event) -> Re /// Address every stored community message to `actor`, so the channel's /// mention count is exactly its unread top-level count. async fn mention_everywhere(pool: &PgPool, community: CommunityId, actor: &Keys) { - sqlx::query("UPDATE events SET tags=$2 WHERE community_id=$1") - .bind(community.as_uuid()) - .bind(serde_json::json!([["p", actor.public_key().to_hex()]])) - .execute(pool) - .await - .unwrap(); + sqlx::query( + "INSERT INTO event_mentions + (community_id,pubkey_hex,event_id,event_created_at,channel_id,event_kind) + SELECT community_id,$2,id,created_at,channel_id,kind FROM events WHERE community_id=$1 + ON CONFLICT DO NOTHING", + ) + .bind(community.as_uuid()) + .bind(actor.public_key().to_hex()) + .execute(pool) + .await + .unwrap(); } async fn add_history(db: &Db, community: CommunityId, channel: Uuid, count: usize) -> nostr::Event { @@ -134,7 +181,7 @@ async fn add_history(db: &Db, community: CommunityId, channel: Uuid, count: usiz #[ignore = "requires Postgres"] async fn personal_read_sidebar_marked_history_past_the_scan_cap_is_read_and_keeps_latest() { let (db, _pool, community, channel, actor, _) = fixture().await; - let last = add_history(&db, community, channel, MAX_UNREAD_SCAN + 44).await; + let last = add_history(&db, community, channel, MAX_TIMELINE_SCAN + 44).await; let outcome = db .apply_personal_read_intent( community, @@ -156,7 +203,7 @@ async fn personal_read_sidebar_marked_history_past_the_scan_cap_is_read_and_keep #[ignore = "requires Postgres"] async fn personal_read_sidebar_author_window_excludes_expired_and_late_old_messages() { let (db, pool, community, channel, actor, _) = fixture().await; - let latest = add_history(&db, community, channel, MAX_UNREAD_SCAN + 44).await; + let latest = add_history(&db, community, channel, MAX_TIMELINE_SCAN + 44).await; expire(&pool, community).await; let row = &sidebar(&db, community, &actor).await.channels[0]; assert!( @@ -183,34 +230,30 @@ async fn personal_read_sidebar_author_window_excludes_expired_and_late_old_messa #[tokio::test] #[ignore = "requires Postgres"] -async fn personal_read_sidebar_window_budget_counts_boundary_and_ineligible_tail() { +async fn personal_read_counts_stop_at_the_cap_after_eligibility() { let (db, pool, community, channel, actor, _) = fixture().await; - // The fixture contributes one event, so this is exactly the evidence budget. - add_history(&db, community, channel, MAX_UNREAD_SCAN - 1).await; + // The fixture contributes one message, so the count reaches the cap. + add_history(&db, community, channel, MAX_UNREAD_COUNT - 1).await; mention_everywhere(&pool, community, &actor).await; let mentions = |page: SidebarPage| page.channels[0].mentions; assert_eq!( mentions(sidebar(&db, community, &actor).await), - MAX_UNREAD_SCAN as u32 + MAX_UNREAD_COUNT as u32 ); - let overflow = EventBuilder::new(Kind::Custom(9), "one beyond the budget") - .sign_with_keys(&Keys::generate()) - .unwrap(); - db.insert_event(community, &overflow, Some(channel)) - .await - .unwrap(); + add_history(&db, community, channel, 50).await; mention_everywhere(&pool, community, &actor).await; - // The budget bounds the count; the unexamined message is left out. assert_eq!( mentions(sidebar(&db, community, &actor).await), - MAX_UNREAD_SCAN as u32 + MAX_UNREAD_COUNT as u32, + "a count of the cap means at least that many" ); - // Expire all but 601 messages. Of these, 300 are own and 300 deleted. - // Eligibility is downstream of the bounded unread window, not the old 256 cap. - sqlx::query("WITH ranked AS (SELECT created_at,id,row_number() OVER (ORDER BY created_at DESC,id) AS n FROM events WHERE community_id=$1 AND channel_id=$2) - UPDATE events e SET created_at=CASE WHEN r.n>601 THEN e.created_at-interval '31 days' ELSE e.created_at END, - pubkey=CASE WHEN r.n<=300 THEN $3 ELSE e.pubkey END, - deleted_at=CASE WHEN r.n>300 AND r.n<=600 THEN now() ELSE NULL END + // The last to arrive stays; of the 100 before it, 50 become own and 50 + // deleted; the rest expire. The cap applies after eligibility, so one + // remains. + sqlx::query("WITH ranked AS (SELECT created_at,id,row_number() OVER (ORDER BY received_at DESC,id) AS n FROM events WHERE community_id=$1 AND channel_id=$2) + UPDATE events e SET created_at=CASE WHEN r.n>101 THEN e.created_at-interval '31 days' ELSE e.created_at END, + pubkey=CASE WHEN r.n BETWEEN 2 AND 51 THEN $3 ELSE e.pubkey END, + deleted_at=CASE WHEN r.n BETWEEN 52 AND 101 THEN now() ELSE NULL END FROM ranked r WHERE e.community_id=$1 AND e.created_at=r.created_at AND e.id=r.id") .bind(community.as_uuid()).bind(channel).bind(actor.public_key().to_bytes().as_slice()).execute(&pool).await.unwrap(); let page = sidebar(&db, community, &actor).await; @@ -694,7 +737,7 @@ async fn personal_read_counts_only_arrivals_after_the_account_starts() { async fn personal_read_unstarted_account_is_caught_up_past_the_scan_cap() { let (db, pool, community, channel, actor, _) = fixture().await; unstart(&pool, community).await; - let last = add_history(&db, community, channel, MAX_UNREAD_SCAN + 44).await; + let last = add_history(&db, community, channel, MAX_TIMELINE_SCAN + 44).await; let page = sidebar(&db, community, &actor).await; let row = &page.channels[0]; // The start floor covers every arrival, examined by the scan or not. @@ -710,24 +753,18 @@ async fn personal_read_unstarted_account_is_caught_up_past_the_scan_cap() { async fn personal_read_channel_and_thread_never_inherit_each_other() { let (db, pool, community, channel, actor, root) = fixture().await; let base = root.created_at.as_secs(); - // Directed, so they count outside the actor's conversations. + // Mentioning the actor, who therefore follows the thread. let reply_at = |at: u64| { - let (db, pool) = (db.clone(), pool.clone()); + let db = db.clone(); let mention = Tag::public_key(actor.public_key()); - let root = root.id; + let root = root.clone(); async move { let reply = EventBuilder::new(Kind::Custom(9), "unseen thread reply") .tags([mention]) .custom_created_at(nostr::Timestamp::from(at)) .sign_with_keys(&Keys::generate()) .unwrap(); - db.insert_event(community, &reply, Some(channel)) - .await - .unwrap(); - sqlx::query("INSERT INTO thread_metadata (community_id,event_id,event_created_at,channel_id,root_event_id,parent_event_id,depth) - VALUES ($1,$2,to_timestamp($3),$4,$5,$5,1)") - .bind(community.as_uuid()).bind(reply.id.as_bytes().as_slice()).bind(at as f64) - .bind(channel).bind(root.as_bytes().as_slice()).execute(&pool).await.unwrap(); + insert_reply(&db, community, channel, &root, &reply).await; reply } }; @@ -812,10 +849,9 @@ async fn personal_read_channel_and_thread_never_inherit_each_other() { .fetch_all(&pool) .await .unwrap(); - assert_eq!( - roots, - vec![root.id.as_bytes().to_vec()], - "thread reading never creates a channel frontier" + assert!( + roots.is_empty(), + "reading a thread never follows it or creates a channel frontier" ); } diff --git a/crates/buzz-db/src/store/personal_read/projection.rs b/crates/buzz-db/src/store/personal_read/projection.rs index 38e6e026009..35204ba8ee4 100644 --- a/crates/buzz-db/src/store/personal_read/projection.rs +++ b/crates/buzz-db/src/store/personal_read/projection.rs @@ -1,20 +1,44 @@ -//! Bounded evidence projection. The cap limits evidence, not the definition of -//! unread: an unexamined tail goes uncounted, so counts may undercount. +//! Sidebar read state, counted forward from each read position. A caught-up +//! reader's work is a few index probes per channel and per followed thread; +//! a reader who is behind walks only what arrived after their position, and +//! counts stop at `MAX_UNREAD_COUNT`. -use super::{model::*, participation, writes}; +use super::{model::*, writes}; use buzz_core::CommunityId; use chrono::{DateTime, Utc}; -use serde_json::Value; use sqlx::{Acquire, PgConnection, Row}; -use std::cmp::Reverse; -use std::collections::{hash_map::Entry, HashMap}; use uuid::Uuid; use crate::{observability, Db, DbError, Result}; +/// Event `e` is eligible: a conversation kind, not deleted. +const ELIGIBLE: &str = "e.kind=ANY($5) AND e.deleted_at IS NULL"; + +/// Event `e`, with its `thread_metadata` row `tm`, is on the channel timeline: +/// not a reply, and not a NIP-10 reply whose ancestry was never recorded +/// (neither counted nor a timeline anchor, since a mark cannot validate it). +/// Parity-tested against the shared NIP-10 parser. +const TIMELINE: &str = r#"(tm.root_event_id IS NULL OR tm.root_event_id=e.id) + AND NOT (tm.root_event_id IS NULL AND jsonb_typeof(e.tags)='array' + AND EXISTS (SELECT 1 FROM jsonb_array_elements(e.tags) t(tag) + WHERE jsonb_typeof(tag)='array' AND tag->>0='e' AND tag->>3='reply' + AND (tag->>1) COLLATE "C" ~ '^[0123456789abcdefABCDEF]{64}$'))"#; + +/// What the budgeted timeline walks carry out of their `LIMIT`. +const COLUMNS: &str = + "e.id, e.pubkey, e.created_at, e.received_at, e.kind, e.deleted_at, e.tags, e.channel_id"; + +const EVENT_THREAD_ROW: &str = "LEFT JOIN thread_metadata tm ON tm.community_id=$1 + AND tm.channel_id=e.channel_id AND tm.event_created_at=e.created_at AND tm.event_id=e.id"; + +/// Replies `tm` in followed thread `tf`, with their events `e`. +const THREAD_REPLIES: &str = "thread_metadata tm JOIN events e ON e.community_id=$1 + AND e.created_at=tm.event_created_at AND e.id=tm.event_id + WHERE tm.community_id=$1 AND tm.root_event_id=tf.root_id AND tm.channel_id=r.id + AND tm.event_id<>tf.root_id"; + impl Db { - /// Read a bounded joined roster from the writer. One SQL statement projects - /// channels, event evidence and read authority at a compatible MVCC cut. + /// Read a bounded joined roster from the writer in one read-only snapshot. /// Callers must recheck admission/resource access before releasing this data. pub async fn personal_read_sidebar( &self, @@ -70,7 +94,7 @@ impl Db { ) .await?; let mut tx = conn.begin().await?; - // The horizon and frontier evidence share one read-only cut. + // The horizon and every read position share one read-only cut. sqlx::query("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ READ ONLY") .execute(&mut *tx) .await?; @@ -78,225 +102,176 @@ impl Db { sqlx::query("SET LOCAL jit = off").execute(&mut *tx).await?; let actor_bytes = actor.to_bytes(); let account = read_account(&mut tx, community, &actor_bytes, retention_seconds).await?; - // The inner event LIMIT is deliberately before eligibility filtering. - // This bounds rows/joins even with long deleted or self-authored runs. - // Aggregate equivalent eligible evidence before transfer. Multiplicity - // preserves counts. - // Validate relevant tag parts before compacting directed/ancestry facts. - // PostgreSQL scalar "p" ->> 0 is "p": the type check must reject it too. - // Canonical timeline roots share an empty (present) root sentinel. - // Tags are bounded before transfer; oversized/corrupt evidence is - // skipped, never falsely top-level/unmentioned/read. - // Latest comes from the unread scan, so its ID arrives no earlier than - // anything counted; the shallow probe answers only when the horizon - // holds no top-level message. Thread replies are never the timeline's - // latest: a timeline mark cannot use one as its anchor. - let rows = sqlx::query( + // A position is its frontier floored at the account's start (infinity + // before the actor's first intent, so nothing counts). Positions are + // arrival times and the indexes are by author time, which the relay + // accepts within the skew of arrival, so a walk forward from a + // position starts the skew before it (and never before the horizon). + // + // Timeline, per channel: + // - latest_id: the newest timeline message by author time within the + // scan budget, then the last to arrive among the newest budget of + // events authored up to twice the skew before it, which holds the + // last arrival unless that many arrived out of order; + // - unread: latest is someone else's, inside the horizon, and arrived + // after the position (an own post moves the position past it); + // - mentions: those of them that tag the actor (event_mentions), or + // every one in a DM. + // Threads: each followed thread whose newest reply could have arrived + // after its position (one index-only probe of + // idx_thread_metadata_window), then its last arrival the same way as + // latest_id, and its unread replies by others. + let sql = format!( r#"WITH roster AS MATERIALIZED ( - SELECT c.id,c.channel_type::text AS channel_type + SELECT c.id, c.channel_type='dm' AS dm, + GREATEST(cf.through_timestamp, COALESCE($7::timestamptz, 'infinity')) AS position, + encode(cf.through_message_id,'hex') AS read_through_id FROM channel_members cm JOIN channels c ON c.community_id=cm.community_id AND c.id=cm.channel_id + LEFT JOIN personal_read_frontiers cf ON cf.community_id=$1 AND cf.actor=$2 + AND cf.channel_id=c.id AND cf.root_id=''::bytea WHERE cm.community_id=$1 AND cm.pubkey=$2 AND cm.removed_at IS NULL AND c.deleted_at IS NULL AND ($3::uuid IS NULL OR c.id>$3) - AND ($9::uuid[] IS NULL OR c.id=ANY($9)) - ORDER BY c.id LIMIT $4 + AND ($4::uuid[] IS NULL OR c.id=ANY($4)) + ORDER BY c.id LIMIT $6 ) - SELECT r.*, COALESCE(e.latest_id, latest.latest_id) AS latest_id, - encode(cf.through_message_id,'hex') AS read_through_id, - COALESCE(e.evidence,'[]'::jsonb) AS evidence FROM roster r + SELECT r.id, r.read_through_id, encode(l.id,'hex') AS latest_id, u.unread, + COALESCE(m.mentions, 0) AS mentions, COALESCE(th.items, '[]') AS threads + FROM roster r + CROSS JOIN LATERAL (SELECT GREATEST(r.position-$8, $9) AS walk_from) w LEFT JOIN LATERAL ( - WITH candidates AS MATERIALIZED ( - SELECT id,created_at,received_at,kind,deleted_at,tags FROM events - WHERE community_id=$1 AND channel_id=r.id - ORDER BY created_at DESC,id LIMIT $5 - ) - SELECT (array_agg(encode(c.id,'hex') ORDER BY c.received_at DESC,c.id) - FILTER (WHERE c.kind=ANY($6) AND c.deleted_at IS NULL - AND NOT COALESCE(tm.root_event_id<>c.id,false) - -- Same unresolved-reply test as `threaded` below. - AND NOT (tm.root_event_id IS NULL AND jsonb_typeof(c.tags)='array' - AND EXISTS (SELECT 1 FROM jsonb_array_elements(c.tags) t(tag) - WHERE jsonb_typeof(tag)='array' AND tag->>0='e' - AND tag->>3='reply' - AND (tag->>1) COLLATE "C" ~ '^[0123456789abcdefABCDEF]{64}$')) - ))[1] AS latest_id - FROM candidates c LEFT JOIN thread_metadata tm ON tm.community_id=$1 - AND tm.channel_id=r.id AND tm.event_created_at=c.created_at AND tm.event_id=c.id - ) latest ON true - LEFT JOIN personal_read_frontiers cf ON cf.community_id=$1 AND cf.actor=$2 - AND cf.channel_id=r.id AND cf.root_id=''::bytea + SELECT e.created_at FROM ( + SELECT {COLUMNS} FROM events e WHERE e.community_id=$1 AND e.channel_id=r.id + ORDER BY e.created_at DESC, e.id DESC LIMIT $10 + ) e {EVENT_THREAD_ROW} + WHERE {ELIGIBLE} AND {TIMELINE} + ORDER BY e.created_at DESC, e.id DESC LIMIT 1 + ) n ON true LEFT JOIN LATERAL ( - WITH candidates AS MATERIALIZED ( - SELECT id,pubkey,created_at,received_at,deleted_at,kind,tags - FROM events WHERE community_id=$1 AND channel_id=r.id AND created_at >= $7 - ORDER BY created_at DESC,id LIMIT $8 - ), threaded AS MATERIALIZED ( - SELECT e.*, tm.root_event_id AS root, tm.parent_event_id AS parent, - COALESCE(tm.root_event_id<>e.id,false) AS is_reply, - -- A NIP-10 reply marker without canonical ancestry: neither - -- counted nor a timeline anchor, since a mark cannot validate - -- it. Parity-tested against the shared NIP-10 parser. - tm.root_event_id IS NULL AND jsonb_typeof(e.tags)='array' - AND EXISTS (SELECT 1 FROM jsonb_array_elements(e.tags) t(tag) - WHERE jsonb_typeof(tag)='array' AND tag->>0='e' - AND tag->>3='reply' - AND (tag->>1) COLLATE "C" ~ '^[0123456789abcdefABCDEF]{64}$') - AS unresolved - FROM candidates e - LEFT JOIN thread_metadata tm ON tm.community_id=$1 AND tm.channel_id=r.id - AND tm.event_created_at=e.created_at AND tm.event_id=e.id - WHERE e.kind=ANY($6) AND e.deleted_at IS NULL - ), classified AS ( - SELECT e.*, - -- Every frontier is floored at the account's start. - COALESCE(e.received_at <= GREATEST(COALESCE($10::timestamptz,'infinity'), - CASE WHEN e.is_reply THEN tf.through_timestamp - ELSE cf.through_timestamp END),false) AS covered, - encode(tf.through_message_id,'hex') AS read_through_id - FROM threaded e - LEFT JOIN personal_read_frontiers tf ON tf.community_id=$1 AND tf.actor=$2 - AND tf.channel_id=r.id AND tf.root_id=e.root AND e.is_reply - WHERE e.pubkey<>$2 AND NOT e.unresolved - ), grouped AS ( - SELECT CASE WHEN root IS NULL THEN NULL - WHEN is_reply THEN encode(root,'hex') ELSE '' END AS root, - CASE WHEN is_reply THEN encode(parent,'hex') END AS parent, - is_reply, covered, read_through_id, - -- ->>0 also selects scalar "p"/"e": reject nonarrays first. - -- C collation matches Rust's ASCII case rules. - CASE WHEN octet_length(tags::text)<=8192 - AND jsonb_typeof(tags)='array' THEN - (SELECT CASE WHEN bool_or(jsonb_typeof(tag)<>'array' - OR jsonb_path_exists(tag,'strict $[*] ? (@.type() != "string")')) - THEN NULL ELSE jsonb_build_object( - 'directed',COALESCE(bool_or( - (tag->>0='p' AND lower((tag->>1) COLLATE "C")=encode($2,'hex')) - OR (tag->>0='broadcast' AND tag->>1='1')),false)) END - FROM jsonb_array_elements(tags) t(tag) - WHERE tag->>0 IN ('p','broadcast','e')) ELSE NULL END AS facts, - count(*) AS n, - (extract(epoch FROM max(received_at))*1000000)::bigint AS newest_arrival, - (array_agg(encode(id,'hex') ORDER BY received_at DESC,id))[1] AS newest_id, - (array_agg(extract(epoch FROM created_at)::bigint - ORDER BY received_at DESC,id))[1] AS newest_at - FROM classified - WHERE NOT covered OR root IS NULL - GROUP BY 1,2,3,4,5,6 - ) - SELECT (array_agg(encode(id,'hex') ORDER BY received_at DESC,id) - FILTER (WHERE NOT is_reply AND NOT unresolved))[1] AS latest_id, - (SELECT jsonb_agg(to_jsonb(grouped)) FROM grouped) AS evidence - FROM threaded - ) e ON true ORDER BY r.id"#, - ).bind(community.as_uuid()).bind(actor_bytes.as_slice()).bind(after) - .bind((limit+1) as i64).bind(MAX_CHANNEL_SCAN as i64) - .bind(ELIGIBLE_KINDS.as_slice()) - .bind(DateTime::from_timestamp_millis(account.cutoff_ms) - .ok_or_else(|| DbError::InvalidData("invalid unread cutoff".into()))?) - .bind(MAX_UNREAD_SCAN as i64) + SELECT e.id, e.pubkey, e.created_at, e.received_at FROM ( + SELECT {COLUMNS} FROM events e WHERE e.community_id=$1 AND e.channel_id=r.id + AND e.created_at BETWEEN n.created_at-2*$8 AND n.created_at + ORDER BY e.created_at DESC, e.id DESC LIMIT $10 + ) e {EVENT_THREAD_ROW} + WHERE {ELIGIBLE} AND {TIMELINE} + ORDER BY e.received_at DESC, e.id LIMIT 1 + ) l ON true + -- The last arrival is unread exactly when the timeline is: the + -- actor's own posts read through themselves. + CROSS JOIN LATERAL ( + SELECT COALESCE(l.received_at > r.position AND l.pubkey<>$2 + AND l.created_at >= $9, false) AS unread + ) u + LEFT JOIN LATERAL ( + SELECT count(*) AS mentions FROM ( + SELECT 1 FROM ( + SELECT {COLUMNS} FROM events e WHERE r.dm AND e.community_id=$1 + AND e.channel_id=r.id AND e.created_at >= w.walk_from + ORDER BY e.created_at, e.id LIMIT $10 + ) e {EVENT_THREAD_ROW} + WHERE e.received_at > r.position AND e.pubkey<>$2 AND {ELIGIBLE} AND {TIMELINE} + UNION ALL + SELECT 1 FROM event_mentions em JOIN events e ON e.community_id=$1 + AND e.created_at=em.event_created_at AND e.id=em.event_id {EVENT_THREAD_ROW} + WHERE NOT r.dm AND em.community_id=$1 AND em.pubkey_hex=$11 + AND em.channel_id=r.id AND em.event_created_at >= w.walk_from + AND e.channel_id=r.id AND e.received_at > r.position + AND e.pubkey<>$2 AND {ELIGIBLE} AND {TIMELINE} + LIMIT $12 + ) x WHERE u.unread + ) m ON true + LEFT JOIN LATERAL ( + SELECT jsonb_agg(jsonb_build_object('root_id',encode(t.root_id,'hex'), + 'unread',true,'mentions',t.n,'read_through_id',t.read_through_id, + 'latest_id',t.latest_id) ORDER BY t.latest_at DESC, t.root_id) AS items + FROM ( + SELECT tf.root_id, encode(tf.through_message_id,'hex') AS read_through_id, + encode(tl.id,'hex') AS latest_id, tl.created_at AS latest_at, tc.n + FROM personal_read_frontiers tf + CROSS JOIN LATERAL (SELECT GREATEST(tf.through_timestamp, COALESCE($7, 'infinity')) AS position) p + CROSS JOIN LATERAL ( + SELECT tm.event_created_at AS created_at FROM thread_metadata tm + WHERE tm.community_id=$1 AND tm.root_event_id=tf.root_id + ORDER BY tm.event_created_at DESC, tm.event_id LIMIT 1 + ) tq + CROSS JOIN LATERAL ( + -- Eligibility is a per-row filter, not a join, so the + -- planner walks the window index newest-first and stops + -- at the first eligible reply instead of sorting them all. + SELECT tm.event_created_at AS created_at FROM thread_metadata tm + WHERE tm.community_id=$1 AND tm.root_event_id=tf.root_id + AND tm.channel_id=r.id AND tm.event_id<>tf.root_id + AND (SELECT {ELIGIBLE} FROM events e WHERE e.community_id=$1 + AND e.created_at=tm.event_created_at AND e.id=tm.event_id) + ORDER BY tm.event_created_at DESC, tm.event_id LIMIT 1 + ) tn + CROSS JOIN LATERAL ( + SELECT e.id, e.created_at, e.received_at FROM ( + SELECT e.id, e.created_at, e.received_at FROM {THREAD_REPLIES} + AND tm.event_created_at BETWEEN tn.created_at-2*$8 AND tn.created_at + AND {ELIGIBLE} + ORDER BY tm.event_created_at DESC, tm.event_id LIMIT $10 + ) e + ORDER BY e.received_at DESC, e.id LIMIT 1 + ) tl + CROSS JOIN LATERAL ( + SELECT count(*) AS n FROM ( + SELECT 1 FROM {THREAD_REPLIES} + AND tm.event_created_at >= GREATEST(p.position-$8, $9) + AND e.received_at > p.position AND e.pubkey<>$2 AND {ELIGIBLE} + LIMIT $12 + ) x + ) tc + WHERE tf.community_id=$1 AND tf.actor=$2 AND tf.channel_id=r.id + AND tf.root_id<>''::bytea + AND tq.created_at+$8 > p.position AND tl.received_at > p.position + AND tc.n > 0 + ORDER BY tl.created_at DESC, tf.root_id LIMIT $13 + ) t + ) th ON true + ORDER BY r.id"# + ); + let cutoff = DateTime::from_timestamp_millis(account.cutoff_ms) + .ok_or_else(|| DbError::InvalidData("invalid unread cutoff".into()))?; + // Only compile-time constants are interpolated. + let rows = sqlx::query(sqlx::AssertSqlSafe(sql)) + .bind(community.as_uuid()) + .bind(actor_bytes.as_slice()) + .bind(after) .bind(only) + .bind(ELIGIBLE_KINDS.as_slice()) + .bind((limit + 1) as i64) .bind(account.started_at) - .fetch_all(&mut *tx).await?; + .bind(skew()) + .bind(cutoff) + .bind(MAX_TIMELINE_SCAN as i64) + .bind(actor.to_hex()) + .bind(MAX_UNREAD_COUNT as i64) + .bind(MAX_THREAD_SUMMARIES as i64) + .fetch_all(&mut *tx) + .await?; + tx.commit().await?; let has_more = rows.len() > limit; - // An unstarted account has read every arrival, so nothing is unread. - let started = account.started_at.is_some(); - let mut channels = Vec::new(); - let mut pending = Vec::new(); + let mut channels = Vec::with_capacity(limit); for row in rows.into_iter().take(limit) { - let evidence: Value = row.try_get("evidence")?; - let evidence = evidence - .as_array() - .ok_or_else(|| DbError::InvalidData("invalid sidebar evidence".into()))?; - let evidence = if started { evidence.as_slice() } else { &[] }; - let channel_type: String = row.try_get("channel_type")?; - let mut unread = false; - let mut mentions = 0; - let mut threads: HashMap, Replies> = HashMap::new(); - let mut undirected = Vec::new(); - for e in evidence { - let n = e["n"] - .as_u64() - .filter(|n| *n <= MAX_UNREAD_SCAN as u64) - .ok_or_else(|| DbError::InvalidData("invalid evidence multiplicity".into()))? - as u32; - // Unusable tags prove nothing: the message is left out. - let Some(facts) = e["facts"].as_object() else { - continue; - }; - if e["covered"] == true { - continue; - } - let directed = - channel_type == "dm" || facts.get("directed") == Some(&Value::Bool(true)); - // Roots are timeline messages; descendants belong exclusively - // to their canonical thread. Never inherit the channel prefix. - if e["is_reply"] != true { - unread = true; - if directed { - mentions += n; - } - continue; - } - let Some(root) = e["root"].as_str().and_then(writes::event_id) else { - continue; - }; - let replies = Replies { - n, - newest: ( - e["newest_arrival"].as_i64().ok_or_else(invalid_newest)?, - e["newest_id"] - .as_str() - .ok_or_else(invalid_newest)? - .to_owned(), - ), - newest_at: e["newest_at"].as_i64().ok_or_else(invalid_newest)?, - read_through_id: e["read_through_id"].as_str().map(str::to_owned), - }; - // A directed reply counts whatever its conversation. Any other - // reply counts only in one of the actor's conversations. - if directed { - count(&mut threads, root, replies); - } else if let Some(parent) = e["parent"].as_str().and_then(writes::event_id) { - undirected.push((root, parent, replies)); - } - } - pending.push((threads, undirected)); + let threads: Vec = + serde_json::from_value(row.try_get("threads")?) + .map_err(|_| DbError::InvalidData("invalid thread summaries".into()))?; + let mentions: i64 = row.try_get("mentions")?; channels.push(ChannelReadSummary { channel_id: row.try_get("id")?, - unread, - mentions, + unread: row.try_get("unread")?, + mentions: mentions as u32, read_through_id: row.try_get("read_through_id")?, latest_id: row.try_get("latest_id")?, - // Threads wait for conversation membership below. - threads: Vec::new(), + threads, }); } - let targets: Vec<_> = channels - .iter() - .zip(&pending) - .flat_map(|(channel, (_, undirected))| { - undirected - .iter() - .map(|(_, parent, _)| (channel.channel_id, parent.clone())) - }) - .collect(); - let members = participation::resolve(&mut tx, community, &actor_bytes, &targets).await?; - for (channel, (mut threads, undirected)) in channels.iter_mut().zip(pending) { - // Relevance is decided before the thread cap. An undecided - // membership leaves the reply out. - for (root, parent, replies) in undirected { - if members.get(&(channel.channel_id, parent)) == Some(&true) { - count(&mut threads, root, replies); - } - } - channel.threads = summarize(threads); - } let next_cursor = if has_more { channels.last().map(|c| c.channel_id) } else { None }; - tx.commit().await?; Ok(SidebarPage { channels, next_cursor, @@ -304,61 +279,8 @@ impl Db { } } -/// Uncovered replies in one canonical thread: one evidence group, or the -/// thread's counted total. -struct Replies { - n: u32, - /// Last of them to arrive: (arrival microseconds, lowercase hex ID). - newest: (i64, String), - /// Its author seconds. - newest_at: i64, - /// The thread's frontier anchor, shared by every group of the thread. - read_through_id: Option, -} - -/// Add replies that count to their thread. -fn count(threads: &mut HashMap, Replies>, root: Vec, replies: Replies) { - match threads.entry(root) { - Entry::Vacant(slot) => { - slot.insert(replies); - } - Entry::Occupied(mut slot) => { - let thread = slot.get_mut(); - thread.n += replies.n; - // Latest arrival first; equal arrivals break toward the smaller ID. - if (replies.newest.0, Reverse(&replies.newest.1)) - > (thread.newest.0, Reverse(&thread.newest.1)) - { - thread.newest = replies.newest; - thread.newest_at = replies.newest_at; - } - } - } -} - -fn invalid_newest() -> DbError { - DbError::InvalidData("invalid thread evidence".into()) -} - -/// Order newest unread reply first (root ID breaks ties) and cap the list. -fn summarize(threads: HashMap, Replies>) -> Vec { - let mut threads: Vec<_> = threads.into_iter().collect(); - threads.sort_unstable_by(|(a_root, a), (b_root, b)| { - b.newest_at - .cmp(&a.newest_at) - .then_with(|| a_root.cmp(b_root)) - }); - threads - .into_iter() - .take(MAX_THREAD_SUMMARIES) - .map(|(root, thread)| ThreadReadSummary { - root_id: hex::encode(root), - unread: true, - mentions: thread.n, - read_through_id: thread.read_through_id, - latest_id: thread.newest.1, - }) - .collect() +fn skew() -> chrono::Duration { + chrono::Duration::seconds(i64::from(MAX_ARRIVAL_SKEW_SECONDS)) } /// Read-time horizon and the account's start, in the caller's snapshot. @@ -384,37 +306,3 @@ pub(super) async fn read_account( started_at: row.try_get("started_at")?, }) } - -#[cfg(test)] -mod tests { - use super::*; - - fn replies(at: i64) -> Replies { - Replies { - n: 1, - newest: (at, String::new()), - newest_at: at, - read_through_id: None, - } - } - - #[test] - fn thread_rows_order_newest_first_break_ties_by_root_and_cap_at_twenty_five() { - // Literal contract boundaries deliberately do not derive from the constant. - for count in [0_u8, 1, 24, 25, 26, 27] { - // Pairwise-equal times exercise the tie-break. - let threads = (0..count) - .map(|i| (vec![i; 32], replies(i64::from(i / 2)))) - .collect(); - let roots: Vec<_> = summarize(threads).into_iter().map(|t| t.root_id).collect(); - let mut expected: Vec<_> = (0..count).map(|i| (i64::from(i / 2), i)).collect(); - expected.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(&b.1))); - let expected: Vec<_> = expected - .into_iter() - .take(25) - .map(|(_, i)| hex::encode([i; 32])) - .collect(); - assert_eq!(roots, expected, "count {count}"); - } - } -} diff --git a/crates/buzz-db/src/store/personal_read/projection_postgres_tests.rs b/crates/buzz-db/src/store/personal_read/projection_postgres_tests.rs index 3dc5c131b36..f4fb0976776 100644 --- a/crates/buzz-db/src/store/personal_read/projection_postgres_tests.rs +++ b/crates/buzz-db/src/store/personal_read/projection_postgres_tests.rs @@ -1,4 +1,6 @@ use super::{postgres_tests::fixture, *}; +use crate::Db; +use nostr::{EventBuilder, Keys, Kind, Tag}; use serde_json::json; #[tokio::test] @@ -9,6 +11,16 @@ async fn sidebar_sql_eligibility_follows_kind_author_deletion_and_horizon() { let horizon = i64::from(DEFAULT_RETENTION_SECONDS) * 1000; // Addressed to the actor, so a counted message is also exactly one mention. let tags = json!([["p", actor.public_key().to_hex()]]); + sqlx::query( + "INSERT INTO event_mentions + (community_id,pubkey_hex,event_id,event_created_at,channel_id,event_kind) + SELECT community_id,$2,id,created_at,channel_id,kind FROM events WHERE community_id=$1", + ) + .bind(community.as_uuid()) + .bind(actor.public_key().to_hex()) + .execute(&pool) + .await + .unwrap(); for (kind, eligible_kind) in [ (9, true), (40002, true), @@ -32,6 +44,8 @@ async fn sidebar_sql_eligibility_follows_kind_author_deletion_and_horizon() { .bind(community.as_uuid()).bind(kind) .bind(if own { actor.public_key().to_bytes() } else { event.pubkey.to_bytes() }.as_slice()) .bind(deleted).bind(created as f64).bind(&tags).execute(&pool).await.unwrap(); + sqlx::query("UPDATE event_mentions SET event_created_at=to_timestamp($2::double precision/1000) WHERE community_id=$1") + .bind(community.as_uuid()).bind(created as f64).execute(&pool).await.unwrap(); let page = db .personal_read_sidebar( community, @@ -55,56 +69,6 @@ async fn sidebar_sql_eligibility_follows_kind_author_deletion_and_horizon() { } } -#[tokio::test] -#[ignore = "requires Postgres"] -async fn sidebar_compacted_tags_preserve_directed_and_corruption_rules() { - let (db, pool, community, _, actor, _) = fixture().await; - let actor_hex = actor.public_key().to_hex().to_uppercase(); - // Unusable tags prove nothing: that message is left out, neither unread - // nor a mention. - for (tags, unread, mentions) in [ - (json!([]), true, 0), - (json!(["p"]), false, 0), - (json!(["e"]), false, 0), - (json!(["broadcast"]), false, 0), - (json!([["p", actor_hex, "relay", "petname"]]), true, 1), - (json!([["p", "00".repeat(32)]]), true, 0), - (json!([["broadcast", "1", "extra"]]), true, 1), - (json!([["broadcast", "0"]]), true, 0), - (json!([["p", "00".repeat(32), 42]]), false, 0), - (json!([["broadcast", "0", 42]]), false, 0), - (json!([["e", "00".repeat(32), "", "root", 42]]), false, 0), - (json!([["p", actor_hex], ["p", "other", 42]]), false, 0), - (json!([["e", "00".repeat(32), "", "reply"]]), false, 0), - (json!([["x", 42]]), true, 0), - (json!({"p":actor_hex}), false, 0), - (json!([["p", "x".repeat(8193)]]), false, 0), - ] { - sqlx::query("UPDATE events SET tags=$2 WHERE community_id=$1") - .bind(community.as_uuid()) - .bind(&tags) - .execute(&pool) - .await - .unwrap(); - let page = db - .personal_read_sidebar( - community, - &actor.public_key(), - DEFAULT_RETENTION_SECONDS, - 20, - None, - ) - .await - .unwrap(); - let row = &page.channels[0]; - assert_eq!( - (row.unread, row.mentions), - (unread, mentions), - "tags={tags}" - ); - } -} - #[tokio::test] #[ignore = "requires Postgres"] async fn sidebar_ancestry_fact_matches_shared_nip10_parser() { @@ -169,7 +133,7 @@ async fn sidebar_ancestry_fact_matches_shared_nip10_parser() { #[tokio::test] #[ignore = "requires Postgres"] -async fn sidebar_directed_fact_follows_dm_mention_and_broadcast_rules() { +async fn sidebar_mentions_are_p_tags_in_a_stream_and_everything_in_a_dm() { let (db, pool, community, channel, actor, _) = fixture().await; let actor_hex = actor.public_key().to_hex(); let fullwidth: String = actor_hex @@ -177,54 +141,60 @@ async fn sidebar_directed_fact_follows_dm_mention_and_broadcast_rules() { .map(|c| if c.is_ascii_alphabetic() { 'Ａ' } else { c }) .collect(); assert_ne!(fullwidth, actor_hex); - // Whether each tag set is directed in a stream; in a DM every one is. + let tag = |parts: &[&str]| Tag::parse(parts.iter().copied()).unwrap(); + // Whether each message mentions the actor in a stream. A top-level + // broadcast is not a mention. let cases = [ - (json!([]), false), - (json!([["p", actor_hex]]), true), - (json!([["p", actor_hex.to_uppercase()]]), true), - (json!([["p", fullwidth]]), false), - (json!([["p"]]), false), - (json!([["broadcast", "1"]]), true), - (json!([["broadcast", "true"]]), false), - (json!([["p", "00".repeat(32)]]), false), + (vec![], false), + (vec![tag(&["p", &actor_hex])], true), + (vec![tag(&["p", &actor_hex.to_uppercase()])], true), + (vec![tag(&["p", &fullwidth])], false), + (vec![tag(&["broadcast", "1"])], false), + (vec![tag(&["p", &"00".repeat(32)])], false), ( - json!([["broadcast", "0"], ["p", actor_hex.to_uppercase()]]), + vec![tag(&["broadcast", "0"]), tag(&["p", &actor_hex])], true, ), ]; - for channel_type in ["stream", "dm"] { - sqlx::query( - "UPDATE channels SET channel_type=$3::channel_type WHERE community_id=$1 AND id=$2", - ) + let mut expected = 0; + for (tags, mentioned) in &cases { + let event = EventBuilder::new(Kind::Custom(9), "case") + .tags(tags.clone()) + .sign_with_keys(&Keys::generate()) + .unwrap(); + db.insert_event(community, &event, Some(channel)) + .await + .unwrap(); + expected += u32::from(*mentioned); + assert_eq!( + mentions(&db, community, &actor).await, + expected, + "tags={tags:?}" + ); + } + sqlx::query("UPDATE channels SET channel_type='dm' WHERE community_id=$1 AND id=$2") .bind(community.as_uuid()) .bind(channel) - .bind(channel_type) .execute(&pool) .await .unwrap(); - for (tags, in_stream) in &cases { - let expected = u32::from(channel_type == "dm" || *in_stream); - sqlx::query("UPDATE events SET tags=$2 WHERE community_id=$1") - .bind(community.as_uuid()) - .bind(tags) - .execute(&pool) - .await - .unwrap(); - let page = db - .personal_read_sidebar( - community, - &actor.public_key(), - DEFAULT_RETENTION_SECONDS, - 20, - None, - ) - .await - .unwrap(); - assert!(page.channels[0].unread); - assert_eq!( - page.channels[0].mentions, expected, - "channel_type={channel_type} tags={tags}" - ); - } - } + // Every message by someone else, the fixture's included. + assert_eq!( + mentions(&db, community, &actor).await, + cases.len() as u32 + 1 + ); +} + +async fn mentions(db: &Db, community: buzz_core::CommunityId, actor: &Keys) -> u32 { + db.personal_read_sidebar( + community, + &actor.public_key(), + DEFAULT_RETENTION_SECONDS, + 20, + None, + ) + .await + .unwrap() + .channels[0] + .mentions } diff --git a/crates/buzz-db/src/store/personal_read/threads_postgres_tests.rs b/crates/buzz-db/src/store/personal_read/threads_postgres_tests.rs index 2bb86aab8da..b366279eea8 100644 --- a/crates/buzz-db/src/store/personal_read/threads_postgres_tests.rs +++ b/crates/buzz-db/src/store/personal_read/threads_postgres_tests.rs @@ -1,12 +1,14 @@ //! Thread summaries, anchor validation and targeted refresh. -use super::{postgres_tests::fixture, *}; +use super::{ + postgres_tests::{fixture, insert_reply}, + *, +}; use crate::{ channel::{ChannelType, ChannelVisibility}, Db, }; use buzz_core::CommunityId; use nostr::{EventBuilder, Keys, Kind, Tag}; -use sqlx::PgPool; use uuid::Uuid; async fn post( @@ -38,48 +40,41 @@ async fn post_as( event } -/// A canonical reply to the root: stored event plus its thread metadata. +/// A canonical reply to the root, through ingest. async fn reply( db: &Db, - pool: &PgPool, community: CommunityId, channel: Uuid, root: &nostr::Event, at: u64, tags: Vec, ) -> nostr::Event { - let event = post(db, community, channel, at, tags).await; - link(pool, community, channel, root, &event).await; - event + reply_as(db, community, channel, root, &Keys::generate(), at, tags).await } -async fn link( - pool: &PgPool, +async fn reply_as( + db: &Db, community: CommunityId, channel: Uuid, root: &nostr::Event, - event: &nostr::Event, -) { - sqlx::query("INSERT INTO thread_metadata (community_id,event_id,event_created_at,channel_id,root_event_id,parent_event_id,depth) - VALUES ($1,$2,to_timestamp($3),$4,$5,$5,1)") - .bind(community.as_uuid()).bind(event.id.as_bytes().as_slice()) - .bind(event.created_at.as_secs() as f64) - .bind(channel).bind(root.id.as_bytes().as_slice()).execute(pool).await.unwrap(); + author: &Keys, + at: u64, + tags: Vec, +) -> nostr::Event { + let event = EventBuilder::new(Kind::Custom(9), format!("reply at {at}")) + .tags(tags) + .custom_created_at(nostr::Timestamp::from(at)) + .sign_with_keys(author) + .unwrap(); + insert_reply(db, community, channel, root, &event).await; + event } -/// Put the actor in the root's conversation, so that plain replies to it -/// count. The actor's own reply is never unread. -async fn join( - db: &Db, - pool: &PgPool, - community: CommunityId, - channel: Uuid, - actor: &Keys, - root: &nostr::Event, -) { +/// The actor replies to the root, and so follows its thread. The actor's own +/// reply is never unread. +async fn join(db: &Db, community: CommunityId, channel: Uuid, actor: &Keys, root: &nostr::Event) { let at = root.created_at.as_secs(); - let own = post_as(db, community, channel, actor, at, vec![]).await; - link(pool, community, channel, root, &own).await; + reply_as(db, community, channel, root, actor, at, vec![]).await; } async fn sidebar(db: &Db, community: CommunityId, actor: &Keys) -> ChannelReadSummary { @@ -162,35 +157,6 @@ async fn mark_through_rejects_unresolved_missing_auxiliary_and_malformed_anchors assert_eq!(row.read_through_id, None, "nothing was marked"); } -#[tokio::test] -#[ignore = "requires Postgres"] -async fn channel_frontier_never_covers_an_epoch_zero_reply() { - let (db, pool, community, channel, actor, root) = fixture().await; - join(&db, &pool, community, channel, &actor, &root).await; - let ancient = reply(&db, &pool, community, channel, &root, 0, vec![]).await; - // A channel frontier exists, with no thread frontier: absent must not - // read as epoch zero or as the channel's. - apply( - &db, - community, - &actor, - channel_mark(channel, root.id.to_hex()), - ) - .await; - // Widen the horizon to reach the epoch, so that only a misread absent - // frontier could hide this reply. - let row = db - .personal_read_sidebar(community, &actor.public_key(), u32::MAX, 20, None) - .await - .unwrap() - .channels - .remove(0); - assert!(!row.unread); - assert_eq!(row.threads.len(), 1, "epoch-zero reply stays unread"); - assert_eq!(row.threads[0].mentions, 1); - assert_eq!(row.threads[0].latest_id, ancient.id.to_hex()); -} - #[tokio::test] #[ignore = "requires Postgres"] async fn thread_summaries_order_cap_and_anchor() { @@ -202,7 +168,7 @@ async fn thread_summaries_order_cap_and_anchor() { let mut roots = Vec::new(); for i in 0..n { let root = post(&db, community, channel, base + i, vec![]).await; - join(&db, &pool, community, channel, &actor, &root).await; + join(&db, community, channel, &actor, &root).await; roots.push(root); } let newest_root = roots.last().unwrap().clone(); @@ -220,11 +186,10 @@ async fn thread_summaries_order_cap_and_anchor() { let mut anchors = Vec::new(); for (i, root) in (0..).zip(&roots) { let at = base + 100 + offset(i); - anchors.push(reply(&db, &pool, community, channel, root, at, vec![]).await); + anchors.push(reply(&db, community, channel, root, at, vec![]).await); } reply( &db, - &pool, community, channel, &roots[2], @@ -234,7 +199,6 @@ async fn thread_summaries_order_cap_and_anchor() { .await; let twin = reply( &db, - &pool, community, channel, &roots[2], @@ -305,7 +269,7 @@ impl ThreadReadSummary { #[tokio::test] #[ignore = "requires Postgres"] async fn thread_on_a_never_unread_root_is_selectable_and_readable_by_itself() { - let (db, pool, community, channel, actor, root) = fixture().await; + let (db, _pool, community, channel, actor, root) = fixture().await; let base = root.created_at.as_secs(); let mention = || vec![Tag::parse(["p", &actor.public_key().to_hex()]).unwrap()]; // A diff is never unread, not even one addressed to the actor. @@ -317,8 +281,8 @@ async fn thread_on_a_never_unread_root_is_selectable_and_readable_by_itself() { db.insert_event(community, &diff, Some(channel)) .await .unwrap(); - let on_diff = reply(&db, &pool, community, channel, &diff, base + 10, mention()).await; - let elsewhere = reply(&db, &pool, community, channel, &root, base + 20, mention()).await; + let on_diff = reply(&db, community, channel, &diff, base + 10, mention()).await; + let elsewhere = reply(&db, community, channel, &root, base + 20, mention()).await; let row = sidebar(&db, community, &actor).await; // The undirected fixture root is unread; the directed diff is not counted. diff --git a/crates/buzz-db/src/store/personal_read/writes.rs b/crates/buzz-db/src/store/personal_read/writes.rs index 9d4c4a6586c..26736ff69a7 100644 --- a/crates/buzz-db/src/store/personal_read/writes.rs +++ b/crates/buzz-db/src/store/personal_read/writes.rs @@ -24,7 +24,8 @@ pub(super) async fn deadlines(conn: &mut PgConnection) -> Result<()> { } /// Start the account if this is the actor's first read intent, then serialize -/// private frontier writes, never shared conversation rows. +/// private frontier writes, never shared conversation rows. NO KEY UPDATE +/// leaves ingest's foreign-key checks (which create thread rows) unblocked. pub(super) async fn lock_account( conn: &mut PgConnection, community: CommunityId, @@ -40,7 +41,8 @@ pub(super) async fn lock_account( .execute(&mut *conn) .await?; sqlx::query( - "SELECT actor FROM personal_read_accounts WHERE community_id=$1 AND actor=$2 FOR UPDATE", + "SELECT actor FROM personal_read_accounts WHERE community_id=$1 AND actor=$2 + FOR NO KEY UPDATE", ) .bind(community.as_uuid()) .bind(actor) @@ -159,7 +161,8 @@ pub(super) async fn valid_target( } /// Advance a monotone frontier. The anchor ID follows the greatest arrival; -/// an equal arrival keeps the existing anchor. +/// an equal arrival keeps the existing anchor. Reading a thread never follows +/// it: only a followed thread has a row to advance. async fn frontier( conn: &mut PgConnection, community: CommunityId, @@ -168,7 +171,7 @@ async fn frontier( root: &[u8], msg: &Message, ) -> Result<()> { - sqlx::query( + let sql = if root.is_empty() { "INSERT INTO personal_read_frontiers (community_id, actor, channel_id, root_id, through_timestamp, through_message_id) VALUES ($1,$2,$3,$4,$5,$6) @@ -176,10 +179,23 @@ async fn frontier( SET through_timestamp=GREATEST(personal_read_frontiers.through_timestamp, EXCLUDED.through_timestamp), through_message_id=CASE WHEN EXCLUDED.through_timestamp > personal_read_frontiers.through_timestamp - THEN EXCLUDED.through_message_id ELSE personal_read_frontiers.through_message_id END", - ).bind(community.as_uuid()).bind(actor).bind(target.channel_id).bind(root) - .bind(msg.received_at).bind(&msg.id) - .execute(&mut *conn).await?; + THEN EXCLUDED.through_message_id ELSE personal_read_frontiers.through_message_id END" + } else { + "UPDATE personal_read_frontiers + SET through_timestamp=GREATEST(through_timestamp, $5), + through_message_id=CASE WHEN $5 > through_timestamp + THEN $6 ELSE through_message_id END + WHERE community_id=$1 AND actor=$2 AND channel_id=$3 AND root_id=$4" + }; + sqlx::query(sql) + .bind(community.as_uuid()) + .bind(actor) + .bind(target.channel_id) + .bind(root) + .bind(msg.received_at) + .bind(&msg.id) + .execute(&mut *conn) + .await?; Ok(()) } diff --git a/crates/buzz-relay/src/api/buzz_v1/postgres_tests.rs b/crates/buzz-relay/src/api/buzz_v1/postgres_tests.rs index 63942726793..6d50c42815a 100644 --- a/crates/buzz-relay/src/api/buzz_v1/postgres_tests.rs +++ b/crates/buzz-relay/src/api/buzz_v1/postgres_tests.rs @@ -437,12 +437,15 @@ async fn accessory_write_revocation_is_terminal_before_persistence() { // Exercise the same per-item function the batch handler uses after admission. let result = super::handlers::write_intent(&state, &headers, &principal, &intent).await; assert_eq!(result, json!({"status":"blocked"})); - let persisted: i64 = - sqlx::query_scalar("SELECT count(*) FROM personal_read_accounts WHERE community_id=$1") - .bind(community.as_uuid()) - .fetch_one(state.db.pool()) - .await - .unwrap(); + // Ingest gave the message's author an account; the actor has none. + let persisted: i64 = sqlx::query_scalar( + "SELECT count(*) FROM personal_read_accounts WHERE community_id=$1 AND actor=$2", + ) + .bind(community.as_uuid()) + .bind(actor.public_key().to_bytes().as_slice()) + .fetch_one(state.db.pool()) + .await + .unwrap(); assert_eq!(persisted, 0, "denied intent must not persist"); } diff --git a/docs/buzz-v1-read-state.md b/docs/buzz-v1-read-state.md index f89e2e07740..5e0aab9a2c6 100644 --- a/docs/buzz-v1-read-state.md +++ b/docs/buzz-v1-read-state.md @@ -75,9 +75,9 @@ marking one never moves another. | Field | Channel row | Thread row | |---|---|---| | `unread` | an unread top-level message counts | an unread reply counts | -| `mentions` | unread top-level messages directed at you | unread replies that count | +| `mentions` | unread top-level messages that tag you with `p`, or every unread one in a DM (at most 100) | unread replies (at most 100) | | `read_through_id` | the anchor of the timeline's frontier, or null | the anchor of the thread's frontier, or null | -| `latest_id` | the last eligible top-level message to arrive, or null | the last counted unread reply to arrive | +| `latest_id` | the last eligible top-level message to arrive, or null | the last reply to arrive | A thread reply never makes the channel row unread and never becomes its `latest_id`, including a reply carrying `broadcast=1`: it shows on its thread @@ -88,31 +88,41 @@ channel `latest_id` counts any author and ignores read progress. Null means the bounded scans found no top-level message, not that the channel is empty. `read_through_id` may name a message that has since been deleted. -`threads` lists threads with unread replies that count, newest unread reply -first by author time (then `root_id`), at most 25. Threads past the 25th are -omitted with their mentions, and nothing signals the omission: after marking -the listed threads read, a refresh can list the next ones. A thread row's -`unread` is always true in this version. No message bytes are included. +`threads` lists the threads you follow that have unread replies, newest +unread reply first by author time (then `root_id`), at most 25. Threads past +the 25th are omitted with their mentions, and nothing signals the omission: +after marking the listed threads read, a refresh can list the next ones. A +thread row's `unread` is always true in this version. No message bytes are +included. A message is eligible when it is non-own, nondeleted, of the advertised `eligible_kinds` and inside the horizon. The same kinds alone define latest activity, so an edit, reaction or diff (40008) neither makes a channel unread nor moves it. Classify live arrivals with the advertised set, not a client copy. -A top-level message is directed at you when its channel is a DM, it tags you -with `p`, or it carries `broadcast=1`. A reply counts only when it is directed -in the same way, or it is in one of your conversations: you wrote its direct -parent or have a reply to that same parent, in that channel. Conversation -membership uses only your live eligible messages (a deleted parent proves -nothing; a surviving reply to it still does), looks at the direct parent only -(owning the root or replying elsewhere in the thread proves nothing), and is -independent of read progress and retention. This is not Desktop notification -policy: follows and mutes do not affect these counts. - -Counts are what the bounded scans could establish, and may undercount. A message -whose tags are unusable, whose ancestry is unresolved, or whose conversation -membership is undecided is left out, as are messages beyond the scan bounds -(see [Bounds](#bounds-and-deployment)). +### Following + +Every top-level message counts on its channel row. A reply counts only in a +thread you follow. The relay records follows when it stores an eligible message, +in the same transaction: + +- Its author has read through it: the timeline for a top-level message, + otherwise the thread, which the author now follows. +- Each channel member it tags with `p` follows its thread (for a top-level + message, the thread rooted at it). +- A reply also makes the root's author follow, and in a DM every member. + +A new follower starts just before the message that made them follow, so it is +unread; one who already follows keeps their position. Reading never follows: +a thread `mark_through` moves an existing follow and otherwise changes nothing. +There is no unfollow yet. `broadcast=1` does not make anyone follow, so a +broadcast reply counts only for followers. Follows start when the relay that +records them is deployed; there is no backfill. Desktop notification follows +and mutes do not affect these counts. + +Counts are what the bounded walks could establish, and may undercount. A +message whose ancestry is unresolved is left out, as are messages beyond the +bounds (see [Bounds](#bounds-and-deployment)). The unread horizon defaults to 30 days (`BUZZ_V1_RETENTION_SECONDS`) and is measured in author time (`created_at`). It filters unread and mentions, not @@ -186,11 +196,21 @@ microsecond resolution. Three limits follow, none of which strands a badge: skew. - Messages with the identical stamp are read together. -A channel's `latest_id` is the last top-level message to arrive among those the -unread count examined: the 4,096 most recent events by author time inside the -horizon. So marking through it reads every top-level message counted. When -that scan holds no top-level message, it is the last top-level message to -arrive among the channel's 256 most recent events. +Counting walks forward from a position. Positions are arrival times but the +event indexes are by author time, which the relay accepts within 15 minutes of +arrival, so a walk starts 15 minutes of author time before the position. A +message backdated further than that arrives unseen; Order above still holds for +what the walk finds. A channel's `latest_id` is found from the newest top-level +message by author time: it is the last top-level message to arrive among those +authored up to 30 minutes before that one, which includes the last arrival. So +marking through it reads every top-level message counted. A thread's +`latest_id` is found the same way among its replies. + +A channel row is unread exactly when its `latest_id` is someone else's, inside +the horizon, and arrived after the timeline's position. Nothing earlier needs +checking: posting reads the timeline through the post, so an own `latest_id` +means everything before it is read, and an unread one is the anchor that clears +the row. A row is never unread without an anchor to mark. There is no import of earlier client read state. The relay records the account's `started_at` at its first applied read intent and never moves it. @@ -204,16 +224,14 @@ device-local. - 20 sidebar rows and 100 intents per request; 25 thread rows per channel, so a POST's rows for 100 channels fit the response limit. - 64 KiB write body; 1 MiB serialized API response. -- 4096 raw events per channel inside the horizon, before eligibility. -- The latest fallback probes 256 events; a long ineligible or reply-only tail - can leave `latest_id` null. -- Tag documents over 8192 bytes or malformed relevant tags leave the message - out. Compact boolean facts cross the database boundary, never raw tag - payloads. -- Conversation membership: at most 1024 unique parents per request, with a - 500 ms savepoint budget. The lookup is exact, so its work grows with the - replies under each parent. Past either bound a reply is undecided and left - out. +- Counts stop at 100 per channel and per thread. +- Each walk reads at most 4096 raw events, before eligibility. The `latest_id` + probe reads the newest 4096 per channel; a longer ineligible or reply-only + tail (reactions, edits, thread replies) leaves it null and the row read. The + last-arrival search reads the newest 4096 of the 30-minute window, so more + than that many backdated arrivals can hide one. +- A caught-up reader costs a few index probes per channel and per followed + thread; a reader who is behind walks only what arrived after their position. - DB statement/lock deadlines and HTTP read deadlines bound work; a write's intents and its row refresh share one eight-second deadline after admission. Limits are containment, not a production capacity claim. @@ -222,9 +240,13 @@ Apply migrations 0056 through 0058 (or the equivalent desired schema). 0056 creates two empty private tables and no index on `events`, 0057 adds the nullable `personal_read_accounts.started_at` column, and 0058 adds the nullable `personal_read_frontiers.through_message_id` column and drops -`threads_through_timestamp`. Both sidebar scans are -served by the existing `idx_events_community_channel_created`. No new -per-message ingest write path or stored unread counters are introduced. +`threads_through_timestamp`. Timeline walks use the existing +`idx_events_community_channel_created`, thread walks +`idx_thread_metadata_window`, and mention counts +`idx_event_mentions_pubkey_created`. Ingest writes a few frontier rows per +eligible channel message (the author, and anyone it makes follow), whether or +not the API is enabled, so follows exist once it is; there are no stored unread +counters. Use existing HTTP route/status/latency metrics for `/buzz/v1/me/sidebar` and `/buzz/v1/me/read-state`, plus database pool/statement metrics. No payload, @@ -236,7 +258,7 @@ local seed is not a DAU/concurrency or p95/p99 production acceptance result. Within `/buzz/v1`, clients must ignore unknown response object fields. The shape is stable: existing fields, their types, status variants and what a `mark_through` does remain. What the counts include is not yet: which messages -are unread or directed, which threads are listed, and the bounds may change +are unread or mentions, which threads are listed, and the bounds may change while v1 is pre-release. A client therefore replaces its rows with each response rather than reconciling them with its own counts. Breaking shape changes require an explicitly negotiated contract or a new API version. @@ -253,8 +275,8 @@ additional capabilities. These typed relational frontiers are signer-private application state, **not self-encrypted**. Database operators can see reading progress; ordinary Nostr queries, search and moderator interfaces do not expose it. No public receipts -are emitted. Storage grows by touched contexts, not observed messages, and has -no fixed context-count ceiling. +are emitted. Storage grows by followed threads and posted-in channels, not +observed messages, and has no fixed context-count ceiling. Leaving/rejoining does not erase progress; revoked access hides it. Soft-deleted channels are inaccessible, while hard channel deletion cascades their frontiers. From 4da2b9f4e4cb21c9c5f7b2d4c9d9194baeee7279 Mon Sep 17 00:00:00 2001 From: Bradley Axen Date: Sun, 11 Oct 2026 16:14:33 -0700 Subject: [PATCH 2/4] feat(read-state): follow intents, threads view, arrival-ordered access paths Extend the merged /buzz/v1 read-state wire with the minimum the follow model needs: - `follow` / `unfollow` intents on a thread target. An unfollow sticks until the actor posts in the thread or is mentioned there again. - `GET /me/threads`: followed threads in joined channels, newest reply arrival first, cursor-paged, at most 50 per page. - Thread rows count `mentions` as direct mentions only. - Broadcast replies count on the channel timeline and are valid channel anchors. - NIP-11 advertises `capabilities: ["follow","threads"]` and `max_threads`. Migration 0061 adds `events.read_timeline`, `thread_metadata.received_at` and `personal_read_frontiers.unfollowed`, plus arrival-ordered partial indexes so the sidebar and threads view probe an index per channel and per follow instead of scanning a bounded window. No backfill. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Bradley Axen --- crates/buzz-db/src/runtime/migration.rs | 23 +- .../tests/thread_window_postgres_tests.rs | 17 +- crates/buzz-db/src/store/event.rs | 61 ++-- .../personal_read/arrival_postgres_tests.rs | 42 +-- .../src/store/personal_read/follows.rs | 116 +++++-- .../personal_read/follows_postgres_tests.rs | 294 ++++++++++++++++- crates/buzz-db/src/store/personal_read/mod.rs | 2 +- .../buzz-db/src/store/personal_read/model.rs | 79 ++++- .../src/store/personal_read/postgres_tests.rs | 27 +- .../src/store/personal_read/projection.rs | 300 ++++++++++-------- .../projection_postgres_tests.rs | 64 ++-- .../personal_read/threads_postgres_tests.rs | 20 +- .../buzz-db/src/store/personal_read/writes.rs | 56 +++- crates/buzz-relay/src/api/buzz_v1/handlers.rs | 69 +++- crates/buzz-relay/src/api/buzz_v1/mod.rs | 1 + .../src/api/buzz_v1/postgres_tests.rs | 112 ++++++- crates/buzz-relay/src/nip11.rs | 9 +- docs/buzz-v1-read-state.md | 162 +++++++--- migrations/0061_read_state_access_paths.sql | 68 ++++ schema/tables/public/event_mentions.sql | 2 + schema/tables/public/events.sql | 6 + .../tables/public/personal_read_frontiers.sql | 2 + schema/tables/public/thread_metadata.sql | 5 + 23 files changed, 1193 insertions(+), 344 deletions(-) create mode 100644 migrations/0061_read_state_access_paths.sql diff --git a/crates/buzz-db/src/runtime/migration.rs b/crates/buzz-db/src/runtime/migration.rs index b6ead54e06f..24830bbf118 100644 --- a/crates/buzz-db/src/runtime/migration.rs +++ b/crates/buzz-db/src/runtime/migration.rs @@ -705,7 +705,7 @@ mod postgres_tests { let mut migrations: Vec<_> = MIGRATOR.iter().collect(); migrations.sort_by_key(|migration| migration.version); - assert_eq!(migrations.len(), 60); + assert_eq!(migrations.len(), 61); assert_eq!(migrations[58].version, 59); assert!(migrations[58].sql.as_str().contains("buzz.push_enabled")); assert_eq!(migrations[59].version, 60); @@ -713,6 +713,11 @@ mod postgres_tests { .sql .as_str() .contains("ALTER TABLE push_leases DROP COLUMN app_profile")); + assert_eq!(migrations[60].version, 61); + assert!(migrations[60] + .sql + .as_str() + .contains("idx_events_read_timeline")); assert_eq!(migrations[55].version, 56); assert!(migrations[55] .sql @@ -2253,6 +2258,16 @@ mod postgres_tests { .sql .as_str(); assert!(through_message.contains("DROP COLUMN threads_through_timestamp")); + // 0061 adds the sticky thread unfollow. + let unfollowed_column = "unfollowed boolean not null default false \ + check (not unfollowed or root_id <> ''::bytea), "; + assert!(MIGRATOR + .iter() + .find(|m| m.version == 61) + .expect("read-state access path migration") + .sql + .as_str() + .contains("$d$BOOLEAN NOT NULL DEFAULT false CHECK (NOT unfollowed OR root_id <> ''::bytea)$d$")); for (table, definition) in personal.tables { let in_schema = schema.tables.get(&table).map(|schema_definition| { if table == "personal_read_accounts" { @@ -2262,7 +2277,11 @@ mod postgres_tests { } }); let definition = if table == "personal_read_frontiers" { - definition.replacen(threads_through_column, through_message_column, 1) + definition.replacen( + threads_through_column, + &format!("{through_message_column}{unfollowed_column}"), + 1, + ) } else { definition }; diff --git a/crates/buzz-db/src/runtime/tests/thread_window_postgres_tests.rs b/crates/buzz-db/src/runtime/tests/thread_window_postgres_tests.rs index beda4b199fc..eca8ea37652 100644 --- a/crates/buzz-db/src/runtime/tests/thread_window_postgres_tests.rs +++ b/crates/buzz-db/src/runtime/tests/thread_window_postgres_tests.rs @@ -270,9 +270,11 @@ async fn migration_schema_thread_window_prebuild_does_not_queue_behind_writer() .await .unwrap(); - // Also bind the public migrator: the advisory schema/destruction lock, - // ledger write and post-migration catalog checks must work with ingestion. - let production_result = migration::run_migrations(&pool).await; + // Also bind the migrator: the advisory schema/destruction lock, ledger + // write and post-migration catalog checks must work with ingestion. 0061 + // adds a column to thread_metadata, which takes a brief exclusive lock + // bounded by its lock_timeout, so it runs after the writer below. + let production_result = migration::run_migrations_through(&pool, 60).await; sqlx::query(insert) .bind(community) .bind(channel) @@ -280,10 +282,6 @@ async fn migration_schema_thread_window_prebuild_does_not_queue_behind_writer() .execute(&mut *witness) .await .unwrap(); - let version: i64 = sqlx::query_scalar("SELECT max(version) FROM _sqlx_migrations") - .fetch_one(&mut *witness) - .await - .unwrap(); let final_oid: i64 = sqlx::query_scalar("SELECT 'idx_thread_metadata_window'::regclass::oid::bigint") .fetch_one(&mut *witness) @@ -297,6 +295,11 @@ async fn migration_schema_thread_window_prebuild_does_not_queue_behind_writer() .unwrap(); drop(witness); writer.rollback().await.unwrap(); + migration::run_migrations(&pool).await.unwrap(); + let version: i64 = sqlx::query_scalar("SELECT max(version) FROM _sqlx_migrations") + .fetch_one(&pool) + .await + .unwrap(); drop_scratch_db(&admin, pool, &name).await; assert!( diff --git a/crates/buzz-db/src/store/event.rs b/crates/buzz-db/src/store/event.rs index 9eec332b44c..a217251d4ed 100644 --- a/crates/buzz-db/src/store/event.rs +++ b/crates/buzz-db/src/store/event.rs @@ -447,15 +447,9 @@ pub async fn insert_event_in_transaction( ) -> Result<(StoredEvent, bool)> { let result = insert_event_on(tx, event, channel_id).await?; if result.1 { - if let Some(channel) = channel_id { - crate::personal_read::record_message( - tx, - event, - channel, - crate::personal_read::Place::TopLevel, - result.0.received_at, - ) - .await?; + if let Some((channel, place)) = crate::personal_read::place(event, channel_id, None) { + crate::personal_read::record_message(tx, event, channel, place, result.0.received_at) + .await?; } crate::operator_listener::enqueue_mentions_in_transaction(tx, event).await?; } @@ -492,8 +486,8 @@ async fn insert_event_on( let not_before = extract_not_before(event); let result = sqlx::query( r#" - INSERT INTO events (community_id, id, pubkey, created_at, kind, tags, content, sig, received_at, channel_id, d_tag, not_before) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) + INSERT INTO events (community_id, id, pubkey, created_at, kind, tags, content, sig, received_at, channel_id, d_tag, not_before, read_timeline) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13) ON CONFLICT DO NOTHING "#, ) @@ -509,6 +503,7 @@ async fn insert_event_on( .bind(channel_id) .bind(d_tag.as_deref()) .bind(not_before) + .bind(crate::personal_read::place(event, channel_id, None).is_some_and(|(_, p)| p.on_timeline())) .execute(tx.conn()) .await?; @@ -1740,11 +1735,22 @@ pub(crate) async fn insert_event_with_thread_metadata_tx( let received_at = Utc::now(); let d_tag = extract_d_tag(event); let not_before = extract_not_before(event); + let place = match &thread_meta { + Some(meta) => crate::personal_read::place( + event, + Some(meta.channel_id), + meta.parent_event_id.map(|parent| match meta.root_event_id { + Some(root) => (root, meta.root_event_created_at, meta.broadcast), + None => (parent, meta.parent_event_created_at, meta.broadcast), + }), + ), + None => crate::personal_read::place(event, channel_id, None), + }; let result = sqlx::query( r#" - INSERT INTO events (community_id, id, pubkey, created_at, kind, tags, content, sig, received_at, channel_id, d_tag, not_before) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) + INSERT INTO events (community_id, id, pubkey, created_at, kind, tags, content, sig, received_at, channel_id, d_tag, not_before, read_timeline) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13) ON CONFLICT DO NOTHING "#, ) @@ -1760,6 +1766,7 @@ pub(crate) async fn insert_event_with_thread_metadata_tx( .bind(channel_id) .bind(d_tag.as_deref()) .bind(not_before) + .bind(place.is_some_and(|(_, p)| p.on_timeline())) .execute(tx.conn()) .await?; @@ -1774,6 +1781,12 @@ pub(crate) async fn insert_event_with_thread_metadata_tx( ) .await?; + // Before the thread counters below: their root row is shared by every + // reply, so its lock must not be held across these writes. + if let Some((channel, place)) = place { + crate::personal_read::record_message(tx, event, channel, place, received_at).await?; + } + if let Some(ref meta) = thread_meta { let broadcast_val: bool = meta.broadcast; @@ -1783,8 +1796,8 @@ pub(crate) async fn insert_event_with_thread_metadata_tx( (community_id, event_created_at, event_id, channel_id, parent_event_id, parent_event_created_at, root_event_id, root_event_created_at, - depth, broadcast) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) + depth, broadcast, received_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) ON CONFLICT DO NOTHING "#, ) @@ -1798,6 +1811,8 @@ pub(crate) async fn insert_event_with_thread_metadata_tx( .bind(meta.root_event_created_at) .bind(meta.depth) .bind(broadcast_val) + // An eligible reply's arrival, for the read-state thread index. + .bind(place.and(meta.parent_event_id).map(|_| received_at)) .execute(tx.conn()) .await?; @@ -1882,22 +1897,6 @@ pub(crate) async fn insert_event_with_thread_metadata_tx( } } - let place = match &thread_meta { - Some(meta) => Some(( - meta.channel_id, - match meta.parent_event_id { - Some(parent) => crate::personal_read::Place::Reply { - root: meta.root_event_id.unwrap_or(parent), - }, - // Workflow messages store a depth-0 metadata row. - None => crate::personal_read::Place::TopLevel, - }, - )), - None => channel_id.map(|channel| (channel, crate::personal_read::Place::TopLevel)), - }; - if let Some((channel, place)) = place { - crate::personal_read::record_message(tx, event, channel, place, received_at).await?; - } crate::operator_listener::enqueue_mentions_in_transaction(tx, event).await?; } diff --git a/crates/buzz-db/src/store/personal_read/arrival_postgres_tests.rs b/crates/buzz-db/src/store/personal_read/arrival_postgres_tests.rs index f8add017bf1..d0bb2ca9006 100644 --- a/crates/buzz-db/src/store/personal_read/arrival_postgres_tests.rs +++ b/crates/buzz-db/src/store/personal_read/arrival_postgres_tests.rs @@ -1,7 +1,7 @@ //! Read progress follows relay arrival (`received_at`), never author time. //! Each case sets every arrival explicitly: back-to-back inserts share a clock. use super::{ - postgres_tests::{fixture, insert_reply}, + postgres_tests::{fixture, insert_reply, sync_reply_arrivals}, *, }; use crate::Db; @@ -23,6 +23,7 @@ async fn arrive(pool: &PgPool, community: CommunityId, event: &nostr::Event, arr .unwrap() .rows_affected(); assert_eq!(updated, 1, "arrival must land on exactly one stored event"); + sync_reply_arrivals(pool).await; } /// A message authored at `authored` that arrives at `arrived`. It mentions the @@ -254,10 +255,9 @@ async fn mark_thread_read_with_the_sidebar_anchor_clears_a_late_reply() { assert!(row.threads.is_empty()); } -/// The latest probe reads the newest `MAX_TIMELINE_SCAN` events by author -/// time, of any kind. A late arrival with an older author time than the -/// uncounted events after it must still be the sidebar anchor, or Mark as read -/// with that anchor leaves it unread. +/// A late arrival with an older author time than the uncounted events after +/// it must still be the sidebar anchor, or Mark as read with that anchor +/// leaves it unread. async fn reactions( pool: &PgPool, db: &Db, @@ -310,37 +310,6 @@ async fn mark_as_read_clears_a_late_arrival_behind_reactions() { assert_eq!(counts(&db, community, &actor).await, (false, 0)); } -/// Past the budget the probe finds no anchor, and nothing counts: a badge -/// with no latest to mark could never be cleared. -#[tokio::test] -#[ignore = "requires Postgres"] -async fn a_message_behind_a_full_budget_of_reactions_is_not_a_badge_without_an_anchor() { - let (db, pool, community, channel, actor, first) = fixture().await; - let now = first.created_at.as_secs(); - let demoted = sqlx::query("UPDATE events SET kind=7 WHERE community_id=$1 AND id=$2") - .bind(community.as_uuid()) - .bind(first.id.as_bytes().as_slice()) - .execute(&pool) - .await - .unwrap() - .rows_affected(); - assert_eq!(demoted, 1); - post(&db, &pool, community, channel, &actor, now - 600, now - 10).await; - reactions( - &pool, - &db, - community, - channel, - MAX_TIMELINE_SCAN as u64, - now, - ) - .await; - - let row = sidebar(&db, community, &actor).await; - assert_eq!(row.latest_id, None); - assert_eq!((row.unread, row.mentions), (false, 0)); -} - /// Store `event` as having arrived at exactly `seconds` plus `micros`. Built /// from integers, never a float, so microsecond order cannot hinge on rounding. async fn arrive_exact( @@ -360,6 +329,7 @@ async fn arrive_exact( .unwrap() .rows_affected(); assert_eq!(updated, 1, "arrival must land on exactly one stored event"); + sync_reply_arrivals(pool).await; let stored: chrono::DateTime = sqlx::query_scalar("SELECT received_at FROM events WHERE community_id=$1 AND id=$2") .bind(community.as_uuid()) diff --git a/crates/buzz-db/src/store/personal_read/follows.rs b/crates/buzz-db/src/store/personal_read/follows.rs index 680ccfb9dcc..6681582efb9 100644 --- a/crates/buzz-db/src/store/personal_read/follows.rs +++ b/crates/buzz-db/src/store/personal_read/follows.rs @@ -1,5 +1,5 @@ -//! Read state that ingest writes. A thread frontier row exists only for a -//! thread its actor follows; only this module creates one. +//! Read state that ingest writes. A thread frontier row exists for a thread +//! its actor follows or has unfollowed; ingest and follow intents create one. use super::model::ELIGIBLE_KINDS; use crate::{AdmittedTx, Result}; use chrono::{DateTime, Utc}; @@ -11,19 +11,75 @@ use uuid::Uuid; pub(crate) enum Place<'a> { /// A top-level timeline message. TopLevel, - /// A reply in the thread rooted at `root`. - Reply { root: &'a [u8] }, + /// A reply in the thread rooted at `root`, authored at `root_created_at`; + /// a broadcast reply is also on the channel timeline. + Reply { + root: &'a [u8], + root_created_at: DateTime, + broadcast: bool, + }, +} + +impl Place<'_> { + /// Whether the message counts on its channel timeline (`read_timeline`). + pub(crate) fn on_timeline(&self) -> bool { + matches!( + self, + Place::TopLevel + | Place::Reply { + broadcast: true, + .. + } + ) + } +} + +/// A reply's recorded ancestry: its root, the root's author time, and whether +/// it is a broadcast. +pub(crate) type Ancestry<'a> = (&'a [u8], Option>, bool); + +/// Where an eligible channel message sits, from its recorded ancestry. None +/// for other events, and for a reply whose ancestry was never (fully) +/// recorded: it is on neither read path. +pub(crate) fn place<'a>( + event: &Event, + channel: Option, + reply: Option>, +) -> Option<(Uuid, Place<'a>)> { + let channel = channel?; + if !ELIGIBLE_KINDS.contains(&i32::from(event.kind.as_u16())) { + return None; + } + match reply { + Some((root, root_created_at, broadcast)) => Some(( + channel, + Place::Reply { + root, + root_created_at: root_created_at?, + broadcast, + }, + )), + None if buzz_core::nip10::parse_thread_markers(&event.tags) + .resolve() + .is_some() => + { + None + } + None => Some((channel, Place::TopLevel)), + } } /// Record what a newly stored eligible message changes, in its transaction: /// - its author has read through it: the timeline for a top-level message, -/// otherwise the thread, which the author then follows; +/// otherwise the thread, which the author then follows (and the timeline +/// too for a broadcast reply); /// - each channel member it mentions follows its thread (for a top-level /// message, the thread rooted at it); /// - a reply makes the root's author, and in a DM every member, follow too. /// /// Everyone but the author starts just before the message, so it is unread -/// to them. A follower keeps an existing position. +/// to them. A follower keeps an existing position. An unfollow sticks until +/// its actor posts in the thread or is mentioned there again. pub(crate) async fn record_message( tx: &mut AdmittedTx, event: &Event, @@ -41,40 +97,49 @@ pub(crate) async fn record_message( .public_keys() .map(|key| key.to_bytes().to_vec()) .collect(); - let (read, thread, reply) = match place { - Place::TopLevel => (&[][..], id, false), - Place::Reply { root } => (root, root, true), + let (read, thread, reply, root_created_at, broadcast) = match place { + Place::TopLevel => (&[][..], id, false, None, false), + Place::Reply { + root, + root_created_at, + broadcast, + } => (root, root, true, Some(root_created_at), broadcast), }; // Accounts, then frontiers, each in key order, so concurrent messages that // involve the same people cannot deadlock: the frontier insert joins the // finished account insert. Foreign-key checks run at the end of the // statement, after the account rows exist. sqlx::query( - "WITH involved (actor, root_id, through, message) AS ( - SELECT $4::bytea, $5::bytea, $6::timestamptz, $3::bytea + "WITH involved (actor, root_id, through, message, refollow) AS ( + SELECT $4::bytea, $5::bytea, $6::timestamptz, $3::bytea, true + UNION ALL + SELECT $4, ''::bytea, $6, $3, true WHERE $11 UNION ALL - SELECT cm.pubkey, $7, $6-interval '1 microsecond', NULL FROM channel_members cm + SELECT cm.pubkey, $7, $6-interval '1 microsecond', NULL, true FROM channel_members cm WHERE cm.community_id=$1 AND cm.channel_id=$2 AND cm.removed_at IS NULL AND cm.pubkey=ANY($8) AND cm.pubkey<>$4 UNION ALL - SELECT e.pubkey, $7, $6-interval '1 microsecond', NULL FROM events e - WHERE $9 AND e.community_id=$1 AND e.channel_id=$2 AND e.id=$7 AND e.pubkey<>$4 + SELECT e.pubkey, $7, $6-interval '1 microsecond', NULL, false FROM events e + WHERE $9 AND e.community_id=$1 AND e.created_at=$10 AND e.id=$7 + AND e.channel_id=$2 AND e.pubkey<>$4 UNION ALL - SELECT cm.pubkey, $7, $6-interval '1 microsecond', NULL + SELECT cm.pubkey, $7, $6-interval '1 microsecond', NULL, false FROM channel_members cm JOIN channels c ON c.community_id=cm.community_id AND c.id=cm.channel_id WHERE $9 AND cm.community_id=$1 AND cm.channel_id=$2 AND cm.removed_at IS NULL AND c.channel_type='dm' AND cm.pubkey<>$4 ), rows AS ( - -- An existing follower keeps their position, so is not touched. - -- Existence probes are LIMIT 1 laterals: always one index lookup. + -- An existing follower keeps their position, so is not touched; + -- nor is an unfollow, unless this message re-follows (the author + -- posting, or a mention). Existence probes are LIMIT 1 laterals: + -- always one index lookup. SELECT DISTINCT ON (i.actor, i.root_id) i.* FROM involved i LEFT JOIN LATERAL ( - SELECT true AS found FROM personal_read_frontiers f WHERE f.community_id=$1 + SELECT f.unfollowed FROM personal_read_frontiers f WHERE f.community_id=$1 AND f.actor=i.actor AND f.channel_id=$2 AND f.root_id=i.root_id LIMIT 1 ) f ON true - WHERE i.message IS NOT NULL OR f.found IS NULL - ORDER BY i.actor, i.root_id + WHERE i.message IS NOT NULL OR f.unfollowed IS NULL OR (i.refollow AND f.unfollowed) + ORDER BY i.actor, i.root_id, i.message IS NULL, NOT i.refollow ), accounts AS ( INSERT INTO personal_read_accounts (community_id, actor) SELECT $1, r.actor FROM (SELECT DISTINCT actor FROM rows) r @@ -85,6 +150,9 @@ pub(crate) async fn record_message( WHERE a.found IS NULL ORDER BY r.actor ON CONFLICT DO NOTHING RETURNING 1 ) + -- A re-follow starts just before this message, like a new follow, + -- unless the unfollow is later. A row that a concurrent message + -- created is only advanced when it carries an anchor. INSERT INTO personal_read_frontiers (community_id, actor, channel_id, root_id, through_timestamp, through_message_id) SELECT $1, actor, $2, root_id, through, message @@ -95,8 +163,10 @@ pub(crate) async fn record_message( excluded.through_timestamp), through_message_id=CASE WHEN excluded.through_timestamp > personal_read_frontiers.through_timestamp - THEN excluded.through_message_id ELSE personal_read_frontiers.through_message_id END - WHERE excluded.through_message_id IS NOT NULL", + THEN excluded.through_message_id ELSE personal_read_frontiers.through_message_id END, + unfollowed=false + WHERE excluded.through_message_id IS NOT NULL OR (personal_read_frontiers.unfollowed + AND excluded.through_timestamp > personal_read_frontiers.through_timestamp)", ) .bind(community.as_uuid()) .bind(channel) @@ -107,6 +177,8 @@ pub(crate) async fn record_message( .bind(thread) .bind(&mentioned) .bind(reply) + .bind(root_created_at) + .bind(broadcast) .execute(tx.conn()) .await?; Ok(()) diff --git a/crates/buzz-db/src/store/personal_read/follows_postgres_tests.rs b/crates/buzz-db/src/store/personal_read/follows_postgres_tests.rs index 33a2e75929e..f1928326fd8 100644 --- a/crates/buzz-db/src/store/personal_read/follows_postgres_tests.rs +++ b/crates/buzz-db/src/store/personal_read/follows_postgres_tests.rs @@ -95,7 +95,7 @@ async fn follow( .unwrap() } -/// (root, unread replies) for each listed thread. +/// (root, mentions) for each listed thread. fn threads(row: &ChannelReadSummary) -> Vec<(String, u32)> { row.threads .iter() @@ -129,7 +129,8 @@ async fn replying_follows_the_thread_and_reads_through_the_reply() { let other = reply(&db, community, channel, &root, &Keys::generate(), vec![]).await; let summary = row(&db, community, channel, &actor).await; - assert_eq!(threads(&summary), [(root.id.to_hex(), 1)]); + // Unread, but not directed at the actor. + assert_eq!(threads(&summary), [(root.id.to_hex(), 0)]); assert_eq!(summary.threads[0].latest_id, other.id.to_hex()); assert_eq!(summary.threads[0].read_through_id, Some(own.id.to_hex())); } @@ -148,9 +149,9 @@ async fn a_mention_follows_a_member_from_just_before_it() { assert_eq!(follow(&pool, community, &bee, &root).await, Some(None)); assert_eq!(follow(&pool, community, &outsider, &root).await, None); - // The mention and what came after it; not the reply before it. + // The mention and what came after it are unread; not the reply before it. let summary = row(&db, community, channel, &bee).await; - assert_eq!(threads(&summary), [(root.id.to_hex(), 2)]); + assert_eq!(threads(&summary), [(root.id.to_hex(), 1)]); assert_ne!(summary.threads[0].latest_id, earlier.id.to_hex()); assert_ne!(summary.threads[0].latest_id, mentioning.id.to_hex()); } @@ -175,7 +176,7 @@ async fn a_later_mention_keeps_the_followers_position() { Some(Some(own.id.to_hex())) ); let summary = row(&db, community, channel, &actor).await; - assert_eq!(threads(&summary), [(root.id.to_hex(), 2)]); + assert_eq!(threads(&summary), [(root.id.to_hex(), 1)]); } #[tokio::test] @@ -188,7 +189,7 @@ async fn the_root_author_follows_on_the_first_reply() { reply(&db, community, channel, &root, &Keys::generate(), vec![]).await; assert_eq!(follow(&pool, community, &actor, &root).await, Some(None)); let summary = row(&db, community, channel, &actor).await; - assert_eq!(threads(&summary), [(root.id.to_hex(), 1)]); + assert_eq!(threads(&summary), [(root.id.to_hex(), 0)]); } #[tokio::test] @@ -202,7 +203,8 @@ async fn a_top_level_mention_follows_the_thread_rooted_at_it() { let summary = row(&db, community, channel, &actor).await; // The fixture message and the mention are unread; one is a mention. assert_eq!((summary.unread, summary.mentions), (true, 1)); - assert_eq!(threads(&summary), [(root.id.to_hex(), 1)]); + // The reply is unread but does not mention the actor. + assert_eq!(threads(&summary), [(root.id.to_hex(), 0)]); } #[tokio::test] @@ -236,8 +238,9 @@ async fn an_unfollowed_thread_never_counts_and_marking_it_follows_nothing() { let last = reply(&db, community, channel, &root, &Keys::generate(), broadcast).await; let summary = row(&db, community, channel, &actor).await; assert!(summary.threads.is_empty()); - // Replies are not on the timeline; only the fixture message is unread. + // Only the broadcast reply is also on the timeline. assert_eq!((summary.unread, summary.mentions), (true, 0)); + assert_eq!(summary.latest_id, Some(last.id.to_hex())); let outcome = db .apply_personal_read_intent( @@ -270,7 +273,7 @@ async fn thread_counts_stop_at_the_cap() { reply(&db, community, channel, &root, &actor, vec![]).await; let other = Keys::generate(); for _ in 0..=MAX_UNREAD_COUNT { - reply(&db, community, channel, &root, &other, vec![]).await; + reply(&db, community, channel, &root, &other, mention(&actor)).await; } let summary = row(&db, community, channel, &actor).await; assert_eq!( @@ -355,7 +358,7 @@ async fn a_nested_reply_follows_the_root_thread() { ); assert_eq!(follow(&pool, community, &bee, &parent).await, None); let summary = row(&db, community, channel, &actor).await; - assert_eq!(threads(&summary), [(root.id.to_hex(), 2)]); + assert_eq!(threads(&summary), [(root.id.to_hex(), 0)]); assert_eq!(summary.threads[0].latest_id, nested.id.to_hex()); } @@ -371,3 +374,274 @@ async fn a_message_stored_without_thread_metadata_reads_the_timeline() { assert!(!summary.unread); assert_eq!(summary.read_through_id, Some(own.id.to_hex())); } + +fn follow_intent(channel: Uuid, root: Option<&nostr::Event>, follow: bool) -> ReadIntent { + let target = ReadTarget { + channel_id: channel, + root_id: root.map(|r| r.id.to_hex()), + }; + if follow { + ReadIntent::Follow { target } + } else { + ReadIntent::Unfollow { target } + } +} + +async fn intent( + db: &Db, + community: CommunityId, + actor: &Keys, + intent: ReadIntent, +) -> IntentOutcome { + db.apply_personal_read_intent(community, &actor.public_key(), &intent) + .await + .unwrap() +} + +#[tokio::test] +#[ignore = "requires Postgres"] +async fn following_starts_now_and_needs_a_thread() { + let (db, pool, community, channel, actor, root) = fixture().await; + reply(&db, community, channel, &root, &Keys::generate(), vec![]).await; + assert_eq!( + intent(&db, community, &actor, follow_intent(channel, None, true)).await, + IntentOutcome::Invalid + ); + let missing = message(&Keys::generate(), vec![]); + assert_eq!( + intent( + &db, + community, + &actor, + follow_intent(channel, Some(&missing), true) + ) + .await, + IntentOutcome::Blocked + ); + assert_eq!( + intent( + &db, + community, + &actor, + follow_intent(channel, Some(&root), true) + ) + .await, + IntentOutcome::Applied + ); + assert_eq!(follow(&pool, community, &actor, &root).await, Some(None)); + // The reply before the follow is read; the next one is not. + assert!(row(&db, community, channel, &actor) + .await + .threads + .is_empty()); + let next = reply(&db, community, channel, &root, &Keys::generate(), vec![]).await; + let summary = row(&db, community, channel, &actor).await; + assert_eq!(threads(&summary), [(root.id.to_hex(), 0)]); + assert_eq!(summary.threads[0].latest_id, next.id.to_hex()); + // Following again keeps the position. + intent( + &db, + community, + &actor, + follow_intent(channel, Some(&root), true), + ) + .await; + assert_eq!( + threads(&row(&db, community, channel, &actor).await).len(), + 1 + ); +} + +#[tokio::test] +#[ignore = "requires Postgres"] +async fn an_unfollow_sticks_until_a_mention_or_own_post() { + let (db, pool, community, channel, actor, _) = fixture().await; + // The actor's own root: its author follows on every reply. + let root = post(&db, community, channel, &actor, vec![]).await; + reply(&db, community, channel, &root, &Keys::generate(), vec![]).await; + assert_eq!( + threads(&row(&db, community, channel, &actor).await).len(), + 1 + ); + assert_eq!( + intent( + &db, + community, + &actor, + follow_intent(channel, Some(&root), false) + ) + .await, + IntentOutcome::Applied + ); + reply(&db, community, channel, &root, &Keys::generate(), vec![]).await; + assert!(row(&db, community, channel, &actor) + .await + .threads + .is_empty()); + + // A mention follows again, from just before it. + let mentioning = reply( + &db, + community, + channel, + &root, + &Keys::generate(), + mention(&actor), + ) + .await; + let summary = row(&db, community, channel, &actor).await; + assert_eq!(threads(&summary), [(root.id.to_hex(), 1)]); + assert_eq!(summary.threads[0].latest_id, mentioning.id.to_hex()); + assert_eq!(summary.threads[0].read_through_id, None); + + // So does posting, which reads through the post. + intent( + &db, + community, + &actor, + follow_intent(channel, Some(&root), false), + ) + .await; + let own = reply(&db, community, channel, &root, &actor, vec![]).await; + assert_eq!( + follow(&pool, community, &actor, &root).await, + Some(Some(own.id.to_hex())) + ); + reply(&db, community, channel, &root, &Keys::generate(), vec![]).await; + assert_eq!( + threads(&row(&db, community, channel, &actor).await).len(), + 1 + ); +} + +#[tokio::test] +#[ignore = "requires Postgres"] +async fn a_broadcast_reply_is_a_timeline_message_and_anchor() { + let (db, _pool, community, channel, actor, root) = fixture().await; + apply_mark(&db, community, channel, &actor, &root).await; + let broadcast = vec![ + Tag::parse(["broadcast", "1"]).unwrap(), + Tag::public_key(actor.public_key()), + ]; + let last = reply(&db, community, channel, &root, &Keys::generate(), broadcast).await; + let summary = row(&db, community, channel, &actor).await; + assert_eq!((summary.unread, summary.mentions), (true, 1)); + assert_eq!(summary.latest_id, Some(last.id.to_hex())); + assert_eq!( + apply_mark(&db, community, channel, &actor, &last).await, + IntentOutcome::Applied + ); + assert!(!row(&db, community, channel, &actor).await.unread); + + // A broadcast by the reader reads the timeline through it, as a post does. + post(&db, community, channel, &Keys::generate(), vec![]).await; + let broadcast = vec![Tag::parse(["broadcast", "1"]).unwrap()]; + let own = reply(&db, community, channel, &root, &actor, broadcast).await; + let summary = row(&db, community, channel, &actor).await; + assert!(!summary.unread); + assert_eq!(summary.read_through_id, Some(own.id.to_hex())); +} + +async fn apply_mark( + db: &Db, + community: CommunityId, + channel: Uuid, + actor: &Keys, + message: &nostr::Event, +) -> IntentOutcome { + let mark = ReadIntent::MarkThrough { + target: ReadTarget { + channel_id: channel, + root_id: None, + }, + message_id: message.id.to_hex(), + }; + intent(db, community, actor, mark).await +} + +async fn threads_page( + db: &Db, + community: CommunityId, + actor: &Keys, + limit: usize, + after: Option<&str>, +) -> ThreadsPage { + let cursor = after.map(|c| ThreadsCursor::parse(c).unwrap()); + db.personal_read_threads( + community, + &actor.public_key(), + DEFAULT_RETENTION_SECONDS, + limit, + cursor.as_ref(), + ) + .await + .unwrap() +} + +#[tokio::test] +#[ignore = "requires Postgres"] +async fn the_threads_view_lists_follows_newest_arrival_first() { + let (db, pool, community, channel, actor, _) = fixture().await; + let mut roots = Vec::new(); + for _ in 0..3 { + let root = post(&db, community, channel, &Keys::generate(), vec![]).await; + reply(&db, community, channel, &root, &actor, vec![]).await; + roots.push(root); + } + // The first thread has news, so it comes first; the rest are read. + reply( + &db, + community, + channel, + &roots[0], + &Keys::generate(), + mention(&actor), + ) + .await; + // Neither an unfollowed thread nor one with no reply is listed. + let quiet = post(&db, community, channel, &Keys::generate(), mention(&actor)).await; + let left = post(&db, community, channel, &Keys::generate(), vec![]).await; + reply(&db, community, channel, &left, &actor, vec![]).await; + intent( + &db, + community, + &actor, + follow_intent(channel, Some(&left), false), + ) + .await; + assert_eq!(follow(&pool, community, &actor, &quiet).await, Some(None)); + + let first = threads_page(&db, community, &actor, 2, None).await; + let rows: Vec<_> = first + .threads + .iter() + .map(|t| (t.thread.root_id.clone(), t.thread.unread, t.thread.mentions)) + .collect(); + assert_eq!( + rows, + [ + (roots[0].id.to_hex(), true, 1), + (roots[2].id.to_hex(), false, 0) + ] + ); + assert!(first.threads.iter().all(|t| t.channel_id == channel)); + let rest = threads_page(&db, community, &actor, 2, first.next_cursor.as_deref()).await; + let rest: Vec<_> = rest + .threads + .iter() + .map(|t| t.thread.root_id.clone()) + .collect(); + assert_eq!(rest, [roots[1].id.to_hex()]); + + // Leaving the channel hides its threads. + sqlx::query("UPDATE channel_members SET removed_at=now() WHERE community_id=$1 AND pubkey=$2") + .bind(community.as_uuid()) + .bind(actor.public_key().to_bytes().as_slice()) + .execute(&pool) + .await + .unwrap(); + assert!(threads_page(&db, community, &actor, 2, None) + .await + .threads + .is_empty()); +} diff --git a/crates/buzz-db/src/store/personal_read/mod.rs b/crates/buzz-db/src/store/personal_read/mod.rs index 130b55d81c4..6df99443fc8 100644 --- a/crates/buzz-db/src/store/personal_read/mod.rs +++ b/crates/buzz-db/src/store/personal_read/mod.rs @@ -10,7 +10,7 @@ mod model; mod projection; mod writes; -pub(crate) use follows::{record_message, Place}; +pub(crate) use follows::{place, record_message}; pub use model::*; #[cfg(test)] diff --git a/crates/buzz-db/src/store/personal_read/model.rs b/crates/buzz-db/src/store/personal_read/model.rs index 1b276918f6c..c6dd551bb14 100644 --- a/crates/buzz-db/src/store/personal_read/model.rs +++ b/crates/buzz-db/src/store/personal_read/model.rs @@ -28,6 +28,29 @@ pub enum ReadIntent { /// Fixed anchor; retry must not substitute the latest message. message_id: String, }, + /// Follow a thread. A new follow, or one after an unfollow, starts now; + /// an existing follow keeps its position. + Follow { + /// The thread; a channel-only target is invalid. + target: ReadTarget, + }, + /// Stop following a thread until the actor posts in it or is mentioned + /// there again. + Unfollow { + /// The thread; a channel-only target is invalid. + target: ReadTarget, + }, +} + +impl ReadIntent { + /// The intent's target. + pub fn target(&self) -> &ReadTarget { + match self { + ReadIntent::MarkThrough { target, .. } + | ReadIntent::Follow { target } + | ReadIntent::Unfollow { target } => target, + } + } } /// Outcome for one independent transaction, never acknowledged before commit. @@ -57,16 +80,11 @@ pub const MAX_CHANNELS: usize = 20; /// Maximum thread rows per channel row. A POST can return rows for /// `MAX_INTENTS` channels, and they must fit the 1 MiB response limit. pub const MAX_THREAD_SUMMARIES: usize = 25; -/// Events each walk examines before eligibility: newest-first for the -/// newest timeline message, then for the last arrival among those authored up -/// to twice the skew before it (in a thread too), and forward from the read -/// position for DM counts. What lies past the budget goes unseen. -pub const MAX_TIMELINE_SCAN: usize = 4096; /// Counts stop here: a count of 100 means at least 100. pub const MAX_UNREAD_COUNT: usize = 100; /// The relay accepts author times up to this far from arrival either way. -/// Read positions are arrival times and the indexes are by author time, so -/// every forward walk starts this much before the position. +/// Read positions are arrival times and mentions are indexed by author time, +/// so a mention walk starts this much before the position. pub const MAX_ARRIVAL_SKEW_SECONDS: u32 = 15 * 60; /// Conversation kinds eligible for ordinary unread state (not edits/reactions). pub const ELIGIBLE_KINDS: [i32; 4] = [9, 40002, 45001, 45003]; @@ -89,7 +107,7 @@ pub struct ChannelReadSummary { /// Last eligible top-level message to arrive, whatever its author or read /// progress: marking the timeline through it reads the timeline. pub latest_id: Option, - /// Followed threads with unread replies, newest unread reply first. + /// Followed threads with unread replies, newest arrival first. pub threads: Vec, } @@ -98,9 +116,10 @@ pub struct ChannelReadSummary { pub struct ThreadReadSummary { /// Canonical thread-root event ID. pub root_id: String, - /// Whether an unread reply that counts was found. + /// Whether the last reply to arrive is unread. pub unread: bool, - /// Unread replies by others in this followed thread. + /// Unread replies directed at the actor: every one in a DM, otherwise + /// those that tag the actor with `p`. pub mentions: u32, /// The anchor of the thread's frontier: the marked or own reply that /// arrived last. None for a follower who has read nothing yet. @@ -117,3 +136,43 @@ pub struct SidebarPage { /// Exclusive UUID roster cursor. None means this roster scan exhausted. pub next_cursor: Option, } + +/// Maximum rows in one threads page. +pub const MAX_THREADS: usize = 50; + +/// One followed thread in the threads view. +#[derive(Debug, Serialize)] +pub struct FollowedThread { + /// Joined channel UUID. + pub channel_id: Uuid, + /// The thread's read state; `unread` may be false here. + #[serde(flatten)] + pub thread: ThreadReadSummary, +} + +/// A page of followed threads, newest arrival first. +#[derive(Debug, Serialize)] +pub struct ThreadsPage { + /// Followed threads in joined channels. + pub threads: Vec, + /// Opaque exclusive cursor. None means this scan exhausted. + pub next_cursor: Option, +} + +/// Where a threads page resumes: after this arrival and root. +#[derive(Clone, Debug)] +pub struct ThreadsCursor { + pub(super) latest_at: DateTime, + pub(super) root_id: Vec, +} + +impl ThreadsCursor { + /// Parse a `next_cursor` value; None if it is not one. + pub fn parse(value: &str) -> Option { + let (micros, root) = value.split_once('.')?; + Some(Self { + latest_at: DateTime::from_timestamp_micros(micros.parse().ok()?)?, + root_id: super::writes::event_id(root)?, + }) + } +} diff --git a/crates/buzz-db/src/store/personal_read/postgres_tests.rs b/crates/buzz-db/src/store/personal_read/postgres_tests.rs index ad850d219c6..53dc50ff100 100644 --- a/crates/buzz-db/src/store/personal_read/postgres_tests.rs +++ b/crates/buzz-db/src/store/personal_read/postgres_tests.rs @@ -62,6 +62,20 @@ pub(super) async fn start_before_everything( } /// Store `event` as a canonical reply in `root`'s thread, through ingest. +/// Copy rewritten event arrivals onto their replies' thread rows, as ingest +/// stores them. +pub(super) async fn sync_reply_arrivals(pool: &PgPool) { + sqlx::query( + "UPDATE thread_metadata tm SET received_at=e.received_at FROM events e + WHERE e.community_id=tm.community_id AND e.created_at=tm.event_created_at + AND e.id=tm.event_id AND tm.received_at IS NOT NULL + AND tm.received_at<>e.received_at", + ) + .execute(pool) + .await + .unwrap(); +} + pub(super) async fn insert_reply( db: &Db, community: CommunityId, @@ -179,9 +193,9 @@ async fn add_history(db: &Db, community: CommunityId, channel: Uuid, count: usiz #[tokio::test] #[ignore = "requires Postgres"] -async fn personal_read_sidebar_marked_history_past_the_scan_cap_is_read_and_keeps_latest() { +async fn personal_read_sidebar_marked_history_is_read_and_keeps_latest() { let (db, _pool, community, channel, actor, _) = fixture().await; - let last = add_history(&db, community, channel, MAX_TIMELINE_SCAN + 44).await; + let last = add_history(&db, community, channel, 300).await; let outcome = db .apply_personal_read_intent( community, @@ -191,7 +205,6 @@ async fn personal_read_sidebar_marked_history_past_the_scan_cap_is_read_and_keep .await .unwrap(); assert_eq!(outcome, IntentOutcome::Applied); - // The scan never reaches the oldest 45 messages; nothing is unread. let row = &sidebar(&db, community, &actor).await.channels[0]; assert!(!row.unread); assert_eq!(row.mentions, 0); @@ -203,7 +216,7 @@ async fn personal_read_sidebar_marked_history_past_the_scan_cap_is_read_and_keep #[ignore = "requires Postgres"] async fn personal_read_sidebar_author_window_excludes_expired_and_late_old_messages() { let (db, pool, community, channel, actor, _) = fixture().await; - let latest = add_history(&db, community, channel, MAX_TIMELINE_SCAN + 44).await; + let latest = add_history(&db, community, channel, 300).await; expire(&pool, community).await; let row = &sidebar(&db, community, &actor).await.channels[0]; assert!( @@ -734,13 +747,13 @@ async fn personal_read_counts_only_arrivals_after_the_account_starts() { #[tokio::test] #[ignore = "requires Postgres"] -async fn personal_read_unstarted_account_is_caught_up_past_the_scan_cap() { +async fn personal_read_unstarted_account_is_caught_up() { let (db, pool, community, channel, actor, _) = fixture().await; unstart(&pool, community).await; - let last = add_history(&db, community, channel, MAX_TIMELINE_SCAN + 44).await; + let last = add_history(&db, community, channel, 300).await; let page = sidebar(&db, community, &actor).await; let row = &page.channels[0]; - // The start floor covers every arrival, examined by the scan or not. + // The start floor covers every arrival. assert!(!row.unread); assert_eq!(row.mentions, 0); assert!(row.threads.is_empty()); diff --git a/crates/buzz-db/src/store/personal_read/projection.rs b/crates/buzz-db/src/store/personal_read/projection.rs index 35204ba8ee4..a3bd0569dcd 100644 --- a/crates/buzz-db/src/store/personal_read/projection.rs +++ b/crates/buzz-db/src/store/personal_read/projection.rs @@ -1,7 +1,7 @@ -//! Sidebar read state, counted forward from each read position. A caught-up -//! reader's work is a few index probes per channel and per followed thread; -//! a reader who is behind walks only what arrived after their position, and -//! counts stop at `MAX_UNREAD_COUNT`. +//! Sidebar and threads read state, counted forward from each read position. +//! Each channel timeline and each followed thread is one index probe for its +//! last arrival; counts walk only what arrived after the position and stop at +//! `MAX_UNREAD_COUNT`. use super::{model::*, writes}; use buzz_core::CommunityId; @@ -11,31 +11,37 @@ use uuid::Uuid; use crate::{observability, Db, DbError, Result}; -/// Event `e` is eligible: a conversation kind, not deleted. -const ELIGIBLE: &str = "e.kind=ANY($5) AND e.deleted_at IS NULL"; +/// The last eligible reply `tl` to arrive in thread `tf` (idx_thread_metadata_read_arrival). +/// Deletion is a per-row filter, not a join, so the walk stops at the first +/// living reply instead of sorting them all. +const LATEST_REPLY: &str = "SELECT tm.event_id AS id, tm.received_at, e.pubkey, e.created_at + FROM (SELECT tm.event_id, tm.event_created_at, tm.received_at FROM thread_metadata tm + WHERE tm.community_id=$1 AND tm.root_event_id=tf.root_id AND tm.received_at IS NOT NULL + AND tm.channel_id=tf.channel_id + AND (SELECT e.deleted_at IS NULL FROM events e WHERE e.community_id=$1 + AND e.created_at=tm.event_created_at AND e.id=tm.event_id) + ORDER BY tm.received_at DESC, tm.event_id LIMIT 1) tm + JOIN events e ON e.community_id=$1 AND e.created_at=tm.event_created_at AND e.id=tm.event_id"; -/// Event `e`, with its `thread_metadata` row `tm`, is on the channel timeline: -/// not a reply, and not a NIP-10 reply whose ancestry was never recorded -/// (neither counted nor a timeline anchor, since a mark cannot validate it). -/// Parity-tested against the shared NIP-10 parser. -const TIMELINE: &str = r#"(tm.root_event_id IS NULL OR tm.root_event_id=e.id) - AND NOT (tm.root_event_id IS NULL AND jsonb_typeof(e.tags)='array' - AND EXISTS (SELECT 1 FROM jsonb_array_elements(e.tags) t(tag) - WHERE jsonb_typeof(tag)='array' AND tag->>0='e' AND tag->>3='reply' - AND (tag->>1) COLLATE "C" ~ '^[0123456789abcdefABCDEF]{64}$'))"#; - -/// What the budgeted timeline walks carry out of their `LIMIT`. -const COLUMNS: &str = - "e.id, e.pubkey, e.created_at, e.received_at, e.kind, e.deleted_at, e.tags, e.channel_id"; - -const EVENT_THREAD_ROW: &str = "LEFT JOIN thread_metadata tm ON tm.community_id=$1 - AND tm.channel_id=e.channel_id AND tm.event_created_at=e.created_at AND tm.event_id=e.id"; - -/// Replies `tm` in followed thread `tf`, with their events `e`. -const THREAD_REPLIES: &str = "thread_metadata tm JOIN events e ON e.community_id=$1 - AND e.created_at=tm.event_created_at AND e.id=tm.event_id - WHERE tm.community_id=$1 AND tm.root_event_id=tf.root_id AND tm.channel_id=r.id - AND tm.event_id<>tf.root_id"; +/// Unread replies in thread `t` at position `t.position` that are directed at +/// the actor: every one in a DM (`t.dm`), otherwise those that tag the actor. +/// Nothing is walked unless `t.unread`. +const THREAD_MENTIONS: &str = "SELECT count(*) AS n FROM ( + SELECT 1 FROM thread_metadata tm JOIN events e ON e.community_id=$1 + AND e.created_at=tm.event_created_at AND e.id=tm.event_id + WHERE t.unread AND t.dm AND tm.community_id=$1 AND tm.root_event_id=t.root_id + AND tm.received_at > t.position AND tm.channel_id=t.channel_id + AND e.deleted_at IS NULL AND e.pubkey<>$2 AND e.created_at >= $4 + UNION ALL + SELECT 1 FROM event_mentions em JOIN thread_metadata tm ON tm.community_id=$1 + AND tm.event_created_at=em.event_created_at AND tm.event_id=em.event_id + JOIN events e ON e.community_id=$1 AND e.created_at=em.event_created_at AND e.id=em.event_id + WHERE t.unread AND NOT t.dm AND em.community_id=$1 AND em.pubkey_hex=$5 AND em.channel_id=t.channel_id + AND em.event_created_at >= GREATEST(t.position-$6, $4) + AND tm.root_event_id=t.root_id AND tm.received_at > t.position + AND e.deleted_at IS NULL AND e.pubkey<>$2 + LIMIT $7 + ) x"; impl Db { /// Read a bounded joined roster from the writer in one read-only snapshot. @@ -103,131 +109,77 @@ impl Db { let actor_bytes = actor.to_bytes(); let account = read_account(&mut tx, community, &actor_bytes, retention_seconds).await?; // A position is its frontier floored at the account's start (infinity - // before the actor's first intent, so nothing counts). Positions are - // arrival times and the indexes are by author time, which the relay - // accepts within the skew of arrival, so a walk forward from a - // position starts the skew before it (and never before the horizon). + // before the actor's first intent, so nothing counts). // // Timeline, per channel: - // - latest_id: the newest timeline message by author time within the - // scan budget, then the last to arrive among the newest budget of - // events authored up to twice the skew before it, which holds the - // last arrival unless that many arrived out of order; + // - latest_id: the last timeline message to arrive (idx_events_read_timeline); // - unread: latest is someone else's, inside the horizon, and arrived // after the position (an own post moves the position past it); - // - mentions: those of them that tag the actor (event_mentions), or - // every one in a DM. - // Threads: each followed thread whose newest reply could have arrived - // after its position (one index-only probe of - // idx_thread_metadata_window), then its last arrival the same way as - // latest_id, and its unread replies by others. + // - mentions: unread timeline messages that tag the actor, found by + // author time (event_mentions) from the skew before the position, + // or every one in a DM. + // Threads: each followed thread whose last reply is unread the same + // way, then its mentions for the listed ones only. let sql = format!( r#"WITH roster AS MATERIALIZED ( SELECT c.id, c.channel_type='dm' AS dm, - GREATEST(cf.through_timestamp, COALESCE($7::timestamptz, 'infinity')) AS position, + GREATEST(cf.through_timestamp, COALESCE($3::timestamptz, 'infinity')) AS position, encode(cf.through_message_id,'hex') AS read_through_id FROM channel_members cm JOIN channels c ON c.community_id=cm.community_id AND c.id=cm.channel_id LEFT JOIN personal_read_frontiers cf ON cf.community_id=$1 AND cf.actor=$2 AND cf.channel_id=c.id AND cf.root_id=''::bytea WHERE cm.community_id=$1 AND cm.pubkey=$2 AND cm.removed_at IS NULL - AND c.deleted_at IS NULL AND ($3::uuid IS NULL OR c.id>$3) - AND ($4::uuid[] IS NULL OR c.id=ANY($4)) - ORDER BY c.id LIMIT $6 + AND c.deleted_at IS NULL AND ($8::uuid IS NULL OR c.id>$8) + AND ($9::uuid[] IS NULL OR c.id=ANY($9)) + ORDER BY c.id LIMIT $10 ) SELECT r.id, r.read_through_id, encode(l.id,'hex') AS latest_id, u.unread, COALESCE(m.mentions, 0) AS mentions, COALESCE(th.items, '[]') AS threads FROM roster r - CROSS JOIN LATERAL (SELECT GREATEST(r.position-$8, $9) AS walk_from) w - LEFT JOIN LATERAL ( - SELECT e.created_at FROM ( - SELECT {COLUMNS} FROM events e WHERE e.community_id=$1 AND e.channel_id=r.id - ORDER BY e.created_at DESC, e.id DESC LIMIT $10 - ) e {EVENT_THREAD_ROW} - WHERE {ELIGIBLE} AND {TIMELINE} - ORDER BY e.created_at DESC, e.id DESC LIMIT 1 - ) n ON true LEFT JOIN LATERAL ( - SELECT e.id, e.pubkey, e.created_at, e.received_at FROM ( - SELECT {COLUMNS} FROM events e WHERE e.community_id=$1 AND e.channel_id=r.id - AND e.created_at BETWEEN n.created_at-2*$8 AND n.created_at - ORDER BY e.created_at DESC, e.id DESC LIMIT $10 - ) e {EVENT_THREAD_ROW} - WHERE {ELIGIBLE} AND {TIMELINE} + SELECT e.id, e.pubkey, e.created_at, e.received_at FROM events e + WHERE e.community_id=$1 AND e.channel_id=r.id AND e.read_timeline + AND e.deleted_at IS NULL ORDER BY e.received_at DESC, e.id LIMIT 1 ) l ON true - -- The last arrival is unread exactly when the timeline is: the - -- actor's own posts read through themselves. CROSS JOIN LATERAL ( SELECT COALESCE(l.received_at > r.position AND l.pubkey<>$2 - AND l.created_at >= $9, false) AS unread + AND l.created_at >= $4, false) AS unread ) u LEFT JOIN LATERAL ( SELECT count(*) AS mentions FROM ( - SELECT 1 FROM ( - SELECT {COLUMNS} FROM events e WHERE r.dm AND e.community_id=$1 - AND e.channel_id=r.id AND e.created_at >= w.walk_from - ORDER BY e.created_at, e.id LIMIT $10 - ) e {EVENT_THREAD_ROW} - WHERE e.received_at > r.position AND e.pubkey<>$2 AND {ELIGIBLE} AND {TIMELINE} + SELECT 1 FROM events e WHERE r.dm AND e.community_id=$1 + AND e.channel_id=r.id AND e.read_timeline AND e.deleted_at IS NULL + AND e.received_at > r.position AND e.pubkey<>$2 AND e.created_at >= $4 UNION ALL SELECT 1 FROM event_mentions em JOIN events e ON e.community_id=$1 - AND e.created_at=em.event_created_at AND e.id=em.event_id {EVENT_THREAD_ROW} - WHERE NOT r.dm AND em.community_id=$1 AND em.pubkey_hex=$11 - AND em.channel_id=r.id AND em.event_created_at >= w.walk_from - AND e.channel_id=r.id AND e.received_at > r.position - AND e.pubkey<>$2 AND {ELIGIBLE} AND {TIMELINE} - LIMIT $12 + AND e.created_at=em.event_created_at AND e.id=em.event_id + WHERE NOT r.dm AND em.community_id=$1 AND em.pubkey_hex=$5 + AND em.channel_id=r.id AND em.event_created_at >= GREATEST(r.position-$6, $4) + AND e.read_timeline AND e.deleted_at IS NULL + AND e.received_at > r.position AND e.pubkey<>$2 + LIMIT $7 ) x WHERE u.unread ) m ON true LEFT JOIN LATERAL ( SELECT jsonb_agg(jsonb_build_object('root_id',encode(t.root_id,'hex'), - 'unread',true,'mentions',t.n,'read_through_id',t.read_through_id, - 'latest_id',t.latest_id) ORDER BY t.latest_at DESC, t.root_id) AS items + 'unread',true,'mentions',tc.n,'read_through_id',t.read_through_id, + 'latest_id',encode(t.latest_id,'hex')) + ORDER BY t.latest_at DESC, t.root_id) AS items FROM ( - SELECT tf.root_id, encode(tf.through_message_id,'hex') AS read_through_id, - encode(tl.id,'hex') AS latest_id, tl.created_at AS latest_at, tc.n + SELECT tf.root_id, tf.channel_id, r.dm, p.position, true AS unread, + encode(tf.through_message_id,'hex') AS read_through_id, + tl.id AS latest_id, tl.received_at AS latest_at FROM personal_read_frontiers tf - CROSS JOIN LATERAL (SELECT GREATEST(tf.through_timestamp, COALESCE($7, 'infinity')) AS position) p - CROSS JOIN LATERAL ( - SELECT tm.event_created_at AS created_at FROM thread_metadata tm - WHERE tm.community_id=$1 AND tm.root_event_id=tf.root_id - ORDER BY tm.event_created_at DESC, tm.event_id LIMIT 1 - ) tq - CROSS JOIN LATERAL ( - -- Eligibility is a per-row filter, not a join, so the - -- planner walks the window index newest-first and stops - -- at the first eligible reply instead of sorting them all. - SELECT tm.event_created_at AS created_at FROM thread_metadata tm - WHERE tm.community_id=$1 AND tm.root_event_id=tf.root_id - AND tm.channel_id=r.id AND tm.event_id<>tf.root_id - AND (SELECT {ELIGIBLE} FROM events e WHERE e.community_id=$1 - AND e.created_at=tm.event_created_at AND e.id=tm.event_id) - ORDER BY tm.event_created_at DESC, tm.event_id LIMIT 1 - ) tn - CROSS JOIN LATERAL ( - SELECT e.id, e.created_at, e.received_at FROM ( - SELECT e.id, e.created_at, e.received_at FROM {THREAD_REPLIES} - AND tm.event_created_at BETWEEN tn.created_at-2*$8 AND tn.created_at - AND {ELIGIBLE} - ORDER BY tm.event_created_at DESC, tm.event_id LIMIT $10 - ) e - ORDER BY e.received_at DESC, e.id LIMIT 1 - ) tl - CROSS JOIN LATERAL ( - SELECT count(*) AS n FROM ( - SELECT 1 FROM {THREAD_REPLIES} - AND tm.event_created_at >= GREATEST(p.position-$8, $9) - AND e.received_at > p.position AND e.pubkey<>$2 AND {ELIGIBLE} - LIMIT $12 - ) x - ) tc + CROSS JOIN LATERAL (SELECT GREATEST(tf.through_timestamp, + COALESCE($3, 'infinity')) AS position) p + CROSS JOIN LATERAL ({LATEST_REPLY}) tl WHERE tf.community_id=$1 AND tf.actor=$2 AND tf.channel_id=r.id - AND tf.root_id<>''::bytea - AND tq.created_at+$8 > p.position AND tl.received_at > p.position - AND tc.n > 0 - ORDER BY tl.created_at DESC, tf.root_id LIMIT $13 - ) t + AND tf.root_id<>''::bytea AND NOT tf.unfollowed + AND tl.received_at > p.position AND tl.pubkey<>$2 AND tl.created_at >= $4 + ORDER BY tl.received_at DESC, tf.root_id LIMIT $11 + ) t CROSS JOIN LATERAL ({THREAD_MENTIONS}) tc ) th ON true ORDER BY r.id"# ); @@ -237,16 +189,14 @@ impl Db { let rows = sqlx::query(sqlx::AssertSqlSafe(sql)) .bind(community.as_uuid()) .bind(actor_bytes.as_slice()) - .bind(after) - .bind(only) - .bind(ELIGIBLE_KINDS.as_slice()) - .bind((limit + 1) as i64) .bind(account.started_at) - .bind(skew()) .bind(cutoff) - .bind(MAX_TIMELINE_SCAN as i64) .bind(actor.to_hex()) + .bind(skew()) .bind(MAX_UNREAD_COUNT as i64) + .bind(after) + .bind(only) + .bind((limit + 1) as i64) .bind(MAX_THREAD_SUMMARIES as i64) .fetch_all(&mut *tx) .await?; @@ -277,6 +227,106 @@ impl Db { next_cursor, }) } + + /// Followed threads in joined channels, newest arrival first, read or + /// not, from the writer in one read-only snapshot. A thread with no reply + /// yet is omitted. Callers must recheck admission/resource access before + /// releasing this data. + pub async fn personal_read_threads( + &self, + community: CommunityId, + actor: &nostr::PublicKey, + retention_seconds: u32, + limit: usize, + after: Option<&ThreadsCursor>, + ) -> Result { + if !(1..=MAX_THREADS).contains(&limit) { + return Err(DbError::InvalidData("invalid threads limit".into())); + } + let mut conn = observability::acquire_writer( + &self.pool, + observability::WriterOperation::SubscriptionHistory, + ) + .await?; + let mut tx = conn.begin().await?; + sqlx::query("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ READ ONLY") + .execute(&mut *tx) + .await?; + writes::deadlines(&mut tx).await?; + sqlx::query("SET LOCAL jit = off").execute(&mut *tx).await?; + let actor_bytes = actor.to_bytes(); + let account = read_account(&mut tx, community, &actor_bytes, retention_seconds).await?; + // Every follow is probed for its last arrival and sorted: the cost + // grows with the actor's lifetime follows, not with community size. + let sql = format!( + r#"SELECT t.channel_id, encode(t.root_id,'hex') AS root_id, t.unread, + tc.n AS mentions, + encode(t.through_message_id,'hex') AS read_through_id, + encode(t.latest_id,'hex') AS latest_id, t.latest_at + FROM ( + SELECT tf.root_id, tf.channel_id, c.channel_type='dm' AS dm, p.position, + tf.through_message_id, tl.id AS latest_id, tl.received_at AS latest_at, + (tl.received_at > p.position AND tl.pubkey<>$2 AND tl.created_at >= $4) AS unread + FROM personal_read_frontiers tf + JOIN channel_members cm ON cm.community_id=$1 AND cm.channel_id=tf.channel_id + AND cm.pubkey=$2 AND cm.removed_at IS NULL + JOIN channels c ON c.community_id=$1 AND c.id=tf.channel_id AND c.deleted_at IS NULL + CROSS JOIN LATERAL (SELECT GREATEST(tf.through_timestamp, + COALESCE($3::timestamptz, 'infinity')) AS position) p + CROSS JOIN LATERAL ({LATEST_REPLY}) tl + WHERE tf.community_id=$1 AND tf.actor=$2 AND tf.root_id<>''::bytea + AND NOT tf.unfollowed AND ($8::timestamptz IS NULL OR tl.received_at < $8 + OR (tl.received_at=$8 AND tf.root_id>$9)) + ORDER BY tl.received_at DESC, tf.root_id LIMIT $10 + ) t CROSS JOIN LATERAL ({THREAD_MENTIONS}) tc + ORDER BY t.latest_at DESC, t.root_id"# + ); + let cutoff = DateTime::from_timestamp_millis(account.cutoff_ms) + .ok_or_else(|| DbError::InvalidData("invalid unread cutoff".into()))?; + // Only compile-time constants are interpolated. + let rows = sqlx::query(sqlx::AssertSqlSafe(sql)) + .bind(community.as_uuid()) + .bind(actor_bytes.as_slice()) + .bind(account.started_at) + .bind(cutoff) + .bind(actor.to_hex()) + .bind(skew()) + .bind(MAX_UNREAD_COUNT as i64) + .bind(after.map(|c| c.latest_at)) + .bind(after.map(|c| c.root_id.as_slice())) + .bind((limit + 1) as i64) + .fetch_all(&mut *tx) + .await?; + tx.commit().await?; + let has_more = rows.len() > limit; + let mut threads = Vec::with_capacity(limit); + let mut last = None; + for row in rows.into_iter().take(limit) { + let mentions: i64 = row.try_get("mentions")?; + let root_id: String = row.try_get("root_id")?; + last = Some(( + row.try_get::, _>("latest_at")?, + root_id.clone(), + )); + threads.push(FollowedThread { + channel_id: row.try_get("channel_id")?, + thread: ThreadReadSummary { + root_id, + unread: row.try_get("unread")?, + mentions: mentions as u32, + read_through_id: row.try_get("read_through_id")?, + latest_id: row.try_get("latest_id")?, + }, + }); + } + let next_cursor = last + .filter(|_| has_more) + .map(|(at, root)| format!("{}.{root}", at.timestamp_micros())); + Ok(ThreadsPage { + threads, + next_cursor, + }) + } } fn skew() -> chrono::Duration { diff --git a/crates/buzz-db/src/store/personal_read/projection_postgres_tests.rs b/crates/buzz-db/src/store/personal_read/projection_postgres_tests.rs index f4fb0976776..77af8d257fe 100644 --- a/crates/buzz-db/src/store/personal_read/projection_postgres_tests.rs +++ b/crates/buzz-db/src/store/personal_read/projection_postgres_tests.rs @@ -1,26 +1,16 @@ use super::{postgres_tests::fixture, *}; use crate::Db; use nostr::{EventBuilder, Keys, Kind, Tag}; -use serde_json::json; +/// Eligibility is classified at ingest (kind) and at read (author, deletion, +/// horizon). Each case is stored through ingest, then rewritten in place for +/// the read-time facts. #[tokio::test] #[ignore = "requires Postgres"] async fn sidebar_sql_eligibility_follows_kind_author_deletion_and_horizon() { - let (db, pool, community, _, actor, event) = fixture().await; + let (db, pool, community, channel, actor, _) = fixture().await; let now = chrono::Utc::now().timestamp_millis(); let horizon = i64::from(DEFAULT_RETENTION_SECONDS) * 1000; - // Addressed to the actor, so a counted message is also exactly one mention. - let tags = json!([["p", actor.public_key().to_hex()]]); - sqlx::query( - "INSERT INTO event_mentions - (community_id,pubkey_hex,event_id,event_created_at,channel_id,event_kind) - SELECT community_id,$2,id,created_at,channel_id,kind FROM events WHERE community_id=$1", - ) - .bind(community.as_uuid()) - .bind(actor.public_key().to_hex()) - .execute(&pool) - .await - .unwrap(); for (kind, eligible_kind) in [ (9, true), (40002, true), @@ -28,9 +18,22 @@ async fn sidebar_sql_eligibility_follows_kind_author_deletion_and_horizon() { (45003, true), (1, false), (7, false), - (39002, false), (40008, false), ] { + sqlx::query("UPDATE events SET deleted_at=now() WHERE community_id=$1") + .bind(community.as_uuid()) + .execute(&pool) + .await + .unwrap(); + // Addressed to the actor, so a counted message is also exactly one mention. + let author = Keys::generate(); + let event = EventBuilder::new(Kind::Custom(kind), "case") + .tags([Tag::public_key(actor.public_key())]) + .sign_with_keys(&author) + .unwrap(); + db.insert_event(community, &event, Some(channel)) + .await + .unwrap(); for own in [false, true] { for deleted in [false, true] { // Now, then one minute inside and one minute outside the horizon. @@ -40,12 +43,13 @@ async fn sidebar_sql_eligibility_follows_kind_author_deletion_and_horizon() { (horizon + 60_000, false), ] { let created = now - age; - sqlx::query("UPDATE events SET kind=$2,pubkey=$3,deleted_at=CASE WHEN $4 THEN now() ELSE NULL END,created_at=to_timestamp($5::double precision/1000),tags=$6 WHERE community_id=$1") - .bind(community.as_uuid()).bind(kind) - .bind(if own { actor.public_key().to_bytes() } else { event.pubkey.to_bytes() }.as_slice()) - .bind(deleted).bind(created as f64).bind(&tags).execute(&pool).await.unwrap(); - sqlx::query("UPDATE event_mentions SET event_created_at=to_timestamp($2::double precision/1000) WHERE community_id=$1") - .bind(community.as_uuid()).bind(created as f64).execute(&pool).await.unwrap(); + sqlx::query("UPDATE events SET pubkey=$3,deleted_at=CASE WHEN $4 THEN now() ELSE NULL END,created_at=to_timestamp($5::double precision/1000) WHERE community_id=$1 AND id=$2") + .bind(community.as_uuid()).bind(event.id.as_bytes().as_slice()) + .bind(if own { actor.public_key() } else { author.public_key() }.to_bytes().as_slice()) + .bind(deleted).bind(created as f64).execute(&pool).await.unwrap(); + sqlx::query("UPDATE event_mentions SET event_created_at=to_timestamp($3::double precision/1000) WHERE community_id=$1 AND event_id=$2") + .bind(community.as_uuid()).bind(event.id.as_bytes().as_slice()) + .bind(created as f64).execute(&pool).await.unwrap(); let page = db .personal_read_sidebar( community, @@ -69,10 +73,12 @@ async fn sidebar_sql_eligibility_follows_kind_author_deletion_and_horizon() { } } +/// A message with a NIP-10 reply marker but no recorded ancestry is on +/// neither read path, by the shared parser's definition of a reply. #[tokio::test] #[ignore = "requires Postgres"] async fn sidebar_ancestry_fact_matches_shared_nip10_parser() { - let (db, pool, community, _, actor, _) = fixture().await; + let (db, pool, community, channel, actor, _) = fixture().await; let ids = [ "a".repeat(64), "A".repeat(64), @@ -104,12 +110,22 @@ async fn sidebar_ancestry_fact_matches_shared_nip10_parser() { buzz_core::nip10::parse_thread_markers_from_parts(tags.iter().map(Vec::as_slice)) .resolve() .is_some(); - sqlx::query("UPDATE events SET tags=$2 WHERE community_id=$1") + sqlx::query("UPDATE events SET deleted_at=now() WHERE community_id=$1") .bind(community.as_uuid()) - .bind(json!(tags)) .execute(&pool) .await .unwrap(); + let parsed: Vec = tags + .iter() + .map(|tag| Tag::parse(tag.iter().map(String::as_str)).unwrap()) + .collect(); + let event = EventBuilder::new(Kind::Custom(9), "case") + .tags(parsed) + .sign_with_keys(&Keys::generate()) + .unwrap(); + db.insert_event(community, &event, Some(channel)) + .await + .unwrap(); let page = db .personal_read_sidebar( community, diff --git a/crates/buzz-db/src/store/personal_read/threads_postgres_tests.rs b/crates/buzz-db/src/store/personal_read/threads_postgres_tests.rs index b366279eea8..2259649b6bc 100644 --- a/crates/buzz-db/src/store/personal_read/threads_postgres_tests.rs +++ b/crates/buzz-db/src/store/personal_read/threads_postgres_tests.rs @@ -1,6 +1,6 @@ //! Thread summaries, anchor validation and targeted refresh. use super::{ - postgres_tests::{fixture, insert_reply}, + postgres_tests::{fixture, insert_reply, sync_reply_arrivals}, *, }; use crate::{ @@ -179,7 +179,7 @@ async fn thread_summaries_order_cap_and_anchor() { channel_mark(channel, newest_root.id.to_hex()), ) .await; - // Threads 0 and 1 tie on newest reply time and the rest follow, newest + // Threads 0 and 1 tie on last arrival and the rest follow, newest // first. Thread 2 has one directed reply and two that arrive last, // together. let offset = |i: u64| if i < 2 { n * 10 } else { (n - i) * 10 }; @@ -206,14 +206,18 @@ async fn thread_summaries_order_cap_and_anchor() { vec![], ) .await; + // Replies arrive in author order, so the twin and thread 2's anchor + // arrive together. sqlx::query( - "UPDATE events SET received_at=(SELECT received_at FROM events WHERE id=$1) WHERE id=$2", + "UPDATE events e SET received_at=e.created_at FROM thread_metadata tm + WHERE tm.community_id=e.community_id AND tm.event_id=e.id + AND tm.received_at IS NOT NULL AND e.community_id=$1", ) - .bind(twin.id.as_bytes().as_slice()) - .bind(anchors[2].id.as_bytes().as_slice()) + .bind(community.as_uuid()) .execute(&pool) .await .unwrap(); + sync_reply_arrivals(&pool).await; let row = sidebar(&db, community, &actor).await; // Every root is read; the eight unread replies count only on threads. @@ -230,7 +234,7 @@ async fn thread_summaries_order_cap_and_anchor() { // The oldest unread thread is past the cap. assert_eq!(order(&row), [tied.to_vec(), ids(&roots[2..last])].concat()); let third = &row.threads[2]; - assert_eq!(third.mentions, 3); + assert_eq!(third.mentions, 1, "only the directed reply is a mention"); assert_eq!( third.latest_id, std::cmp::min(anchors[2].id.to_hex(), twin.id.to_hex()), @@ -249,8 +253,8 @@ async fn thread_summaries_order_cap_and_anchor() { [vec![tied[1].clone()], ids(&roots[2..])].concat() ); let mentions: Vec<_> = row.threads.iter().map(|t| t.mentions).collect(); - let mut expected = vec![1; MAX_THREAD_SUMMARIES]; - expected[1] = 3; + let mut expected = vec![0; MAX_THREAD_SUMMARIES]; + expected[1] = 1; assert_eq!(mentions, expected); } diff --git a/crates/buzz-db/src/store/personal_read/writes.rs b/crates/buzz-db/src/store/personal_read/writes.rs index 26736ff69a7..8c158547994 100644 --- a/crates/buzz-db/src/store/personal_read/writes.rs +++ b/crates/buzz-db/src/store/personal_read/writes.rs @@ -88,6 +88,8 @@ struct Message { id: Vec, received_at: DateTime, root: Option>, + /// A reply that is also on the channel timeline. + broadcast: bool, } /// A channel event and its canonical ancestry. `kinds` bounds the lookup: an @@ -100,7 +102,7 @@ async fn message( kinds: Option<&[i32]>, ) -> Result> { let row = sqlx::query( - "SELECT e.id, e.received_at, e.tags, tm.root_event_id + "SELECT e.id, e.received_at, e.tags, tm.root_event_id, COALESCE(tm.broadcast, false) AS broadcast FROM events e LEFT JOIN thread_metadata tm ON tm.community_id=e.community_id AND tm.event_created_at=e.created_at AND tm.event_id=e.id AND tm.channel_id=e.channel_id WHERE e.community_id=$1 AND e.channel_id=$2 AND e.id=$3 @@ -125,6 +127,7 @@ async fn message( id, received_at, root, + broadcast: row.try_get("broadcast")?, }) }) .transpose() @@ -199,6 +202,38 @@ async fn frontier( Ok(()) } +/// Follow or unfollow a thread. A follow that changes state starts now, so +/// earlier replies are read; an unfollowed row keeps ingest from following +/// again, except as `record_message` allows. +async fn follow( + conn: &mut PgConnection, + community: CommunityId, + actor: &[u8], + channel: Uuid, + root: &[u8], + unfollow: bool, +) -> Result<()> { + sqlx::query( + "INSERT INTO personal_read_frontiers + (community_id, actor, channel_id, root_id, through_timestamp, unfollowed) + VALUES ($1,$2,$3,$4,now(),$5) + ON CONFLICT (community_id, actor, channel_id, root_id) DO UPDATE + SET unfollowed=$5, + through_timestamp=GREATEST(personal_read_frontiers.through_timestamp, now()), + through_message_id=CASE WHEN now() > personal_read_frontiers.through_timestamp + THEN NULL ELSE personal_read_frontiers.through_message_id END + WHERE personal_read_frontiers.unfollowed <> $5", + ) + .bind(community.as_uuid()) + .bind(actor) + .bind(channel) + .bind(root) + .bind(unfollow) + .execute(conn) + .await?; + Ok(()) +} + pub(super) async fn apply( conn: &mut PgConnection, community: CommunityId, @@ -225,13 +260,30 @@ pub(super) async fn apply( return Ok(IntentOutcome::Blocked); }; let is_reply = msg.root.as_ref().is_some_and(|r| r != &msg.id); - if (root.is_empty() && is_reply) + if (root.is_empty() && is_reply && !msg.broadcast) || (!root.is_empty() && msg.id != root && msg.root.as_ref() != Some(&root)) { return Ok(IntentOutcome::Blocked); } frontier(conn, community, actor, target, &root, &msg).await?; } + ReadIntent::Follow { target } | ReadIntent::Unfollow { target } => { + if target.root_id.is_none() { + return Ok(IntentOutcome::Invalid); + } + let Some(root) = valid_target(conn, community, actor, target).await? else { + return Ok(IntentOutcome::Blocked); + }; + follow( + conn, + community, + actor, + target.channel_id, + &root, + matches!(intent, ReadIntent::Unfollow { .. }), + ) + .await?; + } } Ok(IntentOutcome::Applied) } diff --git a/crates/buzz-relay/src/api/buzz_v1/handlers.rs b/crates/buzz-relay/src/api/buzz_v1/handlers.rs index f3dbf095654..8f7d1eb17c1 100644 --- a/crates/buzz-relay/src/api/buzz_v1/handlers.rs +++ b/crates/buzz-relay/src/api/buzz_v1/handlers.rs @@ -7,7 +7,8 @@ use axum::{ response::Response, }; use buzz_db::personal_read::{ - ChannelReadSummary, ReadIntent, SidebarPage, MAX_CHANNELS, MAX_INTENTS, + ChannelReadSummary, ReadIntent, ThreadsCursor, ThreadsPage, MAX_CHANNELS, MAX_INTENTS, + MAX_THREADS, }; use serde::Deserialize; use serde_json::{json, Value}; @@ -74,27 +75,72 @@ pub(super) async fn sidebar( } } .map_err(|_| Error::unavailable())?; - auth::response(release(&state, &headers, &principal, page).await?) + let channels: Vec<_> = page.channels.iter().map(|c| c.channel_id).collect(); + release(&state, &headers, &principal, &channels).await?; + auth::response(page) + }) + .await + .map_err(|_| Error::unavailable())? +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct ThreadsQuery { + limit: Option, + cursor: Option, +} + +pub(super) async fn threads( + State(state): State>, + headers: HeaderMap, + OriginalUri(uri): OriginalUri, + query: Result, axum::extract::rejection::QueryRejection>, +) -> Result { + tokio::time::timeout(Duration::from_secs(8), async { + let principal = auth::authorize(&state, &headers, &uri, "GET", None).await?; + let Query(query) = query.map_err(|_| Error::invalid())?; + let limit = query.limit.unwrap_or(MAX_THREADS); + let cursor = query + .cursor + .map(|c| ThreadsCursor::parse(&c).ok_or_else(Error::invalid)) + .transpose()?; + if !(1..=MAX_THREADS).contains(&limit) { + return Err(Error::invalid()); + } + let page: ThreadsPage = state + .db + .personal_read_threads( + principal.tenant.community(), + &principal.actor, + state.config.buzz_v1_retention_seconds, + limit, + cursor.as_ref(), + ) + .await + .map_err(|_| Error::unavailable())?; + let channels: BTreeSet<_> = page.threads.iter().map(|t| t.channel_id).collect(); + let channels: Vec<_> = channels.into_iter().collect(); + release(&state, &headers, &principal, &channels).await?; + auth::response(page) }) .await .map_err(|_| Error::unavailable())? } /// Release projected rows only after admission and the actor's membership of -/// every row are rechecked outside the projection snapshot. +/// every row's channel are rechecked outside the projection snapshot. async fn release( state: &AppState, headers: &HeaderMap, principal: &auth::Principal, - page: SidebarPage, -) -> Result { + channels: &[Uuid], +) -> Result<(), Error> { auth::recheck(state, headers, principal).await?; - let channels: Vec<_> = page.channels.iter().map(|c| c.channel_id).collect(); let memberships = state .db .membership_pairs( principal.tenant.community(), - &channels, + channels, &[principal.actor.to_bytes().to_vec()], ) .await @@ -102,7 +148,7 @@ async fn release( if memberships.len() != channels.len() { return Err(Error::unavailable()); } - Ok(page) + Ok(()) } /// Updated rows for the channels a batch applied to. A channel the actor has @@ -126,7 +172,9 @@ async fn refreshed( ) .await .map_err(|_| Error::unavailable())?; - rows.extend(release(state, headers, principal, page).await?.channels); + let ids: Vec<_> = page.channels.iter().map(|c| c.channel_id).collect(); + release(state, headers, principal, &ids).await?; + rows.extend(page.channels); } Ok(rows) } @@ -168,8 +216,7 @@ pub(super) async fn write( .await .unwrap_or_else(|_| json!({"status":"unknown","retryable":true})); if outcome["status"] == "applied" { - let ReadIntent::MarkThrough { target, .. } = &intent; - applied.insert(target.channel_id); + applied.insert(intent.target().channel_id); } outcomes.push(outcome); } diff --git a/crates/buzz-relay/src/api/buzz_v1/mod.rs b/crates/buzz-relay/src/api/buzz_v1/mod.rs index d462ac15aae..0236dd7761e 100644 --- a/crates/buzz-relay/src/api/buzz_v1/mod.rs +++ b/crates/buzz-relay/src/api/buzz_v1/mod.rs @@ -17,6 +17,7 @@ pub const BASE_PATH: &str = "/buzz/v1"; pub fn router(state: Arc) -> Router { Router::new() .route("/me/sidebar", get(handlers::sidebar)) + .route("/me/threads", get(handlers::threads)) .route("/me/read-state", post(handlers::write)) .fallback(|| async { auth::Error::new(axum::http::StatusCode::NOT_FOUND, "not_found") }) .with_state(state) diff --git a/crates/buzz-relay/src/api/buzz_v1/postgres_tests.rs b/crates/buzz-relay/src/api/buzz_v1/postgres_tests.rs index 6d50c42815a..c7becba628c 100644 --- a/crates/buzz-relay/src/api/buzz_v1/postgres_tests.rs +++ b/crates/buzz-relay/src/api/buzz_v1/postgres_tests.rs @@ -361,6 +361,115 @@ async fn accessory_batch_outcomes_are_independent_and_return_applied_rows() { assert_ne!(response.status(), StatusCode::OK); } +#[tokio::test] +#[ignore = "requires Postgres"] +async fn accessory_follow_intents_and_threads_view() { + let fixture = crate::api::bridge::postgres_tests::bridge_handler_test_state() + .await + .unwrap(); + let mut state = (*fixture).clone(); + Arc::make_mut(&mut state.config).buzz_v1_enabled = true; + let state = Arc::new(state); + let host = format!("bff-threads-{}.local", uuid::Uuid::new_v4()); + let community = state + .db + .ensure_configured_community(&host) + .await + .unwrap() + .id; + let actor = Keys::generate(); + let channel = state + .db + .create_channel( + community, + "threads", + buzz_db::channel::ChannelType::Stream, + buzz_db::channel::ChannelVisibility::Open, + None, + &actor.public_key().to_bytes(), + None, + ) + .await + .unwrap() + .id; + start_before_everything(&state, community, &actor).await; + let mut roots = Vec::new(); + for _ in 0..2 { + let root = EventBuilder::new(Kind::Custom(9), "root") + .sign_with_keys(&Keys::generate()) + .unwrap(); + state + .db + .insert_event(community, &root, Some(channel)) + .await + .unwrap(); + roots.push(root); + } + let write_path = "/buzz/v1/me/read-state"; + let body = serde_json::to_vec(&json!({"intents":[ + {"type":"follow","target":{"channel_id":channel}}, + {"type":"follow","target":{"channel_id":channel,"root_id":roots[0].id.to_hex()}}, + {"type":"follow","target":{"channel_id":channel,"root_id":roots[1].id.to_hex()}}, + {"type":"unfollow","target":{"channel_id":channel,"root_id":roots[1].id.to_hex()}} + ]})) + .unwrap(); + let auth = proof(&actor, &host, write_path, "POST", Some(&body)); + let result = request(state.clone(), &host, write_path, "POST", Some(&auth), &body).await; + assert_eq!(result.0, StatusCode::OK, "{}", result.1); + assert_eq!( + result.1["outcomes"], + json!([{"status":"invalid"},{"status":"applied"},{"status":"applied"},{"status":"applied"}]) + ); + assert_eq!(result.1["channels"][0]["channel_id"], json!(channel)); + + let mut replies = Vec::new(); + for root in &roots { + let reply = EventBuilder::new(Kind::Custom(9), "reply") + .tags([Tag::parse(["e", &root.id.to_hex(), "", "root"]).unwrap()]) + .sign_with_keys(&Keys::generate()) + .unwrap(); + let at = |e: &nostr::Event| { + chrono::DateTime::from_timestamp(e.created_at.as_secs() as i64, 0).unwrap() + }; + state + .db + .insert_event_with_thread_metadata( + community, + &reply, + Some(channel), + Some(buzz_db::event::ThreadMetadataParams { + event_id: reply.id.as_bytes(), + event_created_at: at(&reply), + channel_id: channel, + parent_event_id: Some(root.id.as_bytes()), + parent_event_created_at: Some(at(root)), + root_event_id: Some(root.id.as_bytes()), + root_event_created_at: Some(at(root)), + depth: 1, + broadcast: false, + }), + ) + .await + .unwrap(); + replies.push(reply); + } + let path = "/buzz/v1/me/threads?limit=1"; + let auth = proof(&actor, &host, path, "GET", None); + let result = request(state.clone(), &host, path, "GET", Some(&auth), b"").await; + assert_eq!(result.0, StatusCode::OK, "{}", result.1); + // Only the followed thread, unread since the follow. + assert_eq!( + result.1, + json!({"threads":[{"channel_id":channel,"root_id":roots[0].id.to_hex(), + "unread":true,"mentions":0,"read_through_id":null, + "latest_id":replies[0].id.to_hex()}],"next_cursor":null}) + ); + let path = "/buzz/v1/me/threads?cursor=not-a-cursor"; + let auth = proof(&actor, &host, path, "GET", None); + let result = request(state.clone(), &host, path, "GET", Some(&auth), b"").await; + assert_eq!(result.0, StatusCode::BAD_REQUEST, "{}", result.1); +} + #[tokio::test] #[ignore = "requires Postgres"] async fn accessory_write_revocation_is_terminal_before_persistence() { @@ -495,7 +604,8 @@ async fn accessory_discovery_is_host_bound_and_opt_in() { doc["buzz_v1"], json!({"version":1,"base_path":"/buzz/v1","retention_seconds":1234, "max_channels":20,"max_intents":100, - "eligible_kinds":[9, 40002, 45001, 45003]}) + "eligible_kinds":[9, 40002, 45001, 45003], + "capabilities":["follow","threads"],"max_threads":50}) ); } } diff --git a/crates/buzz-relay/src/nip11.rs b/crates/buzz-relay/src/nip11.rs index d60771b2bc4..2a885f14b23 100644 --- a/crates/buzz-relay/src/nip11.rs +++ b/crates/buzz-relay/src/nip11.rs @@ -97,6 +97,11 @@ pub struct BuzzV1Descriptor { /// Message kinds that count as unread and as latest activity. Clients /// classify live arrivals with this set instead of keeping a copy. pub eligible_kinds: [i32; 4], + /// Additions beyond the first v1 contract: `follow` (follow and unfollow + /// intents) and `threads` (`GET /me/threads`). Send one only when listed. + pub capabilities: Vec, + /// Maximum followed threads per threads page. + pub max_threads: usize, } /// Public capability descriptor for relay-proxied GIF search. @@ -382,7 +387,7 @@ pub(crate) async fn nip11_document(state: &crate::state::AppState, raw_host: &st ); if let Ok(tenant) = crate::tenant::bind_community(&state.db, raw_host).await { if state.config.buzz_v1_enabled { - use buzz_db::personal_read::{ELIGIBLE_KINDS, MAX_CHANNELS, MAX_INTENTS}; + use buzz_db::personal_read::{ELIGIBLE_KINDS, MAX_CHANNELS, MAX_INTENTS, MAX_THREADS}; info.buzz_v1 = Some(BuzzV1Descriptor { version: 1, base_path: crate::api::buzz_v1::BASE_PATH.to_owned(), @@ -390,6 +395,8 @@ pub(crate) async fn nip11_document(state: &crate::state::AppState, raw_host: &st max_channels: MAX_CHANNELS, max_intents: MAX_INTENTS, eligible_kinds: ELIGIBLE_KINDS, + capabilities: vec!["follow".to_owned(), "threads".to_owned()], + max_threads: MAX_THREADS, }); } info.read_state_snapshot = Some(serde_json::json!({ diff --git a/docs/buzz-v1-read-state.md b/docs/buzz-v1-read-state.md index 5e0aab9a2c6..58b988b1d10 100644 --- a/docs/buzz-v1-read-state.md +++ b/docs/buzz-v1-read-state.md @@ -12,9 +12,14 @@ application/nostr+json`, or `GET /info`) includes `buzz_v1` only when enabled: ```json {"buzz_v1":{"version":1,"base_path":"/buzz/v1","retention_seconds":2592000, -"max_channels":20,"max_intents":100,"eligible_kinds":[9,40002,45001,45003]}} +"max_channels":20,"max_intents":100,"eligible_kinds":[9,40002,45001,45003], +"capabilities":["follow","threads"],"max_threads":50}} ``` +`capabilities` lists additions to the first v1 contract: `follow` for the +`follow` and `unfollow` intents, `threads` for `GET /me/threads`. Use one only +when it is listed. + Use the requesting origin plus this relative prefix. Discovery is a configured capability, not a promise that the next request cannot fail. Unknown hosts and disabled deployments omit it, and a disabled deployment does not mount @@ -74,22 +79,23 @@ marking one never moves another. | Field | Channel row | Thread row | |---|---|---| -| `unread` | an unread top-level message counts | an unread reply counts | -| `mentions` | unread top-level messages that tag you with `p`, or every unread one in a DM (at most 100) | unread replies (at most 100) | +| `unread` | an unread timeline message counts | an unread reply counts | +| `mentions` | unread timeline messages that tag you with `p`, or every unread one in a DM (at most 100) | unread replies that tag you with `p`, or every unread one in a DM (at most 100) | | `read_through_id` | the anchor of the timeline's frontier, or null | the anchor of the thread's frontier, or null | -| `latest_id` | the last eligible top-level message to arrive, or null | the last reply to arrive | - -A thread reply never makes the channel row unread and never becomes its -`latest_id`, including a reply carrying `broadcast=1`: it shows on its thread -row. `latest_id` is the anchor that reads its scope: "mark channel read" sends a -channel `mark_through` at the channel's `latest_id`, plus one thread -`mark_through` at each listed thread's `latest_id` to clear those too. The -channel `latest_id` counts any author and ignores read progress. Null means the -bounded scans found no top-level message, not that the channel is empty. -`read_through_id` may name a message that has since been deleted. +| `latest_id` | the last eligible timeline message to arrive, or null | the last reply to arrive | + +The timeline holds top-level messages and replies carrying `broadcast=1`; a +broadcast reply also counts on its thread row. No other reply makes the channel +row unread or becomes its `latest_id`. `latest_id` is the anchor that reads its +scope: "mark channel read" sends a channel `mark_through` at the channel's +`latest_id`, plus one thread `mark_through` at each listed thread's `latest_id` +to clear those too. The channel `latest_id` counts any author and ignores read +progress. Null means no timeline message was recorded since 0061, not that the +channel is empty. `read_through_id` may name a message that has since been +deleted. `threads` lists the threads you follow that have unread replies, newest -unread reply first by author time (then `root_id`), at most 25. Threads past +arrival first (then `root_id`), at most 25. Threads past the 25th are omitted with their mentions, and nothing signals the omission: after marking the listed threads read, a refresh can list the next ones. A thread row's `unread` is always true in this version. No message bytes are @@ -107,7 +113,8 @@ thread you follow. The relay records follows when it stores an eligible message, in the same transaction: - Its author has read through it: the timeline for a top-level message, - otherwise the thread, which the author now follows. + otherwise the thread, which the author now follows. A broadcast reply reads + only its thread. - Each channel member it tags with `p` follows its thread (for a top-level message, the thread rooted at it). - A reply also makes the root's author follow, and in a DM every member. @@ -115,11 +122,16 @@ in the same transaction: A new follower starts just before the message that made them follow, so it is unread; one who already follows keeps their position. Reading never follows: a thread `mark_through` moves an existing follow and otherwise changes nothing. -There is no unfollow yet. `broadcast=1` does not make anyone follow, so a -broadcast reply counts only for followers. Follows start when the relay that +`broadcast=1` does not make anyone follow. Follows start when the relay that records them is deployed; there is no backfill. Desktop notification follows and mutes do not affect these counts. +The `follow` intent follows a thread explicitly, from now: replies that arrived +before it are read. Following a thread you already follow changes nothing. +`unfollow` stops counting it, and the stored message rules above no longer +follow you again, with two exceptions: posting in the thread follows you through +your post, and a mention follows you from just before it. + Counts are what the bounded walks could establish, and may undercount. A message whose ancestry is unresolved is left out, as are messages beyond the bounds (see [Bounds](#bounds-and-deployment)). @@ -140,6 +152,27 @@ clock, relay arrival (see [Order](#order)). Three consequences: A later configuration expansion can change counts without having lost progress. +## Threads view + +`GET /buzz/v1/me/threads?limit=50&cursor=` lists the threads you follow +in joined, nondeleted channels, newest last reply first by arrival (then +`root_id`), read or not, at most 50 per page. Omit the cursor on the first +request and pass `next_cursor` back unchanged; null means the scan is done. A +thread with no reply yet is not listed. + +```json +{"threads":[{"channel_id":"","root_id":"<64-hex>","unread":false, + "mentions":0,"read_through_id":"<64-hex>","latest_id":"<64-hex>"}], + "next_cursor":""} +``` + +Each row has the sidebar thread row's fields and meanings plus its +`channel_id`, except that `unread` may be false and `mentions` is then 0. Pages +are separate snapshots: a thread that gets a new reply while you page moves +ahead of the cursor, so refresh from the first page after a change. Every +page probes each of your follows, so its cost grows with how many threads you +have ever followed, not with community size. + ## Fixed-operand writes `POST /buzz/v1/me/read-state` accepts 1–100 independent intents: @@ -147,7 +180,9 @@ A later configuration expansion can change counts without having lost progress. ```json {"intents":[ {"type":"mark_through","target":{"channel_id":""},"message_id":"<64-hex-event>"}, - {"type":"mark_through","target":{"channel_id":"","root_id":"<64-hex-root>"},"message_id":"<64-hex-event>"} + {"type":"mark_through","target":{"channel_id":"","root_id":"<64-hex-root>"},"message_id":"<64-hex-event>"}, + {"type":"follow","target":{"channel_id":"","root_id":"<64-hex-root>"}}, + {"type":"unfollow","target":{"channel_id":"","root_id":"<64-hex-root>"}} ]} ``` @@ -165,8 +200,9 @@ until its outcome is resolved. A mark-through validates a fixed message and advances its context's monotone frontier to that message's relay arrival (see [Order](#order)). A channel -anchor must be a top-level message; a thread anchor must be the thread's root -or one of its replies. The frontier records the anchor as `read_through_id` +anchor must be a top-level message or a broadcast reply; a thread anchor must +be the thread's root or one of its replies. `follow` and `unfollow` need a +thread target; a channel-only one is `invalid`. The frontier records the anchor as `read_through_id` when it advances; an anchor that arrived at or before the current frontier changes nothing. Opening a view is not itself a reading action; client dwell/focus policy determines when to send an actual observed anchor. Old or @@ -195,16 +231,16 @@ microsecond resolution. Three limits follow, none of which strands a badge: - Relay processes with different clocks can stamp out of true order by their skew. - Messages with the identical stamp are read together. +- A `follow` and the account's start use the database clock, so they can be + off from relay arrivals by the skew between the two. -Counting walks forward from a position. Positions are arrival times but the -event indexes are by author time, which the relay accepts within 15 minutes of -arrival, so a walk starts 15 minutes of author time before the position. A -message backdated further than that arrives unseen; Order above still holds for -what the walk finds. A channel's `latest_id` is found from the newest top-level -message by author time: it is the last top-level message to arrive among those -authored up to 30 minutes before that one, which includes the last arrival. So -marking through it reads every top-level message counted. A thread's -`latest_id` is found the same way among its replies. +Counting walks forward from a position over indexes in arrival order: the +timeline's messages per channel and each thread's replies. A channel's +`latest_id` is exactly its last timeline arrival and a thread's its last reply +arrival, so marking through it reads everything counted. Mentions are indexed +by author time, which the relay accepts within 15 minutes of arrival, so a +mention walk starts 15 minutes of author time before the position: a mention +backdated further than that is unread but not counted as a mention. A channel row is unread exactly when its `latest_id` is someone else's, inside the horizon, and arrived after the timeline's position. Nothing earlier needs @@ -225,28 +261,62 @@ device-local. a POST's rows for 100 channels fit the response limit. - 64 KiB write body; 1 MiB serialized API response. - Counts stop at 100 per channel and per thread. -- Each walk reads at most 4096 raw events, before eligibility. The `latest_id` - probe reads the newest 4096 per channel; a longer ineligible or reply-only - tail (reactions, edits, thread replies) leaves it null and the row read. The - last-arrival search reads the newest 4096 of the 30-minute window, so more - than that many backdated arrivals can hide one. -- A caught-up reader costs a few index probes per channel and per followed +- A caught-up reader costs one index probe per channel and per followed thread; a reader who is behind walks only what arrived after their position. +- 50 rows per threads page; each page probes every follow of the reader. - DB statement/lock deadlines and HTTP read deadlines bound work; a write's intents and its row refresh share one eight-second deadline after admission. Limits are containment, not a production capacity claim. -Apply migrations 0056 through 0058 (or the equivalent desired schema). 0056 -creates two empty private tables and no index on `events`, 0057 adds the -nullable `personal_read_accounts.started_at` column, and 0058 adds the nullable -`personal_read_frontiers.through_message_id` column and drops -`threads_through_timestamp`. Timeline walks use the existing -`idx_events_community_channel_created`, thread walks -`idx_thread_metadata_window`, and mention counts -`idx_event_mentions_pubkey_created`. Ingest writes a few frontier rows per -eligible channel message (the author, and anyone it makes follow), whether or -not the API is enabled, so follows exist once it is; there are no stored unread -counters. +Apply migrations 0056 through 0058 and 0061 (or the equivalent desired +schema). 0056 creates two empty private tables and no index on `events`, 0057 +adds the nullable `personal_read_accounts.started_at` column, and 0058 adds the +nullable `personal_read_frontiers.through_message_id` column and drops +`threads_through_timestamp`. 0061 adds `events.read_timeline` (set at insert +for timeline messages), `thread_metadata.received_at` (set at insert for +eligible replies) and `personal_read_frontiers.unfollowed`, plus three indexes: +`idx_events_read_timeline` for timeline walks, `idx_thread_metadata_read_arrival` +for thread walks and `idx_event_mentions_pubkey_channel_created` for mention +counts. There is no backfill: messages stored before the new relay serves, +including by older relay processes still running after 0061, are on neither +arrival index, so they never count as unread or become `latest_id`. Ingest +writes a few frontier rows per eligible channel message (the author, and anyone +it makes follow), whether or not the API is enabled, so follows exist once it +is; there are no stored unread counters. + +The column additions are catalog-only but briefly take an exclusive lock on +`events`, `thread_metadata` and `personal_read_frontiers`, bounded by the +migration's one-second `lock_timeout`; a busy writer fails the deployment, +which retries, rather than stalling ingestion. Building the indexes inside the +migration is bounded the same way, so on a database with data, add the columns +and prebuild the indexes before deploying, as for +[0049](thread-window-deployment.md). 0061 then skips both and only validates +the indexes. Run each statement on its own, retrying any that times out; +`CREATE INDEX CONCURRENTLY` cannot run in a transaction block: + +```sql +SET lock_timeout = '1s'; +ALTER TABLE events ADD COLUMN IF NOT EXISTS read_timeline BOOLEAN NOT NULL DEFAULT false; +ALTER TABLE thread_metadata ADD COLUMN IF NOT EXISTS received_at TIMESTAMPTZ; +ALTER TABLE personal_read_frontiers ADD COLUMN IF NOT EXISTS unfollowed BOOLEAN NOT NULL + DEFAULT false CHECK (NOT unfollowed OR root_id <> ''::bytea); +CREATE INDEX CONCURRENTLY idx_thread_metadata_read_arrival ON public.thread_metadata + (community_id, root_event_id, received_at DESC, event_id) WHERE received_at IS NOT NULL; +CREATE INDEX CONCURRENTLY idx_event_mentions_pubkey_channel_created ON public.event_mentions + (community_id, pubkey_hex, channel_id, event_created_at DESC); +-- events is partitioned: an invalid parent, then each partition concurrently. +CREATE INDEX idx_events_read_timeline ON ONLY public.events + (community_id, channel_id, received_at DESC, id) WHERE read_timeline AND deleted_at IS NULL; +-- For each partition p in +-- SELECT inhrelid::regclass FROM pg_inherits WHERE inhparent = 'public.events'::regclass: +CREATE INDEX CONCURRENTLY p_read_timeline ON p + (community_id, channel_id, received_at DESC, id) WHERE read_timeline AND deleted_at IS NULL; +ALTER INDEX idx_events_read_timeline ATTACH PARTITION p_read_timeline; +``` + +The parent becomes valid once every partition's index is attached. Verify the +three like 0049's, against the definitions the migration expects; recover a +failed build the same way. Use existing HTTP route/status/latency metrics for `/buzz/v1/me/sidebar` and `/buzz/v1/me/read-state`, plus database pool/statement metrics. No payload, diff --git a/migrations/0061_read_state_access_paths.sql b/migrations/0061_read_state_access_paths.sql new file mode 100644 index 00000000000..a1ec0677697 --- /dev/null +++ b/migrations/0061_read_state_access_paths.sql @@ -0,0 +1,68 @@ +-- Read-state access paths: one index seek answers "newest countable message +-- here" and "what arrived after this position" for a channel timeline or a +-- thread, and "my mentions in this channel since then". Arrival order +-- (received_at), not author time. No backfill: rows written before this +-- migration are not on the new paths. +-- +-- events.read_timeline: an eligible message that counts on its channel +-- timeline (top-level, or a broadcast reply), set at insert. +-- thread_metadata.received_at: a reply's arrival, set at insert; NULL on root +-- stubs and earlier rows. +-- personal_read_frontiers.unfollowed: a sticky unfollow of a thread. +-- +-- Brownfield operators MUST add the three columns and prebuild the three +-- indexes concurrently as documented in docs/buzz-v1-read-state.md. Startup is +-- bounded like 0049: a large table fails deployment rather than blocking +-- ingestion. +SET LOCAL lock_timeout = '1s'; +SET LOCAL statement_timeout = '5s'; + +DO $$ +DECLARE + spec RECORD; +BEGIN + -- Even ADD COLUMN IF NOT EXISTS requests a writer-conflicting lock: skip + -- a column an operator added beforehand. + FOR spec IN SELECT * FROM (VALUES + ('events', 'read_timeline', 'BOOLEAN NOT NULL DEFAULT false'), + ('thread_metadata', 'received_at', 'TIMESTAMPTZ'), + ('personal_read_frontiers', 'unfollowed', + $d$BOOLEAN NOT NULL DEFAULT false CHECK (NOT unfollowed OR root_id <> ''::bytea)$d$) + ) AS s(tbl, col, definition) LOOP + IF NOT EXISTS ( + SELECT 1 FROM pg_attribute + WHERE attrelid = ('public.' || spec.tbl)::regclass + AND attname = spec.col AND NOT attisdropped + ) THEN + EXECUTE format('ALTER TABLE public.%I ADD COLUMN %I %s', + spec.tbl, spec.col, spec.definition); + END IF; + END LOOP; + + -- A partitioned parent's definition reads ON ONLY; CREATE recurses to + -- every partition regardless. + FOR spec IN SELECT * FROM (VALUES + ('idx_events_read_timeline', + 'CREATE INDEX idx_events_read_timeline ON ONLY public.events USING btree (community_id, channel_id, received_at DESC, id) WHERE (read_timeline AND (deleted_at IS NULL))'), + ('idx_thread_metadata_read_arrival', + 'CREATE INDEX idx_thread_metadata_read_arrival ON public.thread_metadata USING btree (community_id, root_event_id, received_at DESC, event_id) WHERE (received_at IS NOT NULL)'), + ('idx_event_mentions_pubkey_channel_created', + 'CREATE INDEX idx_event_mentions_pubkey_channel_created ON public.event_mentions USING btree (community_id, pubkey_hex, channel_id, event_created_at DESC)') + ) AS s(name, definition) LOOP + -- IF NOT EXISTS still requests a writer-conflicting lock: skip CREATE + -- for a prebuilt index, then validate it. + IF to_regclass('public.' || spec.name) IS NULL THEN + EXECUTE replace(spec.definition, ' ON ONLY ', ' ON '); + END IF; + IF NOT EXISTS ( + SELECT 1 FROM pg_index i + JOIN pg_class c ON c.oid = i.indexrelid + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = 'public' AND c.relname = spec.name + AND i.indisvalid AND i.indisready AND i.indislive + AND pg_get_indexdef(i.indexrelid) = spec.definition + ) THEN + RAISE EXCEPTION '% invalid or wrong definition; see docs/buzz-v1-read-state.md', spec.name; + END IF; + END LOOP; +END $$; diff --git a/schema/tables/public/event_mentions.sql b/schema/tables/public/event_mentions.sql index b0eb9dcb4a7..df0ee76e534 100644 --- a/schema/tables/public/event_mentions.sql +++ b/schema/tables/public/event_mentions.sql @@ -18,5 +18,7 @@ CREATE INDEX idx_event_mentions_pubkey_created ON event_mentions (community_id, pubkey_hex, event_created_at DESC); CREATE INDEX idx_event_mentions_pubkey_kind_created ON event_mentions (community_id, pubkey_hex, event_kind, event_created_at DESC); +CREATE INDEX idx_event_mentions_pubkey_channel_created + ON event_mentions (community_id, pubkey_hex, channel_id, event_created_at DESC); CREATE INDEX idx_event_mentions_community_event ON event_mentions (community_id, event_id); diff --git a/schema/tables/public/events.sql b/schema/tables/public/events.sql index 10c1adc19e9..936d7d737d5 100644 --- a/schema/tables/public/events.sql +++ b/schema/tables/public/events.sql @@ -37,6 +37,8 @@ CREATE TABLE events ( d_tag TEXT, not_before BIGINT, delivered_at BIGINT, + -- An eligible message that counts on its channel's read-state timeline. + read_timeline BOOLEAN NOT NULL DEFAULT false, PRIMARY KEY (community_id, created_at, id) ) PARTITION BY RANGE (created_at); @@ -52,6 +54,10 @@ CREATE INDEX idx_events_community_pubkey_kind_created CREATE INDEX idx_events_community_kind_created ON events (community_id, kind, created_at DESC, id); CREATE INDEX idx_events_community_deleted ON events (community_id, deleted_at); +-- Read state: newest timeline message, and what arrived after a position. +CREATE INDEX idx_events_read_timeline + ON events (community_id, channel_id, received_at DESC, id) + WHERE read_timeline AND deleted_at IS NULL; -- Addressable (replaceable) and NIP-33 parameterized lookups. CREATE INDEX idx_events_addressable ON events (community_id, kind, pubkey, channel_id, deleted_at); diff --git a/schema/tables/public/personal_read_frontiers.sql b/schema/tables/public/personal_read_frontiers.sql index e62eb9c8b94..0aab16ad5c3 100644 --- a/schema/tables/public/personal_read_frontiers.sql +++ b/schema/tables/public/personal_read_frontiers.sql @@ -6,6 +6,8 @@ CREATE TABLE personal_read_frontiers ( through_timestamp TIMESTAMPTZ NOT NULL, -- The message whose arrival is through_timestamp; NULL before 0058. through_message_id BYTEA CHECK (octet_length(through_message_id) = 32), + -- A sticky thread unfollow; thread rows only. + unfollowed BOOLEAN NOT NULL DEFAULT false CHECK (NOT unfollowed OR root_id <> ''::bytea), PRIMARY KEY (community_id, actor, channel_id, root_id), FOREIGN KEY (community_id, actor) REFERENCES personal_read_accounts (community_id, actor) ON DELETE CASCADE, diff --git a/schema/tables/public/thread_metadata.sql b/schema/tables/public/thread_metadata.sql index 2c9ed9800ac..94a162316e0 100644 --- a/schema/tables/public/thread_metadata.sql +++ b/schema/tables/public/thread_metadata.sql @@ -15,6 +15,8 @@ CREATE TABLE thread_metadata ( descendant_count INT NOT NULL DEFAULT 0, last_reply_at TIMESTAMPTZ, broadcast BOOLEAN NOT NULL DEFAULT FALSE, + -- A reply's relay arrival; NULL on root stubs and rows before 0061. + received_at TIMESTAMPTZ, PRIMARY KEY (community_id, event_created_at, event_id), FOREIGN KEY (community_id, channel_id) REFERENCES channels (community_id, id) ); @@ -26,3 +28,6 @@ CREATE INDEX idx_thread_metadata_window CREATE INDEX idx_thread_metadata_channel_depth ON thread_metadata (community_id, channel_id, depth, event_created_at); CREATE INDEX idx_thread_metadata_event_id ON thread_metadata (community_id, event_id); +CREATE INDEX idx_thread_metadata_read_arrival + ON thread_metadata (community_id, root_event_id, received_at DESC, event_id) + WHERE received_at IS NOT NULL; From 9127a1be3d173ee70590eb83a5999ebe913ecfe6 Mon Sep 17 00:00:00 2001 From: Bradley Axen Date: Tue, 13 Oct 2026 21:53:31 -0700 Subject: [PATCH 3/4] fix(read-state): root-first thread mentions, follow dormancy, humans only Three fixes from the unread load study: - THREAD_MENTIONS starts from the thread's unread replies and probes the mention key for each, instead of walking the actor's mentions across the channel and filtering by root. Same counts (pinned against the old query in a test), no new index; it was ~92% of the heaviest sidebar. - Automatic thread follows go dormant after BUZZ_V1_RETENTION_SECONDS (30 days) without the actor's involvement: posting, being mentioned, a reply to their root or in their DM, marking it read, or following it. A dormant follow is neither counted nor listed but keeps its row and position, and any involvement revives it. Follow intents set `explicit`, which never goes dormant. 0061 adds personal_read_frontiers.explicit, .active_at and idx_personal_read_frontiers_active (actor's threads by involvement, for a later history view). This replaces the author-time unread horizon: author time no longer filters unread or mentions. - Agent accounts (users.agent_owner_pubkey set) keep no read state: ingest writes them no positions or follows, and /buzz/v1 answers 403. Their messages still make people follow. Co-authored-by: murderbot <3754f8729004d95654c46dbab3129e4ab9ef05cc2534e2a3fbfc155983bd637b@buzz.block.builderlab.xyz> Co-Authored-By: Claude Opus 5.5 Signed-off-by: Bradley Axen --- crates/buzz-db/src/runtime/migration.rs | 19 +- .../src/store/personal_read/follows.rs | 44 ++-- .../personal_read/follows_postgres_tests.rs | 221 ++++++++++++++++++ crates/buzz-db/src/store/personal_read/mod.rs | 8 +- .../buzz-db/src/store/personal_read/model.rs | 6 +- .../src/store/personal_read/postgres_tests.rs | 86 +++---- .../src/store/personal_read/projection.rs | 61 ++--- .../projection_postgres_tests.rs | 188 ++++++++++++++- .../buzz-db/src/store/personal_read/writes.rs | 25 +- crates/buzz-relay/src/api/buzz_v1/auth.rs | 10 + .../src/api/buzz_v1/postgres_tests.rs | 80 +++++++ crates/buzz-relay/src/config.rs | 3 +- docs/buzz-v1-read-state.md | 65 +++--- migrations/0061_read_state_access_paths.sql | 14 +- .../tables/public/personal_read_frontiers.sql | 10 + 15 files changed, 697 insertions(+), 143 deletions(-) diff --git a/crates/buzz-db/src/runtime/migration.rs b/crates/buzz-db/src/runtime/migration.rs index 24830bbf118..79df1aa9549 100644 --- a/crates/buzz-db/src/runtime/migration.rs +++ b/crates/buzz-db/src/runtime/migration.rs @@ -2258,16 +2258,25 @@ mod postgres_tests { .sql .as_str(); assert!(through_message.contains("DROP COLUMN threads_through_timestamp")); - // 0061 adds the sticky thread unfollow. + // 0061 adds the sticky thread unfollow and follow liveness. let unfollowed_column = "unfollowed boolean not null default false \ - check (not unfollowed or root_id <> ''::bytea), "; - assert!(MIGRATOR + check (not unfollowed or root_id <> ''::bytea), \ + explicit boolean not null default false, \ + active_at timestamptz not null default now(), "; + let access_paths = MIGRATOR .iter() .find(|m| m.version == 61) .expect("read-state access path migration") .sql - .as_str() - .contains("$d$BOOLEAN NOT NULL DEFAULT false CHECK (NOT unfollowed OR root_id <> ''::bytea)$d$")); + .as_str(); + assert!(access_paths.contains( + "$d$BOOLEAN NOT NULL DEFAULT false CHECK (NOT unfollowed OR root_id <> ''::bytea)$d$" + )); + assert!(access_paths + .contains("('personal_read_frontiers', 'explicit', 'BOOLEAN NOT NULL DEFAULT false')")); + assert!(access_paths.contains( + "('personal_read_frontiers', 'active_at', 'TIMESTAMPTZ NOT NULL DEFAULT now()')" + )); for (table, definition) in personal.tables { let in_schema = schema.tables.get(&table).map(|schema_definition| { if table == "personal_read_accounts" { diff --git a/crates/buzz-db/src/store/personal_read/follows.rs b/crates/buzz-db/src/store/personal_read/follows.rs index 6681582efb9..16fd774d789 100644 --- a/crates/buzz-db/src/store/personal_read/follows.rs +++ b/crates/buzz-db/src/store/personal_read/follows.rs @@ -80,6 +80,9 @@ pub(crate) fn place<'a>( /// Everyone but the author starts just before the message, so it is unread /// to them. A follower keeps an existing position. An unfollow sticks until /// its actor posts in the thread or is mentioned there again. +/// +/// Each of these involves its actor in the thread (`active_at`), which keeps +/// an implicit follow live. Agent accounts keep no read state. pub(crate) async fn record_message( tx: &mut AdmittedTx, event: &Event, @@ -129,16 +132,23 @@ pub(crate) async fn record_message( WHERE $9 AND cm.community_id=$1 AND cm.channel_id=$2 AND cm.removed_at IS NULL AND c.channel_type='dm' AND cm.pubkey<>$4 ), rows AS ( - -- An existing follower keeps their position, so is not touched; - -- nor is an unfollow, unless this message re-follows (the author - -- posting, or a mention). Existence probes are LIMIT 1 laterals: - -- always one index lookup. + -- An existing follower keeps their position, so is only touched + -- to keep the follow live, at most daily rather than per message; + -- an unfollow is not touched unless this message re-follows (the + -- author posting, or a mention). Existence probes are LIMIT 1 + -- laterals: always one index lookup. SELECT DISTINCT ON (i.actor, i.root_id) i.* FROM involved i LEFT JOIN LATERAL ( - SELECT f.unfollowed FROM personal_read_frontiers f WHERE f.community_id=$1 - AND f.actor=i.actor AND f.channel_id=$2 AND f.root_id=i.root_id LIMIT 1 + SELECT f.unfollowed, f.active_at FROM personal_read_frontiers f + WHERE f.community_id=$1 AND f.actor=i.actor AND f.channel_id=$2 + AND f.root_id=i.root_id LIMIT 1 ) f ON true - WHERE i.message IS NOT NULL OR f.unfollowed IS NULL OR (i.refollow AND f.unfollowed) + WHERE NOT EXISTS (SELECT 1 FROM users u WHERE u.community_id=$1 + AND u.pubkey=i.actor AND u.agent_owner_pubkey IS NOT NULL) + AND (i.message IS NOT NULL OR f.unfollowed IS NULL + OR (i.refollow AND f.unfollowed) + OR (i.root_id<>''::bytea AND NOT f.unfollowed + AND f.active_at < now()-interval '1 day')) ORDER BY i.actor, i.root_id, i.message IS NULL, NOT i.refollow ), accounts AS ( INSERT INTO personal_read_accounts (community_id, actor) @@ -152,21 +162,29 @@ pub(crate) async fn record_message( ) -- A re-follow starts just before this message, like a new follow, -- unless the unfollow is later. A row that a concurrent message - -- created is only advanced when it carries an anchor. + -- created is only advanced when it carries an anchor; one kept live + -- keeps its position. INSERT INTO personal_read_frontiers (community_id, actor, channel_id, root_id, through_timestamp, through_message_id) SELECT $1, actor, $2, root_id, through, message FROM rows CROSS JOIN (SELECT count(*) FROM accounts) written ORDER BY actor, root_id ON CONFLICT (community_id, actor, channel_id, root_id) DO UPDATE - SET through_timestamp=GREATEST(personal_read_frontiers.through_timestamp, - excluded.through_timestamp), + SET through_timestamp=CASE + WHEN excluded.through_message_id IS NOT NULL OR personal_read_frontiers.unfollowed + THEN GREATEST(personal_read_frontiers.through_timestamp, excluded.through_timestamp) + ELSE personal_read_frontiers.through_timestamp END, through_message_id=CASE - WHEN excluded.through_timestamp > personal_read_frontiers.through_timestamp + WHEN (excluded.through_message_id IS NOT NULL OR personal_read_frontiers.unfollowed) + AND excluded.through_timestamp > personal_read_frontiers.through_timestamp THEN excluded.through_message_id ELSE personal_read_frontiers.through_message_id END, - unfollowed=false + unfollowed=false, + active_at=CASE WHEN excluded.root_id<>''::bytea THEN now() + ELSE personal_read_frontiers.active_at END WHERE excluded.through_message_id IS NOT NULL OR (personal_read_frontiers.unfollowed - AND excluded.through_timestamp > personal_read_frontiers.through_timestamp)", + AND excluded.through_timestamp > personal_read_frontiers.through_timestamp) + OR (excluded.root_id<>''::bytea AND NOT personal_read_frontiers.unfollowed + AND personal_read_frontiers.active_at < now()-interval '1 day')", ) .bind(community.as_uuid()) .bind(channel) diff --git a/crates/buzz-db/src/store/personal_read/follows_postgres_tests.rs b/crates/buzz-db/src/store/personal_read/follows_postgres_tests.rs index f1928326fd8..f5b146c2867 100644 --- a/crates/buzz-db/src/store/personal_read/follows_postgres_tests.rs +++ b/crates/buzz-db/src/store/personal_read/follows_postgres_tests.rs @@ -645,3 +645,224 @@ async fn the_threads_view_lists_follows_newest_arrival_first() { .threads .is_empty()); } + +/// Put every one of the actor's follows past the default window. +async fn go_dormant(pool: &PgPool, community: CommunityId, actor: &Keys) { + sqlx::query( + "UPDATE personal_read_frontiers SET active_at=now()-interval '31 days' + WHERE community_id=$1 AND actor=$2", + ) + .bind(community.as_uuid()) + .bind(actor.public_key().to_bytes().as_slice()) + .execute(pool) + .await + .unwrap(); +} + +/// Roots of the threads listed for the actor, by the sidebar and by the +/// threads view. +async fn listed(db: &Db, community: CommunityId, channel: Uuid, actor: &Keys) -> [Vec; 2] { + let sidebar = row(db, community, channel, actor) + .await + .threads + .into_iter() + .map(|t| t.root_id) + .collect(); + let view = threads_page(db, community, actor, MAX_THREADS, None) + .await + .threads + .into_iter() + .map(|t| t.thread.root_id) + .collect(); + [sidebar, view] +} + +#[tokio::test] +#[ignore = "requires Postgres"] +async fn a_dormant_follow_is_hidden_until_a_mention_or_post_revives_it() { + let (db, pool, community, channel, actor, root) = fixture().await; + let id = root.id.to_hex(); + let own = reply(&db, community, channel, &root, &actor, vec![]).await; + reply(&db, community, channel, &root, &Keys::generate(), vec![]).await; + assert_eq!( + listed(&db, community, channel, &actor).await, + [vec![id.clone()], vec![id.clone()]] + ); + + go_dormant(&pool, community, &actor).await; + // A reply that does not involve the actor leaves it dormant; the row and + // its position stay. + reply(&db, community, channel, &root, &Keys::generate(), vec![]).await; + assert_eq!( + listed(&db, community, channel, &actor).await, + [Vec::::new(), vec![]] + ); + assert_eq!( + follow(&pool, community, &actor, &root).await, + Some(Some(own.id.to_hex())) + ); + + // A mention revives it at its old position. + reply( + &db, + community, + channel, + &root, + &Keys::generate(), + mention(&actor), + ) + .await; + let summary = row(&db, community, channel, &actor).await; + assert_eq!(threads(&summary), [(id.clone(), 1)]); + assert_eq!(summary.threads[0].read_through_id, Some(own.id.to_hex())); + + // So does posting, which reads through the post. + go_dormant(&pool, community, &actor).await; + assert!(row(&db, community, channel, &actor) + .await + .threads + .is_empty()); + let again = reply(&db, community, channel, &root, &actor, vec![]).await; + reply(&db, community, channel, &root, &Keys::generate(), vec![]).await; + let summary = row(&db, community, channel, &actor).await; + assert_eq!(threads(&summary), [(id.clone(), 0)]); + assert_eq!(summary.threads[0].read_through_id, Some(again.id.to_hex())); + + // And reading it. + go_dormant(&pool, community, &actor).await; + let last = reply(&db, community, channel, &root, &Keys::generate(), vec![]).await; + assert!(row(&db, community, channel, &actor) + .await + .threads + .is_empty()); + assert_eq!( + intent( + &db, + community, + &actor, + mark_thread(channel, &root, &last.id.to_hex()) + ) + .await, + IntentOutcome::Applied + ); + reply(&db, community, channel, &root, &Keys::generate(), vec![]).await; + assert_eq!( + listed(&db, community, channel, &actor).await, + [vec![id.clone()], vec![id]] + ); +} + +#[tokio::test] +#[ignore = "requires Postgres"] +async fn an_explicit_follow_never_goes_dormant() { + let (db, pool, community, channel, actor, _) = fixture().await; + // One thread followed only by intent, one followed implicitly first. + let chosen = post(&db, community, channel, &Keys::generate(), vec![]).await; + let joined = post(&db, community, channel, &Keys::generate(), vec![]).await; + reply(&db, community, channel, &joined, &actor, vec![]).await; + for root in [&chosen, &joined] { + assert_eq!( + intent( + &db, + community, + &actor, + follow_intent(channel, Some(root), true) + ) + .await, + IntentOutcome::Applied + ); + } + let implicit = post(&db, community, channel, &Keys::generate(), vec![]).await; + reply(&db, community, channel, &implicit, &actor, vec![]).await; + for root in [&chosen, &joined, &implicit] { + reply(&db, community, channel, root, &Keys::generate(), vec![]).await; + } + go_dormant(&pool, community, &actor).await; + let mut explicit = vec![chosen.id.to_hex(), joined.id.to_hex()]; + explicit.sort(); + let [mut sidebar, mut view] = listed(&db, community, channel, &actor).await; + sidebar.sort(); + view.sort(); + assert_eq!([sidebar, view], [explicit.clone(), explicit]); + + // Unfollowing ends it; a later mention re-follows it implicitly, so it + // can go dormant again. + intent( + &db, + community, + &actor, + follow_intent(channel, Some(&chosen), false), + ) + .await; + reply( + &db, + community, + channel, + &chosen, + &Keys::generate(), + mention(&actor), + ) + .await; + assert_eq!( + threads(&row(&db, community, channel, &actor).await).len(), + 2 + ); + go_dormant(&pool, community, &actor).await; + assert_eq!( + listed(&db, community, channel, &actor).await, + [vec![joined.id.to_hex()], vec![joined.id.to_hex()]] + ); +} + +#[tokio::test] +#[ignore = "requires Postgres"] +async fn an_agent_keeps_no_read_state() { + let (db, pool, community, channel, actor, _) = fixture().await; + let agent = member(&db, &pool, community, channel).await; + sqlx::query("DELETE FROM personal_read_accounts WHERE community_id=$1 AND actor=$2") + .bind(community.as_uuid()) + .bind(agent.public_key().to_bytes().as_slice()) + .execute(&pool) + .await + .unwrap(); + for (who, owner) in [(&actor, None), (&agent, Some(&actor))] { + sqlx::query( + "INSERT INTO users (community_id, pubkey, agent_owner_pubkey) VALUES ($1,$2,$3)", + ) + .bind(community.as_uuid()) + .bind(who.public_key().to_bytes().as_slice()) + .bind(owner.map(|o| o.public_key().to_bytes().to_vec())) + .execute(&pool) + .await + .unwrap(); + } + // The agent posts, replies to its owner's root, is mentioned, and has a + // root replied to: every way a person would gain a position or a follow. + let own_root = post(&db, community, channel, &agent, mention(&actor)).await; + reply(&db, community, channel, &own_root, &actor, mention(&agent)).await; + let owner_root = post(&db, community, channel, &actor, mention(&agent)).await; + reply( + &db, + community, + channel, + &owner_root, + &agent, + mention(&actor), + ) + .await; + let rows: i64 = sqlx::query_scalar( + "SELECT (SELECT count(*) FROM personal_read_frontiers WHERE community_id=$1 AND actor=$2) + + (SELECT count(*) FROM personal_read_accounts WHERE community_id=$1 AND actor=$2)", + ) + .bind(community.as_uuid()) + .bind(agent.public_key().to_bytes().as_slice()) + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!(rows, 0, "an agent has no positions, follows or account"); + // The agent's messages still involve its owner. + assert!(follow(&pool, community, &actor, &own_root).await.is_some()); + assert!(follow(&pool, community, &actor, &owner_root) + .await + .is_some()); +} diff --git a/crates/buzz-db/src/store/personal_read/mod.rs b/crates/buzz-db/src/store/personal_read/mod.rs index 6df99443fc8..21b61571b3d 100644 --- a/crates/buzz-db/src/store/personal_read/mod.rs +++ b/crates/buzz-db/src/store/personal_read/mod.rs @@ -1,9 +1,11 @@ //! Private, signer-owned accessory read progress, separate from NIP-RS events. //! //! A frontier is the relay arrival time of the message a context was read -//! through; the unread horizon alone uses author time. Read intents and the -//! author's own posts advance frontiers. A thread frontier exists only for a -//! followed thread, and the sidebar counts forward from each frontier. +//! through. Read intents and the author's own posts advance frontiers. A +//! thread frontier exists only for a followed thread, and the sidebar counts +//! forward from each frontier. An implicit follow goes dormant after the +//! relay's window without the actor's involvement; an explicit one never does. +//! Agent accounts keep no read state. mod follows; mod model; diff --git a/crates/buzz-db/src/store/personal_read/model.rs b/crates/buzz-db/src/store/personal_read/model.rs index c6dd551bb14..72cd27955e5 100644 --- a/crates/buzz-db/src/store/personal_read/model.rs +++ b/crates/buzz-db/src/store/personal_read/model.rs @@ -4,7 +4,8 @@ use uuid::Uuid; /// Maximum independent operations in one HTTP request. pub const MAX_INTENTS: usize = 100; -/// Default unread-tracking duration, not event or encrypted NIP-RS retention. +/// Default time an implicit thread follow stays live without the actor's +/// involvement; not event or encrypted NIP-RS retention. pub const DEFAULT_RETENTION_SECONDS: u32 = 30 * 24 * 60 * 60; /// A channel or canonical thread; absence of a root denotes only the channel timeline. @@ -68,7 +69,8 @@ pub enum IntentOutcome { /// The tracking boundary for the authenticated account. Not on the wire. #[derive(Clone, Debug)] pub(super) struct ReadAccount { - /// Read-time author-time cutoff (Unix milliseconds), not a discard boundary. + /// Implicit follows not involved since this instant (Unix milliseconds) are + /// dormant; nothing is discarded. pub(super) cutoff_ms: i64, /// Arrival floor of every frontier: the account's first applied read /// intent. `None` until then, and nothing counts as unread. diff --git a/crates/buzz-db/src/store/personal_read/postgres_tests.rs b/crates/buzz-db/src/store/personal_read/postgres_tests.rs index 53dc50ff100..d8e2c5f3e6e 100644 --- a/crates/buzz-db/src/store/personal_read/postgres_tests.rs +++ b/crates/buzz-db/src/store/personal_read/postgres_tests.rs @@ -109,9 +109,9 @@ pub(super) async fn insert_reply( .unwrap(); } -/// Move every community message past the default horizon by author time, the -/// only clock the unread window reads. -async fn expire(pool: &PgPool, community: CommunityId) { +/// Move every community message's author time back past the default window. +/// Unread reads arrival, so nothing it counts may change. +async fn age_author_time(pool: &PgPool, community: CommunityId) { for table in [ "events SET created_at=created_at", "event_mentions SET event_created_at=event_created_at", @@ -214,20 +214,16 @@ async fn personal_read_sidebar_marked_history_is_read_and_keeps_latest() { #[tokio::test] #[ignore = "requires Postgres"] -async fn personal_read_sidebar_author_window_excludes_expired_and_late_old_messages() { +async fn personal_read_author_time_never_hides_unread() { let (db, pool, community, channel, actor, _) = fixture().await; let latest = add_history(&db, community, channel, 300).await; - expire(&pool, community).await; + age_author_time(&pool, community).await; let row = &sidebar(&db, community, &actor).await.channels[0]; - assert!( - !row.unread, - "an empty author-time window has nothing unread" - ); + assert!(row.unread, "old author time is still an unread arrival"); assert_eq!(row.latest_id.as_deref(), Some(latest.id.to_hex().as_str())); - // Acceptance time is irrelevant: a message accepted now with an author - // time beyond the horizon stays out; one authored now is counted. + // A message accepted now counts whatever its author time. let now = nostr::Timestamp::now().as_secs(); - for (age, unread, mentions) in [(40 * 86400, false, 0), (0, true, 1)] { + for (age, mentions) in [(40 * 86400, 1), (0, 2)] { let event = EventBuilder::new(Kind::Custom(9), "accepted now") .tags([Tag::public_key(actor.public_key())]) .custom_created_at(nostr::Timestamp::from(now - age)) @@ -237,7 +233,7 @@ async fn personal_read_sidebar_author_window_excludes_expired_and_late_old_messa .await .unwrap(); let row = &sidebar(&db, community, &actor).await.channels[0]; - assert_eq!((row.unread, row.mentions), (unread, mentions), "age={age}"); + assert_eq!((row.unread, row.mentions), (true, mentions), "age={age}"); } } @@ -260,13 +256,11 @@ async fn personal_read_counts_stop_at_the_cap_after_eligibility() { MAX_UNREAD_COUNT as u32, "a count of the cap means at least that many" ); - // The last to arrive stays; of the 100 before it, 50 become own and 50 - // deleted; the rest expire. The cap applies after eligibility, so one - // remains. + // The last to arrive stays; of the others, 50 become own and the rest + // are deleted. The cap applies after eligibility, so one remains. sqlx::query("WITH ranked AS (SELECT created_at,id,row_number() OVER (ORDER BY received_at DESC,id) AS n FROM events WHERE community_id=$1 AND channel_id=$2) - UPDATE events e SET created_at=CASE WHEN r.n>101 THEN e.created_at-interval '31 days' ELSE e.created_at END, - pubkey=CASE WHEN r.n BETWEEN 2 AND 51 THEN $3 ELSE e.pubkey END, - deleted_at=CASE WHEN r.n BETWEEN 52 AND 101 THEN now() ELSE NULL END + UPDATE events e SET pubkey=CASE WHEN r.n BETWEEN 2 AND 51 THEN $3 ELSE e.pubkey END, + deleted_at=CASE WHEN r.n>51 THEN now() ELSE NULL END FROM ranked r WHERE e.community_id=$1 AND e.created_at=r.created_at AND e.id=r.id") .bind(community.as_uuid()).bind(channel).bind(actor.public_key().to_bytes().as_slice()).execute(&pool).await.unwrap(); let page = sidebar(&db, community, &actor).await; @@ -424,7 +418,7 @@ async fn personal_read_latest_includes_own_and_excludes_deleted_auxiliary() { latest(sidebar(&db, community, &actor).await), Some(own.id.to_hex()) ); - expire(&pool, community).await; + age_author_time(&pool, community).await; assert_eq!( latest(sidebar(&db, community, &actor).await), Some(own.id.to_hex()) @@ -450,8 +444,8 @@ async fn personal_read_latest_skips_a_reply_without_ancestry() { latest(sidebar(&db, community, &actor).await), Some(first.id.to_hex()) ); - // The shallow probe applies the same rule once the horizon is empty. - expire(&pool, community).await; + // Author time plays no part in the rule. + age_author_time(&pool, community).await; assert_eq!( latest(sidebar(&db, community, &actor).await), Some(first.id.to_hex()) @@ -471,7 +465,7 @@ async fn personal_read_latest_skips_a_reply_without_ancestry() { #[ignore = "requires Postgres"] async fn personal_read_latest_old_message_is_not_an_empty_channel() { let (db, pool, community, channel, actor, event) = fixture().await; - expire(&pool, community).await; + age_author_time(&pool, community).await; let empty = db .create_channel( community, @@ -492,7 +486,7 @@ async fn personal_read_latest_old_message_is_not_an_empty_channel() { .find(|c| c.channel_id == channel) .unwrap(); assert_eq!(old.latest_id.as_deref(), Some(event.id.to_hex().as_str())); - assert!(!old.unread); + assert!(old.unread); let empty = page .channels .iter() @@ -870,10 +864,10 @@ async fn personal_read_channel_and_thread_never_inherit_each_other() { #[tokio::test] #[ignore = "requires Postgres"] -async fn personal_read_author_horizon_unresolved_ancestry_and_deletion() { +async fn personal_read_unresolved_ancestry_and_deletion_not_author_age() { let (db, pool, community, channel, actor, root) = fixture().await; let mention = Tag::public_key(actor.public_key()); - let old = EventBuilder::new(Kind::Custom(9), "directed, about to expire") + let old = EventBuilder::new(Kind::Custom(9), "directed, about to age") .tags([mention.clone()]) .sign_with_keys(&Keys::generate()) .unwrap(); @@ -894,20 +888,32 @@ async fn personal_read_author_horizon_unresolved_ancestry_and_deletion() { let row = &sidebar(&db, community, &actor).await.channels[0]; assert_eq!((row.unread, row.mentions), (true, 1)); assert!(row.threads.is_empty()); - sqlx::query( - "UPDATE events SET created_at=created_at-interval '31 days' WHERE community_id=$1 AND id=$2", - ) - .bind(community.as_uuid()) - .bind(old.id.as_bytes().as_slice()) - .execute(&pool) - .await - .unwrap(); - sqlx::query("UPDATE events SET deleted_at=now() WHERE community_id=$1 AND id=$2") - .bind(community.as_uuid()) - .bind(root.id.as_bytes().as_slice()) - .execute(&pool) - .await - .unwrap(); + for table in [ + "events SET created_at=created_at-interval '31 days' WHERE community_id=$1 AND id=$2", + "event_mentions SET event_created_at=event_created_at-interval '31 days' + WHERE community_id=$1 AND event_id=$2", + ] { + sqlx::query(sqlx::AssertSqlSafe(format!("UPDATE {table}"))) + .bind(community.as_uuid()) + .bind(old.id.as_bytes().as_slice()) + .execute(&pool) + .await + .unwrap(); + } + let row = &sidebar(&db, community, &actor).await.channels[0]; + assert_eq!( + (row.unread, row.mentions), + (true, 1), + "author age hides nothing" + ); + for gone in [&root, &old] { + sqlx::query("UPDATE events SET deleted_at=now() WHERE community_id=$1 AND id=$2") + .bind(community.as_uuid()) + .bind(gone.id.as_bytes().as_slice()) + .execute(&pool) + .await + .unwrap(); + } let row = &sidebar(&db, community, &actor).await.channels[0]; assert_eq!((row.unread, row.mentions), (false, 0)); } diff --git a/crates/buzz-db/src/store/personal_read/projection.rs b/crates/buzz-db/src/store/personal_read/projection.rs index a3bd0569dcd..43dc0426a8a 100644 --- a/crates/buzz-db/src/store/personal_read/projection.rs +++ b/crates/buzz-db/src/store/personal_read/projection.rs @@ -25,20 +25,25 @@ const LATEST_REPLY: &str = "SELECT tm.event_id AS id, tm.received_at, e.pubkey, /// Unread replies in thread `t` at position `t.position` that are directed at /// the actor: every one in a DM (`t.dm`), otherwise those that tag the actor. -/// Nothing is walked unless `t.unread`. -const THREAD_MENTIONS: &str = "SELECT count(*) AS n FROM ( +/// Nothing is walked unless `t.unread`. Root first: the thread's unread +/// replies (idx_thread_metadata_read_arrival), each probed against the +/// mention key, never the actor's mentions across the channel. +pub(super) const THREAD_MENTIONS: &str = "SELECT count(*) AS n FROM ( SELECT 1 FROM thread_metadata tm JOIN events e ON e.community_id=$1 AND e.created_at=tm.event_created_at AND e.id=tm.event_id WHERE t.unread AND t.dm AND tm.community_id=$1 AND tm.root_event_id=t.root_id AND tm.received_at > t.position AND tm.channel_id=t.channel_id - AND e.deleted_at IS NULL AND e.pubkey<>$2 AND e.created_at >= $4 + AND e.deleted_at IS NULL AND e.pubkey<>$2 UNION ALL - SELECT 1 FROM event_mentions em JOIN thread_metadata tm ON tm.community_id=$1 - AND tm.event_created_at=em.event_created_at AND tm.event_id=em.event_id - JOIN events e ON e.community_id=$1 AND e.created_at=em.event_created_at AND e.id=em.event_id - WHERE t.unread AND NOT t.dm AND em.community_id=$1 AND em.pubkey_hex=$5 AND em.channel_id=t.channel_id - AND em.event_created_at >= GREATEST(t.position-$6, $4) - AND tm.root_event_id=t.root_id AND tm.received_at > t.position + SELECT 1 FROM ( + SELECT tm.event_id, tm.event_created_at FROM thread_metadata tm + WHERE t.unread AND NOT t.dm AND tm.community_id=$1 AND tm.root_event_id=t.root_id + AND tm.channel_id=t.channel_id AND tm.received_at > t.position + OFFSET 0 + ) tm JOIN event_mentions em ON em.community_id=$1 AND em.pubkey_hex=$5 + AND em.event_created_at=tm.event_created_at AND em.event_id=tm.event_id + JOIN events e ON e.community_id=$1 AND e.created_at=tm.event_created_at AND e.id=tm.event_id + WHERE em.channel_id=t.channel_id AND em.event_created_at >= t.position-$6 AND e.deleted_at IS NULL AND e.pubkey<>$2 LIMIT $7 ) x"; @@ -100,7 +105,7 @@ impl Db { ) .await?; let mut tx = conn.begin().await?; - // The horizon and every read position share one read-only cut. + // The dormancy cutoff and every read position share one read-only cut. sqlx::query("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ READ ONLY") .execute(&mut *tx) .await?; @@ -113,13 +118,14 @@ impl Db { // // Timeline, per channel: // - latest_id: the last timeline message to arrive (idx_events_read_timeline); - // - unread: latest is someone else's, inside the horizon, and arrived - // after the position (an own post moves the position past it); + // - unread: latest is someone else's and arrived after the position + // (an own post moves the position past it); // - mentions: unread timeline messages that tag the actor, found by // author time (event_mentions) from the skew before the position, // or every one in a DM. - // Threads: each followed thread whose last reply is unread the same - // way, then its mentions for the listed ones only. + // Threads: each live follow (explicit, or involved since the cutoff) + // whose last reply is unread the same way, then its mentions for the + // listed ones only. A dormant follow is filtered before its probe. let sql = format!( r#"WITH roster AS MATERIALIZED ( SELECT c.id, c.channel_type='dm' AS dm, @@ -144,19 +150,18 @@ impl Db { ORDER BY e.received_at DESC, e.id LIMIT 1 ) l ON true CROSS JOIN LATERAL ( - SELECT COALESCE(l.received_at > r.position AND l.pubkey<>$2 - AND l.created_at >= $4, false) AS unread + SELECT COALESCE(l.received_at > r.position AND l.pubkey<>$2, false) AS unread ) u LEFT JOIN LATERAL ( SELECT count(*) AS mentions FROM ( SELECT 1 FROM events e WHERE r.dm AND e.community_id=$1 AND e.channel_id=r.id AND e.read_timeline AND e.deleted_at IS NULL - AND e.received_at > r.position AND e.pubkey<>$2 AND e.created_at >= $4 + AND e.received_at > r.position AND e.pubkey<>$2 UNION ALL SELECT 1 FROM event_mentions em JOIN events e ON e.community_id=$1 AND e.created_at=em.event_created_at AND e.id=em.event_id WHERE NOT r.dm AND em.community_id=$1 AND em.pubkey_hex=$5 - AND em.channel_id=r.id AND em.event_created_at >= GREATEST(r.position-$6, $4) + AND em.channel_id=r.id AND em.event_created_at >= r.position-$6 AND e.read_timeline AND e.deleted_at IS NULL AND e.received_at > r.position AND e.pubkey<>$2 LIMIT $7 @@ -177,14 +182,15 @@ impl Db { CROSS JOIN LATERAL ({LATEST_REPLY}) tl WHERE tf.community_id=$1 AND tf.actor=$2 AND tf.channel_id=r.id AND tf.root_id<>''::bytea AND NOT tf.unfollowed - AND tl.received_at > p.position AND tl.pubkey<>$2 AND tl.created_at >= $4 + AND (tf.explicit OR tf.active_at >= $4) + AND tl.received_at > p.position AND tl.pubkey<>$2 ORDER BY tl.received_at DESC, tf.root_id LIMIT $11 ) t CROSS JOIN LATERAL ({THREAD_MENTIONS}) tc ) th ON true ORDER BY r.id"# ); let cutoff = DateTime::from_timestamp_millis(account.cutoff_ms) - .ok_or_else(|| DbError::InvalidData("invalid unread cutoff".into()))?; + .ok_or_else(|| DbError::InvalidData("invalid dormancy cutoff".into()))?; // Only compile-time constants are interpolated. let rows = sqlx::query(sqlx::AssertSqlSafe(sql)) .bind(community.as_uuid()) @@ -256,8 +262,8 @@ impl Db { sqlx::query("SET LOCAL jit = off").execute(&mut *tx).await?; let actor_bytes = actor.to_bytes(); let account = read_account(&mut tx, community, &actor_bytes, retention_seconds).await?; - // Every follow is probed for its last arrival and sorted: the cost - // grows with the actor's lifetime follows, not with community size. + // Every live follow is probed for its last arrival and sorted: the + // cost grows with the actor's recent follows, not with community size. let sql = format!( r#"SELECT t.channel_id, encode(t.root_id,'hex') AS root_id, t.unread, tc.n AS mentions, @@ -266,7 +272,7 @@ impl Db { FROM ( SELECT tf.root_id, tf.channel_id, c.channel_type='dm' AS dm, p.position, tf.through_message_id, tl.id AS latest_id, tl.received_at AS latest_at, - (tl.received_at > p.position AND tl.pubkey<>$2 AND tl.created_at >= $4) AS unread + (tl.received_at > p.position AND tl.pubkey<>$2) AS unread FROM personal_read_frontiers tf JOIN channel_members cm ON cm.community_id=$1 AND cm.channel_id=tf.channel_id AND cm.pubkey=$2 AND cm.removed_at IS NULL @@ -275,14 +281,15 @@ impl Db { COALESCE($3::timestamptz, 'infinity')) AS position) p CROSS JOIN LATERAL ({LATEST_REPLY}) tl WHERE tf.community_id=$1 AND tf.actor=$2 AND tf.root_id<>''::bytea - AND NOT tf.unfollowed AND ($8::timestamptz IS NULL OR tl.received_at < $8 + AND NOT tf.unfollowed AND (tf.explicit OR tf.active_at >= $4) + AND ($8::timestamptz IS NULL OR tl.received_at < $8 OR (tl.received_at=$8 AND tf.root_id>$9)) ORDER BY tl.received_at DESC, tf.root_id LIMIT $10 ) t CROSS JOIN LATERAL ({THREAD_MENTIONS}) tc ORDER BY t.latest_at DESC, t.root_id"# ); let cutoff = DateTime::from_timestamp_millis(account.cutoff_ms) - .ok_or_else(|| DbError::InvalidData("invalid unread cutoff".into()))?; + .ok_or_else(|| DbError::InvalidData("invalid dormancy cutoff".into()))?; // Only compile-time constants are interpolated. let rows = sqlx::query(sqlx::AssertSqlSafe(sql)) .bind(community.as_uuid()) @@ -333,8 +340,8 @@ fn skew() -> chrono::Duration { chrono::Duration::seconds(i64::from(MAX_ARRIVAL_SKEW_SECONDS)) } -/// Read-time horizon and the account's start, in the caller's snapshot. -/// Frontier state is not discarded on expiry. +/// The follow dormancy cutoff and the account's start, in the caller's +/// snapshot. A dormant follow keeps its row and position. pub(super) async fn read_account( conn: &mut PgConnection, community: CommunityId, diff --git a/crates/buzz-db/src/store/personal_read/projection_postgres_tests.rs b/crates/buzz-db/src/store/personal_read/projection_postgres_tests.rs index 77af8d257fe..5af5abe53bf 100644 --- a/crates/buzz-db/src/store/personal_read/projection_postgres_tests.rs +++ b/crates/buzz-db/src/store/personal_read/projection_postgres_tests.rs @@ -1,13 +1,17 @@ -use super::{postgres_tests::fixture, *}; +use super::{ + postgres_tests::{fixture, insert_reply}, + *, +}; use crate::Db; use nostr::{EventBuilder, Keys, Kind, Tag}; +use uuid::Uuid; -/// Eligibility is classified at ingest (kind) and at read (author, deletion, -/// horizon). Each case is stored through ingest, then rewritten in place for -/// the read-time facts. +/// Eligibility is classified at ingest (kind) and at read (author, deletion); +/// author time plays no part. Each case is stored through ingest, then +/// rewritten in place for the read-time facts. #[tokio::test] #[ignore = "requires Postgres"] -async fn sidebar_sql_eligibility_follows_kind_author_deletion_and_horizon() { +async fn sidebar_sql_eligibility_follows_kind_author_and_deletion() { let (db, pool, community, channel, actor, _) = fixture().await; let now = chrono::Utc::now().timestamp_millis(); let horizon = i64::from(DEFAULT_RETENTION_SECONDS) * 1000; @@ -36,12 +40,8 @@ async fn sidebar_sql_eligibility_follows_kind_author_deletion_and_horizon() { .unwrap(); for own in [false, true] { for deleted in [false, true] { - // Now, then one minute inside and one minute outside the horizon. - for (age, inside) in [ - (0, true), - (horizon - 60_000, true), - (horizon + 60_000, false), - ] { + // Now, and one minute past the default follow window. + for age in [0, horizon + 60_000] { let created = now - age; sqlx::query("UPDATE events SET pubkey=$3,deleted_at=CASE WHEN $4 THEN now() ELSE NULL END,created_at=to_timestamp($5::double precision/1000) WHERE community_id=$1 AND id=$2") .bind(community.as_uuid()).bind(event.id.as_bytes().as_slice()) @@ -60,7 +60,7 @@ async fn sidebar_sql_eligibility_follows_kind_author_deletion_and_horizon() { ) .await .unwrap(); - let expected = eligible_kind && !own && !deleted && inside; + let expected = eligible_kind && !own && !deleted; let row = &page.channels[0]; assert_eq!( (row.unread, row.mentions), @@ -214,3 +214,167 @@ async fn mentions(db: &Db, community: buzz_core::CommunityId, actor: &Keys) -> u .channels[0] .mentions } + +/// THREAD_MENTIONS before the root-first rewrite, kept as the oracle: it +/// walked the actor's mentions in the channel and filtered them by root. `$4` +/// was the author-time horizon. +const CHANNEL_FIRST_THREAD_MENTIONS: &str = "SELECT count(*) AS n FROM ( + SELECT 1 FROM thread_metadata tm JOIN events e ON e.community_id=$1 + AND e.created_at=tm.event_created_at AND e.id=tm.event_id + WHERE t.unread AND t.dm AND tm.community_id=$1 AND tm.root_event_id=t.root_id + AND tm.received_at > t.position AND tm.channel_id=t.channel_id + AND e.deleted_at IS NULL AND e.pubkey<>$2 AND e.created_at >= $4 + UNION ALL + SELECT 1 FROM event_mentions em JOIN thread_metadata tm ON tm.community_id=$1 + AND tm.event_created_at=em.event_created_at AND tm.event_id=em.event_id + JOIN events e ON e.community_id=$1 AND e.created_at=em.event_created_at AND e.id=em.event_id + WHERE t.unread AND NOT t.dm AND em.community_id=$1 AND em.pubkey_hex=$5 AND em.channel_id=t.channel_id + AND em.event_created_at >= GREATEST(t.position-$6, $4) + AND tm.root_event_id=t.root_id AND tm.received_at > t.position + AND e.deleted_at IS NULL AND e.pubkey<>$2 + LIMIT $7 + ) x"; + +/// Each of the actor's thread frontiers and its mention count by `query`. +async fn thread_mentions( + pool: &sqlx::PgPool, + community: buzz_core::CommunityId, + actor: &Keys, + query: &str, +) -> Vec<(String, i64)> { + let sql = format!( + "SELECT encode(t.root_id,'hex') AS root, tc.n FROM ( + SELECT tf.root_id, tf.channel_id, c.channel_type='dm' AS dm, true AS unread, + GREATEST(tf.through_timestamp, COALESCE($3::timestamptz, 'infinity')) AS position + FROM personal_read_frontiers tf JOIN channels c + ON c.community_id=tf.community_id AND c.id=tf.channel_id + WHERE tf.community_id=$1 AND tf.actor=$2 AND tf.root_id<>''::bytea + AND $4::timestamptz IS NOT NULL + ) t CROSS JOIN LATERAL ({query}) tc ORDER BY 1" + ); + sqlx::query_as(sqlx::AssertSqlSafe(sql)) + .bind(community.as_uuid()) + .bind(actor.public_key().to_bytes().as_slice()) + .bind(chrono::DateTime::from_timestamp(0, 0).unwrap()) + .bind(chrono::DateTime::from_timestamp(0, 0).unwrap()) + .bind(actor.public_key().to_hex()) + .bind(chrono::Duration::seconds(i64::from( + MAX_ARRIVAL_SKEW_SECONDS, + ))) + .bind(MAX_UNREAD_COUNT as i64) + .fetch_all(pool) + .await + .unwrap() +} + +/// The root-first thread mention count matches the channel-first query it +/// replaced, across positions, own and deleted replies, mentions in sibling +/// threads and channels, and DMs. +#[tokio::test] +#[ignore = "requires Postgres"] +async fn root_first_thread_mentions_match_the_channel_first_query() { + let (db, pool, community, channel, actor, _) = fixture().await; + let tag = || vec![Tag::public_key(actor.public_key())]; + let (bee, wasp) = (Keys::generate(), Keys::generate()); + let keys = [&actor, &bee, &wasp].map(|k| k.public_key().to_bytes()); + let participants: Vec<&[u8]> = keys.iter().map(|k| k.as_slice()).collect(); + let dm = db + .create_dm(community, &participants, &keys[1]) + .await + .unwrap() + .id; + let other = db + .create_channel( + community, + "sibling channel", + crate::channel::ChannelType::Stream, + crate::channel::ChannelVisibility::Open, + None, + &actor.public_key().to_bytes(), + None, + ) + .await + .unwrap() + .id; + let mut expected = Vec::new(); + for (place, mentions) in [ + (channel, 0), + (channel, 1), + (channel, 3), + (other, 2), + (dm, 2), + ] { + let author = if place == dm { + bee.clone() + } else { + Keys::generate() + }; + let root = send(&db, community, place, None, &author, vec![]).await; + // A mention before the actor's own reply is read. + send(&db, community, place, Some(&root), &Keys::generate(), tag()).await; + db.apply_personal_read_intent( + community, + &actor.public_key(), + &ReadIntent::Follow { + target: ReadTarget { + channel_id: place, + root_id: Some(root.id.to_hex()), + }, + }, + ) + .await + .unwrap(); + // Posting reads through the post; own and deleted mentions never count. + send(&db, community, place, Some(&root), &actor, tag()).await; + for _ in 0..mentions { + send(&db, community, place, Some(&root), &wasp, tag()).await; + } + send( + &db, + community, + place, + Some(&root), + &Keys::generate(), + vec![], + ) + .await; + let gone = send(&db, community, place, Some(&root), &Keys::generate(), tag()).await; + sqlx::query("UPDATE events SET deleted_at=now() WHERE community_id=$1 AND id=$2") + .bind(community.as_uuid()) + .bind(gone.id.as_bytes().as_slice()) + .execute(&pool) + .await + .unwrap(); + // In a DM every reply is directed at the actor. + let n = if place == dm { mentions + 1 } else { mentions }; + expected.push((root.id.to_hex(), n)); + } + expected.sort(); + let before = thread_mentions(&pool, community, &actor, CHANNEL_FIRST_THREAD_MENTIONS).await; + let after = thread_mentions(&pool, community, &actor, super::projection::THREAD_MENTIONS).await; + assert_eq!(before, expected); + assert_eq!(after, expected); +} + +async fn send( + db: &Db, + community: buzz_core::CommunityId, + channel: Uuid, + root: Option<&nostr::Event>, + author: &Keys, + tags: Vec, +) -> nostr::Event { + let event = EventBuilder::new(Kind::Custom(9), Uuid::new_v4().to_string()) + .tags(tags) + .sign_with_keys(author) + .unwrap(); + match root { + Some(root) => insert_reply(db, community, channel, root, &event).await, + None => { + db.insert_event(community, &event, Some(channel)) + .await + .unwrap(); + } + } + event +} diff --git a/crates/buzz-db/src/store/personal_read/writes.rs b/crates/buzz-db/src/store/personal_read/writes.rs index 8c158547994..5e9b7be4351 100644 --- a/crates/buzz-db/src/store/personal_read/writes.rs +++ b/crates/buzz-db/src/store/personal_read/writes.rs @@ -165,7 +165,8 @@ pub(super) async fn valid_target( /// Advance a monotone frontier. The anchor ID follows the greatest arrival; /// an equal arrival keeps the existing anchor. Reading a thread never follows -/// it: only a followed thread has a row to advance. +/// it: only a followed thread has a row to advance, and reading involves the +/// actor, which keeps the follow live. async fn frontier( conn: &mut PgConnection, community: CommunityId, @@ -187,7 +188,8 @@ async fn frontier( "UPDATE personal_read_frontiers SET through_timestamp=GREATEST(through_timestamp, $5), through_message_id=CASE WHEN $5 > through_timestamp - THEN $6 ELSE through_message_id END + THEN $6 ELSE through_message_id END, + active_at=now() WHERE community_id=$1 AND actor=$2 AND channel_id=$3 AND root_id=$4" }; sqlx::query(sql) @@ -204,7 +206,8 @@ async fn frontier( /// Follow or unfollow a thread. A follow that changes state starts now, so /// earlier replies are read; an unfollowed row keeps ingest from following -/// again, except as `record_message` allows. +/// again, except as `record_message` allows. An explicit follow, including of +/// a thread already followed implicitly, never goes dormant. async fn follow( conn: &mut PgConnection, community: CommunityId, @@ -215,14 +218,18 @@ async fn follow( ) -> Result<()> { sqlx::query( "INSERT INTO personal_read_frontiers - (community_id, actor, channel_id, root_id, through_timestamp, unfollowed) - VALUES ($1,$2,$3,$4,now(),$5) + (community_id, actor, channel_id, root_id, through_timestamp, unfollowed, explicit) + VALUES ($1,$2,$3,$4,now(),$5,NOT $5) ON CONFLICT (community_id, actor, channel_id, root_id) DO UPDATE - SET unfollowed=$5, - through_timestamp=GREATEST(personal_read_frontiers.through_timestamp, now()), - through_message_id=CASE WHEN now() > personal_read_frontiers.through_timestamp + SET unfollowed=$5, explicit=NOT $5, active_at=now(), + through_timestamp=CASE WHEN personal_read_frontiers.unfollowed <> $5 + THEN GREATEST(personal_read_frontiers.through_timestamp, now()) + ELSE personal_read_frontiers.through_timestamp END, + through_message_id=CASE WHEN personal_read_frontiers.unfollowed <> $5 + AND now() > personal_read_frontiers.through_timestamp THEN NULL ELSE personal_read_frontiers.through_message_id END - WHERE personal_read_frontiers.unfollowed <> $5", + WHERE personal_read_frontiers.unfollowed <> $5 + OR personal_read_frontiers.explicit = $5", ) .bind(community.as_uuid()) .bind(actor) diff --git a/crates/buzz-relay/src/api/buzz_v1/auth.rs b/crates/buzz-relay/src/api/buzz_v1/auth.rs index dd65d3dad66..ed7ed647f4f 100644 --- a/crates/buzz-relay/src/api/buzz_v1/auth.rs +++ b/crates/buzz-relay/src/api/buzz_v1/auth.rs @@ -132,6 +132,16 @@ pub(super) async fn authorize( signed_at, }; recheck(state, headers, &principal).await?; + // Read state is for people: an agent account has none to read or write. + let agent = state + .db + .get_agent_channel_policy(principal.tenant.community(), &actor.to_bytes()) + .await + .map_err(|_| Error::unavailable())? + .is_some_and(|(_, owner)| owner.is_some()); + if agent { + return Err(Error::new(StatusCode::FORBIDDEN, "forbidden")); + } Ok(principal) } diff --git a/crates/buzz-relay/src/api/buzz_v1/postgres_tests.rs b/crates/buzz-relay/src/api/buzz_v1/postgres_tests.rs index c7becba628c..2e913d2478c 100644 --- a/crates/buzz-relay/src/api/buzz_v1/postgres_tests.rs +++ b/crates/buzz-relay/src/api/buzz_v1/postgres_tests.rs @@ -470,6 +470,86 @@ async fn accessory_follow_intents_and_threads_view() { assert_eq!(result.0, StatusCode::BAD_REQUEST, "{}", result.1); } +/// Read state is for people: an agent account is refused every endpoint, +/// while its owner's sidebar is served. +#[tokio::test] +#[ignore = "requires Postgres"] +async fn accessory_refuses_agent_accounts() { + let fixture = crate::api::bridge::postgres_tests::bridge_handler_test_state() + .await + .unwrap(); + let mut state = (*fixture).clone(); + Arc::make_mut(&mut state.config).buzz_v1_enabled = true; + let state = Arc::new(state); + let host = format!("bff-agents-{}.local", uuid::Uuid::new_v4()); + let community = state + .db + .ensure_configured_community(&host) + .await + .unwrap() + .id; + let (owner, agent) = (Keys::generate(), Keys::generate()); + let channel = state + .db + .create_channel( + community, + "agents", + buzz_db::channel::ChannelType::Stream, + buzz_db::channel::ChannelVisibility::Open, + None, + &owner.public_key().to_bytes(), + None, + ) + .await + .unwrap() + .id; + for (who, by) in [(&owner, None), (&agent, Some(&owner))] { + sqlx::query( + "INSERT INTO users (community_id, pubkey, agent_owner_pubkey) VALUES ($1,$2,$3)", + ) + .bind(community.as_uuid()) + .bind(who.public_key().to_bytes().as_slice()) + .bind(by.map(|k| k.public_key().to_bytes().to_vec())) + .execute(state.db.pool()) + .await + .unwrap(); + } + let write = serde_json::to_vec(&json!({"intents":[ + {"type":"mark_through","target":{"channel_id":channel},"message_id":"ab".repeat(32)} + ]})) + .unwrap(); + for (path, method, body) in [ + ("/buzz/v1/me/sidebar", "GET", &b""[..]), + ("/buzz/v1/me/threads", "GET", &b""[..]), + ("/buzz/v1/me/read-state", "POST", write.as_slice()), + ] { + let auth = proof( + &agent, + &host, + path, + method, + (method == "POST").then_some(body), + ); + let (status, json) = request(state.clone(), &host, path, method, Some(&auth), body).await; + assert_eq!(status, StatusCode::FORBIDDEN, "{method} {path}"); + assert_eq!(json["error"]["code"], "forbidden"); + } + let path = "/buzz/v1/me/sidebar"; + let auth = proof(&owner, &host, path, "GET", None); + let (status, json) = request(state.clone(), &host, path, "GET", Some(&auth), b"").await; + assert_eq!(status, StatusCode::OK); + assert_eq!(json["channels"][0]["channel_id"], channel.to_string()); + let accounts: i64 = sqlx::query_scalar( + "SELECT count(*) FROM personal_read_accounts WHERE community_id=$1 AND actor=$2", + ) + .bind(community.as_uuid()) + .bind(agent.public_key().to_bytes().as_slice()) + .fetch_one(state.db.pool()) + .await + .unwrap(); + assert_eq!(accounts, 0); +} + #[tokio::test] #[ignore = "requires Postgres"] async fn accessory_write_revocation_is_terminal_before_persistence() { diff --git a/crates/buzz-relay/src/config.rs b/crates/buzz-relay/src/config.rs index 44662bb9e10..f19a3327ec1 100644 --- a/crates/buzz-relay/src/config.rs +++ b/crates/buzz-relay/src/config.rs @@ -187,7 +187,8 @@ pub struct Config { pub require_auth_token: bool, /// Opt-in private accessory API; disabled until explicitly deployed. pub buzz_v1_enabled: bool, - /// Author-time unread tracking duration, independent of NIP-RS retention. + /// How long an automatic thread follow stays live without the actor's + /// involvement; independent of NIP-RS retention. pub buzz_v1_retention_seconds: u32, /// Comma-separated list of allowed CORS origins. /// If empty, permissive CORS is used (dev mode). diff --git a/docs/buzz-v1-read-state.md b/docs/buzz-v1-read-state.md index 58b988b1d10..c7e17d2e1b8 100644 --- a/docs/buzz-v1-read-state.md +++ b/docs/buzz-v1-read-state.md @@ -18,7 +18,8 @@ application/nostr+json`, or `GET /info`) includes `buzz_v1` only when enabled: `capabilities` lists additions to the first v1 contract: `follow` for the `follow` and `unfollow` intents, `threads` for `GET /me/threads`. Use one only -when it is listed. +when it is listed. `retention_seconds` is how long an automatic thread follow +stays live without your involvement (see [Following](#following)). Use the requesting origin plus this relative prefix. Discovery is a configured capability, not a promise that the next request cannot fail. Unknown hosts and @@ -36,6 +37,8 @@ NIP-FI admission is enforced and its asserted key must match the request signer. The host chooses the community; the signer chooses `/me`. NIP-OA admission does not grant access to the owner's personal state. Relay membership, bans and resource access are enforced; moderation timeouts do not prohibit reading. +Read state is for people: an agent account (one with an owner) gets 403 from +every endpoint, and the relay keeps no positions or follows for it. Responses produced by the v1 handlers are `Cache-Control: private, no-store`. Application errors (including NIP-98 failures with NIP-FI Off or Shadow) use @@ -102,7 +105,7 @@ thread row's `unread` is always true in this version. No message bytes are included. A message is eligible when it is non-own, nondeleted, of the advertised -`eligible_kinds` and inside the horizon. The same kinds alone define latest +`eligible_kinds`; its author time plays no part. The same kinds alone define latest activity, so an edit, reaction or diff (40008) neither makes a channel unread nor moves it. Classify live arrivals with the advertised set, not a client copy. @@ -119,6 +122,9 @@ in the same transaction: message, the thread rooted at it). - A reply also makes the root's author follow, and in a DM every member. +None of this applies to an agent account: it neither reads through its posts +nor follows, though its messages still make people follow. + A new follower starts just before the message that made them follow, so it is unread; one who already follows keeps their position. Reading never follows: a thread `mark_through` moves an existing follow and otherwise changes nothing. @@ -127,7 +133,8 @@ records them is deployed; there is no backfill. Desktop notification follows and mutes do not affect these counts. The `follow` intent follows a thread explicitly, from now: replies that arrived -before it are read. Following a thread you already follow changes nothing. +before it are read. Following a thread you already follow keeps its position +and only makes the follow explicit. `unfollow` stops counting it, and the stored message rules above no longer follow you again, with two exceptions: posting in the thread follows you through your post, and a mention follows you from just before it. @@ -136,21 +143,15 @@ Counts are what the bounded walks could establish, and may undercount. A message whose ancestry is unresolved is left out, as are messages beyond the bounds (see [Bounds](#bounds-and-deployment)). -The unread horizon defaults to 30 days (`BUZZ_V1_RETENTION_SECONDS`) and is -measured in author time (`created_at`). It filters unread and mentions, not -latest activity, event storage or frontier state. Frontiers use a different -clock, relay arrival (see [Order](#order)). Three consequences: - -- A message accepted late with an author time beyond the horizon (an import, a - backfill, a long-offline sender) is excluded under the current horizon, - however recently the relay accepted it. It can still be latest when the - horizon holds no message. -- Unread expires at author time plus the horizon, so a future-dated author - time extends how long a message counts. -- A context's unread set is two tests, not one range: arrived after the - frontier, and author time at or after the cutoff. - -A later configuration expansion can change counts without having lost progress. +An automatic follow goes dormant after 30 days (`BUZZ_V1_RETENTION_SECONDS`) +without your involvement in the thread: posting there, being mentioned there, +a reply to your root or in your DM, marking it read, or following it. A dormant +thread is neither counted nor listed, but keeps its row and position, and any +of those involvements makes it live again at that position. Replies that do not +involve you never revive it. A follow made by the `follow` intent never goes +dormant; an unfollow ends that. Being mentioned, a reply to your root and DM +replies refresh a live follow at most daily, so dormancy is measured to the +day. ## Threads view @@ -170,8 +171,8 @@ Each row has the sidebar thread row's fields and meanings plus its `channel_id`, except that `unread` may be false and `mentions` is then 0. Pages are separate snapshots: a thread that gets a new reply while you page moves ahead of the cursor, so refresh from the first page after a change. Every -page probes each of your follows, so its cost grows with how many threads you -have ever followed, not with community size. +page probes each of your live follows, so its cost grows with how many threads +you are involved in, not with community size or your whole history. ## Fixed-operand writes @@ -242,8 +243,8 @@ by author time, which the relay accepts within 15 minutes of arrival, so a mention walk starts 15 minutes of author time before the position: a mention backdated further than that is unread but not counted as a mention. -A channel row is unread exactly when its `latest_id` is someone else's, inside -the horizon, and arrived after the timeline's position. Nothing earlier needs +A channel row is unread exactly when its `latest_id` is someone else's and +arrived after the timeline's position. Nothing earlier needs checking: posting reads the timeline through the post, so an own `latest_id` means everything before it is read, and an unread one is the anchor that clears the row. A row is never unread without an anchor to mark. @@ -263,7 +264,7 @@ device-local. - Counts stop at 100 per channel and per thread. - A caught-up reader costs one index probe per channel and per followed thread; a reader who is behind walks only what arrived after their position. -- 50 rows per threads page; each page probes every follow of the reader. +- 50 rows per threads page; each page probes every live follow of the reader. - DB statement/lock deadlines and HTTP read deadlines bound work; a write's intents and its row refresh share one eight-second deadline after admission. Limits are containment, not a production capacity claim. @@ -274,10 +275,12 @@ adds the nullable `personal_read_accounts.started_at` column, and 0058 adds the nullable `personal_read_frontiers.through_message_id` column and drops `threads_through_timestamp`. 0061 adds `events.read_timeline` (set at insert for timeline messages), `thread_metadata.received_at` (set at insert for -eligible replies) and `personal_read_frontiers.unfollowed`, plus three indexes: +eligible replies) and `personal_read_frontiers.unfollowed`, `explicit` and +`active_at` (existing rows start at migration time), plus four indexes: `idx_events_read_timeline` for timeline walks, `idx_thread_metadata_read_arrival` -for thread walks and `idx_event_mentions_pubkey_channel_created` for mention -counts. There is no backfill: messages stored before the new relay serves, +for thread walks, `idx_event_mentions_pubkey_channel_created` for mention +counts and `idx_personal_read_frontiers_active` for an actor's threads by +involvement. There is no backfill: messages stored before the new relay serves, including by older relay processes still running after 0061, are on neither arrival index, so they never count as unread or become `latest_id`. Ingest writes a few frontier rows per eligible channel message (the author, and anyone @@ -300,6 +303,12 @@ ALTER TABLE events ADD COLUMN IF NOT EXISTS read_timeline BOOLEAN NOT NULL DEFAU ALTER TABLE thread_metadata ADD COLUMN IF NOT EXISTS received_at TIMESTAMPTZ; ALTER TABLE personal_read_frontiers ADD COLUMN IF NOT EXISTS unfollowed BOOLEAN NOT NULL DEFAULT false CHECK (NOT unfollowed OR root_id <> ''::bytea); +ALTER TABLE personal_read_frontiers ADD COLUMN IF NOT EXISTS explicit BOOLEAN NOT NULL + DEFAULT false; +ALTER TABLE personal_read_frontiers ADD COLUMN IF NOT EXISTS active_at TIMESTAMPTZ NOT NULL + DEFAULT now(); +CREATE INDEX CONCURRENTLY idx_personal_read_frontiers_active ON public.personal_read_frontiers + (community_id, actor, active_at DESC) WHERE root_id <> ''::bytea; CREATE INDEX CONCURRENTLY idx_thread_metadata_read_arrival ON public.thread_metadata (community_id, root_event_id, received_at DESC, event_id) WHERE received_at IS NOT NULL; CREATE INDEX CONCURRENTLY idx_event_mentions_pubkey_channel_created ON public.event_mentions @@ -315,7 +324,7 @@ ALTER INDEX idx_events_read_timeline ATTACH PARTITION p_read_timeline; ``` The parent becomes valid once every partition's index is attached. Verify the -three like 0049's, against the definitions the migration expects; recover a +four like 0049's, against the definitions the migration expects; recover a failed build the same way. Use existing HTTP route/status/latency metrics for `/buzz/v1/me/sidebar` and @@ -354,7 +363,7 @@ Deleting an account row cascades that actor's frontiers in the same community; community erasure inventories both tables under the existing write fence. There is no new public account export/reset endpoint. Operator-assisted erasure/export must use the established authenticated operational process and explicitly scope -both community and actor; never equate the read-time horizon with data erasure. +both community and actor; never equate follow dormancy with data erasure. There is no down migration, and disabling the API is not a rollback. A relay built before 0056 that restarts with `BUZZ_AUTO_MIGRATE=true` (the Helm default) diff --git a/migrations/0061_read_state_access_paths.sql b/migrations/0061_read_state_access_paths.sql index a1ec0677697..7ab79b05a47 100644 --- a/migrations/0061_read_state_access_paths.sql +++ b/migrations/0061_read_state_access_paths.sql @@ -9,8 +9,12 @@ -- thread_metadata.received_at: a reply's arrival, set at insert; NULL on root -- stubs and earlier rows. -- personal_read_frontiers.unfollowed: a sticky unfollow of a thread. +-- personal_read_frontiers.explicit: a follow intent, never dormant. +-- personal_read_frontiers.active_at: the actor's last involvement in a +-- thread; an implicit follow is dormant once it is older than the relay's +-- window. Earlier rows start at migration time. -- --- Brownfield operators MUST add the three columns and prebuild the three +-- Brownfield operators MUST add the five columns and prebuild the four -- indexes concurrently as documented in docs/buzz-v1-read-state.md. Startup is -- bounded like 0049: a large table fails deployment rather than blocking -- ingestion. @@ -27,7 +31,9 @@ BEGIN ('events', 'read_timeline', 'BOOLEAN NOT NULL DEFAULT false'), ('thread_metadata', 'received_at', 'TIMESTAMPTZ'), ('personal_read_frontiers', 'unfollowed', - $d$BOOLEAN NOT NULL DEFAULT false CHECK (NOT unfollowed OR root_id <> ''::bytea)$d$) + $d$BOOLEAN NOT NULL DEFAULT false CHECK (NOT unfollowed OR root_id <> ''::bytea)$d$), + ('personal_read_frontiers', 'explicit', 'BOOLEAN NOT NULL DEFAULT false'), + ('personal_read_frontiers', 'active_at', 'TIMESTAMPTZ NOT NULL DEFAULT now()') ) AS s(tbl, col, definition) LOOP IF NOT EXISTS ( SELECT 1 FROM pg_attribute @@ -47,7 +53,9 @@ BEGIN ('idx_thread_metadata_read_arrival', 'CREATE INDEX idx_thread_metadata_read_arrival ON public.thread_metadata USING btree (community_id, root_event_id, received_at DESC, event_id) WHERE (received_at IS NOT NULL)'), ('idx_event_mentions_pubkey_channel_created', - 'CREATE INDEX idx_event_mentions_pubkey_channel_created ON public.event_mentions USING btree (community_id, pubkey_hex, channel_id, event_created_at DESC)') + 'CREATE INDEX idx_event_mentions_pubkey_channel_created ON public.event_mentions USING btree (community_id, pubkey_hex, channel_id, event_created_at DESC)'), + ('idx_personal_read_frontiers_active', + $d$CREATE INDEX idx_personal_read_frontiers_active ON public.personal_read_frontiers USING btree (community_id, actor, active_at DESC) WHERE (root_id <> '\x'::bytea)$d$) ) AS s(name, definition) LOOP -- IF NOT EXISTS still requests a writer-conflicting lock: skip CREATE -- for a prebuilt index, then validate it. diff --git a/schema/tables/public/personal_read_frontiers.sql b/schema/tables/public/personal_read_frontiers.sql index 0aab16ad5c3..cc2f05e1f92 100644 --- a/schema/tables/public/personal_read_frontiers.sql +++ b/schema/tables/public/personal_read_frontiers.sql @@ -8,9 +8,19 @@ CREATE TABLE personal_read_frontiers ( through_message_id BYTEA CHECK (octet_length(through_message_id) = 32), -- A sticky thread unfollow; thread rows only. unfollowed BOOLEAN NOT NULL DEFAULT false CHECK (NOT unfollowed OR root_id <> ''::bytea), + -- A follow intent: never dormant. Thread rows only. + explicit BOOLEAN NOT NULL DEFAULT false, + -- The actor's last involvement in the thread: posting, being mentioned, + -- reading, following. An implicit follow older than the window is dormant. + active_at TIMESTAMPTZ NOT NULL DEFAULT now(), PRIMARY KEY (community_id, actor, channel_id, root_id), FOREIGN KEY (community_id, actor) REFERENCES personal_read_accounts (community_id, actor) ON DELETE CASCADE, FOREIGN KEY (community_id, channel_id) REFERENCES channels (community_id, id) ON DELETE CASCADE ); + +-- Live follows, and every thread the actor was involved in, newest first. +CREATE INDEX idx_personal_read_frontiers_active + ON personal_read_frontiers (community_id, actor, active_at DESC) + WHERE root_id <> ''::bytea; From f1dfa9745f43645b711c333a49d029b80b4e479f Mon Sep 17 00:00:00 2001 From: Bradley Axen Date: Tue, 13 Oct 2026 22:13:06 -0700 Subject: [PATCH 4/4] fix(read-state): bound the follow refresh interval by the dormancy window Ingest refreshed a live follow's active_at at most daily, so with a dormancy window shorter than a day a mention could not revive a follow that had gone dormant. The refresh interval is now min(1 day, window/4), from the window the relay sets at startup, so it never exceeds the window. Adds codex00's repro: a 1-hour window, a follow last active 2 hours ago, and a fresh mention revives it in the sidebar and the threads view; it fails against the fixed one-day throttle. Co-authored-by: murderbot <3754f8729004d95654c46dbab3129e4ab9ef05cc2534e2a3fbfc155983bd637b@buzz.block.builderlab.xyz> Co-Authored-By: Claude Opus 5.5 Signed-off-by: Bradley Axen --- .../src/store/personal_read/follows.rs | 29 ++++++-- .../personal_read/follows_postgres_tests.rs | 72 +++++++++++++++++++ crates/buzz-db/src/store/personal_read/mod.rs | 1 + crates/buzz-relay/src/main.rs | 1 + docs/buzz-v1-read-state.md | 4 +- 5 files changed, 101 insertions(+), 6 deletions(-) diff --git a/crates/buzz-db/src/store/personal_read/follows.rs b/crates/buzz-db/src/store/personal_read/follows.rs index 16fd774d789..2ed532a3290 100644 --- a/crates/buzz-db/src/store/personal_read/follows.rs +++ b/crates/buzz-db/src/store/personal_read/follows.rs @@ -1,11 +1,30 @@ //! Read state that ingest writes. A thread frontier row exists for a thread //! its actor follows or has unfollowed; ingest and follow intents create one. -use super::model::ELIGIBLE_KINDS; +use super::model::{DEFAULT_RETENTION_SECONDS, ELIGIBLE_KINDS}; use crate::{AdmittedTx, Result}; use chrono::{DateTime, Utc}; use nostr::Event; +use std::sync::atomic::{AtomicU32, Ordering}; use uuid::Uuid; +/// The relay's follow dormancy window, in seconds (`BUZZ_V1_RETENTION_SECONDS`). +static FOLLOW_WINDOW_SECONDS: AtomicU32 = AtomicU32::new(DEFAULT_RETENTION_SECONDS); + +/// Set the follow dormancy window ingest refreshes against: the same window +/// the read projections are given. Process-wide; the relay sets it once at +/// startup. +pub fn set_follow_window_seconds(seconds: u32) { + FOLLOW_WINDOW_SECONDS.store(seconds, Ordering::Relaxed); +} + +/// How stale a live follow's involvement may grow before an involvement that +/// does not otherwise write its row refreshes it: a day, or a quarter of the +/// window when that is shorter. Never longer than the window, so such an +/// involvement always revives a dormant follow. +fn refresh_after_seconds() -> f64 { + f64::from((FOLLOW_WINDOW_SECONDS.load(Ordering::Relaxed) / 4).min(24 * 60 * 60)) +} + /// Where a newly stored channel message sits. #[derive(Clone, Copy)] pub(crate) enum Place<'a> { @@ -133,7 +152,8 @@ pub(crate) async fn record_message( AND c.channel_type='dm' AND cm.pubkey<>$4 ), rows AS ( -- An existing follower keeps their position, so is only touched - -- to keep the follow live, at most daily rather than per message; + -- to keep the follow live, at most once per refresh interval ($12) + -- rather than per message; -- an unfollow is not touched unless this message re-follows (the -- author posting, or a mention). Existence probes are LIMIT 1 -- laterals: always one index lookup. @@ -148,7 +168,7 @@ pub(crate) async fn record_message( AND (i.message IS NOT NULL OR f.unfollowed IS NULL OR (i.refollow AND f.unfollowed) OR (i.root_id<>''::bytea AND NOT f.unfollowed - AND f.active_at < now()-interval '1 day')) + AND f.active_at < now()-make_interval(secs=>$12))) ORDER BY i.actor, i.root_id, i.message IS NULL, NOT i.refollow ), accounts AS ( INSERT INTO personal_read_accounts (community_id, actor) @@ -184,7 +204,7 @@ pub(crate) async fn record_message( WHERE excluded.through_message_id IS NOT NULL OR (personal_read_frontiers.unfollowed AND excluded.through_timestamp > personal_read_frontiers.through_timestamp) OR (excluded.root_id<>''::bytea AND NOT personal_read_frontiers.unfollowed - AND personal_read_frontiers.active_at < now()-interval '1 day')", + AND personal_read_frontiers.active_at < now()-make_interval(secs=>$12))", ) .bind(community.as_uuid()) .bind(channel) @@ -197,6 +217,7 @@ pub(crate) async fn record_message( .bind(reply) .bind(root_created_at) .bind(broadcast) + .bind(refresh_after_seconds()) .execute(tx.conn()) .await?; Ok(()) diff --git a/crates/buzz-db/src/store/personal_read/follows_postgres_tests.rs b/crates/buzz-db/src/store/personal_read/follows_postgres_tests.rs index f5b146c2867..2f89a9ea72b 100644 --- a/crates/buzz-db/src/store/personal_read/follows_postgres_tests.rs +++ b/crates/buzz-db/src/store/personal_read/follows_postgres_tests.rs @@ -866,3 +866,75 @@ async fn an_agent_keeps_no_read_state() { .await .is_some()); } + +/// How many threads the sidebar and the threads view list under `window`. +async fn listed_within( + db: &Db, + community: CommunityId, + channel: Uuid, + actor: &Keys, + window: u32, +) -> (usize, usize) { + let sidebar = db + .personal_read_sidebar_channels(community, &actor.public_key(), window, &[channel]) + .await + .unwrap() + .channels + .remove(0) + .threads + .len(); + let view = db + .personal_read_threads(community, &actor.public_key(), window, MAX_THREADS, None) + .await + .unwrap() + .threads + .len(); + (sidebar, view) +} + +/// A window shorter than a day still revives on a mention: ingest refreshes +/// against the same window the projections read. +#[tokio::test] +#[ignore = "requires Postgres"] +async fn a_mention_revives_a_follow_under_a_sub_day_window() { + const HOUR: u32 = 60 * 60; + set_follow_window_seconds(HOUR); + let (db, pool, community, channel, actor, root) = fixture().await; + reply( + &db, + community, + channel, + &root, + &Keys::generate(), + mention(&actor), + ) + .await; + sqlx::query( + "UPDATE personal_read_frontiers SET active_at=now()-interval '2 hours' + WHERE community_id=$1 AND actor=$2", + ) + .bind(community.as_uuid()) + .bind(actor.public_key().to_bytes().as_slice()) + .execute(&pool) + .await + .unwrap(); + assert_eq!( + listed_within(&db, community, channel, &actor, HOUR).await, + (0, 0), + "dormant under the window" + ); + reply( + &db, + community, + channel, + &root, + &Keys::generate(), + mention(&actor), + ) + .await; + assert_eq!( + listed_within(&db, community, channel, &actor, HOUR).await, + (1, 1), + "a fresh mention revives it" + ); +} diff --git a/crates/buzz-db/src/store/personal_read/mod.rs b/crates/buzz-db/src/store/personal_read/mod.rs index 21b61571b3d..323894adfd2 100644 --- a/crates/buzz-db/src/store/personal_read/mod.rs +++ b/crates/buzz-db/src/store/personal_read/mod.rs @@ -12,6 +12,7 @@ mod model; mod projection; mod writes; +pub use follows::set_follow_window_seconds; pub(crate) use follows::{place, record_message}; pub use model::*; diff --git a/crates/buzz-relay/src/main.rs b/crates/buzz-relay/src/main.rs index e42c6a49756..8ec1a618bc9 100644 --- a/crates/buzz-relay/src/main.rs +++ b/crates/buzz-relay/src/main.rs @@ -388,6 +388,7 @@ async fn run_relay_main(boot: BootTracker) -> anyhow::Result<()> { anyhow::anyhow!("Configuration error: {error}") })?; boot = next_boot; + buzz_db::personal_read::set_follow_window_seconds(config.buzz_v1_retention_seconds); let key_failure = if config.relay_private_key.is_some() { LifecycleReason::RequiredInvalid diff --git a/docs/buzz-v1-read-state.md b/docs/buzz-v1-read-state.md index c7e17d2e1b8..4a96c894cfe 100644 --- a/docs/buzz-v1-read-state.md +++ b/docs/buzz-v1-read-state.md @@ -150,8 +150,8 @@ thread is neither counted nor listed, but keeps its row and position, and any of those involvements makes it live again at that position. Replies that do not involve you never revive it. A follow made by the `follow` intent never goes dormant; an unfollow ends that. Being mentioned, a reply to your root and DM -replies refresh a live follow at most daily, so dormancy is measured to the -day. +replies refresh a live follow at most once a day, or once a quarter of the +window when that is shorter, so dormancy is measured to that interval. ## Threads view