From 75a55b7468cbfc48e4eaa0bbe216bb7bff594118 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 23:52:38 +0000 Subject: [PATCH 1/4] Make rows' external ids as each row is returned components, minimum_spanning_forest, vector_topk and most_similar built every row's external id before returning the first, in the call's multi-call memory context: about 72 bytes a text id, held until the call ended (0.72 and 1.38 GB at 10M nodes). They now return an iterator over the kernel's result, still in the extension's own buffer, and the loaded version, and make each row's ids in next(), in the per-row context Postgres resets. A pg_test for each opens a cursor on the call and checks that what it holds after its first row does not grow with the rows to come: 3.6 to 7.3 MB more for 50,000 rows before, now none. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012Rxfe2L3iVjSwSBPdUu1Ke --- .../pg_warren/src/compute_sql/similarity.rs | 31 +++++++--- crates/pg_warren/src/compute_sql/structure.rs | 43 ++++++++----- crates/pg_warren/src/compute_sql/summary.rs | 35 ++++++++--- crates/pg_warren/src/fixture.rs | 41 +++++++++++++ .../pg_warren/src/traversal_sql/components.rs | 61 +++++++++++++------ 5 files changed, 161 insertions(+), 50 deletions(-) diff --git a/crates/pg_warren/src/compute_sql/similarity.rs b/crates/pg_warren/src/compute_sql/similarity.rs index 7a21bbf3..34a20a6c 100644 --- a/crates/pg_warren/src/compute_sql/similarity.rs +++ b/crates/pg_warren/src/compute_sql/similarity.rs @@ -203,11 +203,11 @@ fn most_similar( let k = rank_k(k)?; let measure = measure(measure_name)?; let resolved = resolve(graph, version, "read")?; - let rows: Vec<(AnyElement, f64)> = with_id_type!(resolved.graph.id_type, E => { + let rows: Box> = with_id_type!(resolved.graph.id_type, E => { let id = element::(graph, "node", node)?; let loaded = cached::(&resolved)?; match loaded.id_map.dense_id(id.borrow()) { - None => Vec::new(), + None => Box::new(std::iter::empty()), Some(dense) => { let selected = relations(&loaded.relations, rels.as_deref()); let neighbourhood = @@ -225,10 +225,11 @@ fn most_similar( dense, k, )?; - similar - .into_iter() - .filter_map(|s| Some((external(&loaded.id_map, s.node)?, s.score))) - .collect() + // Each row's external id is made when it is returned + // (DESIGN.md §12), the ranking staying in its buffer. + Box::new(similar.into_iter().filter_map(move |s| { + Some((external(&loaded.id_map, s.node)?, s.score)) + })) } } }); @@ -310,7 +311,7 @@ mod tests { use pgrx::prelude::*; use pgrx::spi::Error; - use crate::fixture::{failure, rows, small}; + use crate::fixture::{failure, held_after_first_row, iri_star, rows, small}; /// Graph `t` (`small`) and `n`, the same without reverse CSRs: the /// undirected simple graph of both is the square 1–2–3–4–1, with 5 @@ -327,6 +328,22 @@ mod tests { const PAIRS: &str = "SELECT * FROM (VALUES (1, 3), (2, 4), (1, 2), (5, 6), (1, 99), (NULL, 2)) v"; + /// Each similar node's text id is made when its row is returned, so a + /// call holds no more Postgres memory for 49,999 rows to come than for + /// 999 (a star's leaves are all alike). + #[pg_test] + fn the_most_similar_make_each_rows_id_as_it_is_returned() -> Result<(), Error> { + iri_star("few", 1_000)?; + iri_star("many", 50_000)?; + let call = |graph: &str| { + format!("warren.most_similar('{graph}', 'https://example.org/resource/1'::text, k := 100000)") + }; + let few = held_after_first_row(&call("few"))?; + let many = held_after_first_row(&call("many"))?; + assert!(many < few + (256 << 10), "{many} bytes held for 49,999 rows, {few} for 999"); + Ok(()) + } + #[pg_test] fn pairs_are_scored_in_the_query_s_order_skipping_unknown_ids() -> Result<(), Error> { squares()?; diff --git a/crates/pg_warren/src/compute_sql/structure.rs b/crates/pg_warren/src/compute_sql/structure.rs index e43060ec..120a19de 100644 --- a/crates/pg_warren/src/compute_sql/structure.rs +++ b/crates/pg_warren/src/compute_sql/structure.rs @@ -5,8 +5,6 @@ //! on the compute pool within the call's memory budget; all but the last //! write a named vector. -use std::mem::size_of; - use pgrx::prelude::*; use pgrx::AnyElement; use warren_core::compute; @@ -213,7 +211,7 @@ fn forest_rows( resolved: &Resolved, rels: Option<&[Option]>, weighted: Option, -) -> Result, ComputeSqlError> { +) -> Result { let loaded = cached::(resolved)?; let memory = compute_memory(); let selected = relations(&loaded.relations, rels); @@ -225,20 +223,20 @@ fn forest_rows( selection(selected.as_deref()), weighted, )?; - let mut rows = Vec::new(); - let bytes = forest.edges.len().saturating_mul(size_of::<(AnyElement, AnyElement, f64)>()); - rows.try_reserve_exact(forest.edges.len()) - .map_err(|_| compute::ComputeError::OutOfMemory { bytes: bytes as u64 })?; - for &(source, target, weight) in &forest.edges { - if let (Some(source), Some(target)) = - (external(&loaded.id_map, source), external(&loaded.id_map, target)) - { - rows.push((source, target, weight)); - } - } - Ok(rows) + // Each row's external ids are made when it is returned (DESIGN.md + // §12): the forest stays in its charged buffer, and only the current + // row's ids are palloc'd, in the context Postgres resets between rows. + // The rows hold the loaded version, which keeps it mapped until the + // call ends, however it ends. + Ok(Box::new((0..forest.edges.len()).filter_map(move |edge| { + let &(source, target, weight) = forest.edges.get(edge)?; + Some((external(&loaded.id_map, source)?, external(&loaded.id_map, target)?, weight)) + }))) } +/// The rows of a minimum spanning forest, made one at a time. +type ForestRows = Box>; + #[cfg(any(test, feature = "pg_test"))] #[pgrx::pg_schema] #[allow(clippy::disallowed_methods)] // Test code may assign a transaction id. @@ -246,7 +244,7 @@ mod tests { use pgrx::prelude::*; use pgrx::spi::Error; - use crate::fixture::{failure, refresh, rows, small}; + use crate::fixture::{failure, held_after_first_row, iri_path, refresh, rows, small}; /// The live rows of vector `name` of graph `graph`, as `id:value`. fn values(graph: &str, name: &str) -> Result, Error> { @@ -273,6 +271,19 @@ mod tests { ) } + /// Each forest edge's text ids are made when its row is returned, so a + /// call holds no more Postgres memory for 49,999 rows to come than for + /// 999 (built up front, the ids took about 72 bytes each). + #[pg_test] + fn the_forest_makes_each_rows_ids_as_it_is_returned() -> Result<(), Error> { + iri_path("few", 1_000)?; + iri_path("many", 50_000)?; + let few = held_after_first_row("warren.minimum_spanning_forest('few', NULL::text)")?; + let many = held_after_first_row("warren.minimum_spanning_forest('many', NULL::text)")?; + assert!(many < few + (256 << 10), "{many} bytes held for 49,999 rows, {few} for 999"); + Ok(()) + } + #[pg_test] fn core_numbers_ignore_direction_and_self_loops_and_return_the_degeneracy() -> Result<(), Error> { diff --git a/crates/pg_warren/src/compute_sql/summary.rs b/crates/pg_warren/src/compute_sql/summary.rs index 4fd8d4d1..f54901a1 100644 --- a/crates/pg_warren/src/compute_sql/summary.rs +++ b/crates/pg_warren/src/compute_sql/summary.rs @@ -1,6 +1,8 @@ //! Summaries of a vector's live nodes (DESIGN.md §6.2): `vector_topk` and //! `vector_stats`. +use std::sync::Arc; + use pgrx::prelude::*; use pgrx::{AnyElement, JsonB}; use serde_json::{json, Value}; @@ -43,7 +45,7 @@ fn vector_topk( .map_err(|_| ComputeSqlError::InvalidArgument("k must not be negative".into()))?; let descending = descending.ok_or(ComputeSqlError::NullArgument("descending"))?; let resolved = resolve(graph, version, "read")?; - let rows = with_id_type!(resolved.graph.id_type, E => { + let rows: Box> = with_id_type!(resolved.graph.id_type, E => { // SAFETY: fcinfo is this call's, and argument 3 exists. let oid = unsafe { pgrx::fcinfo::pg_getarg_type(fcinfo, 3) }; check_type::(graph, "id_type", oid, false)?; @@ -57,12 +59,12 @@ fn vector_topk( k, descending, )?; - ranked - .into_iter() - .filter_map(|(node, value)| { - external(&vectors.loaded.id_map, node).map(|id| (id, value)) - }) - .collect::>() + // Each row's external id is made when it is returned (DESIGN.md + // §12), the ranking staying in its buffer. + let loaded = Arc::clone(&vectors.loaded); + Box::new(ranked.into_iter().filter_map(move |(node, value)| { + external(&loaded.id_map, node).map(|id| (id, value)) + })) }); Ok(TableIterator::new(rows)) }) @@ -118,7 +120,24 @@ mod tests { use pgrx::prelude::*; use pgrx::spi::Error; - use crate::fixture::{refresh, rows, small}; + use crate::fixture::{held_after_first_row, iri_path, refresh, rows, small}; + + /// Each ranked node's text id is made when its row is returned, so a + /// call holds no more Postgres memory for 50,000 rows to come than for + /// 1,000. + #[pg_test] + fn the_top_k_makes_each_rows_id_as_it_is_returned() -> Result<(), Error> { + for (graph, nodes) in [("few", 1_000), ("many", 50_000)] { + iri_path(graph, nodes)?; + Spi::run(&format!( + "SELECT warren.vector_from_query('{graph}', 'v', 'SELECT id, n::float8 FROM {graph}_nodes')" + ))?; + } + let few = held_after_first_row("warren.vector_topk('few', 'v', 50000, NULL::text)")?; + let many = held_after_first_row("warren.vector_topk('many', 'v', 50000, NULL::text)")?; + assert!(many < few + (256 << 10), "{many} bytes held for 50,000 rows, {few} for 1,000"); + Ok(()) + } #[pg_test] fn top_k_ranks_live_nodes_with_ties_by_dense_id() -> Result<(), Error> { diff --git a/crates/pg_warren/src/fixture.rs b/crates/pg_warren/src/fixture.rs index 7d236be5..b319ab8f 100644 --- a/crates/pg_warren/src/fixture.rs +++ b/crates/pg_warren/src/fixture.rs @@ -346,3 +346,44 @@ pub(crate) fn ownership() -> Result, Error> { FROM warren.vectors v JOIN warren.graphs g USING (graph_id)), '')", ) } + +/// Graph `name`: a path `1 → 2 → … → nodes` over IRIs +/// (`https://example.org/resource/`), refreshed. +pub(crate) fn iri_path(name: &str, nodes: i64) -> Result<(), Error> { + Spi::run(&format!( + "CREATE TABLE {name}_nodes AS + SELECT n, 'https://example.org/resource/' || n AS id FROM generate_series(1, {nodes}) n; + SELECT warren.create_graph('{name}', 'SELECT id FROM {name}_nodes', + 'SELECT a.id, b.id FROM {name}_nodes a JOIN {name}_nodes b ON b.n = a.n + 1'); + SELECT warren.refresh('{name}')" + )) +} + +/// The Postgres memory a set-returning `query` (a call in the select +/// list, so that its rows are made one at a time) holds once it has +/// returned its first row, past what the backend held before: what the +/// call keeps for the rows it has yet to return. Runs `query` to the end +/// once first, so that its version is cached and every later row checked. +pub(crate) fn held_after_first_row(query: &str) -> Result { + Spi::run(&format!("SELECT count(*) FROM (SELECT {query}) q"))?; + Spi::connect(|client| { + let before = allocated(); + let mut cursor = client.try_open_cursor(&format!("SELECT {query}"), &[])?; + let first = cursor.fetch(1)?; + let held = allocated().saturating_sub(before); + assert_eq!(first.len(), 1, "the call returns a row"); + Ok(held) + }) +} + +/// Graph `name`: a star over IRIs (`https://example.org/resource/`), +/// hub 0 with an edge to each of leaves 1 to `leaves`, refreshed. +pub(crate) fn iri_star(name: &str, leaves: i64) -> Result<(), Error> { + Spi::run(&format!( + "CREATE TABLE {name}_nodes AS + SELECT n, 'https://example.org/resource/' || n AS id FROM generate_series(0, {leaves}) n; + SELECT warren.create_graph('{name}', 'SELECT id FROM {name}_nodes', + 'SELECT a.id, b.id FROM {name}_nodes a JOIN {name}_nodes b ON a.n = 0 AND b.n > 0'); + SELECT warren.refresh('{name}')" + )) +} diff --git a/crates/pg_warren/src/traversal_sql/components.rs b/crates/pg_warren/src/traversal_sql/components.rs index 95e45fb1..31ef3ec8 100644 --- a/crates/pg_warren/src/traversal_sql/components.rs +++ b/crates/pg_warren/src/traversal_sql/components.rs @@ -1,13 +1,15 @@ //! `warren.components`: weakly connected components (DESIGN.md §9.3). +use std::sync::Arc; + use pgrx::prelude::*; use pgrx::AnyElement; -use warren_core::graph::GraphError; -use warren_core::traversal::{components as label_components, TraversalError}; +use warren_core::graph::DenseId; +use warren_core::traversal::components as label_components; use super::stats::Recorder; use super::{entry, TraversalSqlError}; -use crate::cache::{cached, resolve}; +use crate::cache::{cached, resolve, CachedIds, LoadedVersion}; use crate::external_ids::{check_type, external}; use crate::interrupts::PostgresInterrupts; use crate::source::with_id_type; @@ -36,32 +38,40 @@ fn components( let graph = graph.ok_or(TraversalSqlError::NullArgument("graph"))?; let resolved = resolve(graph, version, "read")?; let mut recorder = Recorder::start("components", graph, resolved.version); - let rows = with_id_type!(resolved.graph.id_type, E => { + let rows: Box> = with_id_type!(resolved.graph.id_type, E => { // SAFETY: fcinfo is this call's, and argument 1 exists. let witness = unsafe { pgrx::fcinfo::pg_getarg_type(fcinfo, 1) }; check_type::(graph, "id_type", witness, false)?; let loaded = cached::(&resolved)?; let labelled = label_components(&loaded.graph, &mut PostgresInterrupts)?; recorder.record(labelled.stats); - let tombstones = loaded.id_map.tombstones(); - let mut rows = Vec::new(); - let live = usize::try_from(loaded.id_map.node_count()).unwrap_or(usize::MAX); - rows.try_reserve_exact(live) - .map_err(|_| TraversalError::from(GraphError::OutOfMemory { elements: live }))?; - for (node, &component) in (0..).zip(&labelled.result) { - if tombstones.contains(node) { - continue; - } - if let Some(id) = external(&loaded.id_map, node) { - rows.push((id, i64::try_from(component).unwrap_or(i64::MAX))); - } - } - rows + Box::new(live_rows(loaded, labelled.result)) }); Ok(TableIterator::new(rows)) }) } +/// A row of the result: a node's external id and its component. +type Row = (AnyElement, i64); + +/// The rows of each live node in dense id order, each made when it is +/// returned (DESIGN.md §12): the labels stay in the extension's buffer, +/// and only the current row's external id is palloc'd, in the context +/// Postgres resets between rows. The rows hold the loaded version, which +/// keeps it mapped until the call ends, however it ends. +fn live_rows( + loaded: Arc>, + labels: Vec, +) -> impl Iterator { + (0..).zip(labels).filter_map(move |(node, component)| { + if loaded.id_map.tombstones().contains(node) { + return None; + } + let id = external(&loaded.id_map, node)?; + Some((id, i64::try_from(component).unwrap_or(i64::MAX))) + }) +} + #[cfg(any(test, feature = "pg_test"))] #[pgrx::pg_schema] #[allow(clippy::disallowed_methods)] // Test code may assign a transaction id. @@ -69,7 +79,7 @@ mod tests { use pgrx::prelude::*; use pgrx::spi::Error; - use crate::fixture::{refresh, rows, small}; + use crate::fixture::{held_after_first_row, iri_path, refresh, rows, small}; #[pg_test] fn components_label_live_nodes_by_their_smallest_dense_id() -> Result<(), Error> { @@ -89,6 +99,19 @@ mod tests { Ok(()) } + /// Each row's text id is made when the row is returned, so a call + /// holds no more Postgres memory for 50,000 rows to come than for + /// 1,000 (built up front, the ids took about 72 bytes each). + #[pg_test] + fn components_make_each_rows_id_as_it_is_returned() -> Result<(), Error> { + iri_path("few", 1_000)?; + iri_path("many", 50_000)?; + let few = held_after_first_row("warren.components('few', NULL::text)")?; + let many = held_after_first_row("warren.components('many', NULL::text)")?; + assert!(many < few + (256 << 10), "{many} bytes held for 50,000 rows, {few} for 1,000"); + Ok(()) + } + #[pg_test( error = "graph \"t\" has node ids of type bigint, but id_type is of type uuid; cast it to bigint" )] From 8d759e5884629093a408881bc793f002d7683ae0 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 23:52:38 +0000 Subject: [PATCH 2/4] Test lazily made rows across LIMIT, a cursor, an error and a cancel In every shared cache mode: a call stopped early, a cursor read across statements after its version is evicted, a call that errors mid-scan and one cancelled between rows each return what they should and leave no version held. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012Rxfe2L3iVjSwSBPdUu1Ke --- test/sql/shared_cache_lazy_rows.sql | 89 +++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 test/sql/shared_cache_lazy_rows.sql diff --git a/test/sql/shared_cache_lazy_rows.sql b/test/sql/shared_cache_lazy_rows.sql new file mode 100644 index 00000000..ab338004 --- /dev/null +++ b/test/sql/shared_cache_lazy_rows.sql @@ -0,0 +1,89 @@ +-- DESIGN.md §12: `components`, `minimum_spanning_forest`, `vector_topk` +-- and `most_similar` make each row's external id when the row is +-- returned, holding the loaded version between rows. A caller that stops +-- early (LIMIT), reads through a cursor across statements while the +-- version is evicted, or is cancelled mid-scan gets the rows it would +-- have, and the version is released when the call ends, however it ends. +-- Where the shared cache is off or unavailable, the version is private. +\ir include/shared_cache.sql +CREATE TABLE nodes AS SELECT n, 'https://example.org/resource/' || n AS id FROM generate_series(1, 20000) n; +SELECT warren.create_graph('g', 'SELECT id FROM nodes', + $q$SELECT a.id, b.id FROM nodes a JOIN nodes b ON b.n = a.n + 1$q$); +SELECT warren.refresh('g') AS v1 \gset +SELECT warren.vector_from_query('g', 'v', 'SELECT id, n::float8 FROM nodes'); +SELECT CASE WHEN :'shared' THEN 1 ELSE 0 END::bigint AS one \gset + +-- What this backend still holds of the version a call read. +CREATE FUNCTION holding() RETURNS bigint LANGUAGE sql AS $$ + SELECT count(*) FROM warren.cache() WHERE kind = 'version' AND state IN ('in use', 'held') +$$; + +-- Callers that stop early, in the select list and in FROM. +SELECT assert.equal(count(*), 3::bigint, 'components under LIMIT') +FROM (SELECT warren.components('g', NULL::text) LIMIT 3) s; +SELECT assert.equal(count(*), 3::bigint, 'the forest under LIMIT') +FROM (SELECT warren.minimum_spanning_forest('g', NULL::text) LIMIT 3) s; +SELECT assert.equal(count(*), 3::bigint, 'the top k under LIMIT') +FROM (SELECT warren.vector_topk('g', 'v', 20000, NULL::text) LIMIT 3) s; +SELECT assert.equal(count(*), 1::bigint, 'most_similar under LIMIT') +FROM (SELECT warren.most_similar('g', 'https://example.org/resource/2'::text, k := 100) LIMIT 1) s; +SELECT assert.equal(holding(), 0::bigint, 'a call stopped early holds no version'); + +-- The rows of an open cursor, after its statement's pin has ended. +CREATE FUNCTION drain(c refcursor) RETURNS bigint LANGUAGE plpgsql AS $$ +DECLARE + r record; + n bigint := 0; +BEGIN + LOOP + FETCH c INTO r; + EXIT WHEN NOT FOUND; + IF r.node_id LIKE 'https://example.org/resource/%' AND r.component_id = 0 THEN + n := n + 1; + END IF; + END LOOP; + RETURN n; +END +$$; +BEGIN; +DECLARE c CURSOR FOR + SELECT (x).node_id, (x).component_id FROM (SELECT warren.components('g', NULL::text) AS x OFFSET 0) s; +MOVE 2 IN c; +SELECT warren.evict('g'); +SELECT assert.equal(holding() + shared_segments(:v1), 1::bigint, + 'the evicted version is held for the cursor, privately or mapped'); +SELECT assert.equal(drain('c'), 19998::bigint, 'the cursor reads the rest of the evicted version'); +COMMIT; +SELECT assert.equal(holding(), 0::bigint, 'a closed cursor holds no version'); + +-- A call that errors mid-scan, at its 5,000th row. +CREATE SEQUENCE rows_read; +SELECT assert.raises($$SELECT count(1 / (5000 - nextval('rows_read'))) FROM (SELECT warren.components('g', NULL::text) OFFSET 0) s$$, + 'components erroring mid-scan', '22012'); +SELECT setval('rows_read', 1, false); +SELECT assert.raises($$SELECT count(1 / (5000 - nextval('rows_read'))) FROM (SELECT warren.minimum_spanning_forest('g', NULL::text) OFFSET 0) s$$, + 'the forest erroring mid-scan', '22012'); +SELECT assert.equal(holding(), 0::bigint, 'a call that errored holds no version'); + +-- A call cancelled mid-scan, between rows, in another session. +SELECT dblink_connect('b', :'conninfo'); +SELECT pid FROM dblink('b', 'SELECT pg_backend_pid()') AS t(pid int) \gset +SELECT dblink_send_query('b', $$SELECT count(pg_sleep(0.001)) FROM (SELECT warren.components('g', NULL::text) OFFSET 0) s$$); +SELECT pg_sleep(1); +SELECT assert.that(pg_cancel_backend(:pid), 'the scan is cancelled'); +SELECT assert.equal(count(*), 0::bigint, 'the cancelled scan returns no count') +FROM dblink_get_result('b', false) AS t(n bigint); +SELECT assert.that(dblink_error_message('b') LIKE '%canceling statement due to user request%', + 'components cancelled mid-scan: ' || dblink_error_message('b')); +SELECT * FROM dblink_get_result('b') AS t(n bigint); +SELECT assert.equal(h, 0::bigint, 'a cancelled call holds no version') +FROM dblink('b', $$SELECT count(*) FROM warren.cache() WHERE kind = 'version' AND state IN ('in use', 'held')$$) AS t(h bigint); +SELECT assert.equal(n, 20000::bigint, 'and its session''s next call reads every row') +FROM dblink('b', $$SELECT count(*) FROM warren.components('g', NULL::text)$$) AS t(n bigint); +SELECT dblink_disconnect('b'); + +SELECT assert.equal(count(*), 20000::bigint, 'and the next call reads every row') +FROM warren.components('g', NULL::text); +SELECT assert.equal(count(*), 19999::bigint, 'and every forest edge') +FROM warren.minimum_spanning_forest('g', NULL::text); +SELECT assert.equal(shared_segments(:v1), :one, 'the version is shared again where it can be'); From dbdc6fc475affd2516be476e3622198d634e5713 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 24 Oct 2026 00:07:21 +0000 Subject: [PATCH 3/4] =?UTF-8?q?DESIGN=20=C2=A712:=20rows=20made=20as=20the?= =?UTF-8?q?y=20are=20returned,=20measured=20at=20half=20scale?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The text peaks of components and minimum_spanning_forest fall to what the version's size alone accounts for (0.84 and 1.10 GB from 1.28 and 1.77 at 5M nodes). Lists the SRFs that already streamed and the two that still build their rows up front. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012Rxfe2L3iVjSwSBPdUu1Ke --- DESIGN.md | 2 +- bench/results/soak/README.md | 18 + bench/results/soak/a-scale2-m10-lazy.json | 954 ++++++++++++++++++ .../results/soak/a-scale2-text-m10-lazy.json | 954 ++++++++++++++++++ 4 files changed, 1927 insertions(+), 1 deletion(-) create mode 100644 bench/results/soak/a-scale2-m10-lazy.json create mode 100644 bench/results/soak/a-scale2-text-m10-lazy.json diff --git a/DESIGN.md b/DESIGN.md index 14a9e8c5..a9f1339f 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -1587,7 +1587,7 @@ A pooled backend's own memory is 4–12 MB with the shared cache against the ver - *Cold load:* 1.03 / 0.77 s privately, 1.06 / 0.79 s into the shared cache. - *Lookups:* `neighbours` of one IRI 0.1 / 0.1 ms (median of 20 calls), `most_similar` 1.1 / 1.0 ms, `similarity` of 100,000 pairs 1.31 / 0.41 s (past 65,536 lookups, so through a per-call index, which costs 40 MB at 5M nodes). - *Traversal:* `bfs` to depth 6 (1.07M rows) 0.95 / 0.49 s, `reachable_set` (5.0M rows) 3.72 / 2.22 s. Every row's `text` datum is built and checked against the encoding, so row emission, already most of these calls' time (above), about doubles. The kernels that return no ids are unchanged: `components` 2.77 / 2.74 s, `eigen` for 200 steps 75.9 / 77.3 s, and the library within noise (`clustering` 27.0 / 28.8 s, betweenness 40.5 / 40.1 s, `minimum_spanning_forest` 22.5 / 22.0 s). -- *Peaks:* the stages that read the version peak about 0.12 GB higher privately, the difference in the versions' sizes (`pagerank` 1.28 / 1.16 GB, `eigen` 2.28 / 2.17 GB). The calls that return a row per node peak higher still: `components` 1.28 / 0.80 GB and `minimum_spanning_forest` 1.77 / 0.98 GB, for their rows of text ids. Not investigated. +- *Peaks:* the stages that read the version peak about 0.12 GB higher privately, the difference in the versions' sizes (`pagerank` 1.28 / 1.16 GB, `eigen` 2.28 / 2.17 GB). The calls that return a row per node peak higher still: `components` 1.28 / 0.80 GB and `minimum_spanning_forest` 1.77 / 0.98 GB, for their rows of text ids. *Fixed since:* both built every row before returning the first, a `Vec` of rows whose text ids were each `palloc`ed (`cstring_to_text_with_len`) in the set-returning function's multi-call memory context, which lives until the call ends: an IRI and its 4-byte header round to a 64-byte AllocSet chunk plus an 8-byte header, about 72 bytes an id (68–78 measured here, 69–72 at full scale, where it was 0.72 and 1.38 GB). They, `vector_topk` and `most_similar` now return an iterator over the kernel's result, which stays in the extension's own buffer (charged as before), and the loaded version, and make each row's external ids in `next()`. pgrx calls `next()` after the first row in the caller's memory context, the per-tuple context Postgres resets before each call (`ExecMakeTableFunctionResult` for a call in `FROM`, `ExecProjectSet` in the select list), so only the current row's ids are held; the iterator's `Arc` keeps the version alive, private or mapped from the shared cache, until the call ends, by its last row, a `LIMIT`, an error or a cancel (`shared_cache_lazy_rows.sql`; a cursor read across statements keeps it after its statement's pin has ended, even once it is evicted). Each has a `pg_test` that opens a cursor on the call and checks that what it holds after its first row does not grow with the rows to come (3.6–7.3 MB more for 50,000 rows before, none since). Measured again at half scale (`bench/results/soak/a-scale2*-m10-lazy.json`, 8d759e5, the same flags without the second version), text / `bigint`, peak RssAnon and what the call added to the backend: `components` 0.84 / 0.73 GB, +112 / +63 MB (was 1.28 / 0.80 GB, +555 / +166 MB); `minimum_spanning_forest` 1.10 / 0.98 GB, +326 / +326 MB (was 1.77 / 0.98 GB, +1,002 / +326 MB), what remains being the kernel's own result and scratch: the forest's text peak now differs from `bigint`'s by the versions' sizes alone, as the other stages' do, and `components`' by 49 MB more (about 10 bytes a node). Times are unchanged within noise (`components` 2.87 / 2.00 s, `minimum_spanning_forest` 17.0 / 16.4 s, a load average of 1–3). The SRFs that already made each row as it is returned are `bfs`, `reachable_set`, `vector_values`, `matrix_rows`, `id_map`, `similarity`, `similar_pairs` and `gather`. Two still build their rows up front: `walk`, whose rows (bounded by `max_results` and `warren.max_walk_paths`) are made as the walks run, and `neighbours`, whose rows are one node's neighbours. - *Shared cache:* the shared version is 719.5 / 600.6 MB, and the private peaks fall by the version, as for `bigint` ids: `pagerank` 0.56 / 0.56 GB, `bfs` 0.12 / 0.12 GB, `components` 0.56 / 0.20 GB. Every §12 target that ran was met at this scale except `reachable_set`, which misses for both id types (12–13 / 22–23M edges/s). diff --git a/bench/results/soak/README.md b/bench/results/soak/README.md index 89d4e54a..312ebfb8 100644 --- a/bench/results/soak/README.md +++ b/bench/results/soak/README.md @@ -374,3 +374,21 @@ Half scale, without the recursive CTE baseline, because the machine had about 8 GB of disk free, shared with other work; the full-scale text run needs about 16 GB (DESIGN.md §12). Both private runs skipped their second version for the disk floor, so `diff` and `distance` were not run. + +## Rows made as they are returned, at half scale (§5.7, §12) + +`a-scale2-text-m10-lazy.json` then `a-scale2-m10-lazy.json`: case a at +half scale (5M nodes, 50M edges) with IRIs and with `bigint` ids, after +`components`, `minimum_spanning_forest`, `vector_topk` and +`most_similar` were changed to make each row's external ids as it is +returned, to compare their peaks with `a-scale2-text-m10.json` and +`a-scale2-m10.json`. PostgreSQL 16.15 from a private tree, a release +build of 8d759e5, harness 8d759e5, the same 4-vCPU Xeon, one after the +other on 2026-10-24, load average 1.0–2.9 (another agent's work during +the bigint run): + + bench/soak.py a --ids text --label=-m10-lazy --scale 2 --no-cte --no-eigen-steps \ + --eigen-max-iter 200 --skip communities --skip second-version --disk-floor 3221225472 \ + --pg-config /var/tmp/m10-pgtree/usr/lib/postgresql/16/bin/pg_config --parent /var/tmp/m10-soak \ + --extension-revision 8d759e5 --extension-profile release --threads 4 --statement-timeout 20min + bench/soak.py a --ids bigint --label=-m10-lazy … # otherwise as above diff --git a/bench/results/soak/a-scale2-m10-lazy.json b/bench/results/soak/a-scale2-m10-lazy.json new file mode 100644 index 00000000..e7155647 --- /dev/null +++ b/bench/results/soak/a-scale2-m10-lazy.json @@ -0,0 +1,954 @@ +{ + "provenance": { + "argv": [ + "bench/soak.py", + "a", + "--scale", + "2", + "--no-cte", + "--no-eigen-steps", + "--eigen-max-iter", + "200", + "--skip", + "communities", + "--skip", + "second-version", + "--disk-floor", + "3221225472", + "--pg-config", + "/var/tmp/m10-pgtree/usr/lib/postgresql/16/bin/pg_config", + "--parent", + "/var/tmp/m10-soak", + "--results", + "/var/tmp/m10-soak/results", + "--extension-revision", + "8d759e5", + "--extension-profile", + "release", + "--threads", + "4", + "--statement-timeout", + "20min", + "--label=-m10-lazy", + "--ids", + "bigint" + ], + "harness": { + "revision": "8d759e5884629093a408881bc793f002d7683ae0", + "dirty": false + }, + "extension": { + "revision": "8d759e5", + "profile": "release", + "library": "/var/tmp/m10-pgtree/usr/lib/postgresql/16/lib/pg_warren.so", + "library_sha256": "ccd990d97b0f31f3cf32e72f45d6b6f0c65d8566a0de1a6c975fdd97bb68af12" + } + }, + "case": "a", + "nodes": 5000000, + "edges": 50000000, + "weighted": false, + "ids": "bigint", + "relations": 1, + "store_merged": true, + "threads": 4, + "node_order": "id", + "eigen_max_iter": 200, + "statement_timeout": "20min", + "skip": [ + "communities", + "second-version" + ], + "planned_stages": [ + "cold_load_probe", + "default: bfs depth 6", + "default: components", + "default: eigen", + "default: pagerank", + "default: prewarm", + "default: reachable_set", + "library: betweenness_centrality", + "library: closeness_centrality", + "library: clustering", + "library: core_number", + "library: eigenvector_centrality", + "library: harmonic_centrality", + "library: hits", + "library: katz_centrality", + "library: minimum_spanning_forest", + "library: most_similar", + "library: similar_pairs", + "library: similarity", + "library: strong_components_vector", + "refresh" + ], + "hardware": { + "cpus": 4, + "model": "Intel(R) Xeon(R) Processor", + "memory_bytes": 16872804352, + "kernel": "6.18.44-fc-v153", + "load_average": [ + 2.33349609375, + 1.87109375, + 1.42626953125 + ] + }, + "started_at": "2026-10-24T00:01:55+00:00", + "stages": { + "table 0: create": { + "sql": "CREATE UNLOGGED TABLE soak_removed ()", + "seconds": [ + 0.0004 + ], + "median_seconds": 0.0004, + "rows": [], + "first_rows": [ + null + ], + "notices": [], + "warnings": [], + "rss_anon_before": 5611520, + "peak_rss_anon": 5611520, + "vm_hwm": 35119104, + "rss_anon_after": 5611520, + "rss_shmem_after": 15044608 + }, + "table 1: create": { + "sql": "CREATE UNLOGGED TABLE soak_edges (src int4 NOT NULL, dst int4 NOT NULL) WITH (autovacuum_enabled = false)", + "seconds": [ + 0.0004 + ], + "median_seconds": 0.0004, + "rows": [], + "first_rows": [ + null + ], + "notices": [], + "warnings": [], + "rss_anon_before": 5611520, + "peak_rss_anon": 5611520, + "vm_hwm": 35123200, + "rss_anon_after": 5611520, + "rss_shmem_after": 15048704 + }, + "table 2: insert": { + "sql": "INSERT INTO soak_edges SELECT 1 + floor(random() * 5000000)::int4 AS src, 1 + floor(random() * 5000000)::int4 AS dst FROM (SELECT setseed(0.25) OFFSET 0) seeded CROSS JOIN generate_series(1, 50000000)", + "seconds": [ + 15.9302 + ], + "median_seconds": 15.9302, + "rows": [], + "first_rows": [ + null + ], + "notices": [], + "warnings": [], + "rss_anon_before": 5611520, + "peak_rss_anon": 95424512, + "vm_hwm": 1254473728, + "rss_anon_after": 32477184, + "rss_shmem_after": 1095868416 + }, + "table 3: analyze": { + "sql": "ANALYZE soak_edges", + "seconds": [ + 0.1948 + ], + "median_seconds": 0.1948, + "rows": [], + "first_rows": [ + null + ], + "notices": [], + "warnings": [], + "rss_anon_before": 32477184, + "peak_rss_anon": 32485376, + "vm_hwm": 1192103936, + "rss_anon_after": 32485376, + "rss_shmem_after": 1095868416 + }, + "create_graph": { + "sql": "SELECT warren.create_graph('soak', $n$SELECT id FROM generate_series(1::bigint, 5000000) id$n$, $q$SELECT src::bigint, dst::bigint FROM soak_edges$q$, weighted := false, has_relations := false, store_merged := true, node_order := 'id')", + "seconds": [ + 0.0322 + ], + "median_seconds": 0.0322, + "rows": [ + "" + ], + "first_rows": [ + "" + ], + "notices": [], + "warnings": [], + "rss_anon_before": 3198976, + "peak_rss_anon": 9715712, + "vm_hwm": 74575872, + "rss_anon_after": 9715712, + "rss_shmem_after": 10899456 + }, + "refresh": { + "sql": "SELECT warren.refresh('soak')", + "seconds": [ + 36.1536 + ], + "median_seconds": 36.1536, + "rows": [ + "1" + ], + "first_rows": [ + "1" + ], + "notices": [], + "warnings": [], + "rss_anon_before": 9715712, + "peak_rss_anon": 790708224, + "vm_hwm": 1949499392, + "rss_anon_after": 48058368, + "rss_shmem_after": 1111142400 + }, + "refresh from the generator": { + "sql": "SELECT warren.refresh('soak_generated')", + "seconds": [ + 45.9605 + ], + "median_seconds": 45.9605, + "rows": [ + "2" + ], + "first_rows": [ + "2" + ], + "notices": [], + "warnings": [], + "rss_anon_before": 9809920, + "peak_rss_anon": 827920384, + "vm_hwm": 1025220608, + "rss_anon_after": 48238592, + "rss_shmem_after": 621809664 + }, + "cold_load_probe": { + "sql": "SELECT warren.prewarm('soak')", + "seconds": [ + 0.6879 + ], + "median_seconds": 0.6879, + "rows": [ + "1" + ], + "first_rows": [ + "1" + ], + "notices": [ + "NOTICE: version 1 (600624716 bytes) cannot be kept within warren.max_cache_memory (65536 bytes, 0 of them held by transient vectors), so it is held only until the statement ends, the bytes past the cap charged to warren.max_compute_memory, and reloaded by each statement that reads it; raise warren.max_cache_memory" + ], + "warnings": [], + "rss_anon_before": 3207168, + "peak_rss_anon": 612888576, + "vm_hwm": 1229762560, + "rss_anon_after": 12259328, + "rss_shmem_after": 599527424 + }, + "default: prewarm": { + "sql": "WITH before AS MATERIALIZED (SELECT (SELECT pg_stat_get_xact_tuples_returned('warren.segments'::regclass) + pg_stat_get_xact_tuples_fetched('warren.segments'::regclass) + sum(pg_stat_get_xact_tuples_fetched(indexrelid))::bigint FROM pg_index WHERE indrelid = 'warren.segments'::regclass AND 1 IS NOT NULL) AS reads), run AS MATERIALIZED (SELECT warren.prewarm('soak') AS result FROM before WHERE before.reads IS NOT NULL), after AS MATERIALIZED (SELECT (SELECT pg_stat_get_xact_tuples_returned('warren.segments'::regclass) + pg_stat_get_xact_tuples_fetched('warren.segments'::regclass) + sum(pg_stat_get_xact_tuples_fetched(indexrelid))::bigint FROM pg_index WHERE indrelid = 'warren.segments'::regclass AND run.result IS NOT NULL) AS reads FROM run) SELECT after.reads - before.reads, run.result FROM before, run, after", + "seconds": [ + 1.1762 + ], + "median_seconds": 1.1762, + "rows": [ + "1120|1" + ], + "first_rows": [ + "1120|1" + ], + "notices": [], + "warnings": [], + "rss_anon_before": 3203072, + "peak_rss_anon": 613228544, + "vm_hwm": 1231134720, + "rss_anon_after": 613208064, + "rss_shmem_after": 600391680 + }, + "default: bfs depth 6": { + "sql": "SELECT bench_seconds($soak$SELECT count(*) FROM warren.bfs('soak', ARRAY[1::bigint], 6)$soak$)", + "seconds": [ + 0.9815, + 0.454, + 0.4606 + ], + "median_seconds": 0.4606, + "rows": [ + "0.459566" + ], + "first_rows": [ + "0.980314", + "0.453461", + "0.459566" + ], + "notices": [], + "warnings": [], + "rss_anon_before": 613285888, + "peak_rss_anon": 719638528, + "vm_hwm": 1339052032, + "rss_anon_after": 608145408, + "rss_shmem_after": 600674304, + "server_seconds": [ + 0.980314, + 0.453461, + 0.459566 + ], + "server_median_seconds": 0.459566 + }, + "default: bfs depth 6 count": { + "sql": "SELECT count(*) FROM warren.bfs('soak', ARRAY[1::bigint], 6)", + "seconds": [ + 0.4536 + ], + "median_seconds": 0.4536, + "rows": [ + "1068779" + ], + "first_rows": [ + "1068779" + ], + "notices": [], + "warnings": [], + "rss_anon_before": 608145408, + "peak_rss_anon": 718974976, + "vm_hwm": 1338384384, + "rss_anon_after": 608145408, + "rss_shmem_after": 600674304 + }, + "default: neighbours": { + "sql": "SELECT bench_seconds($soak$SELECT count(*) FROM warren.neighbours('soak', 1::bigint)$soak$)", + "seconds": [ + 0.0007, + 0.0002, + 0.0002, + 0.0001, + 0.0001, + 0.0001, + 0.0001, + 0.0001, + 0.0001, + 0.0001, + 0.0001, + 0.0001, + 0.0001, + 0.0001, + 0.0001, + 0.0001, + 0.0001, + 0.0001, + 0.0001, + 0.0001 + ], + "median_seconds": 0.0001, + "rows": [ + "4.1e-05" + ], + "first_rows": [ + "0.000421", + "8.8e-05", + "6.1e-05", + "5.5e-05", + "4.9e-05", + "4.5e-05", + "4.4e-05", + "4.3e-05", + "4.2e-05", + "4.3e-05", + "4.1e-05", + "4.2e-05", + "4.2e-05", + "4.3e-05", + "4.2e-05", + "5.8e-05", + "4.6e-05", + "4.3e-05", + "4.2e-05", + "4.1e-05" + ], + "notices": [], + "warnings": [], + "rss_anon_before": 608145408, + "peak_rss_anon": 608165888, + "vm_hwm": 1228005376, + "rss_anon_after": 608165888, + "rss_shmem_after": 600674304, + "server_seconds": [ + 0.000421, + 8.8e-05, + 6.1e-05, + 5.5e-05, + 4.9e-05, + 4.5e-05, + 4.4e-05, + 4.3e-05, + 4.2e-05, + 4.3e-05, + 4.1e-05, + 4.2e-05, + 4.2e-05, + 4.3e-05, + 4.2e-05, + 5.8e-05, + 4.6e-05, + 4.3e-05, + 4.2e-05, + 4.1e-05 + ], + "server_median_seconds": 4.3e-05 + }, + "default: reachable_set": { + "sql": "SELECT count(*) FROM warren.reachable_set('soak', ARRAY[1::bigint])", + "seconds": [ + 2.5128 + ], + "median_seconds": 2.5128, + "rows": [ + "4999752" + ], + "first_rows": [ + "4999752" + ], + "notices": [], + "warnings": [], + "rss_anon_before": 608165888, + "peak_rss_anon": 853004288, + "vm_hwm": 1472913408, + "rss_anon_after": 664428544, + "rss_shmem_after": 600674304 + }, + "default: reachable_set stats": { + "stats": [ + "{\"graph\": \"soak\", \"version\": 1, \"function\": \"reachable_set\", \"truncated\": false, \"nodes_visited\": 4999752, \"edges_traversed\": 11392720}" + ] + }, + "default: components": { + "sql": "SELECT count(*) FROM (SELECT component_id FROM warren.components('soak', NULL::bigint) GROUP BY 1) c", + "seconds": [ + 2.003 + ], + "median_seconds": 2.003, + "rows": [ + "1" + ], + "first_rows": [ + "1" + ], + "notices": [], + "warnings": [], + "rss_anon_before": 664432640, + "peak_rss_anon": 727625728, + "vm_hwm": 1347362816, + "rss_anon_after": 665358336, + "rss_shmem_after": 600682496 + }, + "default: pagerank": { + "sql": "WITH before AS MATERIALIZED (SELECT (SELECT pg_stat_get_xact_tuples_returned('warren.segments'::regclass) + pg_stat_get_xact_tuples_fetched('warren.segments'::regclass) + sum(pg_stat_get_xact_tuples_fetched(indexrelid))::bigint FROM pg_index WHERE indrelid = 'warren.segments'::regclass AND 1 IS NOT NULL) AS reads), run AS MATERIALIZED (SELECT warren.pagerank('soak', 'pr', tol := 0, max_iter := 20) AS result FROM before WHERE before.reads IS NOT NULL), after AS MATERIALIZED (SELECT (SELECT pg_stat_get_xact_tuples_returned('warren.segments'::regclass) + pg_stat_get_xact_tuples_fetched('warren.segments'::regclass) + sum(pg_stat_get_xact_tuples_fetched(indexrelid))::bigint FROM pg_index WHERE indrelid = 'warren.segments'::regclass AND run.result IS NOT NULL) AS reads FROM run) SELECT after.reads - before.reads, run.result FROM before, run, after", + "seconds": [ + 5.8937 + ], + "median_seconds": 5.8937, + "rows": [ + "0|20" + ], + "first_rows": [ + "0|20" + ], + "notices": [], + "warnings": [], + "rss_anon_before": 665358336, + "peak_rss_anon": 1168154624, + "vm_hwm": 1848602624, + "rss_anon_after": 652472320, + "rss_shmem_after": 659558400 + }, + "default: pagerank from cold": { + "sql": "WITH before AS MATERIALIZED (SELECT (SELECT pg_stat_get_xact_tuples_returned('warren.segments'::regclass) + pg_stat_get_xact_tuples_fetched('warren.segments'::regclass) + sum(pg_stat_get_xact_tuples_fetched(indexrelid))::bigint FROM pg_index WHERE indrelid = 'warren.segments'::regclass AND 1 IS NOT NULL) AS reads), run AS MATERIALIZED (SELECT warren.pagerank('soak', 'pr', tol := 0, max_iter := 20) AS result FROM before WHERE before.reads IS NOT NULL), after AS MATERIALIZED (SELECT (SELECT pg_stat_get_xact_tuples_returned('warren.segments'::regclass) + pg_stat_get_xact_tuples_fetched('warren.segments'::regclass) + sum(pg_stat_get_xact_tuples_fetched(indexrelid))::bigint FROM pg_index WHERE indrelid = 'warren.segments'::regclass AND run.result IS NOT NULL) AS reads FROM run) SELECT after.reads - before.reads, run.result FROM before, run, after", + "seconds": [ + 6.2388 + ], + "median_seconds": 6.2388, + "rows": [ + "1120|20" + ], + "first_rows": [ + "1120|20" + ], + "notices": [], + "warnings": [], + "rss_anon_before": 11845632, + "peak_rss_anon": 1160314880, + "vm_hwm": 1881862144, + "rss_anon_after": 669478912, + "rss_shmem_after": 701362176 + }, + "default: eigen": { + "sql": "SELECT count(*), max(residual) FROM warren.eigen('soak', 20, 'normalized_laplacian', 'smallest', tol := 1e-06, max_iter := 200)", + "seconds": [ + 51.7348 + ], + "median_seconds": 51.7348, + "rows": [ + "20|0.005383903658682934" + ], + "first_rows": [ + "20|0.005383903658682934" + ], + "notices": [], + "warnings": [ + "WARNING: eigen did not converge in 200 steps; the largest residual is 5.383903658682934e-3 (raise max_iter or tol)" + ], + "rss_anon_before": 669478912, + "peak_rss_anon": 2169753600, + "vm_hwm": 2891894784, + "rss_anon_after": 669523968, + "rss_shmem_after": 701362176 + }, + "library: eigenvector_centrality": { + "sql": "SELECT warren.eigenvector_centrality('soak', 'library', tol := 0, max_iter := 20)", + "seconds": [ + 5.4872 + ], + "median_seconds": 5.4872, + "rows": [ + "20" + ], + "first_rows": [ + "20" + ], + "notices": [], + "warnings": [ + "WARNING: eigenvector_centrality did not converge in 20 iterations" + ], + "rss_anon_before": 3203072, + "peak_rss_anon": 919154688, + "vm_hwm": 1598599168, + "rss_anon_after": 699940864, + "rss_shmem_after": 660049920 + }, + "library: katz_centrality": { + "sql": "SELECT warren.katz_centrality('soak', 'library', alpha := 0.01, tol := 0, max_iter := 20)", + "seconds": [ + 3.738 + ], + "median_seconds": 3.738, + "rows": [ + "20" + ], + "first_rows": [ + "20" + ], + "notices": [], + "warnings": [ + "WARNING: katz_centrality did not converge in 20 iterations" + ], + "rss_anon_before": 659947520, + "peak_rss_anon": 1001140224, + "vm_hwm": 1722822656, + "rss_anon_after": 693538816, + "rss_shmem_after": 701968384 + }, + "library: hits": { + "sql": "SELECT warren.hits('soak', 'library_hubs', 'library_authorities', tol := 0, max_iter := 20)", + "seconds": [ + 6.3162 + ], + "median_seconds": 6.3162, + "rows": [ + "20" + ], + "first_rows": [ + "20" + ], + "notices": [], + "warnings": [ + "WARNING: hits did not converge in 20 iterations" + ], + "rss_anon_before": 653537280, + "peak_rss_anon": 961159168, + "vm_hwm": 1766301696, + "rss_anon_after": 733560832, + "rss_shmem_after": 785453056 + }, + "library: clustering": { + "sql": "SELECT warren.clustering('soak', 'library')", + "seconds": [ + 19.4569 + ], + "median_seconds": 19.4569, + "rows": [ + "" + ], + "first_rows": [ + "" + ], + "notices": [], + "warnings": [], + "rss_anon_before": 653557760, + "peak_rss_anon": 1493569536, + "vm_hwm": 2298699776, + "rss_anon_after": 693596160, + "rss_shmem_after": 827256832 + }, + "library: core_number": { + "sql": "SELECT warren.core_number('soak', 'library', persist := false)", + "seconds": [ + 7.6962 + ], + "median_seconds": 7.6962, + "rows": [ + "14" + ], + "first_rows": [ + "14" + ], + "notices": [], + "warnings": [], + "rss_anon_before": 653594624, + "peak_rss_anon": 1573605376, + "vm_hwm": 2420527104, + "rss_anon_after": 653598720, + "rss_shmem_after": 827256832 + }, + "library: strong_components_vector": { + "sql": "SELECT warren.strong_components_vector('soak', 'library', persist := false)", + "seconds": [ + 6.504 + ], + "median_seconds": 6.504, + "rows": [ + "476" + ], + "first_rows": [ + "476" + ], + "notices": [], + "warnings": [], + "rss_anon_before": 653598720, + "peak_rss_anon": 799551488, + "vm_hwm": 1646510080, + "rss_anon_after": 652689408, + "rss_shmem_after": 827256832 + }, + "library: minimum_spanning_forest": { + "sql": "SELECT count(*) FROM warren.minimum_spanning_forest('soak', NULL::bigint)", + "seconds": [ + 16.3778 + ], + "median_seconds": 16.3778, + "rows": [ + "4999999" + ], + "first_rows": [ + "4999999" + ], + "notices": [], + "warnings": [], + "rss_anon_before": 652689408, + "peak_rss_anon": 978862080, + "vm_hwm": 1825918976, + "rss_anon_after": 667246592, + "rss_shmem_after": 827256832 + }, + "library: betweenness_centrality": { + "sql": "SELECT warren.betweenness_centrality('soak', 'library', k := 16, persist := false)", + "seconds": [ + 26.2493 + ], + "median_seconds": 26.2493, + "rows": [ + "16" + ], + "first_rows": [ + "16" + ], + "notices": [], + "warnings": [], + "rss_anon_before": 667246592, + "peak_rss_anon": 1307275264, + "vm_hwm": 2154651648, + "rss_anon_after": 667250688, + "rss_shmem_after": 827256832 + }, + "library: closeness_centrality": { + "sql": "SELECT warren.closeness_centrality('soak', 'library', k := 16, persist := false)", + "seconds": [ + 2.5737 + ], + "median_seconds": 2.5737, + "rows": [ + "16" + ], + "first_rows": [ + "16" + ], + "notices": [], + "warnings": [], + "rss_anon_before": 667250688, + "peak_rss_anon": 947269632, + "vm_hwm": 1794678784, + "rss_anon_after": 667271168, + "rss_shmem_after": 827256832 + }, + "library: harmonic_centrality": { + "sql": "SELECT warren.harmonic_centrality('soak', 'library', k := 16, persist := false)", + "seconds": [ + 2.4435 + ], + "median_seconds": 2.4435, + "rows": [ + "16" + ], + "first_rows": [ + "16" + ], + "notices": [], + "warnings": [], + "rss_anon_before": 667271168, + "peak_rss_anon": 907284480, + "vm_hwm": 1754583040, + "rss_anon_after": 707305472, + "rss_shmem_after": 827256832 + }, + "library: most_similar": { + "sql": "SELECT count(*) FROM warren.most_similar('soak', 1::bigint)", + "seconds": [ + 0.0014 + ], + "median_seconds": 0.0014, + "rows": [ + "10" + ], + "first_rows": [ + "10" + ], + "notices": [], + "warnings": [], + "rss_anon_before": 667303936, + "peak_rss_anon": 667308032, + "vm_hwm": 1515429888, + "rss_anon_after": 667308032, + "rss_shmem_after": 827256832 + }, + "library: similarity": { + "sql": "SELECT count(*) FROM warren.similarity('soak', $p$SELECT (1 + (i * 7919) % 5000000)::bigint, (1 + (i * 104729) % 5000000)::bigint FROM generate_series(1::bigint, 100000) i$p$, NULL::bigint)", + "seconds": [ + 0.3587 + ], + "median_seconds": 0.3587, + "rows": [ + "100000" + ], + "first_rows": [ + "100000" + ], + "notices": [], + "warnings": [], + "rss_anon_before": 667308032, + "peak_rss_anon": 667320320, + "vm_hwm": 1515581440, + "rss_anon_after": 667320320, + "rss_shmem_after": 827265024 + }, + "library: similar_pairs": { + "sql": "SELECT count(*) FROM warren.similar_pairs('soak', NULL::bigint, nodes := 'library_mask')", + "seconds": [ + 3.7405 + ], + "median_seconds": 3.7405, + "rows": [ + "500000" + ], + "first_rows": [ + "500000" + ], + "notices": [], + "warnings": [], + "rss_anon_before": 707330048, + "peak_rss_anon": 1794117632, + "vm_hwm": 2642092032, + "rss_anon_after": 716414976, + "rss_shmem_after": 827265024 + } + }, + "skipped": { + "second_version": "--skip second-version", + "communities": "--skip communities" + }, + "pg_warren": "0.6.0", + "postgres": "16.15 (Ubuntu 16.15-0ubuntu0.24.04.1)", + "edge_table_bytes": 1812856832, + "version_1": { + "seq": 1, + "bytes": 560624716, + "label": null, + "staged": false, + "current": true, + "vectors": 0, + "version": 1, + "built_at": "2026-10-24T00:02:11.290415+00:00", + "id_space": 5000000, + "built_lsn": "0/1CAF6D8", + "staleness": "00:01:22.56463", + "edge_count": 49999948, + "node_count": 5000000, + "label_vector": null, + "parent_graph": null, + "target_width": 4, + "dropped_edges": 0, + "compacted_from": null, + "parent_version": null, + "deduplicated_edges": 52 + }, + "cached_bytes": 600624716, + "shared_bytes": 0, + "cache_memory_bytes": 4294967296, + "settings": { + "default": [ + "fsync=off", + "full_page_writes=off", + "jit=on", + "maintenance_work_mem=1GB", + "max_parallel_workers_per_gather=2", + "max_wal_size=1GB", + "shared_buffers=1GB", + "shared_preload_libraries=", + "statement_timeout=20min", + "synchronous_commit=off", + "wal_level=minimal", + "warren.max_build_memory=4GB", + "warren.max_cache_memory=4GB", + "warren.max_compute_memory=4GB", + "warren.shared_cache_memory=4GB", + "warren.threads=4", + "work_mem=64MB" + ] + }, + "library_version": 1, + "finished_at": "2026-10-24T00:06:28+00:00", + "free_disk_after": 16768319488, + "load_average_after": [ + 2.9228515625, + 2.32568359375, + 1.712890625 + ], + "measured": { + "cores": 4, + "bfs_seconds": 0.459566, + "components_seconds": 2.003, + "pagerank_seconds": 5.8937, + "refresh_seconds": 36.1536, + "refresh_peak_bytes": 1949499392, + "cold_load_seconds": 1.1762, + "load_segment_reads": 1120, + "pagerank_segment_reads": 0, + "pagerank_loads": 0.0, + "pagerank_cold_segment_reads": 1120, + "pagerank_cold_seconds": 6.2388, + "pagerank_cold_loads": 1.0, + "eigen_residual": 0.005383903658682934, + "eigen_converged": false, + "eigen_seconds_per_step": 0.258674, + "reachable_row_entries_per_second": 4533874.562241324, + "reachable_edges_per_second": 19898100.923272844, + "cached_bytes": 600624716, + "id_space": 5000000, + "max_cache_memory": 4294967296, + "library": { + "eigenvector_centrality": { + "seconds": 5.4872, + "peak_rss_anon": 919154688 + }, + "katz_centrality": { + "seconds": 3.738, + "peak_rss_anon": 1001140224 + }, + "hits": { + "seconds": 6.3162, + "peak_rss_anon": 961159168 + }, + "clustering": { + "seconds": 19.4569, + "peak_rss_anon": 1493569536 + }, + "core_number": { + "seconds": 7.6962, + "peak_rss_anon": 1573605376 + }, + "strong_components_vector": { + "seconds": 6.504, + "peak_rss_anon": 799551488 + }, + "minimum_spanning_forest": { + "seconds": 16.3778, + "peak_rss_anon": 978862080 + }, + "betweenness_centrality": { + "seconds": 26.2493, + "peak_rss_anon": 1307275264 + }, + "closeness_centrality": { + "seconds": 2.5737, + "peak_rss_anon": 947269632 + }, + "harmonic_centrality": { + "seconds": 2.4435, + "peak_rss_anon": 907284480 + }, + "most_similar": { + "seconds": 0.0014, + "peak_rss_anon": 667308032 + }, + "similarity": { + "seconds": 0.3587, + "peak_rss_anon": 667320320 + }, + "similar_pairs": { + "seconds": 3.7405, + "peak_rss_anon": 1794117632 + } + }, + "communities": {} + }, + "verdicts": [ + { + "workload": "reachable_set, full component", + "target": "> 50M edges/s", + "measured": "20M edges/s", + "met": false + }, + { + "workload": "components()", + "target": "< 10 s", + "measured": "2.0 s", + "met": true + }, + { + "workload": "pagerank, 20 iterations", + "target": "< 30 s (8 threads)", + "measured": "5.9 s", + "met": true + }, + { + "workload": "refresh() incl. reverse CSR", + "target": "< 8 min, < 4 GiB RAM", + "measured": "36.2 s, peak 1.95 GB", + "met": true + }, + { + "workload": "cold load into backend", + "target": "< 15 s", + "measured": "1.2 s", + "met": true + }, + { + "workload": "memory: graph + pagerank scratch within max_cache_memory", + "target": "fits 4 GiB", + "measured": "600.62 MB + 440.00 MB scratch", + "met": true + } + ] +} diff --git a/bench/results/soak/a-scale2-text-m10-lazy.json b/bench/results/soak/a-scale2-text-m10-lazy.json new file mode 100644 index 00000000..8a8e8590 --- /dev/null +++ b/bench/results/soak/a-scale2-text-m10-lazy.json @@ -0,0 +1,954 @@ +{ + "provenance": { + "argv": [ + "bench/soak.py", + "a", + "--scale", + "2", + "--no-cte", + "--no-eigen-steps", + "--eigen-max-iter", + "200", + "--skip", + "communities", + "--skip", + "second-version", + "--disk-floor", + "3221225472", + "--pg-config", + "/var/tmp/m10-pgtree/usr/lib/postgresql/16/bin/pg_config", + "--parent", + "/var/tmp/m10-soak", + "--results", + "/var/tmp/m10-soak/results", + "--extension-revision", + "8d759e5", + "--extension-profile", + "release", + "--threads", + "4", + "--statement-timeout", + "20min", + "--label=-m10-lazy", + "--ids", + "text" + ], + "harness": { + "revision": "8d759e5884629093a408881bc793f002d7683ae0", + "dirty": false + }, + "extension": { + "revision": "8d759e5", + "profile": "release", + "library": "/var/tmp/m10-pgtree/usr/lib/postgresql/16/lib/pg_warren.so", + "library_sha256": "ccd990d97b0f31f3cf32e72f45d6b6f0c65d8566a0de1a6c975fdd97bb68af12" + } + }, + "case": "a", + "nodes": 5000000, + "edges": 50000000, + "weighted": false, + "ids": "text", + "relations": 1, + "store_merged": true, + "threads": 4, + "node_order": "id", + "eigen_max_iter": 200, + "statement_timeout": "20min", + "skip": [ + "communities", + "second-version" + ], + "planned_stages": [ + "cold_load_probe", + "default: bfs depth 6", + "default: components", + "default: eigen", + "default: pagerank", + "default: prewarm", + "default: reachable_set", + "library: betweenness_centrality", + "library: closeness_centrality", + "library: clustering", + "library: core_number", + "library: eigenvector_centrality", + "library: harmonic_centrality", + "library: hits", + "library: katz_centrality", + "library: minimum_spanning_forest", + "library: most_similar", + "library: similar_pairs", + "library: similarity", + "library: strong_components_vector", + "refresh" + ], + "hardware": { + "cpus": 4, + "model": "Intel(R) Xeon(R) Processor", + "memory_bytes": 16872804352, + "kernel": "6.18.44-fc-v153", + "load_average": [ + 0.947265625, + 1.0458984375, + 1.0927734375 + ] + }, + "started_at": "2026-10-23T23:56:36+00:00", + "stages": { + "table 0: create": { + "sql": "CREATE UNLOGGED TABLE soak_removed ()", + "seconds": [ + 0.0003 + ], + "median_seconds": 0.0003, + "rows": [], + "first_rows": [ + null + ], + "notices": [], + "warnings": [], + "rss_anon_before": 5611520, + "peak_rss_anon": 5611520, + "vm_hwm": 35340288, + "rss_anon_after": 5611520, + "rss_shmem_after": 15323136 + }, + "table 1: create": { + "sql": "CREATE UNLOGGED TABLE soak_edges (src int4 NOT NULL, dst int4 NOT NULL) WITH (autovacuum_enabled = false)", + "seconds": [ + 0.0005 + ], + "median_seconds": 0.0005, + "rows": [], + "first_rows": [ + null + ], + "notices": [], + "warnings": [], + "rss_anon_before": 5611520, + "peak_rss_anon": 5611520, + "vm_hwm": 35344384, + "rss_anon_after": 5611520, + "rss_shmem_after": 15327232 + }, + "table 2: insert": { + "sql": "INSERT INTO soak_edges SELECT 1 + floor(random() * 5000000)::int4 AS src, 1 + floor(random() * 5000000)::int4 AS dst FROM (SELECT setseed(0.25) OFFSET 0) seeded CROSS JOIN generate_series(1, 50000000)", + "seconds": [ + 17.021 + ], + "median_seconds": 17.021, + "rows": [], + "first_rows": [ + null + ], + "notices": [], + "warnings": [], + "rss_anon_before": 5611520, + "peak_rss_anon": 95424512, + "vm_hwm": 1253904384, + "rss_anon_after": 32477184, + "rss_shmem_after": 1095917568 + }, + "table 3: analyze": { + "sql": "ANALYZE soak_edges", + "seconds": [ + 0.203 + ], + "median_seconds": 0.203, + "rows": [], + "first_rows": [ + null + ], + "notices": [], + "warnings": [], + "rss_anon_before": 32477184, + "peak_rss_anon": 32485376, + "vm_hwm": 1191460864, + "rss_anon_after": 32485376, + "rss_shmem_after": 1095917568 + }, + "create_graph": { + "sql": "SELECT warren.create_graph('soak', $n$SELECT 'https://example.org/resource/' || id FROM generate_series(1::bigint, 5000000) id$n$, $q$SELECT 'https://example.org/resource/' || src, 'https://example.org/resource/' || dst FROM soak_edges$q$, weighted := false, has_relations := false, store_merged := true, node_order := 'id')", + "seconds": [ + 0.0166 + ], + "median_seconds": 0.0166, + "rows": [ + "" + ], + "first_rows": [ + "" + ], + "notices": [], + "warnings": [], + "rss_anon_before": 3198976, + "peak_rss_anon": 9756672, + "vm_hwm": 74563584, + "rss_anon_after": 9756672, + "rss_shmem_after": 10907648 + }, + "refresh": { + "sql": "SELECT warren.refresh('soak')", + "seconds": [ + 54.5122 + ], + "median_seconds": 54.5122, + "rows": [ + "1" + ], + "first_rows": [ + "1" + ], + "notices": [], + "warnings": [], + "rss_anon_before": 9756672, + "peak_rss_anon": 919130112, + "vm_hwm": 2078478336, + "rss_anon_after": 44630016, + "rss_shmem_after": 1111498752 + }, + "refresh from the generator": { + "sql": "SELECT warren.refresh('soak_generated')", + "seconds": [ + 59.8932 + ], + "median_seconds": 59.8932, + "rows": [ + "2" + ], + "first_rows": [ + "2" + ], + "notices": [], + "warnings": [], + "rss_anon_before": 9838592, + "peak_rss_anon": 990941184, + "vm_hwm": 1148678144, + "rss_anon_after": 44965888, + "rss_shmem_after": 783679488 + }, + "cold_load_probe": { + "sql": "SELECT warren.prewarm('soak')", + "seconds": [ + 0.9402 + ], + "median_seconds": 0.9402, + "rows": [ + "1" + ], + "first_rows": [ + "1" + ], + "notices": [ + "NOTICE: version 1 (719513652 bytes) cannot be kept within warren.max_cache_memory (65536 bytes, 0 of them held by transient vectors), so it is held only until the statement ends, the bytes past the cap charged to warren.max_compute_memory, and reloaded by each statement that reads it; raise warren.max_cache_memory" + ], + "warnings": [], + "rss_anon_before": 3207168, + "peak_rss_anon": 731779072, + "vm_hwm": 1514741760, + "rss_anon_after": 12263424, + "rss_shmem_after": 765431808 + }, + "default: prewarm": { + "sql": "WITH before AS MATERIALIZED (SELECT (SELECT pg_stat_get_xact_tuples_returned('warren.segments'::regclass) + pg_stat_get_xact_tuples_fetched('warren.segments'::regclass) + sum(pg_stat_get_xact_tuples_fetched(indexrelid))::bigint FROM pg_index WHERE indrelid = 'warren.segments'::regclass AND 1 IS NOT NULL) AS reads), run AS MATERIALIZED (SELECT warren.prewarm('soak') AS result FROM before WHERE before.reads IS NOT NULL), after AS MATERIALIZED (SELECT (SELECT pg_stat_get_xact_tuples_returned('warren.segments'::regclass) + pg_stat_get_xact_tuples_fetched('warren.segments'::regclass) + sum(pg_stat_get_xact_tuples_fetched(indexrelid))::bigint FROM pg_index WHERE indrelid = 'warren.segments'::regclass AND run.result IS NOT NULL) AS reads FROM run) SELECT after.reads - before.reads, run.result FROM before, run, after", + "seconds": [ + 0.7873 + ], + "median_seconds": 0.7873, + "rows": [ + "1800|1" + ], + "first_rows": [ + "1800|1" + ], + "notices": [], + "warnings": [], + "rss_anon_before": 3203072, + "peak_rss_anon": 732114944, + "vm_hwm": 1518891008, + "rss_anon_after": 732094464, + "rss_shmem_after": 769245184 + }, + "default: bfs depth 6": { + "sql": "SELECT bench_seconds($soak$SELECT count(*) FROM warren.bfs('soak', ARRAY['https://example.org/resource/1'::text], 6)$soak$)", + "seconds": [ + 0.8726, + 0.8663, + 0.8829 + ], + "median_seconds": 0.8726, + "rows": [ + "0.882213" + ], + "first_rows": [ + "0.871515", + "0.865783", + "0.882213" + ], + "notices": [], + "warnings": [], + "rss_anon_before": 732180480, + "peak_rss_anon": 834211840, + "vm_hwm": 1622552576, + "rss_anon_after": 729165824, + "rss_shmem_after": 769392640, + "server_seconds": [ + 0.871515, + 0.865783, + 0.882213 + ], + "server_median_seconds": 0.871515 + }, + "default: bfs depth 6 count": { + "sql": "SELECT count(*) FROM warren.bfs('soak', ARRAY['https://example.org/resource/1'::text], 6)", + "seconds": [ + 0.883 + ], + "median_seconds": 0.883, + "rows": [ + "1068779" + ], + "first_rows": [ + "1068779" + ], + "notices": [], + "warnings": [], + "rss_anon_before": 729165824, + "peak_rss_anon": 834199552, + "vm_hwm": 1622536192, + "rss_anon_after": 729149440, + "rss_shmem_after": 769392640 + }, + "default: neighbours": { + "sql": "SELECT bench_seconds($soak$SELECT count(*) FROM warren.neighbours('soak', 'https://example.org/resource/1'::text)$soak$)", + "seconds": [ + 0.0008, + 0.0002, + 0.0002, + 0.0002, + 0.0002, + 0.0001, + 0.0001, + 0.0001, + 0.0002, + 0.0001, + 0.0001, + 0.0001, + 0.0001, + 0.0001, + 0.0001, + 0.0001, + 0.0001, + 0.0001, + 0.0001, + 0.0001 + ], + "median_seconds": 0.0001, + "rows": [ + "4.4e-05" + ], + "first_rows": [ + "0.000461", + "8.4e-05", + "5.9e-05", + "5.7e-05", + "6.3e-05", + "4.9e-05", + "4.5e-05", + "4.4e-05", + "6.9e-05", + "4.8e-05", + "4.4e-05", + "4.3e-05", + "4.5e-05", + "4.6e-05", + "4.5e-05", + "4.5e-05", + "4.5e-05", + "4.4e-05", + "4.4e-05", + "4.4e-05" + ], + "notices": [], + "warnings": [], + "rss_anon_before": 729149440, + "peak_rss_anon": 729169920, + "vm_hwm": 1517744128, + "rss_anon_after": 729169920, + "rss_shmem_after": 769392640, + "server_seconds": [ + 0.000461, + 8.4e-05, + 5.9e-05, + 5.7e-05, + 6.3e-05, + 4.9e-05, + 4.5e-05, + 4.4e-05, + 6.9e-05, + 4.8e-05, + 4.4e-05, + 4.3e-05, + 4.5e-05, + 4.6e-05, + 4.5e-05, + 4.5e-05, + 4.5e-05, + 4.4e-05, + 4.4e-05, + 4.4e-05 + ], + "server_median_seconds": 4.5e-05 + }, + "default: reachable_set": { + "sql": "SELECT count(*) FROM warren.reachable_set('soak', ARRAY['https://example.org/resource/1'::text])", + "seconds": [ + 3.6607 + ], + "median_seconds": 3.6607, + "rows": [ + "4999752" + ], + "first_rows": [ + "4999752" + ], + "notices": [], + "warnings": [], + "rss_anon_before": 729169920, + "peak_rss_anon": 980918272, + "vm_hwm": 1769459712, + "rss_anon_after": 727056384, + "rss_shmem_after": 769392640 + }, + "default: reachable_set stats": { + "stats": [ + "{\"graph\": \"soak\", \"version\": 1, \"function\": \"reachable_set\", \"truncated\": false, \"nodes_visited\": 4999752, \"edges_traversed\": 11392106}" + ] + }, + "default: components": { + "sql": "SELECT count(*) FROM (SELECT component_id FROM warren.components('soak', NULL::text) GROUP BY 1) c", + "seconds": [ + 2.8718 + ], + "median_seconds": 2.8718, + "rows": [ + "1" + ], + "first_rows": [ + "1" + ], + "notices": [], + "warnings": [], + "rss_anon_before": 727060480, + "peak_rss_anon": 839213056, + "vm_hwm": 1627668480, + "rss_anon_after": 725000192, + "rss_shmem_after": 769400832 + }, + "default: pagerank": { + "sql": "WITH before AS MATERIALIZED (SELECT (SELECT pg_stat_get_xact_tuples_returned('warren.segments'::regclass) + pg_stat_get_xact_tuples_fetched('warren.segments'::regclass) + sum(pg_stat_get_xact_tuples_fetched(indexrelid))::bigint FROM pg_index WHERE indrelid = 'warren.segments'::regclass AND 1 IS NOT NULL) AS reads), run AS MATERIALIZED (SELECT warren.pagerank('soak', 'pr', tol := 0, max_iter := 20) AS result FROM before WHERE before.reads IS NOT NULL), after AS MATERIALIZED (SELECT (SELECT pg_stat_get_xact_tuples_returned('warren.segments'::regclass) + pg_stat_get_xact_tuples_fetched('warren.segments'::regclass) + sum(pg_stat_get_xact_tuples_fetched(indexrelid))::bigint FROM pg_index WHERE indrelid = 'warren.segments'::regclass AND run.result IS NOT NULL) AS reads FROM run) SELECT after.reads - before.reads, run.result FROM before, run, after", + "seconds": [ + 8.2863 + ], + "median_seconds": 8.2863, + "rows": [ + "0|20" + ], + "first_rows": [ + "0|20" + ], + "notices": [], + "warnings": [], + "rss_anon_before": 725000192, + "peak_rss_anon": 1278947328, + "vm_hwm": 2127749120, + "rss_anon_after": 771342336, + "rss_shmem_after": 828583936 + }, + "default: pagerank from cold": { + "sql": "WITH before AS MATERIALIZED (SELECT (SELECT pg_stat_get_xact_tuples_returned('warren.segments'::regclass) + pg_stat_get_xact_tuples_fetched('warren.segments'::regclass) + sum(pg_stat_get_xact_tuples_fetched(indexrelid))::bigint FROM pg_index WHERE indrelid = 'warren.segments'::regclass AND 1 IS NOT NULL) AS reads), run AS MATERIALIZED (SELECT warren.pagerank('soak', 'pr', tol := 0, max_iter := 20) AS result FROM before WHERE before.reads IS NOT NULL), after AS MATERIALIZED (SELECT (SELECT pg_stat_get_xact_tuples_returned('warren.segments'::regclass) + pg_stat_get_xact_tuples_fetched('warren.segments'::regclass) + sum(pg_stat_get_xact_tuples_fetched(indexrelid))::bigint FROM pg_index WHERE indrelid = 'warren.segments'::regclass AND run.result IS NOT NULL) AS reads FROM run) SELECT after.reads - before.reads, run.result FROM before, run, after", + "seconds": [ + 8.1565 + ], + "median_seconds": 8.1565, + "rows": [ + "1800|20" + ], + "first_rows": [ + "1800|20" + ], + "notices": [], + "warnings": [], + "rss_anon_before": 11829248, + "peak_rss_anon": 1279217664, + "vm_hwm": 2169798656, + "rss_anon_after": 771604480, + "rss_shmem_after": 870367232 + }, + "default: eigen": { + "sql": "SELECT count(*), max(residual) FROM warren.eigen('soak', 20, 'normalized_laplacian', 'smallest', tol := 1e-06, max_iter := 200)", + "seconds": [ + 51.3291 + ], + "median_seconds": 51.3291, + "rows": [ + "20|0.005370926710008592" + ], + "first_rows": [ + "20|0.005370926710008592" + ], + "notices": [], + "warnings": [ + "WARNING: eigen did not converge in 200 steps; the largest residual is 5.370926710008592e-3 (raise max_iter or tol)" + ], + "rss_anon_before": 771604480, + "peak_rss_anon": 2276741120, + "vm_hwm": 3168100352, + "rss_anon_after": 776511488, + "rss_shmem_after": 870383616 + }, + "library: eigenvector_centrality": { + "sql": "SELECT warren.eigenvector_centrality('soak', 'library', tol := 0, max_iter := 20)", + "seconds": [ + 6.9326 + ], + "median_seconds": 6.9326, + "rows": [ + "20" + ], + "first_rows": [ + "20" + ], + "notices": [], + "warnings": [ + "WARNING: eigenvector_centrality did not converge in 20 iterations" + ], + "rss_anon_before": 3203072, + "peak_rss_anon": 1037426688, + "vm_hwm": 1886441472, + "rss_anon_after": 818827264, + "rss_shmem_after": 829116416 + }, + "library: katz_centrality": { + "sql": "SELECT warren.katz_centrality('soak', 'library', alpha := 0.01, tol := 0, max_iter := 20)", + "seconds": [ + 5.0378 + ], + "median_seconds": 5.0378, + "rows": [ + "20" + ], + "first_rows": [ + "20" + ], + "notices": [], + "warnings": [ + "WARNING: katz_centrality did not converge in 20 iterations" + ], + "rss_anon_before": 778833920, + "peak_rss_anon": 1120018432, + "vm_hwm": 2010591232, + "rss_anon_after": 812417024, + "rss_shmem_after": 871034880 + }, + "library: hits": { + "sql": "SELECT warren.hits('soak', 'library_hubs', 'library_authorities', tol := 0, max_iter := 20)", + "seconds": [ + 7.5114 + ], + "median_seconds": 7.5114, + "rows": [ + "20" + ], + "first_rows": [ + "20" + ], + "notices": [], + "warnings": [ + "WARNING: hits did not converge in 20 iterations" + ], + "rss_anon_before": 772415488, + "peak_rss_anon": 1077866496, + "vm_hwm": 2054066176, + "rss_anon_after": 852430848, + "rss_shmem_after": 954617856 + }, + "library: clustering": { + "sql": "SELECT warren.clustering('soak', 'library')", + "seconds": [ + 19.4043 + ], + "median_seconds": 19.4043, + "rows": [ + "" + ], + "first_rows": [ + "" + ], + "notices": [], + "warnings": [], + "rss_anon_before": 772427776, + "peak_rss_anon": 1612439552, + "vm_hwm": 2586464256, + "rss_anon_after": 812466176, + "rss_shmem_after": 996552704 + }, + "library: core_number": { + "sql": "SELECT warren.core_number('soak', 'library', persist := false)", + "seconds": [ + 7.6501 + ], + "median_seconds": 7.6501, + "rows": [ + "14" + ], + "first_rows": [ + "14" + ], + "notices": [], + "warnings": [], + "rss_anon_before": 772464640, + "peak_rss_anon": 1692475392, + "vm_hwm": 2708725760, + "rss_anon_after": 772468736, + "rss_shmem_after": 996552704 + }, + "library: strong_components_vector": { + "sql": "SELECT warren.strong_components_vector('soak', 'library', persist := false)", + "seconds": [ + 6.6672 + ], + "median_seconds": 6.6672, + "rows": [ + "476" + ], + "first_rows": [ + "476" + ], + "notices": [], + "warnings": [], + "rss_anon_before": 772468736, + "peak_rss_anon": 918396928, + "vm_hwm": 1934778368, + "rss_anon_after": 771559424, + "rss_shmem_after": 996552704 + }, + "library: minimum_spanning_forest": { + "sql": "SELECT count(*) FROM warren.minimum_spanning_forest('soak', NULL::text)", + "seconds": [ + 16.9852 + ], + "median_seconds": 16.9852, + "rows": [ + "4999999" + ], + "first_rows": [ + "4999999" + ], + "notices": [], + "warnings": [], + "rss_anon_before": 771559424, + "peak_rss_anon": 1097732096, + "vm_hwm": 2114187264, + "rss_anon_after": 777728000, + "rss_shmem_after": 996552704 + }, + "library: betweenness_centrality": { + "sql": "SELECT warren.betweenness_centrality('soak', 'library', k := 16, persist := false)", + "seconds": [ + 24.699 + ], + "median_seconds": 24.699, + "rows": [ + "16" + ], + "first_rows": [ + "16" + ], + "notices": [], + "warnings": [], + "rss_anon_before": 777728000, + "peak_rss_anon": 1417756672, + "vm_hwm": 2434347008, + "rss_anon_after": 777732096, + "rss_shmem_after": 996552704 + }, + "library: closeness_centrality": { + "sql": "SELECT warren.closeness_centrality('soak', 'library', k := 16, persist := false)", + "seconds": [ + 2.4121 + ], + "median_seconds": 2.4121, + "rows": [ + "16" + ], + "first_rows": [ + "16" + ], + "notices": [], + "warnings": [], + "rss_anon_before": 777732096, + "peak_rss_anon": 1057751040, + "vm_hwm": 2074312704, + "rss_anon_after": 777797632, + "rss_shmem_after": 996552704 + }, + "library: harmonic_centrality": { + "sql": "SELECT warren.harmonic_centrality('soak', 'library', k := 16, persist := false)", + "seconds": [ + 2.3884 + ], + "median_seconds": 2.3884, + "rows": [ + "16" + ], + "first_rows": [ + "16" + ], + "notices": [], + "warnings": [], + "rss_anon_before": 777797632, + "peak_rss_anon": 1017810944, + "vm_hwm": 2034266112, + "rss_anon_after": 817811456, + "rss_shmem_after": 996560896 + }, + "library: most_similar": { + "sql": "SELECT count(*) FROM warren.most_similar('soak', 'https://example.org/resource/1'::text)", + "seconds": [ + 0.0012 + ], + "median_seconds": 0.0012, + "rows": [ + "10" + ], + "first_rows": [ + "10" + ], + "notices": [], + "warnings": [], + "rss_anon_before": 777809920, + "peak_rss_anon": 777814016, + "vm_hwm": 1795256320, + "rss_anon_after": 777814016, + "rss_shmem_after": 996560896 + }, + "library: similarity": { + "sql": "SELECT count(*) FROM warren.similarity('soak', $p$SELECT 'https://example.org/resource/' || (1 + (i * 7919) % 5000000), 'https://example.org/resource/' || (1 + (i * 104729) % 5000000) FROM generate_series(1::bigint, 100000) i$p$, NULL::text)", + "seconds": [ + 0.8929 + ], + "median_seconds": 0.8929, + "rows": [ + "100000" + ], + "first_rows": [ + "100000" + ], + "notices": [], + "warnings": [], + "rss_anon_before": 777814016, + "peak_rss_anon": 817840128, + "vm_hwm": 1834917888, + "rss_anon_after": 777838592, + "rss_shmem_after": 996618240 + }, + "library: similar_pairs": { + "sql": "SELECT count(*) FROM warren.similar_pairs('soak', NULL::text, nodes := 'library_mask')", + "seconds": [ + 3.9703 + ], + "median_seconds": 3.9703, + "rows": [ + "500000" + ], + "first_rows": [ + "500000" + ], + "notices": [], + "warnings": [], + "rss_anon_before": 817844224, + "peak_rss_anon": 1904582656, + "vm_hwm": 2921795584, + "rss_anon_after": 858116096, + "rss_shmem_after": 996618240 + } + }, + "skipped": { + "second_version": "--skip second-version", + "communities": "--skip communities" + }, + "pg_warren": "0.6.0", + "postgres": "16.15 (Ubuntu 16.15-0ubuntu0.24.04.1)", + "edge_table_bytes": 1812856832, + "version_1": { + "seq": 1, + "bytes": 719513652, + "label": null, + "staged": false, + "current": true, + "vectors": 0, + "version": 1, + "built_at": "2026-10-23T23:56:53.3472+00:00", + "id_space": 5000000, + "built_lsn": "0/1CAF090", + "staleness": "00:01:55.062826", + "edge_count": 49999948, + "node_count": 5000000, + "label_vector": null, + "parent_graph": null, + "target_width": 4, + "dropped_edges": 0, + "compacted_from": null, + "parent_version": null, + "deduplicated_edges": 52 + }, + "cached_bytes": 719513652, + "shared_bytes": 0, + "cache_memory_bytes": 4294967296, + "settings": { + "default": [ + "fsync=off", + "full_page_writes=off", + "jit=on", + "maintenance_work_mem=1GB", + "max_parallel_workers_per_gather=2", + "max_wal_size=1GB", + "shared_buffers=1GB", + "shared_preload_libraries=", + "statement_timeout=20min", + "synchronous_commit=off", + "wal_level=minimal", + "warren.max_build_memory=4GB", + "warren.max_cache_memory=4GB", + "warren.max_compute_memory=4GB", + "warren.shared_cache_memory=4GB", + "warren.threads=4", + "work_mem=64MB" + ] + }, + "library_version": 1, + "finished_at": "2026-10-24T00:01:53+00:00", + "free_disk_after": 16768356352, + "load_average_after": [ + 2.33349609375, + 1.87109375, + 1.42626953125 + ], + "measured": { + "cores": 4, + "bfs_seconds": 0.871515, + "components_seconds": 2.8718, + "pagerank_seconds": 8.2863, + "refresh_seconds": 54.5122, + "refresh_peak_bytes": 2078478336, + "cold_load_seconds": 0.7873, + "load_segment_reads": 1800, + "pagerank_segment_reads": 0, + "pagerank_loads": 0.0, + "pagerank_cold_segment_reads": 1800, + "pagerank_cold_seconds": 8.1565, + "pagerank_cold_loads": 1.0, + "eigen_residual": 0.005370926710008592, + "eigen_converged": false, + "eigen_seconds_per_step": 0.25664549999999997, + "reachable_row_entries_per_second": 3112002.076105663, + "reachable_edges_per_second": 13658575.682246566, + "cached_bytes": 719513652, + "id_space": 5000000, + "max_cache_memory": 4294967296, + "library": { + "eigenvector_centrality": { + "seconds": 6.9326, + "peak_rss_anon": 1037426688 + }, + "katz_centrality": { + "seconds": 5.0378, + "peak_rss_anon": 1120018432 + }, + "hits": { + "seconds": 7.5114, + "peak_rss_anon": 1077866496 + }, + "clustering": { + "seconds": 19.4043, + "peak_rss_anon": 1612439552 + }, + "core_number": { + "seconds": 7.6501, + "peak_rss_anon": 1692475392 + }, + "strong_components_vector": { + "seconds": 6.6672, + "peak_rss_anon": 918396928 + }, + "minimum_spanning_forest": { + "seconds": 16.9852, + "peak_rss_anon": 1097732096 + }, + "betweenness_centrality": { + "seconds": 24.699, + "peak_rss_anon": 1417756672 + }, + "closeness_centrality": { + "seconds": 2.4121, + "peak_rss_anon": 1057751040 + }, + "harmonic_centrality": { + "seconds": 2.3884, + "peak_rss_anon": 1017810944 + }, + "most_similar": { + "seconds": 0.0012, + "peak_rss_anon": 777814016 + }, + "similarity": { + "seconds": 0.8929, + "peak_rss_anon": 817840128 + }, + "similar_pairs": { + "seconds": 3.9703, + "peak_rss_anon": 1904582656 + } + }, + "communities": {} + }, + "verdicts": [ + { + "workload": "reachable_set, full component", + "target": "> 50M edges/s", + "measured": "14M edges/s", + "met": false + }, + { + "workload": "components()", + "target": "< 10 s", + "measured": "2.9 s", + "met": true + }, + { + "workload": "pagerank, 20 iterations", + "target": "< 30 s (8 threads)", + "measured": "8.3 s", + "met": true + }, + { + "workload": "refresh() incl. reverse CSR", + "target": "< 8 min, < 4 GiB RAM", + "measured": "54.5 s, peak 2.08 GB", + "met": true + }, + { + "workload": "cold load into backend", + "target": "< 15 s", + "measured": "0.8 s", + "met": true + }, + { + "workload": "memory: graph + pagerank scratch within max_cache_memory", + "target": "fits 4 GiB", + "measured": "719.51 MB + 440.00 MB scratch", + "met": true + } + ] +} From 55666e57796e442d8352cd47a1e9501a05862593 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 24 Oct 2026 00:35:28 +0000 Subject: [PATCH 4/4] =?UTF-8?q?DESIGN=20=C2=A712,=20=C2=A713:=20the=20full?= =?UTF-8?q?-scale=20soak=20predates=20the=20lazy=20rows?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The full-scale text paragraph now says its components and minimum_spanning_forest peaks were measured before rows were made as they are returned, and points to the half-scale re-measure. walk and neighbours, which still build their rows up front, are noted with why and listed for M11. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012Rxfe2L3iVjSwSBPdUu1Ke --- DESIGN.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index 929a1039..1294891c 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -1587,12 +1587,12 @@ A pooled backend's own memory is 4–12 MB with the shared cache against the ver - *Cold load:* 1.03 / 0.77 s privately, 1.06 / 0.79 s into the shared cache. - *Lookups:* `neighbours` of one IRI 0.1 / 0.1 ms (median of 20 calls), `most_similar` 1.1 / 1.0 ms, `similarity` of 100,000 pairs 1.31 / 0.41 s (past 65,536 lookups, so through a per-call index, which costs 40 MB at 5M nodes). - *Traversal:* `bfs` to depth 6 (1.07M rows) 0.95 / 0.49 s, `reachable_set` (5.0M rows) 3.72 / 2.22 s. Every row's `text` datum is built and checked against the encoding, so row emission, already most of these calls' time (above), about doubles. The kernels that return no ids are unchanged: `components` 2.77 / 2.74 s, `eigen` for 200 steps 75.9 / 77.3 s, and the library within noise (`clustering` 27.0 / 28.8 s, betweenness 40.5 / 40.1 s, `minimum_spanning_forest` 22.5 / 22.0 s). -- *Peaks:* the stages that read the version peak about 0.12 GB higher privately, the difference in the versions' sizes (`pagerank` 1.28 / 1.16 GB, `eigen` 2.28 / 2.17 GB). The calls that return a row per node peak higher still: `components` 1.28 / 0.80 GB and `minimum_spanning_forest` 1.77 / 0.98 GB, for their rows of text ids. *Fixed since:* both built every row before returning the first, a `Vec` of rows whose text ids were each `palloc`ed (`cstring_to_text_with_len`) in the set-returning function's multi-call memory context, which lives until the call ends: an IRI and its 4-byte header round to a 64-byte AllocSet chunk plus an 8-byte header, about 72 bytes an id (68–78 measured here, 69–72 at full scale, where it was 0.72 and 1.38 GB). They, `vector_topk` and `most_similar` now return an iterator over the kernel's result, which stays in the extension's own buffer (charged as before), and the loaded version, and make each row's external ids in `next()`. pgrx calls `next()` after the first row in the caller's memory context, the per-tuple context Postgres resets before each call (`ExecMakeTableFunctionResult` for a call in `FROM`, `ExecProjectSet` in the select list), so only the current row's ids are held; the iterator's `Arc` keeps the version alive, private or mapped from the shared cache, until the call ends, by its last row, a `LIMIT`, an error or a cancel (`shared_cache_lazy_rows.sql`; a cursor read across statements keeps it after its statement's pin has ended, even once it is evicted). Each has a `pg_test` that opens a cursor on the call and checks that what it holds after its first row does not grow with the rows to come (3.6–7.3 MB more for 50,000 rows before, none since). Measured again at half scale (`bench/results/soak/a-scale2*-m10-lazy.json`, 8d759e5, the same flags without the second version), text / `bigint`, peak RssAnon and what the call added to the backend: `components` 0.84 / 0.73 GB, +112 / +63 MB (was 1.28 / 0.80 GB, +555 / +166 MB); `minimum_spanning_forest` 1.10 / 0.98 GB, +326 / +326 MB (was 1.77 / 0.98 GB, +1,002 / +326 MB), what remains being the kernel's own result and scratch: the forest's text peak now differs from `bigint`'s by the versions' sizes alone, as the other stages' do, and `components`' by 49 MB more (about 10 bytes a node). Times are unchanged within noise (`components` 2.87 / 2.00 s, `minimum_spanning_forest` 17.0 / 16.4 s, a load average of 1–3). The SRFs that already made each row as it is returned are `bfs`, `reachable_set`, `vector_values`, `matrix_rows`, `id_map`, `similarity`, `similar_pairs` and `gather`. Two still build their rows up front: `walk`, whose rows (bounded by `max_results` and `warren.max_walk_paths`) are made as the walks run, and `neighbours`, whose rows are one node's neighbours. +- *Peaks:* the stages that read the version peak about 0.12 GB higher privately, the difference in the versions' sizes (`pagerank` 1.28 / 1.16 GB, `eigen` 2.28 / 2.17 GB). The calls that return a row per node peak higher still: `components` 1.28 / 0.80 GB and `minimum_spanning_forest` 1.77 / 0.98 GB, for their rows of text ids. *Fixed since:* both built every row before returning the first, a `Vec` of rows whose text ids were each `palloc`ed (`cstring_to_text_with_len`) in the set-returning function's multi-call memory context, which lives until the call ends: an IRI and its 4-byte header round to a 64-byte AllocSet chunk plus an 8-byte header, about 72 bytes an id (68–78 measured here, 69–72 at full scale, where it was 0.72 and 1.38 GB). They, `vector_topk` and `most_similar` now return an iterator over the kernel's result, which stays in the extension's own buffer (charged as before), and the loaded version, and make each row's external ids in `next()`. pgrx calls `next()` after the first row in the caller's memory context, the per-tuple context Postgres resets before each call (`ExecMakeTableFunctionResult` for a call in `FROM`, `ExecProjectSet` in the select list), so only the current row's ids are held; the iterator's `Arc` keeps the version alive, private or mapped from the shared cache, until the call ends, by its last row, a `LIMIT`, an error or a cancel (`shared_cache_lazy_rows.sql`; a cursor read across statements keeps it after its statement's pin has ended, even once it is evicted). Each has a `pg_test` that opens a cursor on the call and checks that what it holds after its first row does not grow with the rows to come (3.6–7.3 MB more for 50,000 rows before, none since). Measured again at half scale (`bench/results/soak/a-scale2*-m10-lazy.json`, 8d759e5, the same flags without the second version), text / `bigint`, peak RssAnon and what the call added to the backend: `components` 0.84 / 0.73 GB, +112 / +63 MB (was 1.28 / 0.80 GB, +555 / +166 MB); `minimum_spanning_forest` 1.10 / 0.98 GB, +326 / +326 MB (was 1.77 / 0.98 GB, +1,002 / +326 MB), what remains being the kernel's own result and scratch: the forest's text peak now differs from `bigint`'s by the versions' sizes alone, as the other stages' do, and `components`' by 49 MB more (about 10 bytes a node). Times are unchanged within noise (`components` 2.87 / 2.00 s, `minimum_spanning_forest` 17.0 / 16.4 s, a load average of 1–3). The SRFs that already made each row as it is returned are `bfs`, `reachable_set`, `vector_values`, `matrix_rows`, `id_map`, `similarity`, `similar_pairs` and `gather`. Two still build their rows up front, *left for M11* (§13): `walk`, whose rows are made as the walks run, into a reserved buffer of datums bounded by `max_results` and `warren.max_walk_paths`, so streaming it means keeping each row as dense ids (and a path's as a slice) and building its arrays in `next()`; and `neighbours`, whose rows are one node's neighbours, bounded by its degree, made inside a closure that borrows the loaded version and its adjacency, so streaming it needs an adjacency iterator that owns what it reads. Neither is the small change the four above were, and both are bounded by their arguments or by one node. - *Shared cache:* the shared version is 719.5 / 600.6 MB, and the private peaks fall by the version, as for `bigint` ids: `pagerank` 0.56 / 0.56 GB, `bfs` 0.12 / 0.12 GB, `components` 0.56 / 0.20 GB. Every §12 target that ran was met at this scale except `reachable_set`, which misses for both id types (12–13 / 22–23M edges/s). -*At full scale* (10M nodes, 100M edges; `bench/results/soak/a-text-m10.json` and `a-m10.json`): case a with every stage, the recursive CTE baseline, the second version (`diff`, `distance`), the library and both community detections included, a release build of 7934941 on the soak's machine and settings, text then `bigint` back to back with no other agent building (load average 1.0–1.5, the soak's own). Neither run came within `--disk-floor` (3 GiB, of about 20 GB free). Figures are text / `bigint`; peaks are private RssAnon: +*At full scale* (10M nodes, 100M edges; `bench/results/soak/a-text-m10.json` and `a-m10.json`): case a with every stage, the recursive CTE baseline, the second version (`diff`, `distance`), the library and both community detections included, a release build of 7934941 on the soak's machine and settings, text then `bigint` back to back with no other agent building (load average 1.0–1.5, the soak's own). Neither run came within `--disk-floor` (3 GiB, of about 20 GB free). Both ran *before the lazy-rows fix* (below), so the `components` and `minimum_spanning_forest` peaks are the eager ones. Figures are text / `bigint`; peaks are private RssAnon: | Stage | Text | `bigint` | Text / `bigint` | |---|---|---|---| @@ -1619,7 +1619,7 @@ Every §12 target that ran was met at this scale except `reachable_set`, which m - *Size and build.* The text version is 28% larger, its id map the IRIs' bytes plus offsets and order; cached, 20% larger, since it builds no dense-id direction when it loads. The build costs 31–39% more (the half-scale 35%) and peaks 0.25 GB higher, the larger id map it holds. Every stage that reads the version peaks about 0.24 GB higher, the difference in the versions' cached sizes. - *Where text costs.* Calls that emit a row per reached node pay for each id's `text` datum (built and checked against the encoding): `bfs` and `components` take twice as long, `reachable_set` 42% longer. `similarity`'s 100,000 pairs pass 65,536 lookups, so it builds its per-call index (80 MB at 10M nodes, the 8 bytes a node of §5.7) and maps each pair through it, 1.7 s more. Kernels that map no ids are within 12% (`pagerank`, `eigen`) or unchanged (the library, the community detections, which return a modularity). -- *Why `components` and `minimum_spanning_forest` peak higher.* Both build all their rows before they return the first: a `Vec` of `(AnyElement, …)` whose text datums are each `palloc`ed (`cstring_to_text_with_len`) in the set-returning function's multi-call memory context, which lives until the call ends. An IRI here is 33–36 bytes and its 4-byte varlena header, so AllocSet rounds it to a 64-byte chunk plus its 8-byte header: 72 bytes an id. Measured past each call's starting RssAnon, `components` holds 1,031 / 314 MB (717 MB more for 10M ids: 72 bytes each) and `minimum_spanning_forest` 2,030 / 646 MB (1,384 MB more for 20.0M ids: 69 bytes each); at half scale 78 and 68 bytes. With `bigint` ids the datum is by value and only the `Vec` is held (24 and 40 bytes a row). *The fix* (not built here): return the rows lazily, a `TableIterator` over the kernel's result (the labels, 4 bytes a node, or the forest's edges) and the loaded version, which makes each row's external ids in `next()`. pgrx calls `next()` after the first row in the caller's memory context, which for a function in `FROM` is the per-tuple context that `ExecMakeTableFunctionResult` resets before each call, and the row is then copied into the tuplestore (bounded by `work_mem`, then spilled). The datums would then live one row each, and no `Vec` of rows would be held: here the text peaks would fall by about 1.0 and 1.8 GB (the datums and the `Vec`) and the `bigint` ones by about 0.24 and 0.40 GB (the `Vec`), which leaves the kernel's result (4 bytes a node, or 16 bytes a forest edge). Other functions that build every row up front before they return one could be made lazy the same way. +- *Why `components` and `minimum_spanning_forest` peak higher.* Both build all their rows before they return the first: a `Vec` of `(AnyElement, …)` whose text datums are each `palloc`ed (`cstring_to_text_with_len`) in the set-returning function's multi-call memory context, which lives until the call ends. An IRI here is 33–36 bytes and its 4-byte varlena header, so AllocSet rounds it to a 64-byte chunk plus its 8-byte header: 72 bytes an id. Measured past each call's starting RssAnon, `components` holds 1,031 / 314 MB (717 MB more for 10M ids: 72 bytes each) and `minimum_spanning_forest` 2,030 / 646 MB (1,384 MB more for 20.0M ids: 69 bytes each); at half scale 78 and 68 bytes. With `bigint` ids the datum is by value and only the `Vec` is held (24 and 40 bytes a row). *Fixed since* (§5.7, the half-scale *Peaks* note above, which has the mechanism and the tests): `components`, `minimum_spanning_forest`, `vector_topk` and `most_similar` now make each row's external ids as the row is returned, so only the current row's are held. The figures in this table are from before the fix and were not re-measured at full scale. At half scale the text peaks fell from 1.28 and 1.77 GB to 0.84 and 1.10 GB (`bench/results/soak/a-scale2*-m10-lazy.json`); here they should fall by about the datums' 0.72 and 1.38 GB. - *Disk.* The community detections wrote, text / `bigint`: `louvain` 4.31 / 4.64 GB of segments and 6.14 / 6.14 GB of WAL, `leiden` 9.51 / 13.10 GB and 15.09 / 15.06 GB (the harness drops the edge table and the first version first), within the disk. At full scale every §12 target was met for both id types except `reachable_set` (13 / 19M edges/s, against 50M) and `bfs` against the recursive CTE (5.2 / 10.5×, against 20×); `components` (10.6 s) also misses its 10 s target with text ids only, which the lazy rows above would not change (it is time, not memory: the datums are built either way). @@ -1742,7 +1742,8 @@ Test it by installing as a non-superuser in CI and running the suite. **Make the - many backends refreshing and querying one graph at once; - physical streaming replicas, which §5.1 claims and nothing tests; - crash recovery during `refresh`; - - SQL-surface fuzzing (sqlsmith-style), beside the segment decoder fuzzing that exists. + - SQL-surface fuzzing (sqlsmith-style), beside the segment decoder fuzzing that exists; + - the two set-returning functions that still build every row's external ids before returning the first, `walk` and `neighbours` (§12, text node ids): made as each row is returned, as the others are. - Distribution: CI builds and tests on arm64 as well as x86_64 (most hosted Postgres runs on arm64), prebuilt packages for PG 15–18 on both, and a PGXN release. **M12 — 1.0.0: the stable surface.** A review of the whole SQL surface before it is frozen: