From 48a35ade7ed7bb6260b86f691c78641d27a02790 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 13 Nov 2026 19:31:00 +0000 Subject: [PATCH 01/10] wip(recall): write one row per statement when its start and end share a batch Incomplete: part 1 of the per-run footprint change. Recall schema version 6 adds recall_events.started_at; the statements views read it. Verified with cargo test -p sqlbuild-recall only. --- .../src/records/_helpers/events.rs | 5 ++ .../sqlbuild-recall/src/records/constants.rs | 6 ++ .../src/schema/_helpers/clickhouse_views.rs | 8 ++- .../src/schema/_helpers/relational_views.rs | 9 ++- .../sqlbuild-recall/src/schema/constants.rs | 6 +- .../src/schema/tests/test_ddl.rs | 4 +- .../src/writer/_helpers/materialize.rs | 59 ++++++++++++++++++- .../src/writer/tests/helpers.rs | 57 ++++++++++++++++++ .../src/writer/tests/test_artifacts.rs | 42 ++++++++++++- .../src/writer/tests/test_types.rs | 5 ++ 10 files changed, 188 insertions(+), 13 deletions(-) diff --git a/crates/sqlbuild-recall/src/records/_helpers/events.rs b/crates/sqlbuild-recall/src/records/_helpers/events.rs index 904f592754..52e2a213e3 100644 --- a/crates/sqlbuild-recall/src/records/_helpers/events.rs +++ b/crates/sqlbuild-recall/src/records/_helpers/events.rs @@ -7,6 +7,7 @@ use sqlbuild_core::lifecycle_events::main::catalog::catalog; use crate::records::_helpers::rows::{optional_text, text}; use crate::records::constants::{ CORRELATION_FIELDS, PROMOTED_INTEGER_FIELDS, PROMOTED_REAL_FIELDS, PROMOTED_TEXT_FIELDS, + STARTED_AT_FIELD, }; use crate::records::errors::RecordError; use crate::records::models::RecallValue; @@ -76,6 +77,10 @@ pub(crate) fn envelope_columns(event: &Map) -> Result ("severity", text(policy.severity.value())), ("producer", text(required_text(event, "producer")?)), ("event_schema_version", RecallValue::Integer(schema_version)), + ( + STARTED_AT_FIELD, + optional_text(event.get(STARTED_AT_FIELD).and_then(Value::as_str)), + ), ]; columns.extend(CORRELATION_FIELDS.iter().map(|field| { ( diff --git a/crates/sqlbuild-recall/src/records/constants.rs b/crates/sqlbuild-recall/src/records/constants.rs index b4f54d98a5..5b874ea9cf 100644 --- a/crates/sqlbuild-recall/src/records/constants.rs +++ b/crates/sqlbuild-recall/src/records/constants.rs @@ -16,6 +16,12 @@ pub const AUDIT_EVIDENCE_MEDIA_TYPE: &str = "application/json"; pub const AUDIT_EVIDENCE_ROLE: &str = "audit_evidence"; /// The event whose payload may carry evidence rows. pub const AUDIT_COMPLETED_EVENT: &str = "audit_completed"; +/// A statement's start, folded into its end's row when both reach the same batch. +pub const STATEMENT_STARTED_EVENT: &str = "statement_started"; +/// The events that end a statement. +pub const STATEMENT_TERMINAL_EVENTS: [&str; 2] = ["statement_completed", "statement_failed"]; +/// The event field of a folded end that keeps its statement's start time. +pub const STARTED_AT_FIELD: &str = "started_at"; /// The payload field holding evidence rows, which Recall stores as an artifact instead. pub const EVIDENCE_PAYLOAD_FIELD: &str = "evidence"; /// The payload field naming the evidence artifact's digest, without its `sha256:` prefix. diff --git a/crates/sqlbuild-recall/src/schema/_helpers/clickhouse_views.rs b/crates/sqlbuild-recall/src/schema/_helpers/clickhouse_views.rs index 638fe61891..8c6f7fbe29 100644 --- a/crates/sqlbuild-recall/src/schema/_helpers/clickhouse_views.rs +++ b/crates/sqlbuild-recall/src/schema/_helpers/clickhouse_views.rs @@ -49,7 +49,7 @@ const INVOCATION_SOURCE_COLUMNS: [(&str, &str, &str); 35] = [ ]; /// The statement columns each statement event carries into the second fold, in tuple order. -const STATEMENT_FACT_COLUMNS: [&str; 18] = [ +const STATEMENT_FACT_COLUMNS: [&str; 19] = [ "statement_id", "event_type", "occurred_at", @@ -68,6 +68,7 @@ const STATEMENT_FACT_COLUMNS: [&str; 18] = [ "row_count", "affected_rows", "outcome", + "started_at", ]; /// The columns the `output_invocations` pass reads: the chunk's own values default to what adds nothing to a sum, and the delivery and context values to NULL. @@ -386,7 +387,8 @@ folded AS ( argMin(statement_kind, {order}) AS first_statement_kind, argMin(purpose, {order}) AS first_purpose, argMin(sql_digest, {order}) AS first_sql_digest, - minIf(occurred_at, event_type = 'statement_started') AS started_at, + coalesce(min(started_at), minIf(occurred_at, event_type = 'statement_started')) + AS statement_started_at, minIf(occurred_at, event_type = 'statement_submitted') AS submitted_at, max(query_id) AS any_query_id, countIf({terminal}) > 0 AS has_terminal, @@ -415,7 +417,7 @@ SELECT first_statement_kind AS statement_kind, first_purpose AS purpose, if(has_terminal, coalesce(last_query_id, any_query_id), any_query_id) AS query_id, - started_at, + statement_started_at AS started_at, submitted_at, if(has_terminal, last_ended_at, NULL) AS ended_at, multiIf( diff --git a/crates/sqlbuild-recall/src/schema/_helpers/relational_views.rs b/crates/sqlbuild-recall/src/schema/_helpers/relational_views.rs index 321bdb28a8..f6329fa510 100644 --- a/crates/sqlbuild-recall/src/schema/_helpers/relational_views.rs +++ b/crates/sqlbuild-recall/src/schema/_helpers/relational_views.rs @@ -23,8 +23,13 @@ firsts AS ( FROM facts ), started AS ( - SELECT statement_id, MIN(occurred_at) AS started_at - FROM facts WHERE event_type = 'statement_started' GROUP BY statement_id + SELECT statement_id, MIN(started_at) AS started_at + FROM ( + SELECT statement_id, occurred_at AS started_at FROM facts WHERE event_type = 'statement_started' + UNION ALL + SELECT statement_id, started_at FROM facts WHERE started_at IS NOT NULL + ) AS starts + GROUP BY statement_id ), submitted AS ( SELECT statement_id, MIN(occurred_at) AS submitted_at diff --git a/crates/sqlbuild-recall/src/schema/constants.rs b/crates/sqlbuild-recall/src/schema/constants.rs index 05c9c15d98..1e8c9f737e 100644 --- a/crates/sqlbuild-recall/src/schema/constants.rs +++ b/crates/sqlbuild-recall/src/schema/constants.rs @@ -3,7 +3,7 @@ use crate::schema::models::{ColumnSpec, ColumnType, Table, TableSpec}; /// The Recall schema version this build writes. Raise it when a column or view is added. -pub const RECALL_SCHEMA_VERSION: i64 = 5; +pub const RECALL_SCHEMA_VERSION: i64 = 6; /// Text that only older schema versions wrote into ClickHouse view definitions, with the last version that wrote it. Version 3 changed only view bodies, so view columns cannot tell version 2 views, which set `join_use_nulls`, from current ones. pub const SUPERSEDED_CLICKHOUSE_VIEW_MARKERS: [(&str, i64); 1] = [("join_use_nulls", 2)]; /// The oldest SQLite library the store's views run on (ordered `group_concat`). @@ -83,7 +83,7 @@ const OUTPUT_COLUMNS: [ColumnSpec; 11] = [ ColumnSpec::optional("omitted_sequences", ColumnType::Integer), ]; -const EVENT_COLUMNS: [ColumnSpec; 29] = [ +const EVENT_COLUMNS: [ColumnSpec; 30] = [ ColumnSpec::required("event_id", ColumnType::Text), ColumnSpec::optional("invocation_sequence", ColumnType::Integer), ColumnSpec::required("occurred_at", ColumnType::Timestamp), @@ -113,6 +113,8 @@ const EVENT_COLUMNS: [ColumnSpec; 29] = [ ColumnSpec::optional("row_count", ColumnType::Integer), ColumnSpec::optional("affected_rows", ColumnType::Integer), ColumnSpec::required("payload", ColumnType::Json), + // A statement end whose start was written with it keeps the start time here instead of a start row. + ColumnSpec::optional("started_at", ColumnType::Timestamp), ]; const DIAGNOSTIC_COLUMNS: [ColumnSpec; 14] = [ diff --git a/crates/sqlbuild-recall/src/schema/tests/test_ddl.rs b/crates/sqlbuild-recall/src/schema/tests/test_ddl.rs index a3cf64d08e..32c8e20587 100644 --- a/crates/sqlbuild-recall/src/schema/tests/test_ddl.rs +++ b/crates/sqlbuild-recall/src/schema/tests/test_ddl.rs @@ -154,8 +154,8 @@ fn given_clickhouse_statement_sets_when_rendered_then_every_statement_has_the_sh .collect(), required_fragments: &[ "INSERT INTO `observability`.`recall_meta`", - "SELECT 'schema', 'schema_version', '5'", - "5, '9.8.7', now64(6)", + "SELECT 'schema', 'schema_version', '6'", + "6, '9.8.7', now64(6)", ], expected_count: 1, }, diff --git a/crates/sqlbuild-recall/src/writer/_helpers/materialize.rs b/crates/sqlbuild-recall/src/writer/_helpers/materialize.rs index 54c22db4f2..3f463feeaa 100644 --- a/crates/sqlbuild-recall/src/writer/_helpers/materialize.rs +++ b/crates/sqlbuild-recall/src/writer/_helpers/materialize.rs @@ -1,13 +1,14 @@ //! Rows of queued input, built on the delivery thread so producers only enqueue. -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use serde_json::{Map, Value}; use crate::records::constants::{ ARTIFACT_DIGEST_PREFIX, AUDIT_COMPLETED_EVENT, AUDIT_EVIDENCE_KIND, AUDIT_EVIDENCE_MEDIA_TYPE, AUDIT_EVIDENCE_ROLE, EVIDENCE_PAYLOAD_FIELD, EXECUTED_SQL_ROLE, SAMPLE_DIGEST_PAYLOAD_FIELD, - SQL_ARTIFACT_KIND, SQL_MEDIA_TYPE, + SQL_ARTIFACT_KIND, SQL_MEDIA_TYPE, STARTED_AT_FIELD, STATEMENT_STARTED_EVENT, + STATEMENT_TERMINAL_EVENTS, }; use crate::records::main::artifact_rows::artifact_rows; use crate::records::main::event_object_row::event_object_row; @@ -228,9 +229,18 @@ impl Materializer { ) -> Built { let mut built = Built::default(); let mut pending_events: Vec<(u64, Box)> = Vec::new(); + let starts: HashMap = ended_starts(&batch); for record in batch { let stream: DeliveryStream = record.stream; match record.payload { + Payload::Event(event) if starts.contains_key(statement_of(&event, true)) => { + built.settled.push(stream); + } + Payload::Event(mut event) if starts.contains_key(statement_of(&event, false)) => { + let started_at = starts[statement_of(&event, false)].clone(); + event.insert(STARTED_AT_FIELD.to_owned(), Value::from(started_at)); + built.add_event(stream, self.event_rows(event)); + } Payload::Rows(rows) if stream == DeliveryStream::Output => { match self.output_rows(rows, replacement) { Ok(rows) => { @@ -276,6 +286,51 @@ impl Materializer { } } +/// The statement an event starts (`start`) or ends; empty for any other event. +fn statement_of(event: &Map, start: bool) -> &str { + let text = |field: &str| event.get(field).and_then(Value::as_str); + let event_type: &str = text("event_type").unwrap_or_default(); + let matches: bool = if start { + event_type == STATEMENT_STARTED_EVENT + } else { + STATEMENT_TERMINAL_EVENTS.contains(&event_type) + }; + if matches { + text("statement_id").unwrap_or_default() + } else { + "" + } +} + +/// Start times of the statements that both start and end in `batch`, by statement ID: one row records each. A batch takes higher priorities first, so an end may precede its start. +fn ended_starts(batch: &[Queued]) -> HashMap { + let events = || { + batch.iter().filter_map(|record| match &record.payload { + Payload::Event(event) => Some(event), + _ => None, + }) + }; + let mut starts: HashMap<&str, &str> = HashMap::new(); + for event in events() { + let started: &str = statement_of(event, true); + let at: &str = event + .get("occurred_at") + .and_then(Value::as_str) + .unwrap_or_default(); + if !started.is_empty() && !at.is_empty() { + starts.entry(started).or_insert(at); + } + } + events() + .filter_map(|event| { + let ending: &str = statement_of(event, false); + starts + .get(ending) + .map(|at| (ending.to_owned(), (*at).to_owned())) + }) + .collect() +} + /// An artifact of `content`, referenced by its statement or, without one, by its digest. fn artifact_input( content: Vec, diff --git a/crates/sqlbuild-recall/src/writer/tests/helpers.rs b/crates/sqlbuild-recall/src/writer/tests/helpers.rs index fdf3dbe55d..ee089fca63 100644 --- a/crates/sqlbuild-recall/src/writer/tests/helpers.rs +++ b/crates/sqlbuild-recall/src/writer/tests/helpers.rs @@ -632,6 +632,63 @@ fn audit_event(index: usize, outcome: &str) -> serde_json::Map serde_json::Map { + let event: serde_json::Value = serde_json::json!({ + "event_id": format!("{statement_id}-{event_type}"), + "event_type": event_type, + "schema_version": 2, + "producer": "sqlbuild", + "producer_version": "1", + "occurred_at": occurred_at, + "invocation_id": "fold", + "invocation_sequence": sequence, + "statement_id": statement_id, + "payload": {"intent": "execute", "statement_kind": "SELECT", "duration_ms": 2.5}, + }); + serde_json::from_value(event).expect("the statement event is an object") +} + +/// Per statement: its stored event types, the start time its rows keep, and the `statements` view's start, end and outcome, after one batch where `done` started and completed and `running` only started. +pub(crate) fn folded_statements() -> Vec> { + let directory = tempfile::tempdir().expect("a temporary directory can be created"); + let path = directory.path().join("recall.sqlite3"); + let session = start_recall_session(config(path.clone(), "fold")); + // The first batch leaves at once with the first heartbeat; the statements share the next one. + std::thread::sleep(Duration::from_millis(100)); + for event in [ + statement_event( + ("done", "statement_started"), + (0, "2026-10-19T00:00:01.000000Z"), + ), + statement_event( + ("running", "statement_started"), + (1, "2026-10-19T00:00:02.000000Z"), + ), + statement_event( + ("done", "statement_completed"), + (2, "2026-10-19T00:00:03.000000Z"), + ), + ] { + session.publish_event_object(event); + } + drop(session.close()); + let store = open_sqlite_store(&path, OpenMode::ReadOnly).expect("the store opens"); + query_store( + &store, + "SELECT s.statement_id, \ + (SELECT group_concat(event_type, ',' ORDER BY event_type) FROM recall_events AS e \ + WHERE e.statement_id = s.statement_id), \ + (SELECT MAX(started_at) FROM recall_events AS e WHERE e.statement_id = s.statement_id), \ + s.started_at, s.ended_at, s.outcome FROM statements AS s ORDER BY s.statement_id", + None, + ) + .expect("the query is valid") + .rows +} + /// Per audit outcome: whether the stored event still holds its evidence, its `sample_digest`, and the artifact text that digest names, after a session keeping `audit_evidence`. pub(crate) fn stored_evidence(audit_evidence: AuditEvidence) -> Vec> { let directory = tempfile::tempdir().expect("a temporary directory can be created"); diff --git a/crates/sqlbuild-recall/src/writer/tests/test_artifacts.rs b/crates/sqlbuild-recall/src/writer/tests/test_artifacts.rs index 9b236a0322..2248b28b85 100644 --- a/crates/sqlbuild-recall/src/writer/tests/test_artifacts.rs +++ b/crates/sqlbuild-recall/src/writer/tests/test_artifacts.rs @@ -1,9 +1,12 @@ use crate::records::models::RecallValue; use crate::settings::models::AuditEvidence; use crate::writer::tests::helpers::{ - failed_statement_sql, kept_evidence, left_out_evidence, recall_text, stored_evidence, + failed_statement_sql, folded_statements, kept_evidence, left_out_evidence, recall_text, + stored_evidence, +}; +use crate::writer::tests::test_types::{ + AuditEvidenceTestCase, FailedStatementTestCase, StatementFoldTestCase, }; -use crate::writer::tests::test_types::{AuditEvidenceTestCase, FailedStatementTestCase}; #[test] fn given_audit_evidence_settings_when_audits_complete_then_kept_evidence_is_an_artifact_and_none_stays_in_the_event() @@ -58,3 +61,38 @@ fn given_statements_sharing_sql_and_a_failed_one_when_recorded_then_text_is_stor ); } } + +#[test] +fn given_a_statement_started_and_ended_in_one_batch_when_recorded_then_one_row_keeps_its_start_and_a_running_one_keeps_its_start_row() + { + let test_cases = [StatementFoldTestCase { + description: "the ended statement is one completed row; the running one stays a start row", + expected_rows: vec![ + vec![ + recall_text("done"), + recall_text("statement_completed"), + recall_text("2026-10-19T00:00:01.000000Z"), + recall_text("2026-10-19T00:00:01.000000Z"), + recall_text("2026-10-19T00:00:03.000000Z"), + recall_text("completed"), + ], + vec![ + recall_text("running"), + recall_text("statement_started"), + RecallValue::Null, + recall_text("2026-10-19T00:00:02.000000Z"), + RecallValue::Null, + recall_text("no_terminal"), + ], + ], + }]; + + for test_case in test_cases { + assert_eq!( + folded_statements(), + test_case.expected_rows, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-recall/src/writer/tests/test_types.rs b/crates/sqlbuild-recall/src/writer/tests/test_types.rs index 9d16bfe4f4..6f42c0c016 100644 --- a/crates/sqlbuild-recall/src/writer/tests/test_types.rs +++ b/crates/sqlbuild-recall/src/writer/tests/test_types.rs @@ -357,6 +357,11 @@ pub(crate) struct AuditEvidenceTestCase { pub(crate) expected_rows: Vec>, } +pub(crate) struct StatementFoldTestCase { + pub(crate) description: &'static str, + pub(crate) expected_rows: Vec>, +} + pub(crate) struct FailedStatementTestCase { pub(crate) description: &'static str, pub(crate) expected_rows: Vec>, From ef50887677a47920482f66f504ce1e324668226f Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 13 Nov 2026 20:12:00 +0000 Subject: [PATCH 02/10] perf(recall): keep promoted lifecycle fields out of stored payloads Relational stores drop payload fields that already have their own column from sqlbuild lifecycle events; the events view merges them back so the payload reads as written. Server-produced events keep their payloads. --- .../src/store/_helpers/recall_rows.rs | 4 +- .../src/schema/_helpers/relational_views.rs | 56 +++++++++++++++++-- .../schema/main/control_view_statements.rs | 31 +++------- .../src/schema/main/view_statements.rs | 2 +- .../src/store/_helpers/rows.rs | 4 +- crates/sqlbuild-recall/src/store/main/mod.rs | 1 + .../src/store/main/relational_row.rs | 48 ++++++++++++++++ .../src/store/tests/test_views.rs | 30 ++++++++++ 8 files changed, 144 insertions(+), 32 deletions(-) create mode 100644 crates/sqlbuild-recall/src/store/main/relational_row.rs diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/recall_rows.rs b/crates/sqlbuild-orchestration/src/store/_helpers/recall_rows.rs index 2d92e03869..40131d9a9b 100644 --- a/crates/sqlbuild-orchestration/src/store/_helpers/recall_rows.rs +++ b/crates/sqlbuild-orchestration/src/store/_helpers/recall_rows.rs @@ -9,6 +9,7 @@ use sqlbuild_recall::schema::models::{ColumnType, Table}; use sqlbuild_recall::store::main::conflict_evidence::conflict_evidence; use sqlbuild_recall::store::main::relational_insert::relational_insert; use sqlbuild_recall::store::main::relational_lookup::relational_lookup; +use sqlbuild_recall::store::main::relational_row::relational_row; use crate::store::errors::ControlError; use crate::store::main::query::query; @@ -58,7 +59,8 @@ pub(crate) async fn write_row( .into_iter() .map(|column| column.name.to_owned()) .collect(); - let insert = relational_insert((row, &columns), &ingested, postgres); + let stored = relational_row(row); + let insert = relational_insert((&stored, &columns), &ingested, postgres); let returning = if row.table == Table::Events { "storage_id" } else { diff --git a/crates/sqlbuild-recall/src/schema/_helpers/relational_views.rs b/crates/sqlbuild-recall/src/schema/_helpers/relational_views.rs index f6329fa510..8e4c25fde6 100644 --- a/crates/sqlbuild-recall/src/schema/_helpers/relational_views.rs +++ b/crates/sqlbuild-recall/src/schema/_helpers/relational_views.rs @@ -1,12 +1,58 @@ //! Shared relational views emit dialect differences at each expression's definition. -use crate::schema::_helpers::rendering::{json_field, relational_cast}; -use crate::schema::constants::OUTPUT_TAIL_RECORDS; +use sqlbuild_core::lifecycle_events::constants::DEFAULT_PRODUCER; + +use crate::records::constants::{ + PROMOTED_INTEGER_FIELDS, PROMOTED_REAL_FIELDS, PROMOTED_TEXT_FIELDS, +}; +use crate::schema::_helpers::rendering::{all_columns, json_field, relational_cast}; +use crate::schema::constants::{EVENTS_TABLE, OUTPUT_TAIL_RECORDS}; -const EVENTS: &str = "SELECT * FROM recall_events"; const DIAGNOSTICS: &str = "SELECT * FROM recall_diagnostics"; const CONFLICTS: &str = "SELECT * FROM recall_conflicts"; +/// Relational stores leave promoted fields out of lifecycle payloads; the view merges them back from their columns. A null column adds nothing, so a field the event wrote as null stays null. Control stores also expose their `storage_id` cursor. +fn events(postgres: bool, control: bool) -> String { + let promoted: Vec<&str> = PROMOTED_TEXT_FIELDS + .iter() + .chain(&PROMOTED_INTEGER_FIELDS) + .chain(&PROMOTED_REAL_FIELDS) + .copied() + .collect(); + let pairs: String = promoted + .iter() + .map(|field| format!("'{field}', {field}")) + .collect::>() + .join(", "); + let merged: String = if postgres { + format!("payload || jsonb_strip_nulls(jsonb_build_object({pairs}))") + } else { + // `json_remove` drops each null column's key; a present column removes an absent key instead. + let absent: String = promoted + .iter() + .map(|field| { + format!(", CASE WHEN {field} IS NULL THEN '$.{field}' ELSE '$.payload' END") + }) + .collect(); + format!("json_patch(payload, json_remove(json_object({pairs}){absent}))") + }; + let mut columns: Vec = all_columns(&EVENTS_TABLE) + .iter() + .map(|column| { + if column.name == "payload" { + format!("CASE WHEN producer = '{DEFAULT_PRODUCER}' THEN {merged} ELSE payload END AS payload") + } else { + column.name.to_owned() + } + }) + .collect(); + if control { + columns.push("storage_id".to_owned()); + } + let columns: String = columns.join(",\n "); + format!("SELECT\n {columns}\nFROM recall_events") +} + fn statements(postgres: bool) -> String { let available = relational_cast(postgres, ("artifact.digest IS NOT NULL", "INTEGER")); format!("WITH facts AS ( @@ -516,7 +562,7 @@ WHERE position <= {OUTPUT_TAIL_RECORDS}" } /// Each supported view's name and relational body, in creation order. -pub(crate) fn relational_view_bodies(postgres: bool) -> Vec<(&'static str, String)> { +pub(crate) fn relational_view_bodies(postgres: bool, control: bool) -> Vec<(&'static str, String)> { vec![ ("invocations", invocations(postgres)), ("statements", statements(postgres)), @@ -524,7 +570,7 @@ pub(crate) fn relational_view_bodies(postgres: bool) -> Vec<(&'static str, Strin ("output_invocations", output_invocations(postgres)), ("output_tail", output_tail(postgres)), ("output_transcripts", output_transcripts(postgres)), - ("events", EVENTS.to_owned()), + ("events", events(postgres, control)), ("diagnostics", DIAGNOSTICS.to_owned()), ("artifacts", artifacts(postgres)), ("conflicts", CONFLICTS.to_owned()), diff --git a/crates/sqlbuild-recall/src/schema/main/control_view_statements.rs b/crates/sqlbuild-recall/src/schema/main/control_view_statements.rs index da02bd1f31..128dc7b6b9 100644 --- a/crates/sqlbuild-recall/src/schema/main/control_view_statements.rs +++ b/crates/sqlbuild-recall/src/schema/main/control_view_statements.rs @@ -3,34 +3,17 @@ use sqlbuild_core::identifier_text::main::quote_identifier::quote_identifier as sqlite_identifier; use crate::schema::_helpers::relational_views::relational_view_bodies; -use crate::schema::main::view_statements::view_statements; -use crate::schema::models::{DdlOptions, Dialect}; -/// PostgreSQL validates each view at migration time; SQLite retains its existing view contract. +/// PostgreSQL validates each view at migration time; SQLite creates each missing view. #[must_use] pub fn control_view_statements(postgres: bool) -> Vec { - if postgres { - relational_view_bodies(true) - .into_iter() - .map(|(name, body)| { - format!( - "CREATE OR REPLACE VIEW {} AS\n{body}", - sqlite_identifier(name) - ) - }) - .collect() + let create = if postgres { + "CREATE OR REPLACE VIEW" } else { - view_statements( - Dialect::Sqlite, - &DdlOptions { - database: None, - retention_days: 30, - cluster: None, - producer_version: String::new(), - }, - ) + "CREATE VIEW IF NOT EXISTS" + }; + relational_view_bodies(postgres, true) .into_iter() - .map(|(_, statement)| statement) + .map(|(name, body)| format!("{create} {} AS\n{body}", sqlite_identifier(name))) .collect() - } } diff --git a/crates/sqlbuild-recall/src/schema/main/view_statements.rs b/crates/sqlbuild-recall/src/schema/main/view_statements.rs index 0351938b81..05526fe1d1 100644 --- a/crates/sqlbuild-recall/src/schema/main/view_statements.rs +++ b/crates/sqlbuild-recall/src/schema/main/view_statements.rs @@ -11,7 +11,7 @@ use crate::schema::models::{DdlOptions, Dialect}; #[must_use] pub fn view_statements(dialect: Dialect, options: &DdlOptions) -> Vec<(&'static str, String)> { match dialect { - Dialect::Sqlite => relational_view_bodies(false) + Dialect::Sqlite => relational_view_bodies(false, false) .into_iter() .map(|(name, body)| { ( diff --git a/crates/sqlbuild-recall/src/store/_helpers/rows.rs b/crates/sqlbuild-recall/src/store/_helpers/rows.rs index aa5b4d0b42..20670cb098 100644 --- a/crates/sqlbuild-recall/src/store/_helpers/rows.rs +++ b/crates/sqlbuild-recall/src/store/_helpers/rows.rs @@ -15,6 +15,7 @@ use crate::store::errors::StoreError; use crate::store::main::conflict_evidence::conflict_evidence; use crate::store::main::relational_insert::relational_insert; use crate::store::main::relational_lookup::relational_lookup; +use crate::store::main::relational_row::relational_row; use crate::store::models::{ConflictEvidence, InsertOutcome}; fn text_of(row: &RecallRow, column: &str) -> String { @@ -29,7 +30,8 @@ fn write_row( ingested_at: &str, ) -> Result, StoreError> { let ingested: RecallValue = RecallValue::Text(ingested_at.to_owned()); - let insert = relational_insert((row, columns), &ingested, false); + let stored = relational_row(row); + let insert = relational_insert((&stored, columns), &ingested, false); let parameters: Vec> = insert .values .iter() diff --git a/crates/sqlbuild-recall/src/store/main/mod.rs b/crates/sqlbuild-recall/src/store/main/mod.rs index 78f9b89028..040662b9ad 100644 --- a/crates/sqlbuild-recall/src/store/main/mod.rs +++ b/crates/sqlbuild-recall/src/store/main/mod.rs @@ -13,5 +13,6 @@ pub mod query_store; pub mod read_output_transcript; pub mod relational_insert; pub mod relational_lookup; +pub mod relational_row; pub mod resolve_invocation; pub mod store_location_problem; diff --git a/crates/sqlbuild-recall/src/store/main/relational_row.rs b/crates/sqlbuild-recall/src/store/main/relational_row.rs new file mode 100644 index 0000000000..a618e033c8 --- /dev/null +++ b/crates/sqlbuild-recall/src/store/main/relational_row.rs @@ -0,0 +1,48 @@ +//! `relational_row`: a row as relational stores keep it; a lifecycle event leaves out payload fields its own columns already hold. + +use std::borrow::Cow; + +use serde_json::Value; +use sqlbuild_core::lifecycle_events::constants::DEFAULT_PRODUCER; + +use crate::records::_helpers::events::payload_columns; +use crate::records::models::{RecallRow, RecallValue}; +use crate::schema::models::Table; + +/// The views put the left-out fields back from their columns, so a payload reads as written; the content digest still covers the whole event. Server-owned events keep their payloads, which the server reads back whole. +#[must_use] +pub fn relational_row(row: &RecallRow) -> Cow<'_, RecallRow> { + let lifecycle = row.value("producer") == Some(&RecallValue::Text(DEFAULT_PRODUCER.to_owned())); + let Some(RecallValue::Text(payload)) = row + .value("payload") + .filter(|_| row.table == Table::Events && lifecycle) + else { + return Cow::Borrowed(row); + }; + let Ok(Value::Object(mut fields)) = serde_json::from_str::(payload) else { + return Cow::Borrowed(row); + }; + let held: Vec<&'static str> = payload_columns(&fields) + .into_iter() + .filter(|(name, value)| { + *name != "payload" + && !matches!(value, RecallValue::Null) + && row.value(name) == Some(value) + }) + .map(|(name, _)| name) + .collect(); + if held.is_empty() { + return Cow::Borrowed(row); + } + for name in held { + fields.remove(name); + } + let stripped: String = Value::Object(fields).to_string(); + let mut owned: RecallRow = row.clone(); + for (name, value) in &mut owned.values { + if *name == "payload" { + *value = RecallValue::Text(stripped.clone()); + } + } + Cow::Owned(owned) +} diff --git a/crates/sqlbuild-recall/src/store/tests/test_views.rs b/crates/sqlbuild-recall/src/store/tests/test_views.rs index 742d993e6d..10688472ab 100644 --- a/crates/sqlbuild-recall/src/store/tests/test_views.rs +++ b/crates/sqlbuild-recall/src/store/tests/test_views.rs @@ -463,3 +463,33 @@ fn given_the_same_sql_in_two_invocations_when_written_then_artifacts_are_shared_ ); } } + +#[test] +fn given_lifecycle_payloads_when_stored_then_promoted_fields_are_kept_once_and_read_back_whole() { + let env = envelope("strip", NOW); + let written = r#"{"intent":"execute","statement_kind":"SELECT","duration_ms":12.5,"affected_rows":3,"query_id":null,"attempt_number":2}"#; + let rows = projected( + &[vec![event( + &env, + ("e0", "statement_completed", 0), + r#","statement_id":"s1""#, + written, + )]], + "SELECT recall_events.payload, events.payload FROM recall_events \ + JOIN events ON events.event_id = recall_events.event_id", + ); + let parsed = |value: &RecallValue| match value { + RecallValue::Text(json) => serde_json::from_str::(json).expect("JSON"), + other => panic!("payload is text, got {other:?}"), + }; + assert_eq!( + parsed(&rows[0][0]), + serde_json::json!({"query_id": null, "attempt_number": 2}), + "the table keeps only fields without a column of their own" + ); + assert_eq!( + parsed(&rows[0][1]), + serde_json::from_str::(written).expect("JSON"), + "the events view returns the payload as written, explicit nulls included" + ); +} From 3cafde17b0cef9457773c298f219f49b9ddda05f Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 13 Nov 2026 20:25:31 +0000 Subject: [PATCH 03/10] perf(recall): compact worker IDs, content digests and SQL references Worker IDs namespaced by invocation are stored as a 128-bit digest of the pair instead of the JSON pair. Content digests keep their first 128 bits; a full-length digest stored earlier still matches. Each invocation writes one reference per artifact digest; statements and audits find their SQL and evidence by digest. --- .../src/runs/_helpers/lifecycle/recall.rs | 6 +++-- .../src/runs/constants.rs | 3 +++ .../warehouse_history/_helpers/run_recall.rs | 22 ++++++++++++++----- .../src/clickhouse/_helpers/planning.rs | 2 +- .../src/records/_helpers/ids.rs | 6 ++--- .../sqlbuild-recall/src/records/constants.rs | 2 ++ crates/sqlbuild-recall/src/records/models.rs | 7 ++++++ .../src/store/main/conflict_evidence.rs | 2 +- .../src/store/tests/test_views.rs | 21 +++++++++++++++++- .../src/writer/_helpers/materialize.rs | 21 +++++------------- .../src/writer/_helpers/session.rs | 2 +- .../src/writer/tests/test_artifacts.rs | 8 +++---- .../cli/recall/test_recall_run_facts.py | 3 ++- website/src/content/docs/docs/cli/recall.mdx | 3 ++- .../docs/concepts/observability/recall.mdx | 8 +++---- 15 files changed, 75 insertions(+), 41 deletions(-) diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/recall.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/recall.rs index ecd6740f79..f40e59a373 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/recall.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/recall.rs @@ -7,13 +7,14 @@ use sqlbuild_recall::records::models::{RecallRow, RecallValue, RowEnvelope}; use sqlbuild_recall::schema::models::Table; use sqlbuild_recall::store::models::InsertOutcome; +use crate::runs::constants::NAMESPACED_ID_HEX; use crate::runs::models::RecallRecord; use crate::store::errors::ControlError; use crate::store::main::append_recall::append_recall; use crate::store::main::query::query; use crate::store::models::{ControlTransaction, Parameter}; -/// Recall statement projections join correlations globally, not only by invocation. +/// Recall statement projections join correlations globally, not only by invocation, so each ID is namespaced by its invocation. A 128-bit digest of the pair keeps it unique without storing the invocation in every ID column. pub(crate) fn claim_record( envelope: &RowEnvelope, record: &RecallRecord, @@ -28,7 +29,8 @@ pub(crate) fn claim_record( "statement_id", ] { if let Some(RecallValue::Text(id)) = values.get_mut(name) { - *id = serde_json::json!([envelope.invocation_id, id]).to_string(); + let namespaced: String = serde_json::json!([envelope.invocation_id, id]).to_string(); + *id = sha256_hex(namespaced.as_bytes())[..NAMESPACED_ID_HEX].to_owned(); } } match record.table { diff --git a/crates/sqlbuild-orchestration/src/runs/constants.rs b/crates/sqlbuild-orchestration/src/runs/constants.rs index 6842fddad0..e5cdfb08bf 100644 --- a/crates/sqlbuild-orchestration/src/runs/constants.rs +++ b/crates/sqlbuild-orchestration/src/runs/constants.rs @@ -26,3 +26,6 @@ pub(crate) const PLAN_REASON: &str = "plan"; /// A plan only reads warehouse metadata, so it never takes a warehouse a viewer names; `run.warehouse` gates that for runs. pub(crate) const PLAN_IGNORED_SETTING: &str = "warehouse"; + +/// Hex digits kept of a worker's invocation-namespaced Recall ID: 128 bits. +pub(crate) const NAMESPACED_ID_HEX: usize = 32; diff --git a/crates/sqlbuild-orchestration/src/warehouse_history/_helpers/run_recall.rs b/crates/sqlbuild-orchestration/src/warehouse_history/_helpers/run_recall.rs index eae24f60d4..5282f763ae 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_history/_helpers/run_recall.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_history/_helpers/run_recall.rs @@ -205,16 +205,26 @@ pub(crate) async fn statement_sql_in( parameters.push(Parameter::Text(statement_id.into())); let (transaction, statements) = query( transaction, - &format!("SELECT statement_id FROM statements WHERE {filter} AND statement_id=${next}"), + &format!("SELECT sql_digest FROM statements WHERE {filter} AND statement_id=${next}"), parameters.clone(), ) .await?; - if statements.is_empty() { + let Some(statement) = statements.first() else { return Ok((transaction, None)); - } - let (transaction, rows) = query(transaction, - &format!("SELECT text FROM artifacts WHERE {filter} AND statement_id=${next} AND kind='sql' LIMIT 1"), - parameters).await?; + }; + // One reference names each digest per invocation, so a statement finds its SQL by digest. + parameters[next - 1] = Parameter::Text(format!( + "sha256:{}", + statement["sql_digest"].as_str().unwrap_or_default() + )); + let (transaction, rows) = query( + transaction, + &format!( + "SELECT text FROM artifacts WHERE {filter} AND digest=${next} AND kind='sql' LIMIT 1" + ), + parameters, + ) + .await?; let sql = rows .first() .map(|row| display_text(&row["text"])) diff --git a/crates/sqlbuild-recall/src/clickhouse/_helpers/planning.rs b/crates/sqlbuild-recall/src/clickhouse/_helpers/planning.rs index 0ee4da1649..8096529543 100644 --- a/crates/sqlbuild-recall/src/clickhouse/_helpers/planning.rs +++ b/crates/sqlbuild-recall/src/clickhouse/_helpers/planning.rs @@ -130,7 +130,7 @@ impl Planner { self.held.insert(key, Held::fresh(hash)); return Some(true); }; - if !known.hashes.iter().any(|stored| stored == hash) { + if !known.hashes.iter().any(|stored| row.same_content(stored)) { let (evidence, recorded) = conflict(row, (key.1.clone(), known), &self.detected_at); self.inserts .entry(Table::Conflicts) diff --git a/crates/sqlbuild-recall/src/records/_helpers/ids.rs b/crates/sqlbuild-recall/src/records/_helpers/ids.rs index 97ef2a2302..bcb0568fd1 100644 --- a/crates/sqlbuild-recall/src/records/_helpers/ids.rs +++ b/crates/sqlbuild-recall/src/records/_helpers/ids.rs @@ -1,7 +1,7 @@ //! Deterministic record IDs and content digests. use crate::records::constants::{ - CONTENT_HASH_BUFFER_BYTES, DIAGNOSTIC_RECORD_ID_INFIX, OUTPUT_RECORD_ID_INFIX, + CONTENT_HASH_BUFFER_BYTES, CONTENT_HASH_HEX, DIAGNOSTIC_RECORD_ID_INFIX, OUTPUT_RECORD_ID_INFIX, }; use crate::records::models::RecallValue; use crate::schema::constants::UNHASHED_COLUMNS; @@ -40,7 +40,7 @@ fn valued(mut buffer: Vec, value: &RecallValue) -> Vec { buffer } -/// Hex BLAKE3 over every non-NULL named value except `ingested_at` and `content_hash`, by column name; a NULL column hashes like an absent one. +/// The first 128 bits of a hex BLAKE3 over every non-NULL named value except `ingested_at` and `content_hash`, by column name; a NULL column hashes like an absent one. pub(crate) fn content_hash(values: &[(&str, RecallValue)]) -> String { let mut sorted: Vec<&(&str, RecallValue)> = values .iter() @@ -53,5 +53,5 @@ pub(crate) fn content_hash(values: &[(&str, RecallValue)]) -> String { Vec::with_capacity(CONTENT_HASH_BUFFER_BYTES), |buffer, (name, value)| valued(framed(buffer, b'n', name.as_bytes()), value), ); - blake3::hash(&encoded).to_hex().to_string() + blake3::hash(&encoded).to_hex()[..CONTENT_HASH_HEX].to_owned() } diff --git a/crates/sqlbuild-recall/src/records/constants.rs b/crates/sqlbuild-recall/src/records/constants.rs index 5b874ea9cf..3b3820e034 100644 --- a/crates/sqlbuild-recall/src/records/constants.rs +++ b/crates/sqlbuild-recall/src/records/constants.rs @@ -63,6 +63,8 @@ pub(crate) const MARCH_BASED_JANUARY_INDEX: i64 = 10; pub const DROPPED_BEFORE_COLUMN: &str = "dropped_before"; /// Initial capacity of the buffer a row's content hash is computed over. pub(crate) const CONTENT_HASH_BUFFER_BYTES: usize = 1_024; +/// Hex digits kept of a row's content digest; rows stored before carry all 64, of which these are the prefix. +pub(crate) const CONTENT_HASH_HEX: usize = 32; /// The origin of an invocation an integration such as Dagster started. pub const INTEGRATION_ORIGIN: &str = "integration"; /// The origin of an invocation a CI system ran. diff --git a/crates/sqlbuild-recall/src/records/models.rs b/crates/sqlbuild-recall/src/records/models.rs index eaab7e2c60..35c2db948f 100644 --- a/crates/sqlbuild-recall/src/records/models.rs +++ b/crates/sqlbuild-recall/src/records/models.rs @@ -61,6 +61,13 @@ impl RecallRow { _ => None, } } + + /// Whether `stored` is this row's content digest, including a full-length digest an earlier version stored. + #[must_use] + pub fn same_content(&self, stored: &str) -> bool { + let written: &str = self.content_hash().unwrap_or_default(); + !written.is_empty() && stored.starts_with(written) + } } /// What every row of one invocation carries. diff --git a/crates/sqlbuild-recall/src/store/main/conflict_evidence.rs b/crates/sqlbuild-recall/src/store/main/conflict_evidence.rs index a4c04a92e6..fb53bd68fa 100644 --- a/crates/sqlbuild-recall/src/store/main/conflict_evidence.rs +++ b/crates/sqlbuild-recall/src/store/main/conflict_evidence.rs @@ -8,7 +8,7 @@ use crate::store::models::ConflictEvidence; #[must_use] pub fn conflict_evidence(row: &RecallRow, existing: &str) -> Option { let rejected = row.content_hash().unwrap_or_default(); - (existing != rejected).then(|| ConflictEvidence { + (!row.same_content(existing)).then(|| ConflictEvidence { table: row.table, record_key: table_spec(row.table) .key diff --git a/crates/sqlbuild-recall/src/store/tests/test_views.rs b/crates/sqlbuild-recall/src/store/tests/test_views.rs index 10688472ab..02df08dec9 100644 --- a/crates/sqlbuild-recall/src/store/tests/test_views.rs +++ b/crates/sqlbuild-recall/src/store/tests/test_views.rs @@ -138,6 +138,15 @@ fn given_duplicates_and_conflicts_when_written_then_first_row_is_kept_and_confli vec![original.clone()], vec![conflicting.clone()], ]; + // An earlier version stored the full 64-digit digest, of which today's is the prefix. + let mut legacy = original.clone(); + for (name, value) in &mut legacy.values { + if *name == "content_hash" + && let RecallValue::Text(hash) = value + { + hash.push_str(&"0".repeat(32)); + } + } let test_cases = [ ProjectionTestCase { description: "the first row wins", @@ -153,10 +162,20 @@ fn given_duplicates_and_conflicts_when_written_then_first_row_is_kept_and_confli }, ProjectionTestCase { description: "identical rewrites are not conflicts", - batches: vec![vec![original.clone()], vec![original]], + batches: vec![vec![original.clone()], vec![original.clone()]], sql: "SELECT COUNT(*) FROM conflicts", expected_rows: vec![vec![integer(0)]], }, + ProjectionTestCase { + description: "a row stored with a full-length digest matches its rewrite, not different content", + batches: vec![ + vec![legacy.clone()], + vec![original.clone()], + vec![conflicting], + ], + sql: "SELECT record_key FROM conflicts", + expected_rows: vec![vec![text("dup:0")]], + }, ]; for test_case in test_cases { diff --git a/crates/sqlbuild-recall/src/writer/_helpers/materialize.rs b/crates/sqlbuild-recall/src/writer/_helpers/materialize.rs index 3f463feeaa..6cf49e9ed7 100644 --- a/crates/sqlbuild-recall/src/writer/_helpers/materialize.rs +++ b/crates/sqlbuild-recall/src/writer/_helpers/materialize.rs @@ -69,9 +69,8 @@ impl Built { #[derive(Debug)] pub(crate) struct Materializer { envelope: RowEnvelope, + /// Digests whose content row and one reference row are written. artifacts_seen: HashSet, - /// Digests and the references already written for them. - refs_seen: HashSet<(String, String)>, /// Output bytes stored per stream, for the offsets of replaced chunks. output_bytes: [u64; 2], audit_evidence: AuditEvidence, @@ -83,7 +82,6 @@ impl Materializer { audit_evidence, envelope, artifacts_seen: HashSet::new(), - refs_seen: HashSet::new(), output_bytes: [0; 2], } } @@ -130,7 +128,7 @@ impl Materializer { Ok(stored) } - /// The content row the first time a digest is seen, and a reference row for each statement that ran it; `None` when both are already written. + /// The content row and its reference row the first time a digest is seen; `None` after that. fn sql_rows(&mut self, input: SqlInput) -> Option> { let artifact: ArtifactInput = artifact_input( input.sql.into_bytes(), @@ -141,21 +139,12 @@ impl Materializer { self.artifact_rows(artifact) } - /// `artifact`'s content row the first time its digest is seen and its reference row the first time it is named; `None` when both are written. + /// `artifact`'s content row and one reference row per invocation, named by the first statement or event written with it; statements and audits find it by digest. `None` once its digest is written. fn artifact_rows(&mut self, artifact: ArtifactInput) -> Option> { - if !self - .refs_seen - .insert((artifact.digest.clone(), artifact.ref_id.clone())) - { + if !self.artifacts_seen.insert(artifact.digest.clone()) { return None; } - let first_sighting: bool = self.artifacts_seen.insert(artifact.digest.clone()); - let [content, reference] = artifact_rows(&self.envelope, &artifact); - Some(if first_sighting { - vec![content, reference] - } else { - vec![reference] - }) + Some(artifact_rows(&self.envelope, &artifact).to_vec()) } /// The row of `event`, then the artifact rows of its evidence: an `audit_completed` event's evidence rows leave its payload and, when the outcome keeps them, become an artifact its `sample_digest` names. diff --git a/crates/sqlbuild-recall/src/writer/_helpers/session.rs b/crates/sqlbuild-recall/src/writer/_helpers/session.rs index ee88de2643..98cd39ad4d 100644 --- a/crates/sqlbuild-recall/src/writer/_helpers/session.rs +++ b/crates/sqlbuild-recall/src/writer/_helpers/session.rs @@ -146,7 +146,7 @@ impl RecallSession { Ok(()) } - /// Queue the SQL a statement executed: its text once per digest per invocation, and a reference per statement. + /// Queue the SQL a statement executed: its text and one reference once per digest per invocation; statements find it by `sql_digest`. pub fn publish_sql(&self, sql: &str, statement_id: Option<&str>, resource_id: Option<&str>) { self.queue_sql(sql, (statement_id, resource_id), Priority::Artifact); } diff --git a/crates/sqlbuild-recall/src/writer/tests/test_artifacts.rs b/crates/sqlbuild-recall/src/writer/tests/test_artifacts.rs index 2248b28b85..f08f122442 100644 --- a/crates/sqlbuild-recall/src/writer/tests/test_artifacts.rs +++ b/crates/sqlbuild-recall/src/writer/tests/test_artifacts.rs @@ -40,15 +40,15 @@ fn given_audit_evidence_settings_when_audits_complete_then_kept_evidence_is_an_a } #[test] -fn given_statements_sharing_sql_and_a_failed_one_when_recorded_then_text_is_stored_once_and_every_statement_is_referenced() +fn given_statements_sharing_sql_and_a_failed_one_when_recorded_then_each_text_is_stored_and_referenced_once() { let test_cases = [FailedStatementTestCase { - description: "two texts, three references, no conflict", + description: "two texts, one reference each; failed SQL is written first, so it names the shared text", expected_rows: vec![vec![ RecallValue::Integer(2), - RecallValue::Integer(3), + RecallValue::Integer(2), RecallValue::Integer(0), - recall_text("s1,s2,s3"), + recall_text("s2,s3"), ]], }]; diff --git a/tests/integration/src/sqlbuild/cli/recall/test_recall_run_facts.py b/tests/integration/src/sqlbuild/cli/recall/test_recall_run_facts.py index a1c9043988..42ca4741b9 100644 --- a/tests/integration/src/sqlbuild/cli/recall/test_recall_run_facts.py +++ b/tests/integration/src/sqlbuild/cli/recall/test_recall_run_facts.py @@ -210,7 +210,8 @@ def test_given_a_failed_statement_when_recalled_then_its_sql_is_readable( recorded: list[list[object]] = recall_rows( project_dir, "SELECT s.sql_available, s.sql_unavailable_reason, a.text FROM statements s " - "LEFT JOIN artifacts a USING (invocation_id, statement_id) WHERE s.outcome = 'failed'", + "LEFT JOIN artifacts a ON a.invocation_id = s.invocation_id " + "AND a.digest = 'sha256:' || s.sql_digest WHERE s.outcome = 'failed'", ) assert built.returncode == 1 diff --git a/website/src/content/docs/docs/cli/recall.mdx b/website/src/content/docs/docs/cli/recall.mdx index b5883f38d5..d7e3797b1a 100644 --- a/website/src/content/docs/docs/cli/recall.mdx +++ b/website/src/content/docs/docs/cli/recall.mdx @@ -63,7 +63,8 @@ The slowest statements of past builds, with their SQL: ```bash sqb recall query --max-cell-bytes 60 \ "SELECT s.resource_name, round(s.duration_ms, 1) AS ms, a.text AS sql - FROM statements s JOIN artifacts a USING (invocation_id, statement_id) + FROM statements s JOIN artifacts a + ON a.invocation_id = s.invocation_id AND a.digest = 'sha256:' || s.sql_digest WHERE s.statement_kind = 'CREATE' ORDER BY s.duration_ms DESC LIMIT 2" ``` diff --git a/website/src/content/docs/docs/concepts/observability/recall.mdx b/website/src/content/docs/docs/concepts/observability/recall.mdx index c32df9d52b..83a2adabbc 100644 --- a/website/src/content/docs/docs/concepts/observability/recall.mdx +++ b/website/src/content/docs/docs/concepts/observability/recall.mdx @@ -42,7 +42,7 @@ See the rows the audit flagged, which Recall keeps as the audit's evidence: ```bash sqb recall query \ "SELECT r.text FROM audit_results a JOIN artifacts r - ON r.invocation_id = a.invocation_id AND r.ref_id = a.event_id + ON r.invocation_id = a.invocation_id AND r.digest = 'sha256:' || a.sample_digest WHERE a.audit_name = 'revenue_is_non_negative' AND a.outcome <> 'pass' ORDER BY a.invocation_started_at DESC LIMIT 1" ``` @@ -151,11 +151,11 @@ and `selection_truncated` is `1`, while `selection_count` still counts all of th When an audit warns or fails, Recall keeps its [evidence](/docs/concepts/audits#evidence): up to `evidence_limit` of the rows it flagged. `audit_results` counts them in `evidence_count`, sets `evidence_truncated` to `1` when the limit cut rows off, and names the stored rows by -`sample_digest`. Read them from `artifacts`, joined on the audit's `event_id`: +`sample_digest`. Read them from `artifacts`, joined on that digest: ```sql SELECT r.text FROM audit_results a JOIN artifacts r - ON r.invocation_id = a.invocation_id AND r.ref_id = a.event_id + ON r.invocation_id = a.invocation_id AND r.digest = 'sha256:' || a.sample_digest WHERE a.outcome <> 'pass' ``` @@ -336,7 +336,7 @@ in the store's SQL dialect, against these views: | `audit_results` | Evaluated audit, with its evidence count | | `events` | Lifecycle event | | `diagnostics` | Retained diagnostic | -| `artifacts` | Reference from an invocation to captured SQL, with its text | +| `artifacts` | Captured SQL or audit evidence an invocation used, with its text; join on `digest` | | `output_invocations` | Invocation's output, with completeness | | `output_transcripts` | Invocation's stdout, stderr, and combined output | | `output_tail` | Recent output chunk | From e96acfb9ebe403bb5387f9f69755607b9e01cce8 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 13 Nov 2026 20:46:32 +0000 Subject: [PATCH 04/10] feat(control): prune finished runs' logs after 14 days or 200 runs and report store size A finished run's worker log (events, statements, SQL references and output) goes whole once it is older than --run-log-retention-days or more than --run-log-retention-count newer runs finished in its location target. The run, its node results and the scheduler's events stay. Administrators see both limits and the store's size; heartbeats report the size too. --- .../orchestration/run_log_retention.json | 8 + .../sqlbuild-cli/src/argv/command_table.json | 22 ++ .../src/native_commands/_helpers/decision.rs | 7 + .../src/native_commands/_helpers/execution.rs | 8 +- .../src/native_commands/main/_run_control.rs | 7 +- .../api/_helpers/resources/administrators.rs | 2 + .../sqlbuild-orchestration/src/api/models.rs | 5 + .../src/platform/_helpers/retention.rs | 10 +- .../src/platform/_helpers/watchdog.rs | 10 +- .../src/platform/main/read_installation.rs | 3 + .../src/platform/main/run_maintenance.rs | 8 +- .../src/platform/models.rs | 4 + .../src/runs/constants.rs | 9 + .../src/runs/main/mod.rs | 1 + .../src/runs/main/prune_run_logs.rs | 104 +++++++++ .../sqlbuild-orchestration/src/runs/models.rs | 41 ++++ .../src/store/main/mod.rs | 1 + .../src/store/main/store_bytes.rs | 22 ++ .../tests/platform/helpers.rs | 7 +- .../tests/platform/test_problems.rs | 1 + .../tests/runs/test_contracts.rs | 2 + .../tests/runs/test_run_logs.rs | 221 ++++++++++++++++++ .../tests/sensors/helpers.rs | 3 + 23 files changed, 500 insertions(+), 6 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/run_log_retention.json create mode 100644 crates/sqlbuild-orchestration/src/runs/main/prune_run_logs.rs create mode 100644 crates/sqlbuild-orchestration/src/store/main/store_bytes.rs create mode 100644 crates/sqlbuild-orchestration/tests/runs/test_run_logs.rs diff --git a/benchmarks/duckshop/edge_cases/orchestration/run_log_retention.json b/benchmarks/duckshop/edge_cases/orchestration/run_log_retention.json new file mode 100644 index 0000000000..6a70f782f1 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/run_log_retention.json @@ -0,0 +1,8 @@ +{ + "description": "Four ad-hoc runs of one location target, each with a worker log at its claim and a nested session that shares one SQL text with the others and keeps one of its own. Runs ended 20 days ago, 2 hours ago and 1 hour ago; the fourth is still running. Keeping 14 days and 1 run drops the first two logs whole, keeps the shared SQL, and leaves every run, its nodes and the scheduler's events.", + "keep_days": 14, + "keep_runs": 1, + "ended_hours_ago": [480, 2, 1, null], + "expected_pruned": 2, + "expected_logged_runs": [3, 4] +} diff --git a/crates/sqlbuild-cli/src/argv/command_table.json b/crates/sqlbuild-cli/src/argv/command_table.json index 97cddf35e6..3a3925b9ea 100644 --- a/crates/sqlbuild-cli/src/argv/command_table.json +++ b/crates/sqlbuild-cli/src/argv/command_table.json @@ -1124,6 +1124,28 @@ "help": "published snapshots to keep per location and target; older snapshots are deleted unless a run, trigger, job or other retained record still references them (default: 10)", "group": 1 }, + { + "kind": "store", + "options": [ + "--run-log-retention-days" + ], + "dest": "control_run_log_retention_days", + "default": 14, + "type": "int", + "help": "days a finished run keeps its log of events, statements, SQL and output; the run, its node results and history stay (default: 14)", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--run-log-retention-count" + ], + "dest": "control_run_log_retention_count", + "default": 200, + "type": "int", + "help": "finished runs per location and target that keep their logs; older runs lose theirs, whichever limit comes first (default: 200)", + "group": 1 + }, { "kind": "store", "options": ["--trusted-proxy-config"], diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs index 75fbb92064..0986d05d67 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs @@ -9,6 +9,7 @@ use sqlbuild_orchestration::api::models::{ApiAuthentication, HumanLogin, ServerF use sqlbuild_orchestration::credentials::models::{AuditRetention, CredentialScope}; use sqlbuild_orchestration::declarations::models::TriggerChange; use sqlbuild_orchestration::platform::models::{LocationWork, LocationWorkRequest}; +use sqlbuild_orchestration::runs::models::RunLogRetention; use sqlbuild_orchestration::runs::models::{JobRunRequest, RerunRequest, RunWindow}; use sqlbuild_orchestration::snapshots::models::SnapshotRetention; use sqlbuild_orchestration::store::errors::ControlError; @@ -280,6 +281,11 @@ pub(crate) fn control_api_settings( Some(count) => SnapshotRetention::count(count)?, None => SnapshotRetention::default(), }; + let defaults = RunLogRetention::default(); + let run_logs = RunLogRetention::new( + retention_setting(namespace, "run-log-retention-days")?.unwrap_or(defaults.days()), + retention_setting(namespace, "run-log-retention-count")?.unwrap_or(defaults.count()), + )?; Ok(( control_authentication(namespace)?, ServerFacts { @@ -288,6 +294,7 @@ pub(crate) fn control_api_settings( audit, statistics, snapshots, + run_logs, }, )) } diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs index efec73492f..b5d7568a05 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs @@ -29,6 +29,7 @@ use sqlbuild_orchestration::platform::models::{ LocationHealth, LocationHealthReport, LocationWorkClient, LocationWorkPoll, WarehouseSyncFailure, }; +use sqlbuild_orchestration::runs::models::RunLogRetention; use sqlbuild_orchestration::runs::types::RunHandoff; use sqlbuild_orchestration::scheduler::main::run_scheduler::run_scheduler; use sqlbuild_orchestration::scheduler::models::SchedulerState; @@ -891,7 +892,12 @@ pub(crate) fn location_child(options: &LocationServe, script: &Path) -> Command pub(crate) fn start_platform_roles( tasks: &mut JoinSet>, (store, shutdown): (&ControlStore, &watch::Receiver), - retention: (AuditRetention, StatisticsRetention, SnapshotRetention), + retention: ( + AuditRetention, + StatisticsRetention, + SnapshotRetention, + RunLogRetention, + ), ) -> Result<(), ControlError> { let mut sinks: Vec<(Webhook, &str)> = Vec::new(); for (variable, source, label) in [ diff --git a/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs b/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs index 60753b43f3..a85da56a3c 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs @@ -56,7 +56,12 @@ pub(crate) async fn run_control( start_platform_roles( &mut tasks, (&store, &shutdown), - (facts.audit, facts.statistics, facts.snapshots), + ( + facts.audit, + facts.statistics, + facts.snapshots, + facts.run_logs, + ), )?; } if facts.scheduler { diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/administrators.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/administrators.rs index f06b608797..f807ba8d0f 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/resources/administrators.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/administrators.rs @@ -113,6 +113,8 @@ pub(crate) async fn server( audit_days: i64::from(facts.audit.kept_days()), statistics_days: i64::from(facts.statistics.kept_days()), snapshots_kept: i64::from(facts.snapshots.kept_count()), + run_log_days: i64::from(facts.run_logs.days()), + run_logs_kept: i64::from(facts.run_logs.count()), platform_alert_days: PLATFORM_ALERT_RETENTION_DAYS, heartbeat_days: HEARTBEAT_RETENTION_DAYS, location_health_days: LOCATION_RETENTION_DAYS, diff --git a/crates/sqlbuild-orchestration/src/api/models.rs b/crates/sqlbuild-orchestration/src/api/models.rs index e892515a8a..55bd16862f 100644 --- a/crates/sqlbuild-orchestration/src/api/models.rs +++ b/crates/sqlbuild-orchestration/src/api/models.rs @@ -520,6 +520,7 @@ pub struct ServerFacts { pub audit: crate::credentials::models::AuditRetention, pub statistics: crate::warehouse_metadata::models::StatisticsRetention, pub snapshots: crate::snapshots::models::SnapshotRetention, + pub run_logs: crate::runs::models::RunLogRetention, } /// The server's version, storage, sign-in and retention, for administrators. @@ -540,6 +541,10 @@ pub struct ServerRetention { pub statistics_days: i64, /// Publications kept per location target. pub snapshots_kept: i64, + /// Days a finished run keeps its log of events, statements and output. + pub run_log_days: i64, + /// Finished runs per location target that keep their logs. + pub run_logs_kept: i64, pub platform_alert_days: i64, pub heartbeat_days: i64, pub location_health_days: i64, diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/retention.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/retention.rs index 05bd7f95eb..84941a898c 100644 --- a/crates/sqlbuild-orchestration/src/platform/_helpers/retention.rs +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/retention.rs @@ -8,6 +8,8 @@ use crate::platform::constants::{ HEARTBEAT_PAGE, HEARTBEAT_RETENTION_DAYS, LOCATION_RETENTION_DAYS, PLATFORM_ALERT_RETENTION_DAYS, SENSOR_EVALUATION_RETENTION_DAYS, }; +use crate::runs::main::prune_run_logs::prune_run_logs; +use crate::runs::models::RunLogRetention; use crate::snapshots::main::prune_snapshots::prune_snapshots; use crate::snapshots::models::SnapshotRetention; use crate::store::errors::ControlError; @@ -40,9 +42,15 @@ const LOCATION_INSTANCES: &str = "DELETE FROM sqb_location_instances WHERE insta /// Returns how many audit rows outlive their horizon because a sink has not acknowledged them. pub(crate) async fn retain( store: &ControlStore, - (audit, statistics, snapshots): (AuditRetention, StatisticsRetention, SnapshotRetention), + (audit, statistics, snapshots, run_logs): ( + AuditRetention, + StatisticsRetention, + SnapshotRetention, + RunLogRetention, + ), sinks: &[String], ) -> Result { + prune_run_logs(store, run_logs).await?; prune_snapshots(store, snapshots).await?; while prune_statistics(store, statistics).await? == MAX_HISTORY_RECORDS { tokio::task::yield_now().await; diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/watchdog.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/watchdog.rs index a85f056c46..c310f512a1 100644 --- a/crates/sqlbuild-orchestration/src/platform/_helpers/watchdog.rs +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/watchdog.rs @@ -13,6 +13,7 @@ use crate::store::main::begin::begin; use crate::store::main::commit::commit; use crate::store::main::decode_row::decode_row; use crate::store::main::query::query; +use crate::store::main::store_bytes::store_bytes; use crate::store::models::{ControlStore, Parameter, TransactionMode}; pub(crate) async fn scan_health( @@ -61,11 +62,18 @@ pub(crate) async fn scan_health( return Ok(()); } let transaction = begin(store, TransactionMode::Write).await?; + let (transaction, store_bytes) = store_bytes(transaction).await?; commit( observe( transaction, health.installation_id, - (PlatformEvent::Heartbeat { replica_id }, None), + ( + PlatformEvent::Heartbeat { + replica_id, + store_bytes, + }, + None, + ), ) .await?, ) diff --git a/crates/sqlbuild-orchestration/src/platform/main/read_installation.rs b/crates/sqlbuild-orchestration/src/platform/main/read_installation.rs index 6e20822bba..4411c1a33b 100644 --- a/crates/sqlbuild-orchestration/src/platform/main/read_installation.rs +++ b/crates/sqlbuild-orchestration/src/platform/main/read_installation.rs @@ -6,6 +6,7 @@ use crate::authorization::main::begin_authorized::begin_authorized; use crate::credentials::models::{Action, AuditContext, DefaultRole, PermissionScope}; use crate::platform::models::{InstallationRecord, StorageKind, StoredInstallation}; use crate::store::errors::ControlError; +use crate::store::main::store_bytes::store_bytes; use crate::store::main::{commit::commit, query_records::query_records}; use crate::store::models::{ControlStore, ControlTransaction}; @@ -28,6 +29,7 @@ pub async fn read_installation( Vec::new(), ) .await?; + let (transaction, store_bytes) = store_bytes(transaction).await?; commit(transaction).await?; let stored = rows.into_iter().next().ok_or_else(|| { ControlError::Invalid("the control store has no installation record".into()) @@ -41,5 +43,6 @@ pub async fn read_installation( installed_at: stored.installed_at, schema_checksum: stored.schema_checksum, schema_applied_at: stored.schema_applied_at, + store_bytes, }) } diff --git a/crates/sqlbuild-orchestration/src/platform/main/run_maintenance.rs b/crates/sqlbuild-orchestration/src/platform/main/run_maintenance.rs index d977873af7..ec1517789f 100644 --- a/crates/sqlbuild-orchestration/src/platform/main/run_maintenance.rs +++ b/crates/sqlbuild-orchestration/src/platform/main/run_maintenance.rs @@ -8,6 +8,7 @@ use uuid::Uuid; use crate::credentials::models::AuditRetention; use crate::platform::_helpers::retention::retain; use crate::platform::_helpers::watchdog::scan_health; +use crate::runs::models::RunLogRetention; use crate::snapshots::models::SnapshotRetention; use crate::store::errors::ControlError; use crate::store::models::ControlStore; @@ -18,7 +19,12 @@ use crate::webhooks::models::{Webhook, WebhookSource}; pub async fn run_maintenance( (store, replica_id): (&ControlStore, Uuid), (retention, sinks): ( - (AuditRetention, StatisticsRetention, SnapshotRetention), + ( + AuditRetention, + StatisticsRetention, + SnapshotRetention, + RunLogRetention, + ), &[Webhook], ), (report, mut shutdown): (impl Fn(i64), watch::Receiver), diff --git a/crates/sqlbuild-orchestration/src/platform/models.rs b/crates/sqlbuild-orchestration/src/platform/models.rs index 77f1d5ad40..cb584da5b5 100644 --- a/crates/sqlbuild-orchestration/src/platform/models.rs +++ b/crates/sqlbuild-orchestration/src/platform/models.rs @@ -197,6 +197,8 @@ pub enum PlatformEventKind { pub enum PlatformEvent { Heartbeat { replica_id: Uuid, + /// The control store's database size on disk when the replica checked in. + store_bytes: i64, }, SchedulerNoLeader, TickOverdue { @@ -420,6 +422,8 @@ pub struct InstallationRecord { /// Digest of the schema; a server starts only on the exact schema it creates. pub schema_checksum: String, pub schema_applied_at: DateTime, + /// The database's size on disk now: SQLite's main file, or the whole PostgreSQL database. + pub store_bytes: i64, } /// The stored installation and schema rows. diff --git a/crates/sqlbuild-orchestration/src/runs/constants.rs b/crates/sqlbuild-orchestration/src/runs/constants.rs index e5cdfb08bf..574237df98 100644 --- a/crates/sqlbuild-orchestration/src/runs/constants.rs +++ b/crates/sqlbuild-orchestration/src/runs/constants.rs @@ -29,3 +29,12 @@ pub(crate) const PLAN_IGNORED_SETTING: &str = "warehouse"; /// Hex digits kept of a worker's invocation-namespaced Recall ID: 128 bits. pub(crate) const NAMESPACED_ID_HEX: usize = 32; + +/// Days a finished run keeps its worker log by default. +pub(crate) const DEFAULT_RUN_LOG_RETENTION_DAYS: u16 = 14; +/// Finished runs per location target that keep their worker logs by default. +pub(crate) const DEFAULT_RUN_LOG_RETENTION_COUNT: u16 = 200; +/// Runs whose logs one writer transaction deletes. +pub(crate) const RUN_LOG_RETENTION_BATCH: usize = 10; +/// Artifact digests checked for remaining references per statement. +pub(crate) const RUN_LOG_DIGEST_CHUNK: usize = 500; diff --git a/crates/sqlbuild-orchestration/src/runs/main/mod.rs b/crates/sqlbuild-orchestration/src/runs/main/mod.rs index a062e002e1..4a88c6893b 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/mod.rs @@ -11,6 +11,7 @@ pub mod list_run_nodes; pub mod list_runs; pub mod monitor_runs; pub mod observe_run; +pub(crate) mod prune_run_logs; pub mod publish_run_output; pub mod publish_run_recall; pub(crate) mod queue_run; diff --git a/crates/sqlbuild-orchestration/src/runs/main/prune_run_logs.rs b/crates/sqlbuild-orchestration/src/runs/main/prune_run_logs.rs new file mode 100644 index 0000000000..92a2e02d22 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/main/prune_run_logs.rs @@ -0,0 +1,104 @@ +//! A finished run's worker log goes whole once it is too old or too many newer runs finished; the run, its nodes and the scheduler's node history stay. + +use std::collections::BTreeSet; + +use sqlbuild_recall::schema::constants::INVOCATION_TABLES; +use sqlbuild_recall::schema::main::table_spec::table_spec; + +use crate::runs::constants::{RUN_LOG_DIGEST_CHUNK, RUN_LOG_RETENTION_BATCH}; +use crate::runs::models::RunLogRetention; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{ControlStore, ControlTransaction, Parameter, TransactionMode}; + +/// Claims of finished runs past either limit whose worker log is still stored, oldest first. Worker invocations are `worker/` and `worker//`; the stored logs, not every run ever finished, bound the join. +const EXPIRED_CLAIMS: &str = "WITH logged AS (\ + SELECT DISTINCT substr(invocation_id, 8, 36) AS claim FROM recall_invocations WHERE invocation_id LIKE 'worker/%' \ + UNION SELECT DISTINCT substr(invocation_id, 8, 36) AS claim FROM recall_output WHERE invocation_id LIKE 'worker/%'), \ + ranked AS (SELECT worker_instance, ended_at, ROW_NUMBER() OVER (PARTITION BY location_id, target \ + ORDER BY ended_at DESC, run_id DESC) AS newest FROM sqb_runs \ + WHERE ended_at IS NOT NULL AND status IN ('success', 'warn', 'partial', 'failed', 'cancelled')) \ + SELECT logged.claim FROM ranked JOIN logged ON logged.claim = CAST(ranked.worker_instance AS TEXT) \ + WHERE ranked.newest > $1 OR ranked.ended_at < $2 ORDER BY ranked.ended_at LIMIT $3"; + +/// Batches commit independently; a repeated or interrupted pass finds only logs still stored. +pub(crate) async fn prune_run_logs( + store: &ControlStore, + retention: RunLogRetention, +) -> Result { + let mut deleted = 0; + loop { + let transaction = begin(store, TransactionMode::Write).await?; + let now = transaction.now; + let (transaction, rows) = query( + transaction, + EXPIRED_CLAIMS, + vec![ + Parameter::Integer(i64::from(retention.count())), + Parameter::Time(retention.cutoff(now)), + Parameter::Integer(RUN_LOG_RETENTION_BATCH as i64), + ], + ) + .await?; + let claims: Vec = rows + .iter() + .filter_map(|row| row["claim"].as_str().map(str::to_owned)) + .collect(); + let transaction = delete_logs(transaction, &claims).await?; + commit(transaction).await?; + deleted += claims.len(); + if claims.len() < RUN_LOG_RETENTION_BATCH { + return Ok(deleted); + } + } +} + +/// Every Recall row of the claims' invocations, then stored artifacts no remaining reference names. `worker//` sorts between `worker//` and `worker/0`, so a key range finds nested invocations by index. +async fn delete_logs( + mut transaction: ControlTransaction, + claims: &[String], +) -> Result { + let mut digests: BTreeSet = BTreeSet::new(); + for claim in claims { + let namespace = format!("worker/{claim}"); + let bounds = vec![ + Parameter::Text(namespace.clone()), + Parameter::Text(format!("{namespace}/")), + Parameter::Text(format!("{namespace}0")), + ]; + let owned = "(invocation_id=$1 OR (invocation_id>$2 AND invocation_id<$3))"; + let rows; + (transaction, rows) = query( + transaction, + &format!("SELECT DISTINCT digest FROM recall_artifact_refs WHERE {owned}"), + bounds.clone(), + ) + .await?; + digests.extend( + rows.iter() + .filter_map(|row| row["digest"].as_str().map(str::to_owned)), + ); + for table in INVOCATION_TABLES { + (transaction, _) = query( + transaction, + &format!("DELETE FROM {} WHERE {owned}", table_spec(table).name), + bounds.clone(), + ) + .await?; + } + } + let digests: Vec = digests.into_iter().collect(); + for chunk in digests.chunks(RUN_LOG_DIGEST_CHUNK) { + (transaction, _) = query( + transaction, + &format!( + "DELETE FROM recall_artifacts WHERE digest IN ({}) AND NOT EXISTS \ + (SELECT 1 FROM recall_artifact_refs AS ref WHERE ref.digest = recall_artifacts.digest)", + Parameter::placeholders(1..=chunk.len()) + ), + chunk.iter().cloned().map(Parameter::Text).collect(), + ) + .await?; + } + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/runs/models.rs b/crates/sqlbuild-orchestration/src/runs/models.rs index ebeb34b8ca..5a582a2449 100644 --- a/crates/sqlbuild-orchestration/src/runs/models.rs +++ b/crates/sqlbuild-orchestration/src/runs/models.rs @@ -790,3 +790,44 @@ pub enum PersonRunRequest<'a> { (Uuid, &'a str, &'a crate::platform::models::PlanRequest), ), } + +/// Installation configuration: how long a finished run keeps its worker log, by age and by the number of newer finished runs in its location target, whichever comes first. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct RunLogRetention { + days: u16, + count: u16, +} + +impl RunLogRetention { + pub fn new(days: u16, count: u16) -> Result { + if days == 0 || count == 0 { + return Err(crate::store::errors::ControlError::Invalid( + "run log retention must keep at least one day and one run".into(), + )); + } + Ok(Self { days, count }) + } + + pub(crate) fn cutoff(self, now: DateTime) -> DateTime { + now - chrono::TimeDelta::days(i64::from(self.days)) + } + + #[must_use] + pub const fn days(self) -> u16 { + self.days + } + + #[must_use] + pub const fn count(self) -> u16 { + self.count + } +} + +impl Default for RunLogRetention { + fn default() -> Self { + Self { + days: crate::runs::constants::DEFAULT_RUN_LOG_RETENTION_DAYS, + count: crate::runs::constants::DEFAULT_RUN_LOG_RETENTION_COUNT, + } + } +} diff --git a/crates/sqlbuild-orchestration/src/store/main/mod.rs b/crates/sqlbuild-orchestration/src/store/main/mod.rs index c708818031..f3ff29914a 100644 --- a/crates/sqlbuild-orchestration/src/store/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/store/main/mod.rs @@ -15,6 +15,7 @@ pub mod read_recall_view; pub(crate) mod read_records; pub(crate) mod refresh_clock; pub mod set_release_version; +pub(crate) mod store_bytes; pub(crate) mod store_source_text; pub(crate) mod subscribe_changes; pub(crate) mod write; diff --git a/crates/sqlbuild-orchestration/src/store/main/store_bytes.rs b/crates/sqlbuild-orchestration/src/store/main/store_bytes.rs new file mode 100644 index 0000000000..3e7459a3ce --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/main/store_bytes.rs @@ -0,0 +1,22 @@ +//! `store_bytes`: how large the control store's database is on disk. + +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{Connection, ControlTransaction}; + +/// SQLite's main file without its write-ahead log, or the whole PostgreSQL database. +pub(crate) async fn store_bytes( + transaction: ControlTransaction, +) -> Result<(ControlTransaction, i64), ControlError> { + let sql = if matches!(transaction.connection, Some(Connection::Postgres(_))) { + "SELECT pg_database_size(current_database()) AS bytes" + } else { + "SELECT page_count * page_size AS bytes FROM pragma_page_count(), pragma_page_size()" + }; + let (transaction, rows) = query(transaction, sql, Vec::new()).await?; + let bytes = rows + .first() + .and_then(|row| row["bytes"].as_i64()) + .ok_or_else(|| ControlError::Invalid("control store size is unavailable".into()))?; + Ok((transaction, bytes)) +} diff --git a/crates/sqlbuild-orchestration/tests/platform/helpers.rs b/crates/sqlbuild-orchestration/tests/platform/helpers.rs index b5a4e301a4..d658c71fcc 100644 --- a/crates/sqlbuild-orchestration/tests/platform/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/platform/helpers.rs @@ -109,9 +109,14 @@ pub(crate) async fn wait_for_scans( .collect(); cursor = batch.last().map_or(cursor, |row| row.storage_id); scanned.extend(batch.iter().filter_map(|row| { - let PlatformEvent::Heartbeat { replica_id } = row.payload.platform else { + let PlatformEvent::Heartbeat { + replica_id, + store_bytes, + } = row.payload.platform + else { return None; }; + assert!(store_bytes > 0, "a heartbeat reports the store's size"); Some(replica_id) })); observations.extend(batch); diff --git a/crates/sqlbuild-orchestration/tests/platform/test_problems.rs b/crates/sqlbuild-orchestration/tests/platform/test_problems.rs index 6e2ed8a3a8..25afe12c83 100644 --- a/crates/sqlbuild-orchestration/tests/platform/test_problems.rs +++ b/crates/sqlbuild-orchestration/tests/platform/test_problems.rs @@ -212,6 +212,7 @@ fn given_broken_locations_and_late_ticks_when_people_read_problems_then_only_lis PlatformEvent::SchedulerNoLeader, PlatformEvent::Heartbeat { replica_id: Uuid::now_v7(), + store_bytes: 0, }, PlatformEvent::LocationBroken { location_id: locations[1], diff --git a/crates/sqlbuild-orchestration/tests/runs/test_contracts.rs b/crates/sqlbuild-orchestration/tests/runs/test_contracts.rs index 04088727a6..d211383226 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_contracts.rs @@ -36,5 +36,7 @@ mod test_permissions; mod test_requests; #[path = "test_reruns.rs"] mod test_reruns; +#[path = "test_run_logs.rs"] +mod test_run_logs; #[path = "test_types.rs"] pub(crate) mod test_types; diff --git a/crates/sqlbuild-orchestration/tests/runs/test_run_logs.rs b/crates/sqlbuild-orchestration/tests/runs/test_run_logs.rs new file mode 100644 index 0000000000..3d271197d8 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/runs/test_run_logs.rs @@ -0,0 +1,221 @@ +//! Run log retention deletes whole worker logs by age or count and keeps runs and their history. + +use chrono::TimeDelta; +use serde_json::{Value, json}; +use sqlbuild_recall::records::main::artifact_rows::artifact_rows; +use sqlbuild_recall::records::main::event_object_row::event_object_row; +use sqlbuild_recall::records::main::output_row::output_row; +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; +use sqlbuild_recall::records::models::{ + ArtifactInput, OutputChunk, OutputStream, RecallRow, RowEnvelope, +}; +use uuid::Uuid; + +use crate::credentials::tests::location_helpers::create_location; +use crate::runs::main::prune_run_logs::prune_run_logs; +use crate::runs::models::RunLogRetention; +use crate::runs::tests::helpers::{request, request_run, snapshot}; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::append_recall::append_recall; +use crate::store::main::migrate_store::migrate_store; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +const SHARED_SQL: &str = "select order_id from orders"; + +fn envelope(invocation_id: String) -> RowEnvelope { + RowEnvelope { + invocation_id, + invocation_started_at: "2026-01-02T03:04:05Z".into(), + producer_version: "test".into(), + } +} + +fn sql_rows(envelope: &RowEnvelope, sql: &str) -> [RecallRow; 2] { + artifact_rows( + envelope, + &ArtifactInput { + digest: format!("sha256:{}", sha256_hex(sql.as_bytes())), + kind: "sql".into(), + media_type: "text/x-sql".into(), + content: sql.as_bytes().to_vec(), + seen_at: "2026-01-02T03:04:05Z".into(), + ref_id: "statement".into(), + role: "executed_sql".into(), + statement_id: Some("statement".into()), + resource_id: None, + }, + ) +} + +/// The claim's output, and a nested session's event and SQL: one text shared by every run, one its own. +fn worker_log(claim: Uuid, index: u128) -> Vec { + let worker = envelope(format!("worker/{claim}")); + let session = envelope(format!("worker/{claim}/session")); + let mut rows = vec![output_row( + &worker, + &OutputChunk { + sequence: 0, + stream: OutputStream::Stdout, + occurred_at: "2026-01-02T03:04:05Z".into(), + data: b"building orders\n".to_vec(), + byte_offset: 0, + dropped_before: 0, + omitted: None, + }, + )]; + rows.push( + event_object_row( + &session, + json!({"event_id": "completed", "event_type": "statement_completed", "schema_version": 1, + "occurred_at": "2026-01-02T03:04:05Z", "producer": "sqlbuild", "invocation_sequence": 1, + "statement_id": "statement", "payload": {"duration_ms": 1.5}}) + .as_object() + .expect("event object"), + ) + .expect("statement event"), + ); + rows.extend(sql_rows(&session, SHARED_SQL)); + rows.extend(sql_rows(&session, &format!("select {index} as order_id"))); + rows +} + +async fn read(store: &ControlStore, sql: &str) -> Vec { + let transaction = begin(store, TransactionMode::Read).await.expect("begin"); + let (transaction, rows) = query(transaction, sql, Vec::new()).await.expect("query"); + commit(transaction).await.expect("commit"); + rows.into_iter().map(|row| json!(row)).collect() +} + +#[test] +fn given_finished_runs_past_age_or_count_when_logs_are_pruned_then_whole_logs_go_and_runs_stay() { + for backend in [Backend::Sqlite, Backend::Postgres] { + run(async { + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../benchmarks/duckshop/edge_cases/orchestration/run_log_retention.json"); + let case: Value = + serde_json::from_str(&std::fs::read_to_string(path).expect("run log fixture")) + .expect("run log case"); + let Some(fixture) = open_test_store(backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let published = publish_snapshot(store, &snapshot(location)) + .await + .expect("publish"); + let ended = case["ended_hours_ago"].as_array().expect("runs"); + let mut transaction = begin(store, TransactionMode::Write).await.expect("begin"); + let now = transaction.now; + for (position, hours) in ended.iter().enumerate() { + let index = position as u128 + 1; + let run_request = request((location, published.snapshot_id), index, 0, &["orders"]); + let run_id = run_request.run_id; + commit(transaction).await.expect("commit"); + assert!(request_run(store, &run_request).await.expect("request")); + transaction = begin(store, TransactionMode::Write).await.expect("begin"); + let claim = Uuid::new_v5(&run_id, b"run_claimed"); + let (sql, ended_at) = match hours.as_i64() { + Some(hours) => ( + "UPDATE sqb_runs SET status='success', worker_instance=$1, ended_at=$2 WHERE run_id=$3", + Some(Parameter::Time(now - TimeDelta::hours(hours))), + ), + None => ( + "UPDATE sqb_runs SET status='running', worker_instance=$1 WHERE run_id=$2", + None, + ), + }; + let parameters: Vec = [ + Some(Parameter::Id(claim)), + ended_at, + Some(Parameter::Id(run_id)), + ] + .into_iter() + .flatten() + .collect(); + (transaction, _) = query(transaction, sql, parameters) + .await + .expect("finish run"); + let scheduler = event_object_row( + &envelope(format!("scheduler/{run_id}")), + json!({"event_id": format!("queued-{index}"), "event_type": "run_queued", "schema_version": 1, + "occurred_at": "2026-01-02T03:04:05Z", "producer": "scheduler", "invocation_sequence": 1, + "payload": {}}) + .as_object() + .expect("event object"), + ) + .expect("scheduler event"); + for row in worker_log(claim, index).iter().chain([&scheduler]) { + (transaction, _) = append_recall(transaction, row).await.expect("append"); + } + } + commit(transaction).await.expect("commit"); + let nodes = "SELECT COUNT(*) AS count FROM sqb_run_nodes"; + let nodes_before = read(store, nodes).await; + let scheduler = "SELECT COUNT(*) AS count FROM recall_events WHERE invocation_id LIKE 'scheduler/%'"; + let scheduler_before = read(store, scheduler).await; + let retention = RunLogRetention::new( + u16::try_from(case["keep_days"].as_u64().expect("days")).expect("u16"), + u16::try_from(case["keep_runs"].as_u64().expect("runs")).expect("u16"), + ) + .expect("retention"); + + assert_eq!( + prune_run_logs(store, retention).await.expect("prune") as u64, + case["expected_pruned"].as_u64().expect("pruned"), + "{backend:?}" + ); + let logged: Vec = case["expected_logged_runs"] + .as_array() + .expect("logged") + .iter() + .map(|index| { + let run_id = Uuid::from_u128(u128::from(index.as_u64().expect("index"))); + format!("worker/{}", Uuid::new_v5(&run_id, b"run_claimed")) + }) + .collect(); + let mut expected: Vec = logged + .iter() + .flat_map(|worker| [json!(worker), json!(format!("{worker}/session"))]) + .collect(); + expected.sort_by_key(ToString::to_string); + let stored: Vec = read( + store, + "SELECT invocation_id FROM recall_output UNION SELECT invocation_id FROM recall_events \ + WHERE invocation_id LIKE 'worker/%' UNION SELECT invocation_id FROM recall_artifact_refs \ + ORDER BY invocation_id", + ) + .await + .into_iter() + .map(|row| row["invocation_id"].clone()) + .collect(); + assert_eq!( + stored, expected, + "only the kept runs' logs remain, nested sessions included" + ); + assert_eq!( + read(store, "SELECT COUNT(*) AS count FROM recall_artifacts").await[0]["count"], + json!(3), + "the shared SQL and the kept runs' own SQL stay; the pruned runs' own SQL goes" + ); + assert_eq!( + read(store, scheduler).await, + scheduler_before, + "the scheduler's run history stays" + ); + assert_eq!( + read(store, "SELECT COUNT(*) AS count FROM sqb_runs").await[0]["count"], + json!(4) + ); + assert_eq!(read(store, nodes).await, nodes_before, "node results stay"); + assert_eq!(prune_run_logs(store, retention).await.expect("repeat"), 0); + assert!(RunLogRetention::new(0, 1).is_err() && RunLogRetention::new(1, 0).is_err()); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/sensors/helpers.rs b/crates/sqlbuild-orchestration/tests/sensors/helpers.rs index 997f1835e5..334d1b9e2a 100644 --- a/crates/sqlbuild-orchestration/tests/sensors/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/sensors/helpers.rs @@ -60,6 +60,7 @@ pub(crate) async fn append_events( }, PlatformEvent::Heartbeat { replica_id: Uuid::from_u128(8), + store_bytes: 0, }, PlatformEvent::LocationBroken { location_id: Uuid::from_u128(9), @@ -68,6 +69,7 @@ pub(crate) async fn append_events( }, PlatformEvent::Heartbeat { replica_id: Uuid::from_u128(10), + store_bytes: 0, }, PlatformEvent::LaunchFailed { run_id: Uuid::from_u128(11), @@ -76,6 +78,7 @@ pub(crate) async fn append_events( }, PlatformEvent::Heartbeat { replica_id: Uuid::from_u128(13), + store_bytes: 0, }, ]; for index in indices { From 8280b9318ff04e5f9464809131c86611349e02c4 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 13 Nov 2026 20:55:40 +0000 Subject: [PATCH 05/10] docs(control): document run log retention, store size and one-row statements The Settings page shows the control store's size and the run log limits; the control client carries the new fields. --- clients/typescript/src/schema.d.ts | 20 +++++++++++++++++++ ui/src/lib/api/http-people.test.ts | 8 +++++--- ui/src/lib/api/http-people.ts | 3 +++ ui/src/lib/api/mock/platform.ts | 4 +++- ui/src/lib/api/types/platform.ts | 2 ++ ui/src/routes/settings/+page.svelte | 3 ++- website/src/content/docs/docs/cli/server.mdx | 17 ++++++++++++++-- .../docs/concepts/observability/recall.mdx | 6 ++++++ 8 files changed, 56 insertions(+), 7 deletions(-) diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index 2548c57842..d8edb93421 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -1700,6 +1700,11 @@ export interface components { /** @description Digest of the schema; a server starts only on the exact schema it creates. */ schema_checksum: string; storage: components["schemas"]["StorageKind"]; + /** + * Format: int64 + * @description The database's size on disk now: SQLite's main file, or the whole PostgreSQL database. + */ + store_bytes: number; }; /** @description One job or group as declared, for a person who may list the target; free-form setting values are withheld. */ JobDetailRecord: { @@ -2284,6 +2289,11 @@ export interface components { kind: "heartbeat"; /** Format: uuid */ replica_id: string; + /** + * Format: int64 + * @description The control store's database size on disk when the replica checked in. + */ + store_bytes: number; } | { /** @enum {string} */ kind: "scheduler_no_leader"; @@ -2940,6 +2950,16 @@ export interface components { location_health_days: number; /** Format: int64 */ platform_alert_days: number; + /** + * Format: int64 + * @description Days a finished run keeps its log of events, statements and output. + */ + run_log_days: number; + /** + * Format: int64 + * @description Finished runs per location target that keep their logs. + */ + run_logs_kept: number; /** Format: int64 */ sensor_evaluation_days: number; /** diff --git a/ui/src/lib/api/http-people.test.ts b/ui/src/lib/api/http-people.test.ts index c70d6ed5fc..6a3f7d34e2 100644 --- a/ui/src/lib/api/http-people.test.ts +++ b/ui/src/lib/api/http-people.test.ts @@ -43,12 +43,14 @@ describe('toServerSettings', () => { version: '0.205.1', login_mode: 'password', scheduler: false, - installation: { storage: 'sqlite', installed_at: '2026-11-01T00:00:00Z', schema_checksum: 'abc', schema_applied_at: '2026-11-01T00:00:00Z' }, - retention: { audit_days: 400, snapshots_kept: 10, statistics_days: 90, platform_alert_days: 30, location_health_days: 7, sensor_evaluation_days: 6, heartbeat_days: 1 } + installation: { storage: 'sqlite', installed_at: '2026-11-01T00:00:00Z', schema_checksum: 'abc', schema_applied_at: '2026-11-01T00:00:00Z', store_bytes: 52428800 }, + retention: { audit_days: 400, run_log_days: 14, run_logs_kept: 200, snapshots_kept: 10, statistics_days: 90, platform_alert_days: 30, location_health_days: 7, sensor_evaluation_days: 6, heartbeat_days: 1 } }); - expect([settings.authMode, settings.schedulerEnabled, settings.storage]).toEqual(['password', false, 'sqlite']); + expect([settings.authMode, settings.schedulerEnabled, settings.storage, settings.storeBytes]).toEqual(['password', false, 'sqlite', 52428800]); expect(settings.retention.map((r) => [r.name, r.value])).toEqual([ ['Security audit log', '400 days'], + ['Run logs', '14 days'], + ['Logged runs', '200'], ['Published versions', '10'], ['Warehouse row counts', '90 days'], ['Platform problems', '30 days'], diff --git a/ui/src/lib/api/http-people.ts b/ui/src/lib/api/http-people.ts index 5ce06fac92..759f4129c5 100644 --- a/ui/src/lib/api/http-people.ts +++ b/ui/src/lib/api/http-people.ts @@ -68,10 +68,13 @@ export function toServerSettings(wire: Wire['ServerSettings']): ServerSettings { installedAt: wire.installation.installed_at, schemaChecksum: wire.installation.schema_checksum, schemaAppliedAt: wire.installation.schema_applied_at, + storeBytes: wire.installation.store_bytes, authMode: wire.login_mode, schedulerEnabled: wire.scheduler, retention: [ { name: 'Security audit log', value: days(r.audit_days), description: 'Who did what, and every refusal.' }, + { name: 'Run logs', value: days(r.run_log_days), description: 'What finished runs logged. The runs themselves are kept.' }, + { name: 'Logged runs', value: String(r.run_logs_kept), description: 'Newest finished runs per location target that keep their logs.' }, { name: 'Published versions', value: String(r.snapshots_kept), description: 'Kept for each location target; older versions are removed.' }, { name: 'Warehouse row counts', value: days(r.statistics_days), description: 'Row counts and sizes sampled from the warehouse.' }, { name: 'Platform problems', value: days(r.platform_alert_days), description: 'Late schedules, failed launches and locations that stopped checking in.' }, diff --git a/ui/src/lib/api/mock/platform.ts b/ui/src/lib/api/mock/platform.ts index 223fae92da..fcad8859f8 100644 --- a/ui/src/lib/api/mock/platform.ts +++ b/ui/src/lib/api/mock/platform.ts @@ -906,11 +906,13 @@ export class MockPlatformApi implements PlatformApi { installedAt: new Date(this.#ctx.now - 120 * DAY).toISOString(), schemaChecksum: '6f1c2a9e4b7d0c3f8a5e2b1d9c4f7a0e3b6d8c1f5a2e9b4d7c0f3a6e1b8d5c2f', schemaAppliedAt: new Date(this.#ctx.now - 5 * DAY).toISOString(), + storeBytes: 1_288_490_189, authMode: 'oidc', schedulerEnabled: true, retention: [ { name: 'Run history', value: '90 days', description: 'Runs, schedule history and their events.' }, - { name: 'Run logs', value: '30 days', description: 'Captured run output.' }, + { name: 'Run logs', value: '14 days', description: 'What finished runs logged. The runs themselves are kept.' }, + { name: 'Logged runs', value: '200', description: 'Newest finished runs per location target that keep their logs.' }, { name: 'Platform events', value: '30 days', description: 'Problems such as late schedules. Where each sensor is up to is always kept.' }, { name: 'Security audit log', value: '400 days', description: 'Can’t be changed or deleted early. Forward events to keep them longer.' }, { name: 'Warehouse statistics', value: '90 days', description: 'Row counts and table sizes, sampled over time.' }, diff --git a/ui/src/lib/api/types/platform.ts b/ui/src/lib/api/types/platform.ts index 087d374610..a49f1ec624 100644 --- a/ui/src/lib/api/types/platform.ts +++ b/ui/src/lib/api/types/platform.ts @@ -532,6 +532,8 @@ export type ServerSettings = { /** A server starts only on the exact schema it creates, so a running server's schema is always current. */ schemaChecksum: string; schemaAppliedAt: Iso; + /** The control store's database size on disk. */ + storeBytes: number; authMode: AuthMode; /** Whether this server runs the scheduler; another server may hold the lease. */ schedulerEnabled: boolean; diff --git a/ui/src/routes/settings/+page.svelte b/ui/src/routes/settings/+page.svelte index 8165a8a8de..f8e9c2d011 100644 --- a/ui/src/routes/settings/+page.svelte +++ b/ui/src/routes/settings/+page.svelte @@ -7,6 +7,7 @@ import Panel from '$lib/components/platform/panel.svelte'; import type { AuthMode } from '$lib/api'; import { PERMISSIONS_BY_RISK, RISK_LABEL } from '$lib/platform/permission-catalog'; + import { formatBytes } from '$lib/build-area/labels'; import { formatAgo, formatStamp } from '$lib/format/time'; import { app } from '$lib/state/app.svelte'; import { createResource } from '$lib/state/resource.svelte'; @@ -36,7 +37,7 @@ Date: Fri, 13 Nov 2026 21:00:22 +0000 Subject: [PATCH 06/10] test(recall): check stored and merged lifecycle payloads on both control databases and run log flag limits --- .../tests/store/recall/test_views.rs | 35 ++++++++++++++++++- .../publication/test_retention.py | 10 ++++++ 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/crates/sqlbuild-orchestration/tests/store/recall/test_views.rs b/crates/sqlbuild-orchestration/tests/store/recall/test_views.rs index a157ecf787..abfd6645c9 100644 --- a/crates/sqlbuild-orchestration/tests/store/recall/test_views.rs +++ b/crates/sqlbuild-orchestration/tests/store/recall/test_views.rs @@ -13,7 +13,7 @@ use sqlbuild_recall::records::models::{ ArtifactInput, DeliveryCounts, InvocationContext, OutputChunk, OutputStream, RowEnvelope, }; -use crate::store::tests::helpers::publish_recall; +use crate::store::tests::helpers::{publish_recall, read_recall_events}; use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; use crate::store::tests::store_test_types::Backend; use crate::store::tests::test_types::RecallViewTestCase; @@ -110,6 +110,9 @@ fn given_native_context_output_and_audits_when_reading_views_then_both_databases ] { rows.push(event_object_row(&envelope, json!({"event_id": event_id, "event_type": event_type, "schema_version": 1, "occurred_at": "2026-01-02T00:00:03Z", "producer": "worker", "invocation_sequence": 1, "payload": payload}).as_object().expect("event object")).expect("lifecycle fact")); } + // SQLBuild's own events are stored without the fields their columns hold; the view puts them back. + let written = json!({"intent": "execute", "statement_kind": "SELECT", "affected_rows": 3, "duration_ms": 1.5, "query_id": null, "adapter": "duckdb"}); + rows.push(event_object_row(&envelope, json!({"event_id": "orders-statement", "event_type": "statement_completed", "schema_version": 1, "occurred_at": "2026-01-02T00:00:03Z", "producer": "sqlbuild", "invocation_sequence": 2, "statement_id": "orders-statement", "payload": written}).as_object().expect("event object")).expect("statement fact")); rows.extend(artifact_rows( &envelope, &ArtifactInput { @@ -158,6 +161,36 @@ fn given_native_context_output_and_audits_when_reading_views_then_both_databases .expect("invocation projection"); assert_eq!(invocation[0]["status"], "completed"); assert_eq!(invocation[0]["duration_ms"], json!(12.75)); + let payload = |value: &serde_json::Value| match value { + serde_json::Value::String(text) => { + serde_json::from_str(text).expect("JSON payload") + } + other => other.clone(), + }; + let events = read_recall_view(store, ("events", &envelope.invocation_id), 100) + .await + .expect("events view"); + let statement = events + .iter() + .find(|event| event["event_type"] == "statement_completed") + .expect("statement event"); + assert_eq!( + payload(&statement["payload"]), + written, + "the events view returns the payload as written" + ); + let stored = read_recall_events(store, 0, 100) + .await + .expect("stored rows"); + let stored = stored + .iter() + .find(|row| row["event_type"] == "statement_completed") + .expect("stored statement"); + assert_eq!( + payload(&stored["payload"]), + json!({"query_id": null, "adapter": "duckdb"}), + "the table keeps only fields without a column" + ); assert_eq!(invocation[0]["integration_retry"], json!(2)); let audit = read_recall_view(store, ("audit_results", &envelope.invocation_id), 100) .await diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_retention.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_retention.py index cd62a882c6..655cb468b2 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_retention.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_retention.py @@ -102,6 +102,16 @@ def test_given_expired_audit_rows_when_server_runs_then_only_delivered_rows_are_ ("--snapshot-retention-count", "0"), "--snapshot-retention-count must be between 1 and 65535", ), + RetentionRejectionTestCase( + "Zero run log days", + ("--run-log-retention-days", "0"), + "--run-log-retention-days must be between 1 and 65535", + ), + RetentionRejectionTestCase( + "Run log count beyond the supported range", + ("--run-log-retention-count", "65536"), + "--run-log-retention-count must be between 1 and 65535", + ), ], ids=lambda case: case.description, ) From ee2a08c524c1fb27844805e5d725c7b545fbad65 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 13 Nov 2026 21:13:02 +0000 Subject: [PATCH 07/10] fix(control): name the store and the command that recreates it when it predates this release A control store created by an earlier release failed with a generic checksum message. The error now names the SQLite files or the PostgreSQL database and prints the one command that discards them, so the next start creates a new store. BREAKING CHANGE: control stores created by earlier releases are not upgraded, so sqb server and sqb dev start with no run history after upgrading. The server refuses an old store and prints the command that recreates it: for sqb dev, rm -f .sqlbuild/control.sqlite3 .sqlbuild/control.sqlite3-wal .sqlbuild/control.sqlite3-shm in the project; for a PostgreSQL control database, dropdb and createdb on that database. --- .../src/store/_helpers/migration.rs | 45 +++++++++++++++---- .../src/store/main/migrate_store.rs | 2 +- .../tests/snapshots/helpers.rs | 7 ++- 3 files changed, 43 insertions(+), 11 deletions(-) diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs index a572753c5a..148f3cc735 100644 --- a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs +++ b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs @@ -13,19 +13,46 @@ pub(crate) fn schema_digest(store: &ControlStore) -> String { sha256_hex(format!("{}\n{SCHEMA}", recall_schema(store).join(";\n")).as_bytes()) } -/// Accept only the exact schema this server creates. -pub(crate) fn verify_schema(rows: &[Row], digest: &str) -> Result<(), ControlError> { +/// Accept only the exact schema this server creates, or say how to start a new store. +pub(crate) fn verify_schema( + store: &ControlStore, + rows: &[Row], + digest: &str, +) -> Result<(), ControlError> { if rows .iter() - .any(|row| row.get("checksum").and_then(|value| value.as_str()) != Some(digest)) + .all(|row| row.get("checksum").and_then(|value| value.as_str()) == Some(digest)) { - return Err(ControlError::Schema( - "control store schema checksum mismatch; recreate the control store, because \ - pre-release schemas are not upgraded" - .into(), - )); + return Ok(()); } - Ok(()) + let quote = |text: &str| format!("'{}'", text.replace('\'', r"'\''")); + let command = match store { + ControlStore::Sqlite(path) => { + let path = path.to_string_lossy(); + format!( + "rm -f {} {} {}", + quote(&path), + quote(&format!("{path}-wal")), + quote(&format!("{path}-shm")) + ) + } + ControlStore::Postgres(pool) => { + let options = pool.connect_options(); + let target = format!( + "-h {} -p {} -U {} {}", + quote(options.get_host()), + options.get_port(), + quote(options.get_username()), + quote(options.get_database().unwrap_or(options.get_username())) + ); + format!("dropdb {target} && createdb {target}") + } + }; + Err(ControlError::Schema(format!( + "this control store was created by an earlier sqb release and cannot be upgraded; \ + to start a new, empty store, which discards its run history, stop every server \ + using it and run: {command}" + ))) } pub(crate) fn render(store: &ControlStore, sql: &str) -> String { diff --git a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs index 973d70e069..ed230c65a9 100644 --- a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs +++ b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs @@ -23,7 +23,7 @@ pub async fn migrate_store(store: &ControlStore) -> Result<(), ControlError> { ) .await?; let digest = schema_digest(store); - verify_schema(&rows, &digest)?; + verify_schema(store, &rows, &digest)?; if rows.is_empty() { for statement in recall_schema(store) { (transaction, _) = query(transaction, &statement, vec![]).await?; diff --git a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs index 778cfb2028..7497a5a98c 100644 --- a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs @@ -310,9 +310,14 @@ pub(super) async fn sqlite_failed_publish_and_schema_floor() -> i64 { connection .execute_batch(case.update_sql) .expect("apply synthetic schema incompatibility"); + let ControlStore::Sqlite(path) = &store else { + unreachable!("snapshot recovery runs on SQLite") + }; + let path = path.to_string_lossy(); assert!( matches!(migrate_store(&store).await, - Err(ControlError::Schema(message)) if message.contains("recreate the control store")), + Err(ControlError::Schema(message)) + if message.contains(&format!("run: rm -f '{path}' '{path}-wal' '{path}-shm'"))), "{}", case.description ); From 8b4f6753fa5a16f84196aa19a151dcf6395e322e Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 13 Nov 2026 21:32:57 +0000 Subject: [PATCH 08/10] refactor(control): keep run log pruning and store size with the maintenance role Run log pruning and the store size read move into the platform helpers that use them, retention settings travel as one named struct, and the new tests follow the parametrized, branch-free test layout. --- .../src/native_commands/_helpers/execution.rs | 17 +- .../src/native_commands/main/_run_control.rs | 13 +- .../src/platform/_helpers/retention.rs | 118 ++++++++- .../src/platform/_helpers/watchdog.rs | 22 +- .../src/platform/main/read_installation.rs | 2 +- .../src/platform/main/run_maintenance.rs | 15 +- .../src/platform/models.rs | 17 +- .../src/runs/main/mod.rs | 1 - .../src/runs/main/prune_run_logs.rs | 104 -------- .../src/store/main/mod.rs | 1 - .../src/store/main/store_bytes.rs | 22 -- .../warehouse_history/_helpers/run_recall.rs | 2 +- .../tests/runs/helpers.rs | 126 ++++++++++ .../tests/runs/test_run_logs.rs | 231 ++++++------------ .../tests/runs/test_types.rs | 7 + .../tests/store/recall/test_views.rs | 7 +- .../src/schema/_helpers/relational_views.rs | 7 +- .../sqlbuild-recall/src/schema/constants.rs | 5 +- .../src/store/main/relational_row.rs | 36 ++- .../src/store/tests/test_types.rs | 6 + .../src/store/tests/test_views.rs | 72 +++--- .../src/writer/_helpers/materialize.rs | 17 +- 22 files changed, 453 insertions(+), 395 deletions(-) delete mode 100644 crates/sqlbuild-orchestration/src/runs/main/prune_run_logs.rs delete mode 100644 crates/sqlbuild-orchestration/src/store/main/store_bytes.rs diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs index b5d7568a05..a091345c3f 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs @@ -21,20 +21,17 @@ use sqlbuild_config::project::models::ControlConfig; use sqlbuild_orchestration::api::models::{ ApiAuthentication, DevelopmentScope, HumanLogin, ServerFacts, }; -use sqlbuild_orchestration::credentials::models::AuditRetention; use sqlbuild_orchestration::platform::constants::PLATFORM_WEBHOOK_VARIABLE; use sqlbuild_orchestration::platform::main::report_remote_location_health::report_remote_location_health; use sqlbuild_orchestration::platform::main::run_maintenance::run_maintenance; use sqlbuild_orchestration::platform::models::{ - LocationHealth, LocationHealthReport, LocationWorkClient, LocationWorkPoll, + LocationHealth, LocationHealthReport, LocationWorkClient, LocationWorkPoll, RetentionPolicies, WarehouseSyncFailure, }; -use sqlbuild_orchestration::runs::models::RunLogRetention; use sqlbuild_orchestration::runs::types::RunHandoff; use sqlbuild_orchestration::scheduler::main::run_scheduler::run_scheduler; use sqlbuild_orchestration::scheduler::models::SchedulerState; use sqlbuild_orchestration::snapshots::main::next_remote_generation::next_remote_generation; -use sqlbuild_orchestration::snapshots::models::SnapshotRetention; use sqlbuild_orchestration::store::errors::ControlError; use sqlbuild_orchestration::store::main::migrate_store::migrate_store; use sqlbuild_orchestration::store::main::open_store::open_store; @@ -42,9 +39,8 @@ use sqlbuild_orchestration::store::main::set_release_version::set_release_versio use sqlbuild_orchestration::store::models::ControlStore; use sqlbuild_orchestration::warehouse_metadata::constants::MAX_HISTORY_RECORDS; use sqlbuild_orchestration::warehouse_metadata::models::{ - MetadataBatch, MetadataScan, StatisticsDueRead, StatisticsRead, StatisticsRetention, - WarehouseHistoryKind, WarehouseHistoryPage, WarehouseStateLocation, WarehouseSyncDueRead, - WarehouseWatermarkRead, + MetadataBatch, MetadataScan, StatisticsDueRead, StatisticsRead, WarehouseHistoryKind, + WarehouseHistoryPage, WarehouseStateLocation, WarehouseSyncDueRead, WarehouseWatermarkRead, }; use sqlbuild_orchestration::webhooks::constants::SECURITY_AUDIT_WEBHOOK_VARIABLE; use sqlbuild_orchestration::webhooks::main::{ @@ -892,12 +888,7 @@ pub(crate) fn location_child(options: &LocationServe, script: &Path) -> Command pub(crate) fn start_platform_roles( tasks: &mut JoinSet>, (store, shutdown): (&ControlStore, &watch::Receiver), - retention: ( - AuditRetention, - StatisticsRetention, - SnapshotRetention, - RunLogRetention, - ), + retention: RetentionPolicies, ) -> Result<(), ControlError> { let mut sinks: Vec<(Webhook, &str)> = Vec::new(); for (variable, source, label) in [ diff --git a/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs b/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs index a85da56a3c..32fdc07306 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs @@ -1,6 +1,7 @@ //! Supervise native API/scheduler roles together; a failed role stops its siblings. use sqlbuild_orchestration::api::main::serve_api::serve_api; +use sqlbuild_orchestration::platform::models::RetentionPolicies; use sqlbuild_orchestration::scheduler::main::run_scheduler::run_scheduler; use sqlbuild_orchestration::scheduler::models::SchedulerState; use sqlbuild_orchestration::store::errors::ControlError; @@ -56,12 +57,12 @@ pub(crate) async fn run_control( start_platform_roles( &mut tasks, (&store, &shutdown), - ( - facts.audit, - facts.statistics, - facts.snapshots, - facts.run_logs, - ), + RetentionPolicies { + audit: facts.audit, + statistics: facts.statistics, + snapshots: facts.snapshots, + run_logs: facts.run_logs, + }, )?; } if facts.scheduler { diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/retention.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/retention.rs index 84941a898c..fe24739a7c 100644 --- a/crates/sqlbuild-orchestration/src/platform/_helpers/retention.rs +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/retention.rs @@ -1,23 +1,25 @@ //! Every retention policy runs in one bounded pass; each part is safe to repeat. +use std::collections::BTreeSet; + use chrono::{DateTime, TimeDelta, Utc}; +use sqlbuild_recall::schema::constants::INVOCATION_TABLES; +use sqlbuild_recall::schema::main::table_spec::table_spec; use crate::credentials::main::audit::prune_audit::prune_audit; -use crate::credentials::models::AuditRetention; use crate::platform::constants::{ HEARTBEAT_PAGE, HEARTBEAT_RETENTION_DAYS, LOCATION_RETENTION_DAYS, PLATFORM_ALERT_RETENTION_DAYS, SENSOR_EVALUATION_RETENTION_DAYS, }; -use crate::runs::main::prune_run_logs::prune_run_logs; +use crate::platform::models::RetentionPolicies; +use crate::runs::constants::{RUN_LOG_DIGEST_CHUNK, RUN_LOG_RETENTION_BATCH}; use crate::runs::models::RunLogRetention; use crate::snapshots::main::prune_snapshots::prune_snapshots; -use crate::snapshots::models::SnapshotRetention; use crate::store::errors::ControlError; -use crate::store::main::{query::query, write::write}; -use crate::store::models::{ControlStore, Parameter}; +use crate::store::main::{begin::begin, commit::commit, query::query, write::write}; +use crate::store::models::{ControlStore, ControlTransaction, Parameter, TransactionMode}; use crate::warehouse_metadata::constants::MAX_HISTORY_RECORDS; use crate::warehouse_metadata::main::ingest::prune_statistics::prune_statistics; -use crate::warehouse_metadata::models::StatisticsRetention; /// Old evaluation phases go once a later attempt of the same sensor exists. const SENSOR_EVALUATIONS: &str = "DELETE FROM sqb_sensor_evaluation_events WHERE event_id IN (SELECT e.event_id \ @@ -39,15 +41,25 @@ const LOCATION_INSTANCES: &str = "DELETE FROM sqb_location_instances WHERE insta FROM sqb_location_work w WHERE w.session_id=i.instance_id) \ ORDER BY i.last_seen_at LIMIT $2) RETURNING instance_id"; +/// Claims of finished runs past either limit whose worker log is still stored, oldest first. Worker invocations are `worker/` and `worker//`; the stored logs, not every run ever finished, bound the join. +const EXPIRED_CLAIMS: &str = "WITH logged AS (\ + SELECT DISTINCT substr(invocation_id, 8, 36) AS claim FROM recall_invocations WHERE invocation_id LIKE 'worker/%' \ + UNION SELECT DISTINCT substr(invocation_id, 8, 36) AS claim FROM recall_output WHERE invocation_id LIKE 'worker/%'), \ + ranked AS (SELECT worker_instance, ended_at, ROW_NUMBER() OVER (PARTITION BY location_id, target \ + ORDER BY ended_at DESC, run_id DESC) AS newest FROM sqb_runs \ + WHERE ended_at IS NOT NULL AND status IN ('success', 'warn', 'partial', 'failed', 'cancelled')) \ + SELECT logged.claim FROM ranked JOIN logged ON logged.claim = CAST(ranked.worker_instance AS TEXT) \ + WHERE ranked.newest > $1 OR ranked.ended_at < $2 ORDER BY ranked.ended_at LIMIT $3"; + /// Returns how many audit rows outlive their horizon because a sink has not acknowledged them. pub(crate) async fn retain( store: &ControlStore, - (audit, statistics, snapshots, run_logs): ( - AuditRetention, - StatisticsRetention, - SnapshotRetention, - RunLogRetention, - ), + RetentionPolicies { + audit, + statistics, + snapshots, + run_logs, + }: RetentionPolicies, sinks: &[String], ) -> Result { prune_run_logs(store, run_logs).await?; @@ -99,3 +111,85 @@ async fn drain( tokio::task::yield_now().await; } } + +/// A finished run's worker log goes whole once it is too old or too many newer runs finished; the run, its nodes and the scheduler's node history stay. Batches commit independently; a repeated or interrupted pass finds only logs still stored. +pub(crate) async fn prune_run_logs( + store: &ControlStore, + retention: RunLogRetention, +) -> Result { + let mut deleted = 0; + loop { + let transaction = begin(store, TransactionMode::Write).await?; + let now = transaction.now; + let (transaction, rows) = query( + transaction, + EXPIRED_CLAIMS, + vec![ + Parameter::Integer(i64::from(retention.count())), + Parameter::Time(retention.cutoff(now)), + Parameter::Integer(RUN_LOG_RETENTION_BATCH as i64), + ], + ) + .await?; + let claims: Vec = rows + .iter() + .filter_map(|row| row["claim"].as_str().map(str::to_owned)) + .collect(); + let transaction = delete_logs(transaction, &claims).await?; + commit(transaction).await?; + deleted += claims.len(); + if claims.len() < RUN_LOG_RETENTION_BATCH { + return Ok(deleted); + } + } +} + +/// Every Recall row of the claims' invocations, then stored artifacts no remaining reference names. `worker//` sorts between `worker//` and `worker/0`, so a key range finds nested invocations by index. +async fn delete_logs( + mut transaction: ControlTransaction, + claims: &[String], +) -> Result { + let mut digests: BTreeSet = BTreeSet::new(); + for claim in claims { + let namespace = format!("worker/{claim}"); + let bounds = vec![ + Parameter::Text(namespace.clone()), + Parameter::Text(format!("{namespace}/")), + Parameter::Text(format!("{namespace}0")), + ]; + let owned = "(invocation_id=$1 OR (invocation_id>$2 AND invocation_id<$3))"; + let rows; + (transaction, rows) = query( + transaction, + &format!("SELECT DISTINCT digest FROM recall_artifact_refs WHERE {owned}"), + bounds.clone(), + ) + .await?; + digests.extend( + rows.iter() + .filter_map(|row| row["digest"].as_str().map(str::to_owned)), + ); + for table in INVOCATION_TABLES { + (transaction, _) = query( + transaction, + &format!("DELETE FROM {} WHERE {owned}", table_spec(table).name), + bounds.clone(), + ) + .await?; + } + } + let digests: Vec = digests.into_iter().collect(); + for chunk in digests.chunks(RUN_LOG_DIGEST_CHUNK) { + (transaction, _) = query( + transaction, + &format!( + "DELETE FROM recall_artifacts WHERE digest IN ({}) AND NOT EXISTS \ + (SELECT 1 FROM recall_artifact_refs AS ref WHERE ref.digest = recall_artifacts.digest)", + Parameter::placeholders(1..=chunk.len()) + ), + chunk.iter().cloned().map(Parameter::Text).collect(), + ) + .await?; + } + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/watchdog.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/watchdog.rs index c310f512a1..2b5d0ed596 100644 --- a/crates/sqlbuild-orchestration/src/platform/_helpers/watchdog.rs +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/watchdog.rs @@ -13,8 +13,9 @@ use crate::store::main::begin::begin; use crate::store::main::commit::commit; use crate::store::main::decode_row::decode_row; use crate::store::main::query::query; -use crate::store::main::store_bytes::store_bytes; -use crate::store::models::{ControlStore, Parameter, TransactionMode}; +use crate::store::models::{ + Connection, ControlStore, ControlTransaction, Parameter, TransactionMode, +}; pub(crate) async fn scan_health( store: &ControlStore, @@ -142,3 +143,20 @@ async fn scan_trigger_page( commit(transaction).await?; Ok(next) } + +/// SQLite's main file without its write-ahead log, or the whole PostgreSQL database. +pub(crate) async fn store_bytes( + transaction: ControlTransaction, +) -> Result<(ControlTransaction, i64), ControlError> { + let sql = if matches!(transaction.connection, Some(Connection::Postgres(_))) { + "SELECT pg_database_size(current_database()) AS bytes" + } else { + "SELECT page_count * page_size AS bytes FROM pragma_page_count(), pragma_page_size()" + }; + let (transaction, rows) = query(transaction, sql, Vec::new()).await?; + let bytes = rows + .first() + .and_then(|row| row["bytes"].as_i64()) + .ok_or_else(|| ControlError::Invalid("control store size is unavailable".into()))?; + Ok((transaction, bytes)) +} diff --git a/crates/sqlbuild-orchestration/src/platform/main/read_installation.rs b/crates/sqlbuild-orchestration/src/platform/main/read_installation.rs index 4411c1a33b..4c4adb745e 100644 --- a/crates/sqlbuild-orchestration/src/platform/main/read_installation.rs +++ b/crates/sqlbuild-orchestration/src/platform/main/read_installation.rs @@ -4,9 +4,9 @@ use uuid::Uuid; use crate::authorization::main::begin_authorized::begin_authorized; use crate::credentials::models::{Action, AuditContext, DefaultRole, PermissionScope}; +use crate::platform::_helpers::watchdog::store_bytes; use crate::platform::models::{InstallationRecord, StorageKind, StoredInstallation}; use crate::store::errors::ControlError; -use crate::store::main::store_bytes::store_bytes; use crate::store::main::{commit::commit, query_records::query_records}; use crate::store::models::{ControlStore, ControlTransaction}; diff --git a/crates/sqlbuild-orchestration/src/platform/main/run_maintenance.rs b/crates/sqlbuild-orchestration/src/platform/main/run_maintenance.rs index ec1517789f..083b21e08a 100644 --- a/crates/sqlbuild-orchestration/src/platform/main/run_maintenance.rs +++ b/crates/sqlbuild-orchestration/src/platform/main/run_maintenance.rs @@ -5,28 +5,17 @@ use std::time::{Duration, Instant}; use tokio::sync::watch; use uuid::Uuid; -use crate::credentials::models::AuditRetention; use crate::platform::_helpers::retention::retain; use crate::platform::_helpers::watchdog::scan_health; -use crate::runs::models::RunLogRetention; -use crate::snapshots::models::SnapshotRetention; +use crate::platform::models::RetentionPolicies; use crate::store::errors::ControlError; use crate::store::models::ControlStore; -use crate::warehouse_metadata::models::StatisticsRetention; use crate::webhooks::models::{Webhook, WebhookSource}; /// Only this server's security-audit sinks acknowledge; `report` hears each new retained count. pub async fn run_maintenance( (store, replica_id): (&ControlStore, Uuid), - (retention, sinks): ( - ( - AuditRetention, - StatisticsRetention, - SnapshotRetention, - RunLogRetention, - ), - &[Webhook], - ), + (retention, sinks): (RetentionPolicies, &[Webhook]), (report, mut shutdown): (impl Fn(i64), watch::Receiver), ) -> Result<(), ControlError> { let sinks: Vec = sinks diff --git a/crates/sqlbuild-orchestration/src/platform/models.rs b/crates/sqlbuild-orchestration/src/platform/models.rs index cb584da5b5..b34c046cdb 100644 --- a/crates/sqlbuild-orchestration/src/platform/models.rs +++ b/crates/sqlbuild-orchestration/src/platform/models.rs @@ -5,10 +5,23 @@ use serde::{Deserialize, Serialize}; use uuid::Uuid; use crate::api::models::ControlClient; +use crate::credentials::models::AuditRetention; use crate::declarations::models::TriggerDefinition; -use crate::runs::models::{RunKind, RunNodeFailure, RunOutcome}; +use crate::runs::models::{RunKind, RunLogRetention, RunNodeFailure, RunOutcome}; use crate::sensors::models::{CustomSensorOutcome, SensorEvaluationRequest}; -use crate::warehouse_metadata::models::{WarehouseHistoryKind, WarehouseStateLocation}; +use crate::snapshots::models::SnapshotRetention; +use crate::warehouse_metadata::models::{ + StatisticsRetention, WarehouseHistoryKind, WarehouseStateLocation, +}; + +/// Every retention horizon one server's maintenance role enforces. +#[derive(Clone, Copy, Debug, Default)] +pub struct RetentionPolicies { + pub audit: AuditRetention, + pub statistics: StatisticsRetention, + pub snapshots: SnapshotRetention, + pub run_logs: RunLogRetention, +} /// One pooled outbound connection; the location never receives a store handle. #[derive(Clone, Debug)] diff --git a/crates/sqlbuild-orchestration/src/runs/main/mod.rs b/crates/sqlbuild-orchestration/src/runs/main/mod.rs index 4a88c6893b..a062e002e1 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/mod.rs @@ -11,7 +11,6 @@ pub mod list_run_nodes; pub mod list_runs; pub mod monitor_runs; pub mod observe_run; -pub(crate) mod prune_run_logs; pub mod publish_run_output; pub mod publish_run_recall; pub(crate) mod queue_run; diff --git a/crates/sqlbuild-orchestration/src/runs/main/prune_run_logs.rs b/crates/sqlbuild-orchestration/src/runs/main/prune_run_logs.rs deleted file mode 100644 index 92a2e02d22..0000000000 --- a/crates/sqlbuild-orchestration/src/runs/main/prune_run_logs.rs +++ /dev/null @@ -1,104 +0,0 @@ -//! A finished run's worker log goes whole once it is too old or too many newer runs finished; the run, its nodes and the scheduler's node history stay. - -use std::collections::BTreeSet; - -use sqlbuild_recall::schema::constants::INVOCATION_TABLES; -use sqlbuild_recall::schema::main::table_spec::table_spec; - -use crate::runs::constants::{RUN_LOG_DIGEST_CHUNK, RUN_LOG_RETENTION_BATCH}; -use crate::runs::models::RunLogRetention; -use crate::store::errors::ControlError; -use crate::store::main::{begin::begin, commit::commit, query::query}; -use crate::store::models::{ControlStore, ControlTransaction, Parameter, TransactionMode}; - -/// Claims of finished runs past either limit whose worker log is still stored, oldest first. Worker invocations are `worker/` and `worker//`; the stored logs, not every run ever finished, bound the join. -const EXPIRED_CLAIMS: &str = "WITH logged AS (\ - SELECT DISTINCT substr(invocation_id, 8, 36) AS claim FROM recall_invocations WHERE invocation_id LIKE 'worker/%' \ - UNION SELECT DISTINCT substr(invocation_id, 8, 36) AS claim FROM recall_output WHERE invocation_id LIKE 'worker/%'), \ - ranked AS (SELECT worker_instance, ended_at, ROW_NUMBER() OVER (PARTITION BY location_id, target \ - ORDER BY ended_at DESC, run_id DESC) AS newest FROM sqb_runs \ - WHERE ended_at IS NOT NULL AND status IN ('success', 'warn', 'partial', 'failed', 'cancelled')) \ - SELECT logged.claim FROM ranked JOIN logged ON logged.claim = CAST(ranked.worker_instance AS TEXT) \ - WHERE ranked.newest > $1 OR ranked.ended_at < $2 ORDER BY ranked.ended_at LIMIT $3"; - -/// Batches commit independently; a repeated or interrupted pass finds only logs still stored. -pub(crate) async fn prune_run_logs( - store: &ControlStore, - retention: RunLogRetention, -) -> Result { - let mut deleted = 0; - loop { - let transaction = begin(store, TransactionMode::Write).await?; - let now = transaction.now; - let (transaction, rows) = query( - transaction, - EXPIRED_CLAIMS, - vec![ - Parameter::Integer(i64::from(retention.count())), - Parameter::Time(retention.cutoff(now)), - Parameter::Integer(RUN_LOG_RETENTION_BATCH as i64), - ], - ) - .await?; - let claims: Vec = rows - .iter() - .filter_map(|row| row["claim"].as_str().map(str::to_owned)) - .collect(); - let transaction = delete_logs(transaction, &claims).await?; - commit(transaction).await?; - deleted += claims.len(); - if claims.len() < RUN_LOG_RETENTION_BATCH { - return Ok(deleted); - } - } -} - -/// Every Recall row of the claims' invocations, then stored artifacts no remaining reference names. `worker//` sorts between `worker//` and `worker/0`, so a key range finds nested invocations by index. -async fn delete_logs( - mut transaction: ControlTransaction, - claims: &[String], -) -> Result { - let mut digests: BTreeSet = BTreeSet::new(); - for claim in claims { - let namespace = format!("worker/{claim}"); - let bounds = vec![ - Parameter::Text(namespace.clone()), - Parameter::Text(format!("{namespace}/")), - Parameter::Text(format!("{namespace}0")), - ]; - let owned = "(invocation_id=$1 OR (invocation_id>$2 AND invocation_id<$3))"; - let rows; - (transaction, rows) = query( - transaction, - &format!("SELECT DISTINCT digest FROM recall_artifact_refs WHERE {owned}"), - bounds.clone(), - ) - .await?; - digests.extend( - rows.iter() - .filter_map(|row| row["digest"].as_str().map(str::to_owned)), - ); - for table in INVOCATION_TABLES { - (transaction, _) = query( - transaction, - &format!("DELETE FROM {} WHERE {owned}", table_spec(table).name), - bounds.clone(), - ) - .await?; - } - } - let digests: Vec = digests.into_iter().collect(); - for chunk in digests.chunks(RUN_LOG_DIGEST_CHUNK) { - (transaction, _) = query( - transaction, - &format!( - "DELETE FROM recall_artifacts WHERE digest IN ({}) AND NOT EXISTS \ - (SELECT 1 FROM recall_artifact_refs AS ref WHERE ref.digest = recall_artifacts.digest)", - Parameter::placeholders(1..=chunk.len()) - ), - chunk.iter().cloned().map(Parameter::Text).collect(), - ) - .await?; - } - Ok(transaction) -} diff --git a/crates/sqlbuild-orchestration/src/store/main/mod.rs b/crates/sqlbuild-orchestration/src/store/main/mod.rs index f3ff29914a..c708818031 100644 --- a/crates/sqlbuild-orchestration/src/store/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/store/main/mod.rs @@ -15,7 +15,6 @@ pub mod read_recall_view; pub(crate) mod read_records; pub(crate) mod refresh_clock; pub mod set_release_version; -pub(crate) mod store_bytes; pub(crate) mod store_source_text; pub(crate) mod subscribe_changes; pub(crate) mod write; diff --git a/crates/sqlbuild-orchestration/src/store/main/store_bytes.rs b/crates/sqlbuild-orchestration/src/store/main/store_bytes.rs deleted file mode 100644 index 3e7459a3ce..0000000000 --- a/crates/sqlbuild-orchestration/src/store/main/store_bytes.rs +++ /dev/null @@ -1,22 +0,0 @@ -//! `store_bytes`: how large the control store's database is on disk. - -use crate::store::errors::ControlError; -use crate::store::main::query::query; -use crate::store::models::{Connection, ControlTransaction}; - -/// SQLite's main file without its write-ahead log, or the whole PostgreSQL database. -pub(crate) async fn store_bytes( - transaction: ControlTransaction, -) -> Result<(ControlTransaction, i64), ControlError> { - let sql = if matches!(transaction.connection, Some(Connection::Postgres(_))) { - "SELECT pg_database_size(current_database()) AS bytes" - } else { - "SELECT page_count * page_size AS bytes FROM pragma_page_count(), pragma_page_size()" - }; - let (transaction, rows) = query(transaction, sql, Vec::new()).await?; - let bytes = rows - .first() - .and_then(|row| row["bytes"].as_i64()) - .ok_or_else(|| ControlError::Invalid("control store size is unavailable".into()))?; - Ok((transaction, bytes)) -} diff --git a/crates/sqlbuild-orchestration/src/warehouse_history/_helpers/run_recall.rs b/crates/sqlbuild-orchestration/src/warehouse_history/_helpers/run_recall.rs index 5282f763ae..e8fbfca2c9 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_history/_helpers/run_recall.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_history/_helpers/run_recall.rs @@ -195,6 +195,7 @@ pub(crate) async fn run_statements_in( Ok((transaction, records)) } +/// One artifact reference names each digest per invocation, so a statement finds its SQL by digest. pub(crate) async fn statement_sql_in( transaction: ControlTransaction, scope: &RunRecallScope, @@ -212,7 +213,6 @@ pub(crate) async fn statement_sql_in( let Some(statement) = statements.first() else { return Ok((transaction, None)); }; - // One reference names each digest per invocation, so a statement finds its SQL by digest. parameters[next - 1] = Parameter::Text(format!( "sha256:{}", statement["sql_digest"].as_str().unwrap_or_default() diff --git a/crates/sqlbuild-orchestration/tests/runs/helpers.rs b/crates/sqlbuild-orchestration/tests/runs/helpers.rs index 5dec2a4ee4..1e5c296293 100644 --- a/crates/sqlbuild-orchestration/tests/runs/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/runs/helpers.rs @@ -20,9 +20,19 @@ use crate::store::main::begin::begin; use crate::store::main::commit::commit; use crate::store::main::query::query; use crate::store::models::{ControlStore, Parameter, TransactionMode}; +use chrono::TimeDelta; use serde_json::{Value, json}; +use sqlbuild_recall::records::main::artifact_rows::artifact_rows; +use sqlbuild_recall::records::main::event_object_row::event_object_row; +use sqlbuild_recall::records::main::output_row::output_row; +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; +use sqlbuild_recall::records::models::{ + ArtifactInput, OutputChunk, OutputStream, RecallRow, RowEnvelope, +}; use uuid::Uuid; +use crate::store::main::append_recall::append_recall; + pub(crate) async fn request_run( store: &ControlStore, request: &RunRequest, @@ -286,3 +296,119 @@ pub(crate) fn check_projection(check: &AdhocCheck) -> Value { pub(crate) fn launch_outcome(result: Result) -> Result { result.map_err(|error| ApiError::from(error).status.as_u16()) } + +/// One SQL text every logged run shares. +const SHARED_LOG_SQL: &str = "select order_id from orders"; + +fn log_envelope(invocation_id: String) -> RowEnvelope { + RowEnvelope { + invocation_id, + invocation_started_at: "2026-01-02T03:04:05Z".into(), + producer_version: "test".into(), + } +} + +fn log_sql_rows(envelope: &RowEnvelope, sql: &str) -> [RecallRow; 2] { + artifact_rows( + envelope, + &ArtifactInput { + digest: format!("sha256:{}", sha256_hex(sql.as_bytes())), + kind: "sql".into(), + media_type: "text/x-sql".into(), + content: sql.as_bytes().to_vec(), + seen_at: "2026-01-02T03:04:05Z".into(), + ref_id: "statement".into(), + role: "executed_sql".into(), + statement_id: Some("statement".into()), + resource_id: None, + }, + ) +} + +/// The claim's output, and a nested session's event and SQL: one text shared by every run, one its own. +fn worker_log(claim: Uuid, index: u128) -> Vec { + let worker = log_envelope(format!("worker/{claim}")); + let session = log_envelope(format!("worker/{claim}/session")); + let mut rows = vec![output_row( + &worker, + &OutputChunk { + sequence: 0, + stream: OutputStream::Stdout, + occurred_at: "2026-01-02T03:04:05Z".into(), + data: b"building orders\n".to_vec(), + byte_offset: 0, + dropped_before: 0, + omitted: None, + }, + )]; + rows.push( + event_object_row( + &session, + json!({"event_id": "completed", "event_type": "statement_completed", "schema_version": 1, + "occurred_at": "2026-01-02T03:04:05Z", "producer": "sqlbuild", "invocation_sequence": 1, + "statement_id": "statement", "payload": {"duration_ms": 1.5}}) + .as_object() + .expect("event object"), + ) + .expect("statement event"), + ); + rows.extend(log_sql_rows(&session, SHARED_LOG_SQL)); + rows.extend(log_sql_rows( + &session, + &format!("select {index} as order_id"), + )); + rows +} + +/// The worker invocation of run `index`, claimed by the claim ID `seed_logged_runs` gives it. +pub(crate) fn logged_worker(index: u128) -> String { + format!( + "worker/{}", + Uuid::new_v5(&Uuid::from_u128(index), b"run_claimed") + ) +} + +/// Ad-hoc runs 1, 2, ... of `location`, each ended the given hours ago or, for null, still running, with a worker log and a scheduler event. +pub(crate) async fn seed_logged_runs( + store: &ControlStore, + (location, snapshot_id): (Uuid, Uuid), + ended_hours_ago: &[Value], +) { + for (position, hours) in ended_hours_ago.iter().enumerate() { + let index = position as u128 + 1; + let run_request = request((location, snapshot_id), index, 0, &["orders"]); + let run_id = run_request.run_id; + assert!(request_run(store, &run_request).await.expect("request")); + let transaction = begin(store, TransactionMode::Write).await.expect("begin"); + let claim = Uuid::new_v5(&run_id, b"run_claimed"); + let ended_at = hours + .as_i64() + .map(|hours| transaction.now - TimeDelta::hours(hours)); + let status = ended_at.map_or("running", |_| "success"); + let (mut transaction, _) = query( + transaction, + "UPDATE sqb_runs SET status=$1, worker_instance=$2, ended_at=$3 WHERE run_id=$4", + vec![ + Parameter::Text(status.into()), + Parameter::Id(claim), + Parameter::OptionalTime(ended_at), + Parameter::Id(run_id), + ], + ) + .await + .expect("finish run"); + let scheduler = event_object_row( + &log_envelope(format!("scheduler/{run_id}")), + json!({"event_id": format!("queued-{index}"), "event_type": "run_queued", "schema_version": 1, + "occurred_at": "2026-01-02T03:04:05Z", "producer": "scheduler", "invocation_sequence": 1, + "payload": {}}) + .as_object() + .expect("event object"), + ) + .expect("scheduler event"); + for row in worker_log(claim, index).iter().chain([&scheduler]) { + (transaction, _) = append_recall(transaction, row).await.expect("append"); + } + commit(transaction).await.expect("commit"); + } +} diff --git a/crates/sqlbuild-orchestration/tests/runs/test_run_logs.rs b/crates/sqlbuild-orchestration/tests/runs/test_run_logs.rs index 3d271197d8..bc29b86ceb 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_run_logs.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_run_logs.rs @@ -1,104 +1,41 @@ //! Run log retention deletes whole worker logs by age or count and keeps runs and their history. -use chrono::TimeDelta; use serde_json::{Value, json}; -use sqlbuild_recall::records::main::artifact_rows::artifact_rows; -use sqlbuild_recall::records::main::event_object_row::event_object_row; -use sqlbuild_recall::records::main::output_row::output_row; -use sqlbuild_recall::records::main::sha256_hex::sha256_hex; -use sqlbuild_recall::records::models::{ - ArtifactInput, OutputChunk, OutputStream, RecallRow, RowEnvelope, -}; -use uuid::Uuid; use crate::credentials::tests::location_helpers::create_location; -use crate::runs::main::prune_run_logs::prune_run_logs; +use crate::platform::_helpers::retention::prune_run_logs; use crate::runs::models::RunLogRetention; -use crate::runs::tests::helpers::{request, request_run, snapshot}; +use crate::runs::tests::helpers::{logged_worker, rows, seed_logged_runs, snapshot}; +use crate::runs::tests::test_types::RunLogRetentionTestCase; use crate::snapshots::main::publish_snapshot::publish_snapshot; -use crate::store::main::append_recall::append_recall; use crate::store::main::migrate_store::migrate_store; -use crate::store::main::{begin::begin, commit::commit, query::query}; -use crate::store::models::{ControlStore, Parameter, TransactionMode}; use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; use crate::store::tests::store_test_types::Backend; -const SHARED_SQL: &str = "select order_id from orders"; - -fn envelope(invocation_id: String) -> RowEnvelope { - RowEnvelope { - invocation_id, - invocation_started_at: "2026-01-02T03:04:05Z".into(), - producer_version: "test".into(), - } -} - -fn sql_rows(envelope: &RowEnvelope, sql: &str) -> [RecallRow; 2] { - artifact_rows( - envelope, - &ArtifactInput { - digest: format!("sha256:{}", sha256_hex(sql.as_bytes())), - kind: "sql".into(), - media_type: "text/x-sql".into(), - content: sql.as_bytes().to_vec(), - seen_at: "2026-01-02T03:04:05Z".into(), - ref_id: "statement".into(), - role: "executed_sql".into(), - statement_id: Some("statement".into()), - resource_id: None, - }, - ) -} - -/// The claim's output, and a nested session's event and SQL: one text shared by every run, one its own. -fn worker_log(claim: Uuid, index: u128) -> Vec { - let worker = envelope(format!("worker/{claim}")); - let session = envelope(format!("worker/{claim}/session")); - let mut rows = vec![output_row( - &worker, - &OutputChunk { - sequence: 0, - stream: OutputStream::Stdout, - occurred_at: "2026-01-02T03:04:05Z".into(), - data: b"building orders\n".to_vec(), - byte_offset: 0, - dropped_before: 0, - omitted: None, - }, - )]; - rows.push( - event_object_row( - &session, - json!({"event_id": "completed", "event_type": "statement_completed", "schema_version": 1, - "occurred_at": "2026-01-02T03:04:05Z", "producer": "sqlbuild", "invocation_sequence": 1, - "statement_id": "statement", "payload": {"duration_ms": 1.5}}) - .as_object() - .expect("event object"), - ) - .expect("statement event"), - ); - rows.extend(sql_rows(&session, SHARED_SQL)); - rows.extend(sql_rows(&session, &format!("select {index} as order_id"))); - rows -} - -async fn read(store: &ControlStore, sql: &str) -> Vec { - let transaction = begin(store, TransactionMode::Read).await.expect("begin"); - let (transaction, rows) = query(transaction, sql, Vec::new()).await.expect("query"); - commit(transaction).await.expect("commit"); - rows.into_iter().map(|row| json!(row)).collect() -} - #[test] fn given_finished_runs_past_age_or_count_when_logs_are_pruned_then_whole_logs_go_and_runs_stay() { - for backend in [Backend::Sqlite, Backend::Postgres] { + let test_cases = [ + RunLogRetentionTestCase { + description: "SQLite: the shared SQL and the kept runs' own SQL stay", + backend: Backend::Sqlite, + expected_artifacts: 3, + expected_runs: 4, + }, + RunLogRetentionTestCase { + description: "PostgreSQL: the shared SQL and the kept runs' own SQL stay", + backend: Backend::Postgres, + expected_artifacts: 3, + expected_runs: 4, + }, + ]; + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../benchmarks/duckshop/edge_cases/orchestration/run_log_retention.json"); + let case: Value = + serde_json::from_str(&std::fs::read_to_string(path).expect("run log fixture")) + .expect("run log case"); + for test_case in test_cases { run(async { - let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) - .join("../../benchmarks/duckshop/edge_cases/orchestration/run_log_retention.json"); - let case: Value = - serde_json::from_str(&std::fs::read_to_string(path).expect("run log fixture")) - .expect("run log case"); - let Some(fixture) = open_test_store(backend).await else { + let Some(fixture) = open_test_store(test_case.backend).await else { return; }; let store = &fixture.store; @@ -109,56 +46,16 @@ fn given_finished_runs_past_age_or_count_when_logs_are_pruned_then_whole_logs_go let published = publish_snapshot(store, &snapshot(location)) .await .expect("publish"); - let ended = case["ended_hours_ago"].as_array().expect("runs"); - let mut transaction = begin(store, TransactionMode::Write).await.expect("begin"); - let now = transaction.now; - for (position, hours) in ended.iter().enumerate() { - let index = position as u128 + 1; - let run_request = request((location, published.snapshot_id), index, 0, &["orders"]); - let run_id = run_request.run_id; - commit(transaction).await.expect("commit"); - assert!(request_run(store, &run_request).await.expect("request")); - transaction = begin(store, TransactionMode::Write).await.expect("begin"); - let claim = Uuid::new_v5(&run_id, b"run_claimed"); - let (sql, ended_at) = match hours.as_i64() { - Some(hours) => ( - "UPDATE sqb_runs SET status='success', worker_instance=$1, ended_at=$2 WHERE run_id=$3", - Some(Parameter::Time(now - TimeDelta::hours(hours))), - ), - None => ( - "UPDATE sqb_runs SET status='running', worker_instance=$1 WHERE run_id=$2", - None, - ), - }; - let parameters: Vec = [ - Some(Parameter::Id(claim)), - ended_at, - Some(Parameter::Id(run_id)), - ] - .into_iter() - .flatten() - .collect(); - (transaction, _) = query(transaction, sql, parameters) - .await - .expect("finish run"); - let scheduler = event_object_row( - &envelope(format!("scheduler/{run_id}")), - json!({"event_id": format!("queued-{index}"), "event_type": "run_queued", "schema_version": 1, - "occurred_at": "2026-01-02T03:04:05Z", "producer": "scheduler", "invocation_sequence": 1, - "payload": {}}) - .as_object() - .expect("event object"), - ) - .expect("scheduler event"); - for row in worker_log(claim, index).iter().chain([&scheduler]) { - (transaction, _) = append_recall(transaction, row).await.expect("append"); - } - } - commit(transaction).await.expect("commit"); + seed_logged_runs( + store, + (location, published.snapshot_id), + case["ended_hours_ago"].as_array().expect("runs"), + ) + .await; let nodes = "SELECT COUNT(*) AS count FROM sqb_run_nodes"; - let nodes_before = read(store, nodes).await; + let nodes_before = rows(store, nodes, Vec::new()).await; let scheduler = "SELECT COUNT(*) AS count FROM recall_events WHERE invocation_id LIKE 'scheduler/%'"; - let scheduler_before = read(store, scheduler).await; + let scheduler_before = rows(store, scheduler, Vec::new()).await; let retention = RunLogRetention::new( u16::try_from(case["keep_days"].as_u64().expect("days")).expect("u16"), u16::try_from(case["keep_runs"].as_u64().expect("runs")).expect("u16"), @@ -168,27 +65,23 @@ fn given_finished_runs_past_age_or_count_when_logs_are_pruned_then_whole_logs_go assert_eq!( prune_run_logs(store, retention).await.expect("prune") as u64, case["expected_pruned"].as_u64().expect("pruned"), - "{backend:?}" + "{}", + test_case.description ); - let logged: Vec = case["expected_logged_runs"] + let mut expected: Vec = case["expected_logged_runs"] .as_array() .expect("logged") .iter() - .map(|index| { - let run_id = Uuid::from_u128(u128::from(index.as_u64().expect("index"))); - format!("worker/{}", Uuid::new_v5(&run_id, b"run_claimed")) - }) - .collect(); - let mut expected: Vec = logged - .iter() - .flat_map(|worker| [json!(worker), json!(format!("{worker}/session"))]) + .map(|index| logged_worker(u128::from(index.as_u64().expect("index")))) + .flat_map(|worker| [json!(format!("{worker}/session")), json!(worker)]) .collect(); expected.sort_by_key(ToString::to_string); - let stored: Vec = read( + let stored: Vec = rows( store, "SELECT invocation_id FROM recall_output UNION SELECT invocation_id FROM recall_events \ WHERE invocation_id LIKE 'worker/%' UNION SELECT invocation_id FROM recall_artifact_refs \ ORDER BY invocation_id", + Vec::new(), ) .await .into_iter() @@ -196,25 +89,49 @@ fn given_finished_runs_past_age_or_count_when_logs_are_pruned_then_whole_logs_go .collect(); assert_eq!( stored, expected, - "only the kept runs' logs remain, nested sessions included" + "{}: only the kept runs' logs remain, nested sessions included", + test_case.description ); assert_eq!( - read(store, "SELECT COUNT(*) AS count FROM recall_artifacts").await[0]["count"], - json!(3), - "the shared SQL and the kept runs' own SQL stay; the pruned runs' own SQL goes" + rows( + store, + "SELECT COUNT(*) AS count FROM recall_artifacts", + Vec::new() + ) + .await[0]["count"], + json!(test_case.expected_artifacts), + "{}", + test_case.description ); assert_eq!( - read(store, scheduler).await, + rows(store, scheduler, Vec::new()).await, scheduler_before, - "the scheduler's run history stays" + "{}: the scheduler's run history stays", + test_case.description + ); + assert_eq!( + rows(store, "SELECT COUNT(*) AS count FROM sqb_runs", Vec::new()).await[0]["count"], + json!(test_case.expected_runs), + "{}", + test_case.description ); assert_eq!( - read(store, "SELECT COUNT(*) AS count FROM sqb_runs").await[0]["count"], - json!(4) + rows(store, nodes, Vec::new()).await, + nodes_before, + "{}: node results stay", + test_case.description + ); + assert_eq!( + prune_run_logs(store, retention).await.expect("repeat"), + 0, + "{}: a repeated pass finds nothing", + test_case.description + ); + assert!( + RunLogRetention::new(0, 1).is_err() && RunLogRetention::new(1, 0).is_err(), + "{}: retention keeps at least one day and one run", + test_case.description ); - assert_eq!(read(store, nodes).await, nodes_before, "node results stay"); - assert_eq!(prune_run_logs(store, retention).await.expect("repeat"), 0); - assert!(RunLogRetention::new(0, 1).is_err() && RunLogRetention::new(1, 0).is_err()); close_test_store(fixture).await; }); } diff --git a/crates/sqlbuild-orchestration/tests/runs/test_types.rs b/crates/sqlbuild-orchestration/tests/runs/test_types.rs index 017d2d8798..6ed2f50b13 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_types.rs @@ -265,3 +265,10 @@ pub(crate) struct JobDetailTestCase { pub(crate) backend: Backend, pub(crate) expected_order: [&'static str; 2], } + +pub(crate) struct RunLogRetentionTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_artifacts: i64, + pub(crate) expected_runs: i64, +} diff --git a/crates/sqlbuild-orchestration/tests/store/recall/test_views.rs b/crates/sqlbuild-orchestration/tests/store/recall/test_views.rs index abfd6645c9..cee1baa5da 100644 --- a/crates/sqlbuild-orchestration/tests/store/recall/test_views.rs +++ b/crates/sqlbuild-orchestration/tests/store/recall/test_views.rs @@ -161,11 +161,10 @@ fn given_native_context_output_and_audits_when_reading_views_then_both_databases .expect("invocation projection"); assert_eq!(invocation[0]["status"], "completed"); assert_eq!(invocation[0]["duration_ms"], json!(12.75)); - let payload = |value: &serde_json::Value| match value { - serde_json::Value::String(text) => { + let payload = |value: &serde_json::Value| { + value.as_str().map_or(value.clone(), |text| { serde_json::from_str(text).expect("JSON payload") - } - other => other.clone(), + }) }; let events = read_recall_view(store, ("events", &envelope.invocation_id), 100) .await diff --git a/crates/sqlbuild-recall/src/schema/_helpers/relational_views.rs b/crates/sqlbuild-recall/src/schema/_helpers/relational_views.rs index 8e4c25fde6..6464067a09 100644 --- a/crates/sqlbuild-recall/src/schema/_helpers/relational_views.rs +++ b/crates/sqlbuild-recall/src/schema/_helpers/relational_views.rs @@ -6,12 +6,12 @@ use crate::records::constants::{ PROMOTED_INTEGER_FIELDS, PROMOTED_REAL_FIELDS, PROMOTED_TEXT_FIELDS, }; use crate::schema::_helpers::rendering::{all_columns, json_field, relational_cast}; -use crate::schema::constants::{EVENTS_TABLE, OUTPUT_TAIL_RECORDS}; +use crate::schema::constants::{EVENTS_TABLE, OUTPUT_TAIL_RECORDS, PAYLOAD_COLUMN}; const DIAGNOSTICS: &str = "SELECT * FROM recall_diagnostics"; const CONFLICTS: &str = "SELECT * FROM recall_conflicts"; -/// Relational stores leave promoted fields out of lifecycle payloads; the view merges them back from their columns. A null column adds nothing, so a field the event wrote as null stays null. Control stores also expose their `storage_id` cursor. +/// Relational stores leave promoted fields out of lifecycle payloads; the view merges them back from their columns. A null column adds nothing, so a field the event wrote as null stays null; on SQLite, `json_remove` drops each null column's key, and a present column removes an absent key instead. Control stores also expose their `storage_id` cursor. fn events(postgres: bool, control: bool) -> String { let promoted: Vec<&str> = PROMOTED_TEXT_FIELDS .iter() @@ -27,7 +27,6 @@ fn events(postgres: bool, control: bool) -> String { let merged: String = if postgres { format!("payload || jsonb_strip_nulls(jsonb_build_object({pairs}))") } else { - // `json_remove` drops each null column's key; a present column removes an absent key instead. let absent: String = promoted .iter() .map(|field| { @@ -39,7 +38,7 @@ fn events(postgres: bool, control: bool) -> String { let mut columns: Vec = all_columns(&EVENTS_TABLE) .iter() .map(|column| { - if column.name == "payload" { + if column.name == PAYLOAD_COLUMN { format!("CASE WHEN producer = '{DEFAULT_PRODUCER}' THEN {merged} ELSE payload END AS payload") } else { column.name.to_owned() diff --git a/crates/sqlbuild-recall/src/schema/constants.rs b/crates/sqlbuild-recall/src/schema/constants.rs index 1e8c9f737e..2888e41b5e 100644 --- a/crates/sqlbuild-recall/src/schema/constants.rs +++ b/crates/sqlbuild-recall/src/schema/constants.rs @@ -31,6 +31,9 @@ pub const COMMON_COLUMNS: [ColumnSpec; 6] = [ ]; /// The artifact columns its content digest covers: what it is, not who saw it when. pub const ARTIFACT_CONTENT_COLUMNS: [&str; 5] = ["digest", "kind", "media_type", "size", "content"]; +/// The event column holding the lifecycle payload. +pub const PAYLOAD_COLUMN: &str = "payload"; + /// Common columns left out of a row's content digest. pub const UNHASHED_COLUMNS: [&str; 2] = ["ingested_at", "content_hash"]; @@ -83,6 +86,7 @@ const OUTPUT_COLUMNS: [ColumnSpec; 11] = [ ColumnSpec::optional("omitted_sequences", ColumnType::Integer), ]; +/// `started_at` holds the start time of a statement end whose start was written with it, instead of a start row. const EVENT_COLUMNS: [ColumnSpec; 30] = [ ColumnSpec::required("event_id", ColumnType::Text), ColumnSpec::optional("invocation_sequence", ColumnType::Integer), @@ -113,7 +117,6 @@ const EVENT_COLUMNS: [ColumnSpec; 30] = [ ColumnSpec::optional("row_count", ColumnType::Integer), ColumnSpec::optional("affected_rows", ColumnType::Integer), ColumnSpec::required("payload", ColumnType::Json), - // A statement end whose start was written with it keeps the start time here instead of a start row. ColumnSpec::optional("started_at", ColumnType::Timestamp), ]; diff --git a/crates/sqlbuild-recall/src/store/main/relational_row.rs b/crates/sqlbuild-recall/src/store/main/relational_row.rs index a618e033c8..c3649fe13c 100644 --- a/crates/sqlbuild-recall/src/store/main/relational_row.rs +++ b/crates/sqlbuild-recall/src/store/main/relational_row.rs @@ -2,11 +2,14 @@ use std::borrow::Cow; -use serde_json::Value; +use serde_json::{Map, Value}; use sqlbuild_core::lifecycle_events::constants::DEFAULT_PRODUCER; -use crate::records::_helpers::events::payload_columns; +use crate::records::constants::{ + PROMOTED_INTEGER_FIELDS, PROMOTED_REAL_FIELDS, PROMOTED_TEXT_FIELDS, +}; use crate::records::models::{RecallRow, RecallValue}; +use crate::schema::constants::PAYLOAD_COLUMN; use crate::schema::models::Table; /// The views put the left-out fields back from their columns, so a payload reads as written; the content digest still covers the whole event. Server-owned events keep their payloads, which the server reads back whole. @@ -14,7 +17,7 @@ use crate::schema::models::Table; pub fn relational_row(row: &RecallRow) -> Cow<'_, RecallRow> { let lifecycle = row.value("producer") == Some(&RecallValue::Text(DEFAULT_PRODUCER.to_owned())); let Some(RecallValue::Text(payload)) = row - .value("payload") + .value(PAYLOAD_COLUMN) .filter(|_| row.table == Table::Events && lifecycle) else { return Cow::Borrowed(row); @@ -22,14 +25,12 @@ pub fn relational_row(row: &RecallRow) -> Cow<'_, RecallRow> { let Ok(Value::Object(mut fields)) = serde_json::from_str::(payload) else { return Cow::Borrowed(row); }; - let held: Vec<&'static str> = payload_columns(&fields) - .into_iter() - .filter(|(name, value)| { - *name != "payload" - && !matches!(value, RecallValue::Null) - && row.value(name) == Some(value) - }) - .map(|(name, _)| name) + let held: Vec<&'static str> = PROMOTED_TEXT_FIELDS + .iter() + .chain(&PROMOTED_INTEGER_FIELDS) + .chain(&PROMOTED_REAL_FIELDS) + .copied() + .filter(|name| column_holds(row, name, &fields)) .collect(); if held.is_empty() { return Cow::Borrowed(row); @@ -40,9 +41,20 @@ pub fn relational_row(row: &RecallRow) -> Cow<'_, RecallRow> { let stripped: String = Value::Object(fields).to_string(); let mut owned: RecallRow = row.clone(); for (name, value) in &mut owned.values { - if *name == "payload" { + if *name == PAYLOAD_COLUMN { *value = RecallValue::Text(stripped.clone()); } } Cow::Owned(owned) } + +/// Whether the row's `name` column already holds the payload's value of that field. +fn column_holds(row: &RecallRow, name: &str, fields: &Map) -> bool { + let field: Option<&Value> = fields.get(name); + match row.value(name) { + Some(RecallValue::Text(text)) => field.and_then(Value::as_str) == Some(text.as_str()), + Some(RecallValue::Integer(number)) => field.and_then(Value::as_i64) == Some(*number), + Some(RecallValue::Real(number)) => field.and_then(Value::as_f64) == Some(*number), + _ => false, + } +} diff --git a/crates/sqlbuild-recall/src/store/tests/test_types.rs b/crates/sqlbuild-recall/src/store/tests/test_types.rs index 4b16e167d9..189cfeb6e4 100644 --- a/crates/sqlbuild-recall/src/store/tests/test_types.rs +++ b/crates/sqlbuild-recall/src/store/tests/test_types.rs @@ -9,6 +9,12 @@ pub(crate) struct ProjectionTestCase { pub(crate) expected_rows: Vec>, } +pub(crate) struct PayloadTestCase { + pub(crate) description: &'static str, + pub(crate) sql: &'static str, + pub(crate) expected_payload: &'static str, +} + pub(crate) struct ResolveInvocationTestCase { pub(crate) description: &'static str, pub(crate) selector: &'static str, diff --git a/crates/sqlbuild-recall/src/store/tests/test_views.rs b/crates/sqlbuild-recall/src/store/tests/test_views.rs index 02df08dec9..984ffcfdd3 100644 --- a/crates/sqlbuild-recall/src/store/tests/test_views.rs +++ b/crates/sqlbuild-recall/src/store/tests/test_views.rs @@ -7,7 +7,7 @@ use crate::records::models::{ArtifactInput, InvocationContext, OutputStream, Rec use crate::store::tests::helpers::{ NOW, artifact, chunk, chunks, delivered, envelope, event, integer, projected, text, wall_clock, }; -use crate::store::tests::test_types::ProjectionTestCase; +use crate::store::tests::test_types::{PayloadTestCase, ProjectionTestCase}; const BINARY_PIECES: [(OutputStream, &[u8]); 4] = [ (OutputStream::Stdout, b"line one\n\x00\xff"), @@ -140,13 +140,16 @@ fn given_duplicates_and_conflicts_when_written_then_first_row_is_kept_and_confli ]; // An earlier version stored the full 64-digit digest, of which today's is the prefix. let mut legacy = original.clone(); - for (name, value) in &mut legacy.values { - if *name == "content_hash" - && let RecallValue::Text(hash) = value - { - hash.push_str(&"0".repeat(32)); - } - } + let full_hash = RecallValue::Text(format!( + "{}{}", + original.content_hash().expect("content hash"), + "0".repeat(32) + )); + legacy + .values + .iter_mut() + .filter(|(name, _)| *name == "content_hash") + .for_each(|(_, value)| *value = full_hash.clone()); let test_cases = [ ProjectionTestCase { description: "the first row wins", @@ -485,30 +488,33 @@ fn given_the_same_sql_in_two_invocations_when_written_then_artifacts_are_shared_ #[test] fn given_lifecycle_payloads_when_stored_then_promoted_fields_are_kept_once_and_read_back_whole() { - let env = envelope("strip", NOW); let written = r#"{"intent":"execute","statement_kind":"SELECT","duration_ms":12.5,"affected_rows":3,"query_id":null,"attempt_number":2}"#; - let rows = projected( - &[vec![event( - &env, - ("e0", "statement_completed", 0), - r#","statement_id":"s1""#, - written, - )]], - "SELECT recall_events.payload, events.payload FROM recall_events \ - JOIN events ON events.event_id = recall_events.event_id", - ); - let parsed = |value: &RecallValue| match value { - RecallValue::Text(json) => serde_json::from_str::(json).expect("JSON"), - other => panic!("payload is text, got {other:?}"), - }; - assert_eq!( - parsed(&rows[0][0]), - serde_json::json!({"query_id": null, "attempt_number": 2}), - "the table keeps only fields without a column of their own" - ); - assert_eq!( - parsed(&rows[0][1]), - serde_json::from_str::(written).expect("JSON"), - "the events view returns the payload as written, explicit nulls included" - ); + let test_cases = [ + PayloadTestCase { + description: "the table keeps only fields without a column of their own", + sql: "SELECT payload FROM recall_events", + expected_payload: r#"{"query_id":null,"attempt_number":2}"#, + }, + PayloadTestCase { + description: "the events view returns the payload as written, explicit nulls included", + sql: "SELECT payload FROM events", + expected_payload: written, + }, + ]; + let env = envelope("strip", NOW); + let batches = [vec![event( + &env, + ("e0", "statement_completed", 0), + r#","statement_id":"s1""#, + written, + )]]; + for test_case in test_cases { + let rows = projected(&batches, test_case.sql); + assert_eq!( + serde_json::from_str::(&rows[0][0].key_text()).expect("JSON"), + serde_json::from_str::(test_case.expected_payload).expect("JSON"), + "{}", + test_case.description + ); + } } diff --git a/crates/sqlbuild-recall/src/writer/_helpers/materialize.rs b/crates/sqlbuild-recall/src/writer/_helpers/materialize.rs index 6cf49e9ed7..a5dbf9e71e 100644 --- a/crates/sqlbuild-recall/src/writer/_helpers/materialize.rs +++ b/crates/sqlbuild-recall/src/writer/_helpers/materialize.rs @@ -311,15 +311,20 @@ fn ended_starts(batch: &[Queued]) -> HashMap { } } events() - .filter_map(|event| { - let ending: &str = statement_of(event, false); - starts - .get(ending) - .map(|at| (ending.to_owned(), (*at).to_owned())) - }) + .filter_map(|event| start_of_end(&starts, event)) .collect() } +/// The statement ID and start time of `event` when it ends a statement that started in the same batch. +fn start_of_end( + starts: &HashMap<&str, &str>, + event: &Map, +) -> Option<(String, String)> { + let ending: &str = statement_of(event, false); + let at: &str = starts.get(ending)?; + Some((ending.to_owned(), at.to_owned())) +} + /// An artifact of `content`, referenced by its statement or, without one, by its digest. fn artifact_input( content: Vec, From caf73316ed03f8e6f83b7626ef54abd7fb8a1792 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 13 Nov 2026 21:53:03 +0000 Subject: [PATCH 09/10] fix(control): find finished runs' worker logs through their claim events A finished run clears its worker instance, so run log retention matched no run on a real store. Retention now links a run to its worker log through the scheduler's run_claimed event, and the test claims and finishes runs through the real lifecycle. PostgreSQL control stores order invocation_id by bytes, so a worker's prefix range finds its nested invocations through the index under any database collation. Real payload fields stay stored, because SQLite and PostgreSQL JSON functions do not print them back digit for digit. The recreate message says a different release made the store, names the socket directory for socket connections and says that dropdb removes the whole database. The bundled skill joins SQL and audit evidence by digest. --- .../src/platform/_helpers/retention.rs | 81 +++++++++++------- .../src/store/_helpers/migration.rs | 25 ++++-- .../tests/runs/helpers.rs | 83 ++++++++++++------- .../tests/store/recall/test_views.rs | 4 +- .../src/schema/_helpers/relational_tables.rs | 12 ++- .../src/schema/_helpers/relational_views.rs | 5 +- .../sqlbuild-recall/src/schema/constants.rs | 3 + .../src/store/main/relational_row.rs | 8 +- .../src/store/tests/test_views.rs | 4 +- .../sqlbuild/references/docs/cli/recall.md | 3 +- .../sqlbuild/references/docs/cli/server.md | 17 +++- .../docs/concepts/observability/recall.md | 14 +++- .../sqlbuild/references/investigating-runs.md | 13 +-- 13 files changed, 173 insertions(+), 99 deletions(-) diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/retention.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/retention.rs index fe24739a7c..c486dd4af0 100644 --- a/crates/sqlbuild-orchestration/src/platform/_helpers/retention.rs +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/retention.rs @@ -41,14 +41,21 @@ const LOCATION_INSTANCES: &str = "DELETE FROM sqb_location_instances WHERE insta FROM sqb_location_work w WHERE w.session_id=i.instance_id) \ ORDER BY i.last_seen_at LIMIT $2) RETURNING instance_id"; -/// Claims of finished runs past either limit whose worker log is still stored, oldest first. Worker invocations are `worker/` and `worker//`; the stored logs, not every run ever finished, bound the join. +/// Claims of finished runs past either limit whose worker log is still stored, oldest first. Worker invocations are `worker/` and `worker//`; the stored logs, not every run ever finished, bound the join. A finished run no longer names its worker, so the scheduler's `run_claimed` event, whose ID is the claim, links the two. const EXPIRED_CLAIMS: &str = "WITH logged AS (\ - SELECT DISTINCT substr(invocation_id, 8, 36) AS claim FROM recall_invocations WHERE invocation_id LIKE 'worker/%' \ - UNION SELECT DISTINCT substr(invocation_id, 8, 36) AS claim FROM recall_output WHERE invocation_id LIKE 'worker/%'), \ - ranked AS (SELECT worker_instance, ended_at, ROW_NUMBER() OVER (PARTITION BY location_id, target \ + SELECT DISTINCT substr(invocation_id, 8, 36) AS claim FROM recall_invocations \ + WHERE invocation_id > 'worker/' AND invocation_id < 'worker0' \ + AND (length(invocation_id) = 43 OR substr(invocation_id, 44, 1) = '/') \ + UNION SELECT DISTINCT substr(invocation_id, 8, 36) AS claim FROM recall_output \ + WHERE invocation_id > 'worker/' AND invocation_id < 'worker0' \ + AND (length(invocation_id) = 43 OR substr(invocation_id, 44, 1) = '/')), \ + claimed AS (SELECT DISTINCT event_id AS claim, run_id FROM recall_events \ + WHERE event_type = 'run_claimed' AND producer = 'scheduler'), \ + ranked AS (SELECT CAST(run_id AS TEXT) AS run_id, ended_at, ROW_NUMBER() OVER (PARTITION BY location_id, target \ ORDER BY ended_at DESC, run_id DESC) AS newest FROM sqb_runs \ WHERE ended_at IS NOT NULL AND status IN ('success', 'warn', 'partial', 'failed', 'cancelled')) \ - SELECT logged.claim FROM ranked JOIN logged ON logged.claim = CAST(ranked.worker_instance AS TEXT) \ + SELECT logged.claim FROM ranked JOIN claimed ON claimed.run_id = ranked.run_id \ + JOIN logged ON logged.claim = claimed.claim \ WHERE ranked.newest > $1 OR ranked.ended_at < $2 ORDER BY ranked.ended_at LIMIT $3"; /// Returns how many audit rows outlive their horizon because a sink has not acknowledged them. @@ -144,39 +151,51 @@ pub(crate) async fn prune_run_logs( } } -/// Every Recall row of the claims' invocations, then stored artifacts no remaining reference names. `worker//` sorts between `worker//` and `worker/0`, so a key range finds nested invocations by index. +/// Every Recall row of the claims' invocations, then stored artifacts no remaining reference names. `worker//` sorts between `worker//` and `worker/0` in byte order, which SQLite uses and PostgreSQL control stores declare for `invocation_id`, so each range reads the index. async fn delete_logs( mut transaction: ControlTransaction, claims: &[String], ) -> Result { - let mut digests: BTreeSet = BTreeSet::new(); - for claim in claims { - let namespace = format!("worker/{claim}"); - let bounds = vec![ - Parameter::Text(namespace.clone()), - Parameter::Text(format!("{namespace}/")), - Parameter::Text(format!("{namespace}0")), - ]; - let owned = "(invocation_id=$1 OR (invocation_id>$2 AND invocation_id<$3))"; - let rows; - (transaction, rows) = query( + let parameters: Vec = claims + .iter() + .flat_map(|claim| { + let namespace = format!("worker/{claim}"); + [format!("{namespace}/"), format!("{namespace}0"), namespace] + }) + .map(Parameter::Text) + .collect(); + let ranges: Vec = (0..claims.len()) + .map(|index| { + let first = 3 * index + 1; + format!( + "invocation_id=${} OR (invocation_id>${first} AND invocation_id<${})", + first + 2, + first + 1 + ) + }) + .collect(); + if ranges.is_empty() { + return Ok(transaction); + } + let owned = format!("({})", ranges.join(" OR ")); + let rows; + (transaction, rows) = query( + transaction, + &format!("SELECT DISTINCT digest FROM recall_artifact_refs WHERE {owned}"), + parameters.clone(), + ) + .await?; + let digests: BTreeSet = rows + .iter() + .filter_map(|row| row["digest"].as_str().map(str::to_owned)) + .collect(); + for table in INVOCATION_TABLES { + (transaction, _) = query( transaction, - &format!("SELECT DISTINCT digest FROM recall_artifact_refs WHERE {owned}"), - bounds.clone(), + &format!("DELETE FROM {} WHERE {owned}", table_spec(table).name), + parameters.clone(), ) .await?; - digests.extend( - rows.iter() - .filter_map(|row| row["digest"].as_str().map(str::to_owned)), - ); - for table in INVOCATION_TABLES { - (transaction, _) = query( - transaction, - &format!("DELETE FROM {} WHERE {owned}", table_spec(table).name), - bounds.clone(), - ) - .await?; - } } let digests: Vec = digests.into_iter().collect(); for chunk in digests.chunks(RUN_LOG_DIGEST_CHUNK) { diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs index 148f3cc735..980caf2878 100644 --- a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs +++ b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs @@ -26,32 +26,41 @@ pub(crate) fn verify_schema( return Ok(()); } let quote = |text: &str| format!("'{}'", text.replace('\'', r"'\''")); - let command = match store { + let (scope, command) = match store { ControlStore::Sqlite(path) => { let path = path.to_string_lossy(); - format!( + let command = format!( "rm -f {} {} {}", quote(&path), quote(&format!("{path}-wal")), quote(&format!("{path}-shm")) - ) + ); + ("", command) } ControlStore::Postgres(pool) => { let options = pool.connect_options(); + let host = options.get_socket().map_or_else( + || options.get_host().to_owned(), + |socket| socket.to_string_lossy().into_owned(), + ); let target = format!( "-h {} -p {} -U {} {}", - quote(options.get_host()), + quote(&host), options.get_port(), quote(options.get_username()), quote(options.get_database().unwrap_or(options.get_username())) ); - format!("dropdb {target} && createdb {target}") + ( + " and everything else in that database", + format!("dropdb {target} && createdb {target}"), + ) } }; Err(ControlError::Schema(format!( - "this control store was created by an earlier sqb release and cannot be upgraded; \ - to start a new, empty store, which discards its run history, stop every server \ - using it and run: {command}" + "this control store was created by a different sqb release and cannot be upgraded; \ + if a newer release created it, upgrade this server instead. To start a new, empty \ + store, which discards its run history{scope}, stop every server using it and run: \ + {command}" ))) } diff --git a/crates/sqlbuild-orchestration/tests/runs/helpers.rs b/crates/sqlbuild-orchestration/tests/runs/helpers.rs index 1e5c296293..f251460290 100644 --- a/crates/sqlbuild-orchestration/tests/runs/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/runs/helpers.rs @@ -31,6 +31,11 @@ use sqlbuild_recall::records::models::{ }; use uuid::Uuid; +use crate::runs::main::claim_next_run::claim_next_run; +use crate::runs::models::RunClaim; +use crate::runs::tests::claim_helpers::succeed; +use crate::scheduler::models::SchedulerLease; +use crate::scheduler::tests::contracts::helpers::acquire_lease; use crate::store::main::append_recall::append_recall; pub(crate) async fn request_run( @@ -360,7 +365,7 @@ fn worker_log(claim: Uuid, index: u128) -> Vec { rows } -/// The worker invocation of run `index`, claimed by the claim ID `seed_logged_runs` gives it. +/// The worker invocation of run `index`, which its claim names. pub(crate) fn logged_worker(index: u128) -> String { format!( "worker/{}", @@ -368,47 +373,61 @@ pub(crate) fn logged_worker(index: u128) -> String { ) } -/// Ad-hoc runs 1, 2, ... of `location`, each ended the given hours ago or, for null, still running, with a worker log and a scheduler event. -pub(crate) async fn seed_logged_runs( +/// Claims ad-hoc run `index` through the scheduler and stores its worker log. +async fn claim_logged_run( store: &ControlStore, (location, snapshot_id): (Uuid, Uuid), + lease: &SchedulerLease, + index: u128, +) -> RunClaim { + let run_request = request((location, snapshot_id), index, 0, &["orders"]); + assert!(request_run(store, &run_request).await.expect("request")); + let claim = claim_next_run( + store, + lease, + &BTreeMap::new(), + Some((location, "development")), + ) + .await + .expect("claim") + .expect("queued run"); + let mut transaction = begin(store, TransactionMode::Write).await.expect("begin"); + for row in worker_log(claim.claim_id, index) { + (transaction, _) = append_recall(transaction, &row).await.expect("append"); + } + commit(transaction).await.expect("commit"); + claim +} + +/// Ad-hoc runs 1, 2, ... of `location`, each claimed through the scheduler with a worker log. Runs with hours finish through the worker lifecycle, one at a time because each writes `orders`, and are then dated that many hours ago; the null entries, which come last, keep running. +pub(crate) async fn seed_logged_runs( + store: &ControlStore, + scope: (Uuid, Uuid), ended_hours_ago: &[Value], ) { - for (position, hours) in ended_hours_ago.iter().enumerate() { - let index = position as u128 + 1; - let run_request = request((location, snapshot_id), index, 0, &["orders"]); - let run_id = run_request.run_id; - assert!(request_run(store, &run_request).await.expect("request")); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let finished: Vec = ended_hours_ago.iter().filter_map(Value::as_i64).collect(); + for (position, hours) in finished.iter().enumerate() { + let claim = claim_logged_run(store, scope, &lease, position as u128 + 1).await; + succeed(store, &claim).await; let transaction = begin(store, TransactionMode::Write).await.expect("begin"); - let claim = Uuid::new_v5(&run_id, b"run_claimed"); - let ended_at = hours - .as_i64() - .map(|hours| transaction.now - TimeDelta::hours(hours)); - let status = ended_at.map_or("running", |_| "success"); - let (mut transaction, _) = query( + let ended_at = transaction.now - TimeDelta::hours(*hours); + let (transaction, _) = query( transaction, - "UPDATE sqb_runs SET status=$1, worker_instance=$2, ended_at=$3 WHERE run_id=$4", + "UPDATE sqb_runs SET ended_at=$1 WHERE run_id=$2", vec![ - Parameter::Text(status.into()), - Parameter::Id(claim), - Parameter::OptionalTime(ended_at), - Parameter::Id(run_id), + Parameter::Time(ended_at), + Parameter::Id(claim.request.run_id), ], ) .await - .expect("finish run"); - let scheduler = event_object_row( - &log_envelope(format!("scheduler/{run_id}")), - json!({"event_id": format!("queued-{index}"), "event_type": "run_queued", "schema_version": 1, - "occurred_at": "2026-01-02T03:04:05Z", "producer": "scheduler", "invocation_sequence": 1, - "payload": {}}) - .as_object() - .expect("event object"), - ) - .expect("scheduler event"); - for row in worker_log(claim, index).iter().chain([&scheduler]) { - (transaction, _) = append_recall(transaction, row).await.expect("append"); - } + .expect("date run"); commit(transaction).await.expect("commit"); } + for position in finished.len()..ended_hours_ago.len() { + claim_logged_run(store, scope, &lease, position as u128 + 1).await; + } } diff --git a/crates/sqlbuild-orchestration/tests/store/recall/test_views.rs b/crates/sqlbuild-orchestration/tests/store/recall/test_views.rs index cee1baa5da..db78e4bdcf 100644 --- a/crates/sqlbuild-orchestration/tests/store/recall/test_views.rs +++ b/crates/sqlbuild-orchestration/tests/store/recall/test_views.rs @@ -187,8 +187,8 @@ fn given_native_context_output_and_audits_when_reading_views_then_both_databases .expect("stored statement"); assert_eq!( payload(&stored["payload"]), - json!({"query_id": null, "adapter": "duckdb"}), - "the table keeps only fields without a column" + json!({"duration_ms": 1.5, "query_id": null, "adapter": "duckdb"}), + "the table keeps real fields and fields without a column" ); assert_eq!(invocation[0]["integration_retry"], json!(2)); let audit = read_recall_view(store, ("audit_results", &envelope.invocation_id), 100) diff --git a/crates/sqlbuild-recall/src/schema/_helpers/relational_tables.rs b/crates/sqlbuild-recall/src/schema/_helpers/relational_tables.rs index f0f25c8267..442a7ab692 100644 --- a/crates/sqlbuild-recall/src/schema/_helpers/relational_tables.rs +++ b/crates/sqlbuild-recall/src/schema/_helpers/relational_tables.rs @@ -3,7 +3,7 @@ use sqlbuild_core::identifier_text::main::quote_identifier::quote_identifier as sqlite_identifier; use crate::schema::_helpers::rendering::{all_columns, sqlite_type}; -use crate::schema::constants::EVENTS_TABLE; +use crate::schema::constants::{BYTE_ORDERED_COLUMN, EVENTS_TABLE}; use crate::schema::models::{ColumnType, TableSpec}; pub(crate) fn relational_table(spec: &TableSpec, postgres: bool, control: bool) -> Vec { @@ -18,7 +18,15 @@ pub(crate) fn relational_table(spec: &TableSpec, postgres: bool, control: bool) } else { sqlite_type(column.column_type) }; - format!(" {} {kind}{not_null}", sqlite_identifier(column.name)) + let collation = if postgres && column.name == BYTE_ORDERED_COLUMN { + " COLLATE \"C\"" + } else { + "" + }; + format!( + " {} {kind}{collation}{not_null}", + sqlite_identifier(column.name) + ) }) .collect(); if control && spec.name == EVENTS_TABLE.name { diff --git a/crates/sqlbuild-recall/src/schema/_helpers/relational_views.rs b/crates/sqlbuild-recall/src/schema/_helpers/relational_views.rs index 6464067a09..26cfb9721a 100644 --- a/crates/sqlbuild-recall/src/schema/_helpers/relational_views.rs +++ b/crates/sqlbuild-recall/src/schema/_helpers/relational_views.rs @@ -2,9 +2,7 @@ use sqlbuild_core::lifecycle_events::constants::DEFAULT_PRODUCER; -use crate::records::constants::{ - PROMOTED_INTEGER_FIELDS, PROMOTED_REAL_FIELDS, PROMOTED_TEXT_FIELDS, -}; +use crate::records::constants::{PROMOTED_INTEGER_FIELDS, PROMOTED_TEXT_FIELDS}; use crate::schema::_helpers::rendering::{all_columns, json_field, relational_cast}; use crate::schema::constants::{EVENTS_TABLE, OUTPUT_TAIL_RECORDS, PAYLOAD_COLUMN}; @@ -16,7 +14,6 @@ fn events(postgres: bool, control: bool) -> String { let promoted: Vec<&str> = PROMOTED_TEXT_FIELDS .iter() .chain(&PROMOTED_INTEGER_FIELDS) - .chain(&PROMOTED_REAL_FIELDS) .copied() .collect(); let pairs: String = promoted diff --git a/crates/sqlbuild-recall/src/schema/constants.rs b/crates/sqlbuild-recall/src/schema/constants.rs index 2888e41b5e..367fe349d5 100644 --- a/crates/sqlbuild-recall/src/schema/constants.rs +++ b/crates/sqlbuild-recall/src/schema/constants.rs @@ -31,6 +31,9 @@ pub const COMMON_COLUMNS: [ColumnSpec; 6] = [ ]; /// The artifact columns its content digest covers: what it is, not who saw it when. pub const ARTIFACT_CONTENT_COLUMNS: [&str; 5] = ["digest", "kind", "media_type", "size", "content"]; +/// PostgreSQL control stores order this column by bytes, so a `worker//` prefix range finds a claim's nested invocations through the index under any database collation. +pub const BYTE_ORDERED_COLUMN: &str = "invocation_id"; + /// The event column holding the lifecycle payload. pub const PAYLOAD_COLUMN: &str = "payload"; diff --git a/crates/sqlbuild-recall/src/store/main/relational_row.rs b/crates/sqlbuild-recall/src/store/main/relational_row.rs index c3649fe13c..f64ee11f09 100644 --- a/crates/sqlbuild-recall/src/store/main/relational_row.rs +++ b/crates/sqlbuild-recall/src/store/main/relational_row.rs @@ -5,14 +5,12 @@ use std::borrow::Cow; use serde_json::{Map, Value}; use sqlbuild_core::lifecycle_events::constants::DEFAULT_PRODUCER; -use crate::records::constants::{ - PROMOTED_INTEGER_FIELDS, PROMOTED_REAL_FIELDS, PROMOTED_TEXT_FIELDS, -}; +use crate::records::constants::{PROMOTED_INTEGER_FIELDS, PROMOTED_TEXT_FIELDS}; use crate::records::models::{RecallRow, RecallValue}; use crate::schema::constants::PAYLOAD_COLUMN; use crate::schema::models::Table; -/// The views put the left-out fields back from their columns, so a payload reads as written; the content digest still covers the whole event. Server-owned events keep their payloads, which the server reads back whole. +/// Real fields stay, because a database's JSON functions would not print them back digit for digit. The views put the left-out fields back from their columns, so a payload reads as written; the content digest still covers the whole event. Server-owned events keep their payloads, which the server reads back whole. #[must_use] pub fn relational_row(row: &RecallRow) -> Cow<'_, RecallRow> { let lifecycle = row.value("producer") == Some(&RecallValue::Text(DEFAULT_PRODUCER.to_owned())); @@ -28,7 +26,6 @@ pub fn relational_row(row: &RecallRow) -> Cow<'_, RecallRow> { let held: Vec<&'static str> = PROMOTED_TEXT_FIELDS .iter() .chain(&PROMOTED_INTEGER_FIELDS) - .chain(&PROMOTED_REAL_FIELDS) .copied() .filter(|name| column_holds(row, name, &fields)) .collect(); @@ -54,7 +51,6 @@ fn column_holds(row: &RecallRow, name: &str, fields: &Map) -> boo match row.value(name) { Some(RecallValue::Text(text)) => field.and_then(Value::as_str) == Some(text.as_str()), Some(RecallValue::Integer(number)) => field.and_then(Value::as_i64) == Some(*number), - Some(RecallValue::Real(number)) => field.and_then(Value::as_f64) == Some(*number), _ => false, } } diff --git a/crates/sqlbuild-recall/src/store/tests/test_views.rs b/crates/sqlbuild-recall/src/store/tests/test_views.rs index 984ffcfdd3..3e08baa596 100644 --- a/crates/sqlbuild-recall/src/store/tests/test_views.rs +++ b/crates/sqlbuild-recall/src/store/tests/test_views.rs @@ -491,9 +491,9 @@ fn given_lifecycle_payloads_when_stored_then_promoted_fields_are_kept_once_and_r let written = r#"{"intent":"execute","statement_kind":"SELECT","duration_ms":12.5,"affected_rows":3,"query_id":null,"attempt_number":2}"#; let test_cases = [ PayloadTestCase { - description: "the table keeps only fields without a column of their own", + description: "the table keeps real fields and fields without a column of their own", sql: "SELECT payload FROM recall_events", - expected_payload: r#"{"query_id":null,"attempt_number":2}"#, + expected_payload: r#"{"duration_ms":12.5,"query_id":null,"attempt_number":2}"#, }, PayloadTestCase { description: "the events view returns the payload as written, explicit nulls included", diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/recall.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/recall.md index 3068676d84..5c3d6a9229 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/recall.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/recall.md @@ -72,7 +72,8 @@ The slowest statements of past builds, with their SQL: ```bash sqb recall query --max-cell-bytes 60 \ "SELECT s.resource_name, round(s.duration_ms, 1) AS ms, a.text AS sql - FROM statements s JOIN artifacts a USING (invocation_id, statement_id) + FROM statements s JOIN artifacts a + ON a.invocation_id = s.invocation_id AND a.digest = 'sha256:' || s.sql_digest WHERE s.statement_kind = 'CREATE' ORDER BY s.duration_ms DESC LIMIT 2" ``` diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md index 06b138a6f6..46571bf62c 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md @@ -80,7 +80,7 @@ seconds, stops delivery and retries the same event. Deduplicate by `event_id`, w | `kind` | Sent when | Fields | | --- | --- | --- | -| `heartbeat` | Each server replica is alive, periodically | `replica_id` | +| `heartbeat` | Each server replica is alive, periodically | `replica_id`, `store_bytes` (the control database size on disk) | | `scheduler_no_leader` | No server holds the scheduler lease | None | | `tick_overdue` | A schedule tick is late | `trigger_id`, `scheduled_for` | | `calendar_horizon_low` | A running calendar schedule has a day or less of dates left | `trigger_id`, `refresh_at`, `horizon_end` | @@ -118,7 +118,8 @@ the control server is older than this worker and refused its run report; upgrade Set how much history the server keeps: ```bash -sqb server --statistics-retention-days 30 --audit-retention-days 730 --snapshot-retention-count 20 +sqb server --statistics-retention-days 30 --audit-retention-days 730 --snapshot-retention-count 20 \ + --run-log-retention-days 7 --run-log-retention-count 100 ``` | Data | Option | Default | Range | @@ -126,6 +127,8 @@ sqb server --statistics-retention-days 30 --audit-retention-days 730 --snapshot- | Hourly warehouse statistics | `--statistics-retention-days` | `90` | `1`–`65535` | | Security audit records | `--audit-retention-days` | `400` | `1`–`65535` | | Published snapshots per location and target | `--snapshot-retention-count` | `10` | `1`–`65535` | +| Logs of finished runs | `--run-log-retention-days` | `14` | `1`–`65535` | +| Finished runs per location and target that keep their logs | `--run-log-retention-count` | `200` | `1`–`65535` | | Skipped and covered schedule ticks | None | 30 days | Fixed | Statistics samples older than the horizon disappear from statistics reads at once and are then @@ -147,6 +150,16 @@ snapshot only when no retained run, job, trigger, trigger change or location req deletes its node definitions and SQL text when no other snapshot shares them. Repeating or interrupting this cleanup is safe. +A finished run loses its log, the events, statements, SQL and output its worker recorded, when it +is older than `--run-log-retention-days` or when more than `--run-log-retention-count` newer runs +have finished in its location and target, whichever comes first. The run itself, its node results, +and the run history the scheduler recorded stay, so run lists, node status and history are +unchanged; only the run's log, statements and SQL are no longer shown. A running run keeps its log. + +The **Server** panel on the **Settings** page shows the control database's size on disk and both +run log limits, and each `heartbeat` [platform event](#receive-platform-events) carries the size as +`store_bytes`. + The scheduler deletes covered ticks older than 30 days. It keeps each trigger's latest tick and every tick that started a run. A new schedule, a resumed trigger or a cursor reset catches up only on the latest missed time, so deleted ticks never come back. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/recall.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/recall.md index 9a6586ede0..bfeaf3eb62 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/recall.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/recall.md @@ -63,7 +63,7 @@ See the rows the audit flagged, which Recall keeps as the audit's evidence: ```bash sqb recall query \ "SELECT r.text FROM audit_results a JOIN artifacts r - ON r.invocation_id = a.invocation_id AND r.ref_id = a.event_id + ON r.invocation_id = a.invocation_id AND r.digest = 'sha256:' || a.sample_digest WHERE a.audit_name = 'revenue_is_non_negative' AND a.outcome <> 'pass' ORDER BY a.invocation_started_at DESC LIMIT 1" ``` @@ -172,11 +172,11 @@ and `selection_truncated` is `1`, while `selection_count` still counts all of th When an audit warns or fails, Recall keeps its [evidence](../audits.md#evidence): up to `evidence_limit` of the rows it flagged. `audit_results` counts them in `evidence_count`, sets `evidence_truncated` to `1` when the limit cut rows off, and names the stored rows by -`sample_digest`. Read them from `artifacts`, joined on the audit's `event_id`: +`sample_digest`. Read them from `artifacts`, joined on that digest: ```sql SELECT r.text FROM audit_results a JOIN artifacts r - ON r.invocation_id = a.invocation_id AND r.ref_id = a.event_id + ON r.invocation_id = a.invocation_id AND r.digest = 'sha256:' || a.sample_digest WHERE a.outcome <> 'pass' ``` @@ -356,7 +356,7 @@ in the store's SQL dialect, against these views: | `audit_results` | Evaluated audit, with its evidence count | | `events` | Lifecycle event | | `diagnostics` | Retained diagnostic | -| `artifacts` | Reference from an invocation to captured SQL, with its text | +| `artifacts` | Captured SQL or audit evidence an invocation used, with its text; join on `digest` | | `output_invocations` | Invocation's output, with completeness | | `output_transcripts` | Invocation's stdout, stderr, and combined output | | `output_tail` | Recent output chunk | @@ -381,6 +381,12 @@ is `0`, `sql_unavailable_reason` says why: `not_captured` when Recall never rece `expired_or_not_captured` when the statement references SQL text that is no longer stored, because [retention](#retention) removed it or it was dropped before it was written. +A statement that ends shortly after it starts is one row in `events`: its `statement_completed` or +`statement_failed` row holds the start time in `started_at`, and it has no `statement_started` row. +A statement that is still running, or ran longer, keeps its `statement_started` row. +`statements.started_at` reads both, so take statement timings from `statements` rather than pairing +rows in `events`. + Every view has `invocation_id` and `invocation_started_at`. Filter on them to keep a query cheap: on a ClickHouse store, a filter on either reaches the table reads, so a query about one invocation or the last day reads that part of the store instead of folding the whole retention window. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/investigating-runs.md b/src/sqlbuild/.agents/skills/sqlbuild/references/investigating-runs.md index 6c7648ec1e..4ae00c287e 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/investigating-runs.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/investigating-runs.md @@ -36,11 +36,12 @@ what they printed. `sqb recall query` runs one read-only `SELECT` over a set of and `output_invocations.complete = 0` with a `gap_reason` when output is incomplete. A command stopped by SIGTERM or Ctrl-C is `cancelled`, with `signal_number` 15 or 2 and its real end time. - **SQL is stored once by digest.** `statements.sql_digest` joins `artifacts` on - `(invocation_id, statement_id)` for the SQL text (`artifacts.text`), also for failed and - cancelled statements. `statements.sql_available = 0` means the text is missing, and + `invocation_id` and `artifacts.digest = 'sha256:' || statements.sql_digest` for the SQL text + (`artifacts.text`), also for failed and cancelled statements. `statements.sql_available = 0` means the text is missing, and `sql_unavailable_reason` says why. - **Audit evidence is an artifact.** A warning or failing audit's flagged rows (up to - `evidence_limit`) are in `artifacts` with `ref_id = audit_results.event_id`; `sample_digest` is + `evidence_limit`) are in `artifacts` with `digest = 'sha256:' || audit_results.sample_digest` + in the same invocation; `sample_digest` is `NULL` when `[recall] audit_evidence` kept none. Evidence holds warehouse data values. - **Output is a byte-for-byte copy** of stdout and stderr, up to `max_output_bytes` per stream (the local store keeps the first and last MiB). Use `sqb recall output` rather than querying @@ -137,7 +138,8 @@ What did a model actually execute, with its SQL? Find the run, then its statemen sqb recall query "SELECT invocation_id, invocation_started_at, status FROM invocations WHERE command = 'build' ORDER BY invocation_started_at DESC LIMIT 1" sqb recall query "SELECT s.statement_kind, s.outcome, s.duration_ms, a.text - FROM statements s JOIN artifacts a USING (invocation_id, statement_id) + FROM statements s JOIN artifacts a + ON a.invocation_id = s.invocation_id AND a.digest = 'sha256:' || s.sql_digest WHERE s.invocation_id = '' AND s.resource_name = 'orders' ORDER BY s.started_at" ``` @@ -301,7 +303,8 @@ events (view): every lifecycle fact outcome TEXT row_count INTEGER affected_rows INTEGER - payload TEXT + payload ANY + started_at TEXT invocation_id TEXT join key invocation_started_at TEXT schema_version INTEGER From f28c34172b3dd63bff948f2cfac0576e87c7958d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 13 Nov 2026 22:52:36 +0000 Subject: [PATCH 10/10] test(recall): read the folded statement start in the previous ClickHouse statements view --- .../e2e/src/sqlbuild/cli/commands/main/clickhouse/helpers.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/helpers.py index dcf43b1d8c..d7bb7a2df0 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/helpers.py @@ -748,7 +748,8 @@ def read(table: str, expiry: str = "invocation_started_at", extra_days: int = 0) argMin(statement_kind, {order}) AS first_statement_kind, argMin(purpose, {order}) AS first_purpose, argMin(sql_digest, {order}) AS first_sql_digest, - minIf(occurred_at, event_type = 'statement_started') AS started_at, + coalesce(min(started_at), minIf(occurred_at, event_type = 'statement_started')) + AS statement_started_at, minIf(occurred_at, event_type = 'statement_submitted') AS submitted_at, max(query_id) AS any_query_id, countIf({terminal}) AS terminal_count, @@ -784,7 +785,7 @@ def read(table: str, expiry: str = "invocation_started_at", extra_days: int = 0) folded.first_purpose AS purpose, if(folded.terminal_count > 0, coalesce(folded.last_query_id, folded.any_query_id), folded.any_query_id) AS query_id, - folded.started_at AS started_at, + folded.statement_started_at AS started_at, folded.submitted_at AS submitted_at, if(folded.terminal_count > 0, folded.last_ended_at, NULL) AS ended_at, multiIf(