diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index 09e918d5d4..23f2ec18a7 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -2398,6 +2398,8 @@ export interface components { }; /** @description What the mirror knows as of `synced_at`; `None` when a required table never synced cleanly. */ PreviewCopy: { + /** @description Every mirrored consumption row when asked for; omitted otherwise, as older locations reject it. */ + consumption?: components["schemas"]["WarehouseConsumptionRecord"][]; fingerprints: components["schemas"]["WarehouseFingerprintRecord"][]; latest_runs: components["schemas"]["PreviewLatestRun"][]; observations: components["schemas"]["WarehouseObservationRecord"][]; @@ -2407,6 +2409,8 @@ export interface components { }; /** @description A location asks for the mirrored copy a plan preview plans from, as its fenced reader. */ PreviewCopyRead: { + /** @description Whether the copy carries consumption history; sent only when set, as older servers reject it. */ + consumption?: boolean; /** Format: int64 */ generation: number; /** Format: uuid */ @@ -3258,6 +3262,27 @@ export interface components { state_database?: string | null; state_schema: string; }; + /** @description One input version an execution consumed, or the row marking that the execution started. */ + WarehouseConsumptionRecord: { + execution_id?: string | null; + /** Format: int64 */ + input_count?: number | null; + input_kind?: string | null; + input_name?: string | null; + input_version?: string | null; + node_name?: string | null; + node_type?: string | null; + phase?: string | null; + /** Format: date-time */ + recorded_at: string; + run_id?: string | null; + state_database?: string | null; + state_schema: string; + target_database?: string | null; + target_name?: string | null; + target_schema?: string | null; + version_hash?: string | null; + }; /** @description Fingerprints remain history here; latest-row selection belongs to the existing planner reader. */ WarehouseFingerprintRecord: { definition: string; diff --git a/crates/sqlbuild-cli/src/native_commands/main/_location_plan.rs b/crates/sqlbuild-cli/src/native_commands/main/_location_plan.rs index bf7c4e627c..2460b9980a 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_location_plan.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_location_plan.rs @@ -31,6 +31,9 @@ pub(super) async fn plan_location( let read = PreviewCopyRead { session_id, generation, + consumption: settings + .get("changed") + .is_some_and(|changed| !matches!(changed, Value::Null | Value::Bool(false))), }; let copy = match client.preview_copy(&options.target, &read).await { Ok(copy) => copy, diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/mirror_pages.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/mirror_pages.rs index 57732bff90..e23899c68d 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/mirror_pages.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/mirror_pages.rs @@ -36,6 +36,9 @@ const LATEST_RUNS_SQL: &str = "SELECT record_json FROM (SELECT record_json, ROW_ WHERE location_id=$1 AND target=$2 AND record_json ->> 'phase'='succeeded' \ AND record_json ->> 'run_id' IS NOT NULL) runs WHERE newest=1 \ ORDER BY record_json ->> 'node_type', record_json ->> 'node_name'"; +/// Every consumption row the warehouse still keeps, which complete scans bound. +const CONSUMPTION_SQL: &str = "SELECT record_json FROM sqb_warehouse_consumption \ + WHERE location_id=$1 AND target=$2 ORDER BY observed_at, event_id"; const OBSERVATIONS_SQL: &str = "SELECT record_json FROM (SELECT record_json, kind, node, ROW_NUMBER() OVER \ (PARTITION BY state_database, state_schema, kind, node ORDER BY observed_at DESC, event_id DESC) AS newest \ FROM sqb_warehouse_observations WHERE location_id=$1 AND target=$2) observations WHERE newest=1 \ @@ -147,6 +150,14 @@ pub(crate) async fn preview_latest_runs( Ok((transaction, runs)) } +/// Every mirrored consumption row: a build reads the newest executions and any older one by ID. +pub(crate) async fn preview_consumption( + transaction: ControlTransaction, + scope: (Uuid, &str), +) -> Result<(ControlTransaction, Vec), ControlError> { + mirrored(transaction, CONSUMPTION_SQL, scope).await +} + /// Each node's newest observation of each kind. pub(crate) async fn preview_observations( transaction: ControlTransaction, diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/read_preview_copy.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/read_preview_copy.rs index c72203c06b..46c9305908 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/read_preview_copy.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/read_preview_copy.rs @@ -7,8 +7,8 @@ use crate::store::main::commit::commit; use crate::store::models::ControlStore; use crate::warehouse_metadata::_helpers::ingestion::begin_ingestion; use crate::warehouse_metadata::_helpers::mirror_pages::{ - preview_fingerprints, preview_latest_runs, preview_observations, preview_relations, - preview_synced_at, + preview_consumption, preview_fingerprints, preview_latest_runs, preview_observations, + preview_relations, preview_synced_at, }; use crate::warehouse_metadata::models::{PreviewCopy, PreviewCopyRead}; @@ -23,6 +23,11 @@ pub async fn read_preview_copy( let (transaction, fingerprints) = preview_fingerprints(transaction, scope).await?; let (transaction, latest_runs) = preview_latest_runs(transaction, scope).await?; let (transaction, observations) = preview_observations(transaction, scope).await?; + let (transaction, consumption) = if request.consumption { + preview_consumption(transaction, scope).await? + } else { + (transaction, Vec::new()) + }; let (transaction, synced_at) = preview_synced_at(transaction, scope).await?; commit(transaction).await?; Ok(PreviewCopy { @@ -31,5 +36,6 @@ pub async fn read_preview_copy( fingerprints, latest_runs, observations, + consumption, }) } diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs index feb5fa1814..74df786f88 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs @@ -435,6 +435,9 @@ pub struct WarehouseSyncDue { pub struct PreviewCopyRead { pub session_id: Uuid, pub generation: i64, + /// Whether the copy carries consumption history; sent only when set, as older servers reject it. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub consumption: bool, } /// The newest successful build of a node, which a `cursor_max` observation must come from. @@ -466,6 +469,9 @@ pub struct PreviewCopy { pub fingerprints: Vec, pub latest_runs: Vec, pub observations: Vec, + /// Every mirrored consumption row when asked for; omitted otherwise, as older locations reject it. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub consumption: Vec, } /// Private interpreter input; cursor values are data and receive adapter literal rendering. diff --git a/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/preview_copy/helpers.rs b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/preview_copy/helpers.rs index aad9cb8ab9..aa112308cb 100644 --- a/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/preview_copy/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/preview_copy/helpers.rs @@ -8,6 +8,8 @@ use crate::platform::models::{LocationHealthReport, LocationWorkClient}; use crate::warehouse_metadata::models::{PreviewCopy, PreviewCopyRead}; const NODE: &str = "preview_copy_orders"; +/// A relation with more executions than a build reads recently; read-back can still need them all. +const BUSY_NODE: &str = "preview_copy_customers"; async fn post(client: &Client, (url, bearer): (&str, &str), body: Value) { let response = client @@ -106,6 +108,20 @@ pub(crate) async fn preview_copy_scope( ), ) .await; + let mut busy: Vec = Vec::new(); + for execution in 1..=10 { + for (phase, hour, minute) in [("started", execution, 0), ("succeeded", execution, 30)] { + let mut row = consumption( + &format!("busy-execution-{execution:02}"), + (phase, &format!("busy-run-{execution:02}")), + &format!("2026-03-02T{hour:02}:{minute:02}:00Z"), + ); + row["node_name"] = json!(BUSY_NODE); + row["target_name"] = json!(BUSY_NODE); + busy.push(row); + } + } + post(client, (&metadata, bearer), batch("consumption", busy)).await; post( client, (&metadata, bearer), @@ -142,6 +158,7 @@ pub(crate) async fn preview_copy_scope( let read = PreviewCopyRead { session_id: preview_session, generation: 1, + consumption: false, }; let copy = || async { outbound @@ -174,6 +191,46 @@ pub(crate) async fn preview_copy_scope( ) }; let unsynced = copy().await; + let older_location: Value = client + .post(format!("{metadata}/preview-copy")) + .bearer_auth(bearer) + .json(&json!({"session_id": preview_session, "generation": 1})) + .send() + .await + .expect("copy for an older location") + .json() + .await + .expect("copy body"); + assert!( + unsynced.consumption.is_empty() && older_location.get("consumption").is_none(), + "a read that does not ask for consumption never carries the field, which older locations reject" + ); + let with_consumption = outbound + .preview_copy( + "preview", + &PreviewCopyRead { + consumption: true, + ..read.clone() + }, + ) + .await + .expect("preview copy with consumption"); + let busy: Vec<(String, String)> = with_consumption + .consumption + .iter() + .filter(|row| row.target_name.as_deref() == Some(BUSY_NODE)) + .map(|row| { + ( + row.execution_id.clone().unwrap_or_default(), + row.phase.clone().unwrap_or_default(), + ) + }) + .collect(); + assert_eq!( + busy.len(), + 20, + "every mirrored row, beyond the newest eight executions, so read-back finds older ones: {busy:?}" + ); assert_eq!(unsynced.synced_at, None, "nothing synced cleanly yet"); assert_eq!( ours(&unsynced), @@ -267,6 +324,7 @@ pub(crate) async fn preview_copy_scope( &PreviewCopyRead { session_id, generation: 1, + consumption: false, }, ) .await diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/consumption_state.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/consumption_state.rs index be69f7d915..191071123c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/consumption_state.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/consumption_state.rs @@ -13,6 +13,7 @@ use sqlbuild_state::consumption::constants::{ }; use sqlbuild_state::consumption::main::executions_by_id_sql::executions_by_id_sql; use sqlbuild_state::consumption::main::prune_executions_sql::prune_executions_sql; +use sqlbuild_state::consumption::main::recent_executions::recent_executions; use sqlbuild_state::consumption::main::recent_executions_sql::recent_executions_sql; use sqlbuild_state::consumption::models::{ ConsumedInput, ConsumptionAppend, ConsumptionRow, NodeKey, RelationKey, @@ -22,6 +23,7 @@ use sqlbuild_state::state_tables::main::values_insert_sql::values_insert_sql; use crate::bindings::_helpers::boundary::python_calls::call_with; use crate::bindings::_helpers::contracts::state_literals::render_state_sql_literal; +use crate::bindings::_helpers::native_host::synced_copy::SyncedCopyAdapter; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::state_rows::state_tables::state_table_create_sql; @@ -452,6 +454,14 @@ impl ConsumptionConnection { py: Python<'_>, location: &Location, ) -> PyResult> { + if let Ok(copy) = self.adapter.bind(py).downcast::() { + return Ok(recent_executions( + © + .get() + .consumption((location.database.as_deref(), &location.schema)), + RECENT_EXECUTIONS, + )); + } read_recent(self.adapter.bind(py), &self.connected(py)?, location) } @@ -461,6 +471,13 @@ impl ConsumptionConnection { location: &Location, execution_ids: &[&str], ) -> PyResult> { + if let Ok(copy) = self.adapter.bind(py).downcast::() { + let mut rows: Vec = copy + .get() + .consumption((location.database.as_deref(), &location.schema)); + rows.retain(|row| execution_ids.contains(&row.row.execution_id.as_str())); + return Ok(rows); + } read_executions( self.adapter.bind(py), &self.connected(py)?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/synced_copy.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/synced_copy.rs index 63867edcdc..93fc1dbe40 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/synced_copy.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/synced_copy.rs @@ -17,6 +17,9 @@ use sqlbuild_cache::digest::main::sha256_hex::sha256_hex; use sqlbuild_core::constants::{FINGERPRINT_TABLE_NAME, STATE_TABLE_NAMES}; use sqlbuild_host::entries::models::AdapterHost; use sqlbuild_orchestration::warehouse_metadata::models::{PreviewCopy, WarehouseCatalogRecord}; +use sqlbuild_state::consumption::models::{ + ConsumedInput, ConsumptionAppend, ConsumptionRow, NodeKey, RelationKey, +}; use sqlbuild_state::execution_history::constants::MEMORY_PATH; use sqlbuild_state::observations::constants::{CURSOR_MAX_KIND, SOURCE_COLUMNS_KIND}; @@ -250,6 +253,49 @@ impl SyncedCopyAdapter { None } + /// Every copied consumption row of the table in `(database, schema)`. + pub(crate) fn consumption( + &self, + (database, schema): (Option<&str>, &str), + ) -> Vec { + let mut rows: Vec = Vec::new(); + for record in &self.copy.consumption { + if !same(Some(&record.state_schema), Some(schema)) + || !(database.is_none() + || record.state_database.is_none() + || same(record.state_database.as_deref(), database)) + { + continue; + } + let text = |value: &Option| value.clone().unwrap_or_default(); + rows.push(ConsumptionRow { + row: ConsumptionAppend { + execution_id: text(&record.execution_id), + phase: text(&record.phase), + run_id: text(&record.run_id), + node: NodeKey { + kind: text(&record.node_type), + name: text(&record.node_name), + }, + relation: RelationKey { + database: record.target_database.clone(), + schema: text(&record.target_schema), + name: text(&record.target_name), + }, + version_hash: text(&record.version_hash), + input_count: record.input_count.unwrap_or(-1), + input: record.input_kind.clone().map(|kind| ConsumedInput { + kind, + name: text(&record.input_name), + version: text(&record.input_version), + }), + }, + recorded_at: record.recorded_at.timestamp_micros(), + }); + } + rows + } + /// The source's columns as last described, only when it described exactly `sql`. pub(crate) fn described_columns( &self, diff --git a/crates/sqlbuild-state/src/consumption/main/mod.rs b/crates/sqlbuild-state/src/consumption/main/mod.rs index 31accb8109..0fa927d55d 100644 --- a/crates/sqlbuild-state/src/consumption/main/mod.rs +++ b/crates/sqlbuild-state/src/consumption/main/mod.rs @@ -4,6 +4,7 @@ pub mod consumption_history; pub mod executions_by_id_sql; pub mod prune_executions_sql; pub mod read_back_history; +pub mod recent_executions; pub mod recent_executions_sql; pub mod started_append; pub mod succeeded_appends; diff --git a/crates/sqlbuild-state/src/consumption/main/recent_executions.rs b/crates/sqlbuild-state/src/consumption/main/recent_executions.rs new file mode 100644 index 0000000000..caf764d07e --- /dev/null +++ b/crates/sqlbuild-state/src/consumption/main/recent_executions.rs @@ -0,0 +1,37 @@ +//! Choose the recent executions of every relation from rows already read. + +use std::collections::{BTreeMap, BTreeSet}; + +use crate::consumption::models::{ConsumptionRow, RelationKey}; + +/// Rows of the newest `limit` executions per relation by earliest row, as `recent_executions_sql`. +pub fn recent_executions(rows: &[ConsumptionRow], limit: usize) -> Vec { + let mut started: BTreeMap<(&RelationKey, &str), i64> = BTreeMap::new(); + for row in rows { + let entry = started + .entry((&row.row.relation, row.row.execution_id.as_str())) + .or_insert(row.recorded_at); + *entry = (*entry).min(row.recorded_at); + } + let mut by_relation: BTreeMap<&RelationKey, Vec<(i64, &str)>> = BTreeMap::new(); + for ((relation, execution), time) in started { + by_relation + .entry(relation) + .or_default() + .push((time, execution)); + } + let mut kept: BTreeSet<(&RelationKey, &str)> = BTreeSet::new(); + for (relation, executions) in &mut by_relation { + executions.sort_by(|left, right| right.cmp(left)); + kept.extend( + executions + .iter() + .take(limit) + .map(|(_, execution)| (*relation, *execution)), + ); + } + rows.iter() + .filter(|row| kept.contains(&(&row.row.relation, row.row.execution_id.as_str()))) + .cloned() + .collect() +} diff --git a/crates/sqlbuild-state/src/consumption/tests.rs b/crates/sqlbuild-state/src/consumption/tests.rs index 6027e17bc4..8c354f9bba 100644 --- a/crates/sqlbuild-state/src/consumption/tests.rs +++ b/crates/sqlbuild-state/src/consumption/tests.rs @@ -2,6 +2,8 @@ mod helpers; #[path = "tests/test_decide.rs"] mod test_decide; +#[path = "tests/test_recent_executions.rs"] +mod test_recent_executions; #[path = "tests/test_simulation.rs"] mod test_simulation; #[path = "tests/test_types.rs"] diff --git a/crates/sqlbuild-state/src/consumption/tests/helpers.rs b/crates/sqlbuild-state/src/consumption/tests/helpers.rs index 1ca2aac9c0..591aef47e1 100644 --- a/crates/sqlbuild-state/src/consumption/tests/helpers.rs +++ b/crates/sqlbuild-state/src/consumption/tests/helpers.rs @@ -9,6 +9,7 @@ use crate::consumption::main::changed_run_set::changed_run_set; use crate::consumption::main::consumed_inputs::consumed_inputs; use crate::consumption::main::consumption_history::consumption_history; use crate::consumption::main::read_back_history::read_back_history; +use crate::consumption::main::recent_executions::recent_executions; use crate::consumption::main::started_append::started_append; use crate::consumption::main::succeeded_appends::succeeded_appends; use crate::consumption::models::TriggeredJob; @@ -171,35 +172,7 @@ impl World { /// The reader's SQL: rows of the newest executions per relation by earliest row time. fn read_rows(&self) -> Vec { - let mut started: BTreeMap<(RelationKey, String), i64> = BTreeMap::new(); - for row in &self.rows { - let entry = started - .entry((row.row.relation.clone(), row.row.execution_id.clone())) - .or_insert(row.recorded_at); - *entry = (*entry).min(row.recorded_at); - } - let mut by_relation: BTreeMap> = BTreeMap::new(); - for ((relation, execution), time) in started { - by_relation - .entry(relation) - .or_default() - .push((time, execution)); - } - let mut kept: BTreeSet = BTreeSet::new(); - for executions in by_relation.values_mut() { - executions.sort_by(|left, right| right.cmp(left)); - kept.extend( - executions - .iter() - .take(RECENT_EXECUTIONS) - .map(|(_, execution)| execution.clone()), - ); - } - self.rows - .iter() - .filter(|row| kept.contains(&row.row.execution_id)) - .cloned() - .collect() + recent_executions(&self.rows, RECENT_EXECUTIONS) } /// The janitor's SQL: keep the newest `retain` executions per relation. @@ -788,3 +761,27 @@ pub(crate) fn two_jobs() -> Scenario { pub(crate) const JOB_A: (&str, &[&str]) = ("job_a", &["stg_orders", "int_orders"]); pub(crate) const JOB_B: (&str, &[&str]) = ("job_b", &["fct_revenue", "rpt_pnl"]); + +/// One succeeded row of `execution`, writing the `main` relation named `relation`. +pub(crate) fn recorded_row(relation: &str, execution: &str, recorded_at: i64) -> ConsumptionRow { + ConsumptionRow { + row: ConsumptionAppend { + execution_id: execution.into(), + phase: "succeeded".into(), + run_id: format!("run-{execution}"), + node: NodeKey { + kind: "model".into(), + name: relation.into(), + }, + relation: RelationKey { + database: None, + schema: "main".into(), + name: relation.into(), + }, + version_hash: "version".into(), + input_count: 0, + input: None, + }, + recorded_at, + } +} diff --git a/crates/sqlbuild-state/src/consumption/tests/test_recent_executions.rs b/crates/sqlbuild-state/src/consumption/tests/test_recent_executions.rs new file mode 100644 index 0000000000..da17c10681 --- /dev/null +++ b/crates/sqlbuild-state/src/consumption/tests/test_recent_executions.rs @@ -0,0 +1,61 @@ +use crate::consumption::main::recent_executions::recent_executions; +use crate::consumption::tests::helpers::recorded_row; +use crate::consumption::tests::test_types::RecentExecutionsTestCase; + +#[test] +fn given_more_executions_than_the_limit_when_choosing_recent_then_the_newest_starts_win_per_relation() + { + // `orders-1` started first but wrote its last row after every other execution. + let mut late_finisher = Vec::new(); + for (execution, started, finished) in [(1, 10, 50), (2, 20, 25), (3, 30, 35), (4, 40, 45)] { + late_finisher.push(recorded_row( + "orders", + &format!("orders-{execution}"), + started, + )); + late_finisher.push(recorded_row( + "orders", + &format!("orders-{execution}"), + finished, + )); + } + late_finisher.push(recorded_row("customers", "customers-1", 1)); + let test_cases = [ + RecentExecutionsTestCase { + description: "an execution that started first but wrote last is the oldest; other relations keep theirs", + rows: late_finisher, + limit: 2, + expected_executions: vec!["orders-3", "orders-4", "customers-1"], + expected_rows: 5, + }, + RecentExecutionsTestCase { + description: "equal start times keep the larger execution id, as the warehouse query does", + rows: vec![ + recorded_row("orders", "orders-a", 10), + recorded_row("orders", "orders-b", 10), + ], + limit: 1, + expected_executions: vec!["orders-b"], + expected_rows: 1, + }, + ]; + for test_case in test_cases { + let kept = recent_executions(&test_case.rows, test_case.limit); + let mut executions: Vec<&str> = kept + .iter() + .map(|row| row.row.execution_id.as_str()) + .collect(); + executions.dedup(); + assert_eq!( + executions, test_case.expected_executions, + "{}", + test_case.description + ); + assert_eq!( + kept.len(), + test_case.expected_rows, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-state/src/consumption/tests/test_types.rs b/crates/sqlbuild-state/src/consumption/tests/test_types.rs index 7ed8e9b34d..38e23c0c43 100644 --- a/crates/sqlbuild-state/src/consumption/tests/test_types.rs +++ b/crates/sqlbuild-state/src/consumption/tests/test_types.rs @@ -87,6 +87,14 @@ pub(crate) struct Scenario { pub(crate) clock: i64, } +pub(crate) struct RecentExecutionsTestCase { + pub(crate) description: &'static str, + pub(crate) rows: Vec, + pub(crate) limit: usize, + pub(crate) expected_executions: Vec<&'static str>, + pub(crate) expected_rows: usize, +} + pub(crate) struct DecideTestCase { pub(crate) description: &'static str, pub(crate) outcomes: Vec<(String, crate::consumption::models::ChangedOutcome)>, diff --git a/fensu.toml b/fensu.toml index 086077a723..e425f13b70 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1665,6 +1665,7 @@ allowed_paths = [ "crates/sqlbuild-cli/src/native_commands/main/_location_plan.rs=A location's requested plan names the synced copy for its sqb plan child.", "crates/sqlbuild-core/src/plan_display/constants.rs=Names the synced-copy environment variable.", "crates/sqlbuild-python/src/bindings/_helpers/freshness/planning.rs=Preview branch: a refused freshness observation leaves the source unknown.", + "crates/sqlbuild-python/src/bindings/_helpers/native_host/consumption_state.rs=Preview branch: --changed reads consumption history from the copy instead of the warehouse table.", "crates/sqlbuild-python/src/bindings/_helpers/native_host/registration.rs=Registers the synced copy's refusal error with Python.", "crates/sqlbuild-python/src/bindings/_helpers/native_host/synced_copy.rs=Defines the synced-copy adapter.", "crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs=Only sqb plan activates the synced copy and reports what it could not know.", diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py index b230cebb88..96888e85e6 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -408,6 +408,14 @@ class AdhocRunTestCase: expected_worker_log: str = "" +@dataclass(frozen=True) +class ChangedPlanPreviewTestCase: + description: str + edited_model: str + edit: tuple[str, str] + expected_changed: dict[str, Any] + + @dataclass(frozen=True) class PlanPreviewTestCase: description: str diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/catalog/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/catalog/helpers.py index 417c30573d..0aa5817bea 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/catalog/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/catalog/helpers.py @@ -89,8 +89,10 @@ def wait_for_statistics( return records -def request_plan(control: Path, selection: list[str]) -> str: - """Queue the plan work a person's Plan creates, frozen to exact node keys.""" +def request_plan( + control: Path, selection: list[str], settings: dict[str, object] | None = None +) -> str: + """Queue the plan work a person's Plan creates, frozen to exact node keys and settings.""" request_id: str = str(uuid4()) now: datetime = datetime.now(UTC) with closing(sqlite3.connect(control)) as database: @@ -99,13 +101,14 @@ def request_plan(control: Path, selection: list[str]) -> str: "request_json,requested_by,requested_at,expires_at) SELECT ?,c.location_id,c.target," "c.snapshot_id,s.generation,json_object('request_id',?,'location_id',c.location_id," "'target',c.target,'reason','Plan','work',json_object('kind','plan','command'," - "json('[\"build\"]'),'selection',json(?),'settings',json_object()))," + "json('[\"build\"]'),'selection',json(?),'settings',json(?)))," "'test-admin',?,? FROM sqb_current_snapshots c JOIN sqb_snapshots s " "ON s.snapshot_id=c.snapshot_id", ( request_id, request_id, json.dumps(selection), + json.dumps(settings or {}), now.strftime("%Y-%m-%dT%H:%M:%S.%fZ"), (now + timedelta(minutes=5)).strftime("%Y-%m-%dT%H:%M:%S.%fZ"), ), diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py index abf17ff923..ce40130425 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py @@ -68,6 +68,9 @@ WorkerDeployment, WorkerExecutionTestCase, ) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.catalog.helpers import ( + request_plan, +) from tests.integration.src.sqlbuild.compiler.compiled_project.publication.docker.helpers import ( docker_deployment, ) @@ -1534,12 +1537,14 @@ def run_sqb(project: Path, *arguments: str) -> subprocess.CompletedProcess[str]: ) -def live_plan(project: Path, selection: list[str]) -> subprocess.CompletedProcess[str]: - """Run `sqb plan --json` against the warehouse for node keys such as `model:orders`.""" +def live_plan(project: Path, selection: list[str], *flags: str) -> subprocess.CompletedProcess[str]: + """Run `sqb plan --json` with `flags` on the warehouse for node keys such as `model:orders`.""" selected: list[str] = [] for key in selection: selected.extend(("--select", key.split(":")[1])) - return run_sqb(project, "plan", "--target", "development", "--json", "--no-color", *selected) + return run_sqb( + project, "plan", "--target", "development", "--json", "--no-color", *flags, *selected + ) def hold_write_lock(database: Path) -> duckdb.DuckDBPyConnection: @@ -1599,3 +1604,54 @@ def create_raw_orders(warehouse: Path) -> None: _ = connection.execute( "CREATE TABLE main.raw_orders AS SELECT CAST(1 AS BIGINT) AS order_id" ) + + +def plan_from_copy( + tmp_path: Path, project: Path, selection: list[str], settings: dict[str, object] +) -> tuple[subprocess.CompletedProcess[str], dict[str, Any]]: + """Sync `project` to a location, lock its warehouse, then plan `selection` from the copy. + + Returns the refused live plan run while the lock was held and the location's plan document. + """ + control: Path = tmp_path / "control/.sqlbuild/control.sqlite3" + log: Path = tmp_path / "location.log" + started: datetime = datetime.now(UTC) + with ( + snapshot_api(tmp_path / "control") as api, + location_process( + [ + str(Path(sys.executable).with_name("sqb")), + "--project-dir", + str(project), + "location", + "serve", + "--server", + api.url, + "--location-id", + api.location_id, + "--target", + "development", + ], + log, + tmp_path / "location.stdout", + env=os.environ | {"SQLBUILD_LOCATION_TOKEN": api.bearer}, + ) as location, + ): + _ = wait_for_generation(api, "development", 1, location, log) + _ = wait_for_warehouse_sync_health( + api, + {"failure": None, "tables": ["catalog", "consumption", "fingerprints", "observations"]}, + location, + log, + after=started, + ) + lock: duckdb.DuckDBPyConnection = hold_write_lock(project / "warehouse.db") + try: + refused: subprocess.CompletedProcess[str] = live_plan(project, selection) + request_id: str = request_plan(control, selection, settings) + result: dict[str, Any] = wait_for_location_work(api, request_id, location, log) + finally: + lock.close() + outcome: dict[str, Any] = result["result_json"]["outcome"] + assert outcome["kind"] == "plan", outcome + return refused, outcome["document"] diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_plan_preview.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_plan_preview.py index 9c4f1d7e4f..e79a6f48f2 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_plan_preview.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_plan_preview.py @@ -1,10 +1,7 @@ """A location plans from the server's synced copy and never opens the warehouse to do it.""" import json -import os import subprocess -import sys -from datetime import UTC, datetime from pathlib import Path from textwrap import dedent from typing import Any @@ -13,20 +10,13 @@ import pytest from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + ChangedPlanPreviewTestCase, PlanPreviewTestCase, ) -from tests.integration.src.sqlbuild.compiler.compiled_project.publication.catalog.helpers import ( - request_plan, -) from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( - hold_write_lock, live_plan, - location_process, + plan_from_copy, run_sqb, - snapshot_api, - wait_for_generation, - wait_for_location_work, - wait_for_warehouse_sync_health, ) from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files @@ -189,50 +179,10 @@ def test_given_a_synced_location_when_a_plan_is_requested_then_it_plans_from_the live_run: subprocess.CompletedProcess[str] = live_plan(project, _SELECTION) assert live_run.returncode == 0, live_run.stderr live: dict[str, Any] = json.loads(live_run.stdout) - control: Path = tmp_path / "control/.sqlbuild/control.sqlite3" log: Path = tmp_path / "location.log" - started: datetime = datetime.now(UTC) - with ( - snapshot_api(tmp_path / "control") as api, - location_process( - [ - str(Path(sys.executable).with_name("sqb")), - "--project-dir", - str(project), - "location", - "serve", - "--server", - api.url, - "--location-id", - api.location_id, - "--target", - "development", - ], - log, - tmp_path / "location.stdout", - env=os.environ | {"SQLBUILD_LOCATION_TOKEN": api.bearer}, - ) as location, - ): - _ = wait_for_generation(api, "development", 1, location, log) - _ = wait_for_warehouse_sync_health( - api, - {"failure": None, "tables": ["catalog", "consumption", "fingerprints", "observations"]}, - location, - log, - after=started, - ) - lock: duckdb.DuckDBPyConnection = hold_write_lock(project / "warehouse.db") - try: - refused: subprocess.CompletedProcess[str] = live_plan(project, _SELECTION) - request_id: str = request_plan(control, _SELECTION) - result: dict[str, Any] = wait_for_location_work(api, request_id, location, log) - finally: - lock.close() + refused, preview = plan_from_copy(tmp_path, project, _SELECTION, {}) assert refused.returncode != 0 - outcome: dict[str, Any] = result["result_json"]["outcome"] - assert outcome["kind"] == "plan", outcome - preview: dict[str, Any] = outcome["document"] assert "synced_copy" in preview, log.read_text() synced: dict[str, Any] = preview.pop("synced_copy") assert synced["synced_at"] is not None @@ -269,3 +219,48 @@ def test_given_a_synced_location_when_a_plan_is_requested_then_it_plans_from_the _ = live_models.pop(name) assert preview_models == live_models assert preview == live + + +@pytest.mark.parametrize( + "test_case", + [ + ChangedPlanPreviewTestCase( + description="an edited model is the only change the copy names", + edited_model="models/order_count.sql", + edit=("AS order_count", "AS order_count, 1 AS batch"), + expected_changed={ + "counts": ["definition"], + "reach": "job", + "runs": [ + { + "node": "model:order_count", + "reason": "planner change (query_changed)", + "downstream_of": None, + } + ], + "held_back": [], + "unchanged": ["model:daily_orders", "model:orders", "model:refunds"], + }, + ), + ], + ids=lambda case: case.description, +) +def test_given_an_edited_model_when_a_changed_plan_is_requested_then_the_copy_names_what_changed( + test_case: ChangedPlanPreviewTestCase, tmp_path: Path +) -> None: + project: Path = tmp_path / "project" + write_files(project_dir=project, files=_PROJECT_FILES) + built: subprocess.CompletedProcess[str] = run_sqb(project, "build", "--target", "development") + assert built.returncode == 0, built.stderr + edited: str = _PROJECT_FILES[test_case.edited_model].replace(*test_case.edit) + write_files(project_dir=project, files={test_case.edited_model: edited}) + live_run: subprocess.CompletedProcess[str] = live_plan(project, _SELECTION, "--changed") + assert live_run.returncode == 0, live_run.stderr + + refused, preview = plan_from_copy(tmp_path, project, _SELECTION, {"changed": True}) + + assert refused.returncode != 0 + assert preview.get("changed") == test_case.expected_changed, ( + tmp_path / "location.log" + ).read_text() + assert preview["changed"] == json.loads(live_run.stdout)["changed"]