From aa0f23500b7de9b4f6cd3252f5b11cd93fd512c9 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 00:37:29 +0000 Subject: [PATCH 1/5] fix(server): preview plan --changed from the synced consumption copy A location's plan preview read consumption history through the warehouse adapter, which the preview refuses, so every model and seed counted as changed. The preview copy now carries each relation's newest executions, ranked as a build reads them, and the preview reads them from the copy. --- clients/typescript/src/schema.d.ts | 23 ++++ .../_helpers/mirror_pages.rs | 25 ++++ .../main/location_reads/read_preview_copy.rs | 6 +- .../src/warehouse_metadata/models.rs | 3 + .../preview_copy/helpers.rs | 41 ++++++ .../_helpers/native_host/consumption_state.rs | 9 ++ .../_helpers/native_host/synced_copy.rs | 47 +++++++ .../publication/catalog/helpers.py | 9 +- .../compiled_project/publication/helpers.py | 8 +- .../publication/test_plan_preview.py | 124 ++++++++++++------ 10 files changed, 247 insertions(+), 48 deletions(-) diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index 09e918d5d4..588b260b78 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -2398,6 +2398,8 @@ export interface components { }; /** @description What the mirror knows as of `synced_at`; `None` when a required table never synced cleanly. */ PreviewCopy: { + /** @description Rows of each relation's newest executions, which `--changed` decides from. */ + consumption?: components["schemas"]["WarehouseConsumptionRecord"][]; fingerprints: components["schemas"]["WarehouseFingerprintRecord"][]; latest_runs: components["schemas"]["PreviewLatestRun"][]; observations: components["schemas"]["WarehouseObservationRecord"][]; @@ -3258,6 +3260,27 @@ export interface components { state_database?: string | null; state_schema: string; }; + /** @description One input version an execution consumed, or the row marking that the execution started. */ + WarehouseConsumptionRecord: { + execution_id?: string | null; + /** Format: int64 */ + input_count?: number | null; + input_kind?: string | null; + input_name?: string | null; + input_version?: string | null; + node_name?: string | null; + node_type?: string | null; + phase?: string | null; + /** Format: date-time */ + recorded_at: string; + run_id?: string | null; + state_database?: string | null; + state_schema: string; + target_database?: string | null; + target_name?: string | null; + target_schema?: string | null; + version_hash?: string | null; + }; /** @description Fingerprints remain history here; latest-row selection belongs to the existing planner reader. */ WarehouseFingerprintRecord: { definition: string; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/mirror_pages.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/mirror_pages.rs index 57732bff90..dddd01ecd7 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/mirror_pages.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/mirror_pages.rs @@ -5,6 +5,7 @@ use std::collections::{BTreeMap, BTreeSet}; use chrono::{DateTime, Utc}; use serde::de::DeserializeOwned; use serde_json::Value; +use sqlbuild_state::consumption::constants::RECENT_EXECUTIONS; use uuid::Uuid; use crate::authorization::models::CurrentSnapshotId; @@ -36,6 +37,15 @@ const LATEST_RUNS_SQL: &str = "SELECT record_json FROM (SELECT record_json, ROW_ WHERE location_id=$1 AND target=$2 AND record_json ->> 'phase'='succeeded' \ AND record_json ->> 'run_id' IS NOT NULL) runs WHERE newest=1 \ ORDER BY record_json ->> 'node_type', record_json ->> 'node_name'"; +/// Every row of each relation's newest executions, timing each execution by its earliest row. +const CONSUMPTION_SQL: &str = "SELECT record_json FROM (SELECT record_json, observed_at, DENSE_RANK() OVER \ + (PARTITION BY state_database, state_schema, record_json ->> 'target_database', \ + record_json ->> 'target_schema', record_json ->> 'target_name' \ + ORDER BY started_at DESC, record_json ->> 'execution_id' DESC) AS execution_rank \ + FROM (SELECT record_json, observed_at, state_database, state_schema, MIN(observed_at) OVER \ + (PARTITION BY state_database, state_schema, record_json ->> 'execution_id') AS started_at \ + FROM sqb_warehouse_consumption WHERE location_id=$1 AND target=$2) executions) ranked \ + WHERE execution_rank <= $3 ORDER BY observed_at, record_json ->> 'execution_id'"; const OBSERVATIONS_SQL: &str = "SELECT record_json FROM (SELECT record_json, kind, node, ROW_NUMBER() OVER \ (PARTITION BY state_database, state_schema, kind, node ORDER BY observed_at DESC, event_id DESC) AS newest \ FROM sqb_warehouse_observations WHERE location_id=$1 AND target=$2) observations WHERE newest=1 \ @@ -147,6 +157,21 @@ pub(crate) async fn preview_latest_runs( Ok((transaction, runs)) } +/// The rows of each relation's newest executions, as many as a build reads from the warehouse. +pub(crate) async fn preview_consumption( + transaction: ControlTransaction, + scope: (Uuid, &str), +) -> Result<(ControlTransaction, Vec), ControlError> { + let mut parameters: Vec = scope_parameters(scope); + parameters.push(Parameter::Integer(RECENT_EXECUTIONS as i64)); + let (transaction, rows) = query(transaction, CONSUMPTION_SQL, parameters).await?; + let mut records: Vec = Vec::with_capacity(rows.len()); + for row in rows { + records.push(decoded(row)?); + } + Ok((transaction, records)) +} + /// Each node's newest observation of each kind. pub(crate) async fn preview_observations( transaction: ControlTransaction, diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/read_preview_copy.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/read_preview_copy.rs index c72203c06b..855763d9f4 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/read_preview_copy.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/read_preview_copy.rs @@ -7,8 +7,8 @@ use crate::store::main::commit::commit; use crate::store::models::ControlStore; use crate::warehouse_metadata::_helpers::ingestion::begin_ingestion; use crate::warehouse_metadata::_helpers::mirror_pages::{ - preview_fingerprints, preview_latest_runs, preview_observations, preview_relations, - preview_synced_at, + preview_consumption, preview_fingerprints, preview_latest_runs, preview_observations, + preview_relations, preview_synced_at, }; use crate::warehouse_metadata::models::{PreviewCopy, PreviewCopyRead}; @@ -23,6 +23,7 @@ pub async fn read_preview_copy( let (transaction, fingerprints) = preview_fingerprints(transaction, scope).await?; let (transaction, latest_runs) = preview_latest_runs(transaction, scope).await?; let (transaction, observations) = preview_observations(transaction, scope).await?; + let (transaction, consumption) = preview_consumption(transaction, scope).await?; let (transaction, synced_at) = preview_synced_at(transaction, scope).await?; commit(transaction).await?; Ok(PreviewCopy { @@ -31,5 +32,6 @@ pub async fn read_preview_copy( fingerprints, latest_runs, observations, + consumption, }) } diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs index feb5fa1814..9568c69c7c 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs @@ -466,6 +466,9 @@ pub struct PreviewCopy { pub fingerprints: Vec, pub latest_runs: Vec, pub observations: Vec, + /// Rows of each relation's newest executions, which `--changed` decides from. + #[serde(default)] + pub consumption: Vec, } /// Private interpreter input; cursor values are data and receive adapter literal rendering. diff --git a/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/preview_copy/helpers.rs b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/preview_copy/helpers.rs index aad9cb8ab9..8a0b614d55 100644 --- a/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/preview_copy/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/preview_copy/helpers.rs @@ -8,6 +8,8 @@ use crate::platform::models::{LocationHealthReport, LocationWorkClient}; use crate::warehouse_metadata::models::{PreviewCopy, PreviewCopyRead}; const NODE: &str = "preview_copy_orders"; +/// A relation with more executions than a build reads; only its newest eight are copied. +const BUSY_NODE: &str = "preview_copy_customers"; async fn post(client: &Client, (url, bearer): (&str, &str), body: Value) { let response = client @@ -106,6 +108,21 @@ pub(crate) async fn preview_copy_scope( ), ) .await; + let mut busy: Vec = Vec::new(); + for execution in 1..=10 { + let finished: u32 = if execution == 1 { 20 } else { execution }; + for (phase, hour, minute) in [("started", execution, 0), ("succeeded", finished, 30)] { + let mut row = consumption( + &format!("busy-execution-{execution:02}"), + (phase, &format!("busy-run-{execution:02}")), + &format!("2026-03-02T{hour:02}:{minute:02}:00Z"), + ); + row["node_name"] = json!(BUSY_NODE); + row["target_name"] = json!(BUSY_NODE); + busy.push(row); + } + } + post(client, (&metadata, bearer), batch("consumption", busy)).await; post( client, (&metadata, bearer), @@ -174,6 +191,30 @@ pub(crate) async fn preview_copy_scope( ) }; let unsynced = copy().await; + let mut busy_executions: Vec = unsynced + .consumption + .iter() + .filter(|row| row.target_name.as_deref() == Some(BUSY_NODE)) + .filter_map(|row| row.execution_id.clone()) + .collect(); + busy_executions.dedup(); + assert_eq!( + busy_executions, + (3..=10) + .map(|execution| format!("busy-execution-{execution:02}")) + .collect::>(), + "each relation's newest eight executions by start, every row of each, oldest first; \ + the first execution finished last but started first" + ); + assert_eq!( + unsynced + .consumption + .iter() + .filter(|row| row.target_name.as_deref() == Some(BUSY_NODE)) + .count(), + 16, + "both rows of each copied execution" + ); assert_eq!(unsynced.synced_at, None, "nothing synced cleanly yet"); assert_eq!( ours(&unsynced), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/consumption_state.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/consumption_state.rs index be69f7d915..51d167a052 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/consumption_state.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/consumption_state.rs @@ -22,6 +22,7 @@ use sqlbuild_state::state_tables::main::values_insert_sql::values_insert_sql; use crate::bindings::_helpers::boundary::python_calls::call_with; use crate::bindings::_helpers::contracts::state_literals::render_state_sql_literal; +use crate::bindings::_helpers::native_host::synced_copy::SyncedCopyAdapter; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::state_rows::state_tables::state_table_create_sql; @@ -452,6 +453,9 @@ impl ConsumptionConnection { py: Python<'_>, location: &Location, ) -> PyResult> { + if let Ok(copy) = self.adapter.bind(py).downcast::() { + return Ok(copy.get().consumption(location)); + } read_recent(self.adapter.bind(py), &self.connected(py)?, location) } @@ -461,6 +465,11 @@ impl ConsumptionConnection { location: &Location, execution_ids: &[&str], ) -> PyResult> { + if let Ok(copy) = self.adapter.bind(py).downcast::() { + let mut rows: Vec = copy.get().consumption(location); + rows.retain(|row| execution_ids.contains(&row.row.execution_id.as_str())); + return Ok(rows); + } read_executions( self.adapter.bind(py), &self.connected(py)?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/synced_copy.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/synced_copy.rs index 63867edcdc..7d649fda4b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/synced_copy.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/synced_copy.rs @@ -17,10 +17,14 @@ use sqlbuild_cache::digest::main::sha256_hex::sha256_hex; use sqlbuild_core::constants::{FINGERPRINT_TABLE_NAME, STATE_TABLE_NAMES}; use sqlbuild_host::entries::models::AdapterHost; use sqlbuild_orchestration::warehouse_metadata::models::{PreviewCopy, WarehouseCatalogRecord}; +use sqlbuild_state::consumption::models::{ + ConsumedInput, ConsumptionAppend, ConsumptionRow, NodeKey, RelationKey, +}; use sqlbuild_state::execution_history::constants::MEMORY_PATH; use sqlbuild_state::observations::constants::{CURSOR_MAX_KIND, SOURCE_COLUMNS_KIND}; use crate::bindings::_helpers::boundary::python_calls::{KeywordArguments, call_with}; +use crate::bindings::_helpers::native_host::consumption_state::Location; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::sqlite_connection::{SqliteConnection, SqliteCursor}; use crate::bindings::_helpers::records::fields::field as native_field; @@ -250,6 +254,49 @@ impl SyncedCopyAdapter { None } + /// The copied consumption rows of the table in `location`, each relation's newest executions. + pub(crate) fn consumption(&self, location: &Location) -> Vec { + let mut rows: Vec = Vec::new(); + for record in &self.copy.consumption { + if !same(Some(&record.state_schema), Some(&location.schema)) + || !(location.database.is_none() + || record.state_database.is_none() + || same( + record.state_database.as_deref(), + location.database.as_deref(), + )) + { + continue; + } + let text = |value: &Option| value.clone().unwrap_or_default(); + rows.push(ConsumptionRow { + row: ConsumptionAppend { + execution_id: text(&record.execution_id), + phase: text(&record.phase), + run_id: text(&record.run_id), + node: NodeKey { + kind: text(&record.node_type), + name: text(&record.node_name), + }, + relation: RelationKey { + database: record.target_database.clone(), + schema: text(&record.target_schema), + name: text(&record.target_name), + }, + version_hash: text(&record.version_hash), + input_count: record.input_count.unwrap_or(-1), + input: record.input_kind.clone().map(|kind| ConsumedInput { + kind, + name: text(&record.input_name), + version: text(&record.input_version), + }), + }, + recorded_at: record.recorded_at.timestamp_micros(), + }); + } + rows + } + /// The source's columns as last described, only when it described exactly `sql`. pub(crate) fn described_columns( &self, diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/catalog/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/catalog/helpers.py index 417c30573d..0aa5817bea 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/catalog/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/catalog/helpers.py @@ -89,8 +89,10 @@ def wait_for_statistics( return records -def request_plan(control: Path, selection: list[str]) -> str: - """Queue the plan work a person's Plan creates, frozen to exact node keys.""" +def request_plan( + control: Path, selection: list[str], settings: dict[str, object] | None = None +) -> str: + """Queue the plan work a person's Plan creates, frozen to exact node keys and settings.""" request_id: str = str(uuid4()) now: datetime = datetime.now(UTC) with closing(sqlite3.connect(control)) as database: @@ -99,13 +101,14 @@ def request_plan(control: Path, selection: list[str]) -> str: "request_json,requested_by,requested_at,expires_at) SELECT ?,c.location_id,c.target," "c.snapshot_id,s.generation,json_object('request_id',?,'location_id',c.location_id," "'target',c.target,'reason','Plan','work',json_object('kind','plan','command'," - "json('[\"build\"]'),'selection',json(?),'settings',json_object()))," + "json('[\"build\"]'),'selection',json(?),'settings',json(?)))," "'test-admin',?,? FROM sqb_current_snapshots c JOIN sqb_snapshots s " "ON s.snapshot_id=c.snapshot_id", ( request_id, request_id, json.dumps(selection), + json.dumps(settings or {}), now.strftime("%Y-%m-%dT%H:%M:%S.%fZ"), (now + timedelta(minutes=5)).strftime("%Y-%m-%dT%H:%M:%S.%fZ"), ), diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py index abf17ff923..e39a4ba59d 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py @@ -1534,12 +1534,14 @@ def run_sqb(project: Path, *arguments: str) -> subprocess.CompletedProcess[str]: ) -def live_plan(project: Path, selection: list[str]) -> subprocess.CompletedProcess[str]: - """Run `sqb plan --json` against the warehouse for node keys such as `model:orders`.""" +def live_plan(project: Path, selection: list[str], *flags: str) -> subprocess.CompletedProcess[str]: + """Run `sqb plan --json` with `flags` on the warehouse for node keys such as `model:orders`.""" selected: list[str] = [] for key in selection: selected.extend(("--select", key.split(":")[1])) - return run_sqb(project, "plan", "--target", "development", "--json", "--no-color", *selected) + return run_sqb( + project, "plan", "--target", "development", "--json", "--no-color", *flags, *selected + ) def hold_write_lock(database: Path) -> duckdb.DuckDBPyConnection: diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_plan_preview.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_plan_preview.py index 9c4f1d7e4f..e371d55655 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_plan_preview.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_plan_preview.py @@ -189,6 +189,88 @@ def test_given_a_synced_location_when_a_plan_is_requested_then_it_plans_from_the live_run: subprocess.CompletedProcess[str] = live_plan(project, _SELECTION) assert live_run.returncode == 0, live_run.stderr live: dict[str, Any] = json.loads(live_run.stdout) + log: Path = tmp_path / "location.log" + refused, preview = _plan_from_copy(tmp_path, project, {}) + + assert refused.returncode != 0 + assert "synced_copy" in preview, log.read_text() + synced: dict[str, Any] = preview.pop("synced_copy") + assert synced["synced_at"] is not None + assert synced["unknown_until_run"] == [*_UNKNOWN, *test_case.expected_added_unknowns] + live_models: dict[str, dict[str, Any]] = {model["name"]: model for model in live.pop("models")} + preview_models: dict[str, dict[str, Any]] = { + model["name"]: model for model in preview.pop("models") + } + live_replayed: dict[str, Any] = live_models.pop("daily_orders") + preview_replayed: dict[str, Any] = preview_models.pop("daily_orders") + assert live_replayed["cursor"]["resolved_bounds"]["start"] is not None + assert (preview_replayed["action"], preview_replayed["reason"]) == ( + live_replayed["action"], + live_replayed["reason"], + ) + live_freshness: dict[str, Any] = live["metadata"].pop("direct_source_freshness") + preview_freshness: dict[str, Any] = preview["metadata"].pop("direct_source_freshness") + assert live_freshness["observed_source_names"] == ["raw_refunds"] + assert preview_freshness["unknown_source_details"] == [ + { + "message": "a plan preview does not observe source freshness", + "reason": "error", + "source_name": "raw_refunds", + } + ] + assert synced["cursor_window_starts"] == { + name: live_models[name]["cursor_bounds"]["start"] + for name in test_case.expected_window_models + } + for name in (*test_case.expected_window_models, *test_case.expected_unplanned_cursor_models): + for model in (live_models[name], preview_models.pop(name)): + assert model["action"] == "incremental_append" + assert model["reason"] == "normal_incremental" + _ = live_models.pop(name) + assert preview_models == live_models + assert preview == live + + +def test_given_an_edited_model_when_a_changed_plan_is_requested_then_the_copy_names_what_changed( + tmp_path: Path, +) -> None: + project: Path = tmp_path / "project" + write_files(project_dir=project, files=_PROJECT_FILES) + built: subprocess.CompletedProcess[str] = run_sqb(project, "build", "--target", "development") + assert built.returncode == 0, built.stderr + edited: str = _PROJECT_FILES["models/order_count.sql"].replace( + "AS order_count", "AS order_count, 1 AS batch" + ) + write_files(project_dir=project, files={"models/order_count.sql": edited}) + live_run: subprocess.CompletedProcess[str] = live_plan(project, _SELECTION, "--changed") + assert live_run.returncode == 0, live_run.stderr + + refused, preview = _plan_from_copy(tmp_path, project, {"changed": True}) + + assert refused.returncode != 0 + assert preview.get("changed") == { + "counts": ["definition"], + "reach": "job", + "runs": [ + { + "node": "model:order_count", + "reason": "planner change (query_changed)", + "downstream_of": None, + } + ], + "held_back": [], + "unchanged": ["model:daily_orders", "model:orders", "model:refunds"], + }, (tmp_path / "location.log").read_text() + assert preview["changed"] == json.loads(live_run.stdout)["changed"] + + +def _plan_from_copy( + tmp_path: Path, project: Path, settings: dict[str, object] +) -> tuple[subprocess.CompletedProcess[str], dict[str, Any]]: + """Sync `project` to a location, lock its warehouse, then plan `_SELECTION` from the copy. + + Returns the refused live plan run while the lock was held and the location's plan document. + """ control: Path = tmp_path / "control/.sqlbuild/control.sqlite3" log: Path = tmp_path / "location.log" started: datetime = datetime.now(UTC) @@ -224,48 +306,10 @@ def test_given_a_synced_location_when_a_plan_is_requested_then_it_plans_from_the lock: duckdb.DuckDBPyConnection = hold_write_lock(project / "warehouse.db") try: refused: subprocess.CompletedProcess[str] = live_plan(project, _SELECTION) - request_id: str = request_plan(control, _SELECTION) + request_id: str = request_plan(control, _SELECTION, settings) result: dict[str, Any] = wait_for_location_work(api, request_id, location, log) finally: lock.close() - - assert refused.returncode != 0 outcome: dict[str, Any] = result["result_json"]["outcome"] assert outcome["kind"] == "plan", outcome - preview: dict[str, Any] = outcome["document"] - assert "synced_copy" in preview, log.read_text() - synced: dict[str, Any] = preview.pop("synced_copy") - assert synced["synced_at"] is not None - assert synced["unknown_until_run"] == [*_UNKNOWN, *test_case.expected_added_unknowns] - live_models: dict[str, dict[str, Any]] = {model["name"]: model for model in live.pop("models")} - preview_models: dict[str, dict[str, Any]] = { - model["name"]: model for model in preview.pop("models") - } - live_replayed: dict[str, Any] = live_models.pop("daily_orders") - preview_replayed: dict[str, Any] = preview_models.pop("daily_orders") - assert live_replayed["cursor"]["resolved_bounds"]["start"] is not None - assert (preview_replayed["action"], preview_replayed["reason"]) == ( - live_replayed["action"], - live_replayed["reason"], - ) - live_freshness: dict[str, Any] = live["metadata"].pop("direct_source_freshness") - preview_freshness: dict[str, Any] = preview["metadata"].pop("direct_source_freshness") - assert live_freshness["observed_source_names"] == ["raw_refunds"] - assert preview_freshness["unknown_source_details"] == [ - { - "message": "a plan preview does not observe source freshness", - "reason": "error", - "source_name": "raw_refunds", - } - ] - assert synced["cursor_window_starts"] == { - name: live_models[name]["cursor_bounds"]["start"] - for name in test_case.expected_window_models - } - for name in (*test_case.expected_window_models, *test_case.expected_unplanned_cursor_models): - for model in (live_models[name], preview_models.pop(name)): - assert model["action"] == "incremental_append" - assert model["reason"] == "normal_incremental" - _ = live_models.pop(name) - assert preview_models == live_models - assert preview == live + return refused, outcome["document"] From 81b5ed0d5c13c31f066e5ee74e3c5fb1dd567b96 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 00:53:03 +0000 Subject: [PATCH 2/5] fix(server): copy all consumption rows on request and rank recent ones in memory The preview copy carried only each relation's newest 8 executions, so read-back of an older execution found nothing and its readers counted as changed. The server now sends every mirrored row, but only when a --changed plan asks for them, so older locations never receive a field they reject. The recent read ranks executions with recent_executions, which the consumption simulation now shares. --- clients/typescript/src/schema.d.ts | 7 ++- .../native_commands/main/_location_plan.rs | 3 + .../_helpers/mirror_pages.rs | 24 ++------ .../main/location_reads/read_preview_copy.rs | 6 +- .../src/warehouse_metadata/models.rs | 8 ++- .../preview_copy/helpers.rs | 59 ++++++++++++------- .../_helpers/native_host/consumption_state.rs | 6 +- .../_helpers/native_host/synced_copy.rs | 2 +- .../src/consumption/main/mod.rs | 1 + .../src/consumption/main/recent_executions.rs | 38 ++++++++++++ .../sqlbuild-state/src/consumption/tests.rs | 2 + .../src/consumption/tests/helpers.rs | 31 +--------- .../tests/test_recent_executions.rs | 59 +++++++++++++++++++ 13 files changed, 171 insertions(+), 75 deletions(-) create mode 100644 crates/sqlbuild-state/src/consumption/main/recent_executions.rs create mode 100644 crates/sqlbuild-state/src/consumption/tests/test_recent_executions.rs diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index 588b260b78..04bb09b570 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -2398,7 +2398,10 @@ export interface components { }; /** @description What the mirror knows as of `synced_at`; `None` when a required table never synced cleanly. */ PreviewCopy: { - /** @description Rows of each relation's newest executions, which `--changed` decides from. */ + /** + * @description Every mirrored consumption row when the read asked for them; locations that never ask + * never receive the field. + */ consumption?: components["schemas"]["WarehouseConsumptionRecord"][]; fingerprints: components["schemas"]["WarehouseFingerprintRecord"][]; latest_runs: components["schemas"]["PreviewLatestRun"][]; @@ -2409,6 +2412,8 @@ export interface components { }; /** @description A location asks for the mirrored copy a plan preview plans from, as its fenced reader. */ PreviewCopyRead: { + /** @description Whether the copy carries consumption history, which only `--changed` plans read. */ + consumption?: boolean; /** Format: int64 */ generation: number; /** Format: uuid */ diff --git a/crates/sqlbuild-cli/src/native_commands/main/_location_plan.rs b/crates/sqlbuild-cli/src/native_commands/main/_location_plan.rs index bf7c4e627c..2460b9980a 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_location_plan.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_location_plan.rs @@ -31,6 +31,9 @@ pub(super) async fn plan_location( let read = PreviewCopyRead { session_id, generation, + consumption: settings + .get("changed") + .is_some_and(|changed| !matches!(changed, Value::Null | Value::Bool(false))), }; let copy = match client.preview_copy(&options.target, &read).await { Ok(copy) => copy, diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/mirror_pages.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/mirror_pages.rs index dddd01ecd7..e23899c68d 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/mirror_pages.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/mirror_pages.rs @@ -5,7 +5,6 @@ use std::collections::{BTreeMap, BTreeSet}; use chrono::{DateTime, Utc}; use serde::de::DeserializeOwned; use serde_json::Value; -use sqlbuild_state::consumption::constants::RECENT_EXECUTIONS; use uuid::Uuid; use crate::authorization::models::CurrentSnapshotId; @@ -37,15 +36,9 @@ const LATEST_RUNS_SQL: &str = "SELECT record_json FROM (SELECT record_json, ROW_ WHERE location_id=$1 AND target=$2 AND record_json ->> 'phase'='succeeded' \ AND record_json ->> 'run_id' IS NOT NULL) runs WHERE newest=1 \ ORDER BY record_json ->> 'node_type', record_json ->> 'node_name'"; -/// Every row of each relation's newest executions, timing each execution by its earliest row. -const CONSUMPTION_SQL: &str = "SELECT record_json FROM (SELECT record_json, observed_at, DENSE_RANK() OVER \ - (PARTITION BY state_database, state_schema, record_json ->> 'target_database', \ - record_json ->> 'target_schema', record_json ->> 'target_name' \ - ORDER BY started_at DESC, record_json ->> 'execution_id' DESC) AS execution_rank \ - FROM (SELECT record_json, observed_at, state_database, state_schema, MIN(observed_at) OVER \ - (PARTITION BY state_database, state_schema, record_json ->> 'execution_id') AS started_at \ - FROM sqb_warehouse_consumption WHERE location_id=$1 AND target=$2) executions) ranked \ - WHERE execution_rank <= $3 ORDER BY observed_at, record_json ->> 'execution_id'"; +/// Every consumption row the warehouse still keeps, which complete scans bound. +const CONSUMPTION_SQL: &str = "SELECT record_json FROM sqb_warehouse_consumption \ + WHERE location_id=$1 AND target=$2 ORDER BY observed_at, event_id"; const OBSERVATIONS_SQL: &str = "SELECT record_json FROM (SELECT record_json, kind, node, ROW_NUMBER() OVER \ (PARTITION BY state_database, state_schema, kind, node ORDER BY observed_at DESC, event_id DESC) AS newest \ FROM sqb_warehouse_observations WHERE location_id=$1 AND target=$2) observations WHERE newest=1 \ @@ -157,19 +150,12 @@ pub(crate) async fn preview_latest_runs( Ok((transaction, runs)) } -/// The rows of each relation's newest executions, as many as a build reads from the warehouse. +/// Every mirrored consumption row: a build reads the newest executions and any older one by ID. pub(crate) async fn preview_consumption( transaction: ControlTransaction, scope: (Uuid, &str), ) -> Result<(ControlTransaction, Vec), ControlError> { - let mut parameters: Vec = scope_parameters(scope); - parameters.push(Parameter::Integer(RECENT_EXECUTIONS as i64)); - let (transaction, rows) = query(transaction, CONSUMPTION_SQL, parameters).await?; - let mut records: Vec = Vec::with_capacity(rows.len()); - for row in rows { - records.push(decoded(row)?); - } - Ok((transaction, records)) + mirrored(transaction, CONSUMPTION_SQL, scope).await } /// Each node's newest observation of each kind. diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/read_preview_copy.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/read_preview_copy.rs index 855763d9f4..46c9305908 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/read_preview_copy.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/read_preview_copy.rs @@ -23,7 +23,11 @@ pub async fn read_preview_copy( let (transaction, fingerprints) = preview_fingerprints(transaction, scope).await?; let (transaction, latest_runs) = preview_latest_runs(transaction, scope).await?; let (transaction, observations) = preview_observations(transaction, scope).await?; - let (transaction, consumption) = preview_consumption(transaction, scope).await?; + let (transaction, consumption) = if request.consumption { + preview_consumption(transaction, scope).await? + } else { + (transaction, Vec::new()) + }; let (transaction, synced_at) = preview_synced_at(transaction, scope).await?; commit(transaction).await?; Ok(PreviewCopy { diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs index 9568c69c7c..a2712cec18 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs @@ -435,6 +435,9 @@ pub struct WarehouseSyncDue { pub struct PreviewCopyRead { pub session_id: Uuid, pub generation: i64, + /// Whether the copy carries consumption history, which only `--changed` plans read. + #[serde(default)] + pub consumption: bool, } /// The newest successful build of a node, which a `cursor_max` observation must come from. @@ -466,8 +469,9 @@ pub struct PreviewCopy { pub fingerprints: Vec, pub latest_runs: Vec, pub observations: Vec, - /// Rows of each relation's newest executions, which `--changed` decides from. - #[serde(default)] + /// Every mirrored consumption row when the read asked for them; locations that never ask + /// never receive the field. + #[serde(default, skip_serializing_if = "Vec::is_empty")] pub consumption: Vec, } diff --git a/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/preview_copy/helpers.rs b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/preview_copy/helpers.rs index 8a0b614d55..aa112308cb 100644 --- a/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/preview_copy/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/preview_copy/helpers.rs @@ -8,7 +8,7 @@ use crate::platform::models::{LocationHealthReport, LocationWorkClient}; use crate::warehouse_metadata::models::{PreviewCopy, PreviewCopyRead}; const NODE: &str = "preview_copy_orders"; -/// A relation with more executions than a build reads; only its newest eight are copied. +/// A relation with more executions than a build reads recently; read-back can still need them all. const BUSY_NODE: &str = "preview_copy_customers"; async fn post(client: &Client, (url, bearer): (&str, &str), body: Value) { @@ -110,8 +110,7 @@ pub(crate) async fn preview_copy_scope( .await; let mut busy: Vec = Vec::new(); for execution in 1..=10 { - let finished: u32 = if execution == 1 { 20 } else { execution }; - for (phase, hour, minute) in [("started", execution, 0), ("succeeded", finished, 30)] { + for (phase, hour, minute) in [("started", execution, 0), ("succeeded", execution, 30)] { let mut row = consumption( &format!("busy-execution-{execution:02}"), (phase, &format!("busy-run-{execution:02}")), @@ -159,6 +158,7 @@ pub(crate) async fn preview_copy_scope( let read = PreviewCopyRead { session_id: preview_session, generation: 1, + consumption: false, }; let copy = || async { outbound @@ -191,29 +191,45 @@ pub(crate) async fn preview_copy_scope( ) }; let unsynced = copy().await; - let mut busy_executions: Vec = unsynced + let older_location: Value = client + .post(format!("{metadata}/preview-copy")) + .bearer_auth(bearer) + .json(&json!({"session_id": preview_session, "generation": 1})) + .send() + .await + .expect("copy for an older location") + .json() + .await + .expect("copy body"); + assert!( + unsynced.consumption.is_empty() && older_location.get("consumption").is_none(), + "a read that does not ask for consumption never carries the field, which older locations reject" + ); + let with_consumption = outbound + .preview_copy( + "preview", + &PreviewCopyRead { + consumption: true, + ..read.clone() + }, + ) + .await + .expect("preview copy with consumption"); + let busy: Vec<(String, String)> = with_consumption .consumption .iter() .filter(|row| row.target_name.as_deref() == Some(BUSY_NODE)) - .filter_map(|row| row.execution_id.clone()) + .map(|row| { + ( + row.execution_id.clone().unwrap_or_default(), + row.phase.clone().unwrap_or_default(), + ) + }) .collect(); - busy_executions.dedup(); - assert_eq!( - busy_executions, - (3..=10) - .map(|execution| format!("busy-execution-{execution:02}")) - .collect::>(), - "each relation's newest eight executions by start, every row of each, oldest first; \ - the first execution finished last but started first" - ); assert_eq!( - unsynced - .consumption - .iter() - .filter(|row| row.target_name.as_deref() == Some(BUSY_NODE)) - .count(), - 16, - "both rows of each copied execution" + busy.len(), + 20, + "every mirrored row, beyond the newest eight executions, so read-back finds older ones: {busy:?}" ); assert_eq!(unsynced.synced_at, None, "nothing synced cleanly yet"); assert_eq!( @@ -308,6 +324,7 @@ pub(crate) async fn preview_copy_scope( &PreviewCopyRead { session_id, generation: 1, + consumption: false, }, ) .await diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/consumption_state.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/consumption_state.rs index 51d167a052..3542666348 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/consumption_state.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/consumption_state.rs @@ -13,6 +13,7 @@ use sqlbuild_state::consumption::constants::{ }; use sqlbuild_state::consumption::main::executions_by_id_sql::executions_by_id_sql; use sqlbuild_state::consumption::main::prune_executions_sql::prune_executions_sql; +use sqlbuild_state::consumption::main::recent_executions::recent_executions; use sqlbuild_state::consumption::main::recent_executions_sql::recent_executions_sql; use sqlbuild_state::consumption::models::{ ConsumedInput, ConsumptionAppend, ConsumptionRow, NodeKey, RelationKey, @@ -454,7 +455,10 @@ impl ConsumptionConnection { location: &Location, ) -> PyResult> { if let Ok(copy) = self.adapter.bind(py).downcast::() { - return Ok(copy.get().consumption(location)); + return Ok(recent_executions( + ©.get().consumption(location), + RECENT_EXECUTIONS, + )); } read_recent(self.adapter.bind(py), &self.connected(py)?, location) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/synced_copy.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/synced_copy.rs index 7d649fda4b..afe951466b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/synced_copy.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/synced_copy.rs @@ -254,7 +254,7 @@ impl SyncedCopyAdapter { None } - /// The copied consumption rows of the table in `location`, each relation's newest executions. + /// Every copied consumption row of the table in `location`. pub(crate) fn consumption(&self, location: &Location) -> Vec { let mut rows: Vec = Vec::new(); for record in &self.copy.consumption { diff --git a/crates/sqlbuild-state/src/consumption/main/mod.rs b/crates/sqlbuild-state/src/consumption/main/mod.rs index 31accb8109..0fa927d55d 100644 --- a/crates/sqlbuild-state/src/consumption/main/mod.rs +++ b/crates/sqlbuild-state/src/consumption/main/mod.rs @@ -4,6 +4,7 @@ pub mod consumption_history; pub mod executions_by_id_sql; pub mod prune_executions_sql; pub mod read_back_history; +pub mod recent_executions; pub mod recent_executions_sql; pub mod started_append; pub mod succeeded_appends; diff --git a/crates/sqlbuild-state/src/consumption/main/recent_executions.rs b/crates/sqlbuild-state/src/consumption/main/recent_executions.rs new file mode 100644 index 0000000000..536bbf319c --- /dev/null +++ b/crates/sqlbuild-state/src/consumption/main/recent_executions.rs @@ -0,0 +1,38 @@ +//! Choose the recent executions of every relation from rows already read. + +use std::collections::{BTreeMap, BTreeSet}; + +use crate::consumption::models::{ConsumptionRow, RelationKey}; + +/// Rows of the newest `limit` executions per relation, timing each execution by its earliest row, +/// as `recent_executions_sql` selects them in the warehouse. +pub fn recent_executions(rows: &[ConsumptionRow], limit: usize) -> Vec { + let mut started: BTreeMap<(&RelationKey, &str), i64> = BTreeMap::new(); + for row in rows { + let entry = started + .entry((&row.row.relation, row.row.execution_id.as_str())) + .or_insert(row.recorded_at); + *entry = (*entry).min(row.recorded_at); + } + let mut by_relation: BTreeMap<&RelationKey, Vec<(i64, &str)>> = BTreeMap::new(); + for ((relation, execution), time) in started { + by_relation + .entry(relation) + .or_default() + .push((time, execution)); + } + let mut kept: BTreeSet<(&RelationKey, &str)> = BTreeSet::new(); + for (relation, executions) in &mut by_relation { + executions.sort_by(|left, right| right.cmp(left)); + kept.extend( + executions + .iter() + .take(limit) + .map(|(_, execution)| (*relation, *execution)), + ); + } + rows.iter() + .filter(|row| kept.contains(&(&row.row.relation, row.row.execution_id.as_str()))) + .cloned() + .collect() +} diff --git a/crates/sqlbuild-state/src/consumption/tests.rs b/crates/sqlbuild-state/src/consumption/tests.rs index 6027e17bc4..8c354f9bba 100644 --- a/crates/sqlbuild-state/src/consumption/tests.rs +++ b/crates/sqlbuild-state/src/consumption/tests.rs @@ -2,6 +2,8 @@ mod helpers; #[path = "tests/test_decide.rs"] mod test_decide; +#[path = "tests/test_recent_executions.rs"] +mod test_recent_executions; #[path = "tests/test_simulation.rs"] mod test_simulation; #[path = "tests/test_types.rs"] diff --git a/crates/sqlbuild-state/src/consumption/tests/helpers.rs b/crates/sqlbuild-state/src/consumption/tests/helpers.rs index 1ca2aac9c0..349579917f 100644 --- a/crates/sqlbuild-state/src/consumption/tests/helpers.rs +++ b/crates/sqlbuild-state/src/consumption/tests/helpers.rs @@ -9,6 +9,7 @@ use crate::consumption::main::changed_run_set::changed_run_set; use crate::consumption::main::consumed_inputs::consumed_inputs; use crate::consumption::main::consumption_history::consumption_history; use crate::consumption::main::read_back_history::read_back_history; +use crate::consumption::main::recent_executions::recent_executions; use crate::consumption::main::started_append::started_append; use crate::consumption::main::succeeded_appends::succeeded_appends; use crate::consumption::models::TriggeredJob; @@ -171,35 +172,7 @@ impl World { /// The reader's SQL: rows of the newest executions per relation by earliest row time. fn read_rows(&self) -> Vec { - let mut started: BTreeMap<(RelationKey, String), i64> = BTreeMap::new(); - for row in &self.rows { - let entry = started - .entry((row.row.relation.clone(), row.row.execution_id.clone())) - .or_insert(row.recorded_at); - *entry = (*entry).min(row.recorded_at); - } - let mut by_relation: BTreeMap> = BTreeMap::new(); - for ((relation, execution), time) in started { - by_relation - .entry(relation) - .or_default() - .push((time, execution)); - } - let mut kept: BTreeSet = BTreeSet::new(); - for executions in by_relation.values_mut() { - executions.sort_by(|left, right| right.cmp(left)); - kept.extend( - executions - .iter() - .take(RECENT_EXECUTIONS) - .map(|(_, execution)| execution.clone()), - ); - } - self.rows - .iter() - .filter(|row| kept.contains(&row.row.execution_id)) - .cloned() - .collect() + recent_executions(&self.rows, RECENT_EXECUTIONS) } /// The janitor's SQL: keep the newest `retain` executions per relation. diff --git a/crates/sqlbuild-state/src/consumption/tests/test_recent_executions.rs b/crates/sqlbuild-state/src/consumption/tests/test_recent_executions.rs new file mode 100644 index 0000000000..b0fdb102c8 --- /dev/null +++ b/crates/sqlbuild-state/src/consumption/tests/test_recent_executions.rs @@ -0,0 +1,59 @@ +use crate::consumption::main::recent_executions::recent_executions; +use crate::consumption::models::{ConsumptionAppend, ConsumptionRow, NodeKey, RelationKey}; + +fn row(relation: &str, execution: &str, recorded_at: i64) -> ConsumptionRow { + ConsumptionRow { + row: ConsumptionAppend { + execution_id: execution.into(), + phase: "succeeded".into(), + run_id: format!("run-{execution}"), + node: NodeKey { + kind: "model".into(), + name: relation.into(), + }, + relation: RelationKey { + database: None, + schema: "main".into(), + name: relation.into(), + }, + version_hash: "version".into(), + input_count: 0, + input: None, + }, + recorded_at, + } +} + +#[test] +fn given_more_executions_than_the_limit_when_choosing_recent_then_the_newest_starts_win_per_relation() + { + let mut rows: Vec = Vec::new(); + for execution in 1..=4 { + // `orders-1` started first but wrote its last row after every other execution. + let finished: i64 = if execution == 1 { + 50 + } else { + execution * 10 + 5 + }; + rows.push(row( + "orders", + &format!("orders-{execution}"), + execution * 10, + )); + rows.push(row("orders", &format!("orders-{execution}"), finished)); + } + rows.push(row("customers", "customers-1", 1)); + + let mut kept: Vec = recent_executions(&rows, 2) + .into_iter() + .map(|row| row.row.execution_id) + .collect(); + kept.dedup(); + + assert_eq!( + kept, + ["orders-3", "orders-4", "customers-1"], + "each relation keeps its newest two executions by earliest row, with every row of each" + ); + assert_eq!(recent_executions(&rows, 2).len(), 5); +} From d8c8ff0e429e50169c040e4038e380067140ee62 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 01:00:03 +0000 Subject: [PATCH 3/5] fix(server): send the consumption flag only when it is set A preview-copy request from an upgraded location now omits the flag unless a --changed plan sets it, so other plans still reach a server that predates the flag. --- clients/typescript/src/schema.d.ts | 5 ++++- .../sqlbuild-orchestration/src/warehouse_metadata/models.rs | 5 +++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index 04bb09b570..5c5d8583a7 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -2412,7 +2412,10 @@ export interface components { }; /** @description A location asks for the mirrored copy a plan preview plans from, as its fenced reader. */ PreviewCopyRead: { - /** @description Whether the copy carries consumption history, which only `--changed` plans read. */ + /** + * @description Whether the copy carries consumption history, which only `--changed` plans read; sent only + * when set, so other plans still reach servers that predate it. + */ consumption?: boolean; /** Format: int64 */ generation: number; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs index a2712cec18..bcbccb4abf 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs @@ -435,8 +435,9 @@ pub struct WarehouseSyncDue { pub struct PreviewCopyRead { pub session_id: Uuid, pub generation: i64, - /// Whether the copy carries consumption history, which only `--changed` plans read. - #[serde(default)] + /// Whether the copy carries consumption history, which only `--changed` plans read; sent only + /// when set, so other plans still reach servers that predate it. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] pub consumption: bool, } From b296cb45e9723e22643011963faa37296d972953 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 01:24:04 +0000 Subject: [PATCH 4/5] test(server): table-driven recent executions and preview helpers Move the location plan-from-copy helper out of the test module, drive the --changed preview and recent executions tests from test cases, keep the synced copy's consumption read in its own layer, and list the consumption state reader as a synced-copy preview branch. --- .../src/warehouse_metadata/models.rs | 6 +- .../_helpers/native_host/consumption_state.rs | 8 +- .../_helpers/native_host/synced_copy.rs | 17 ++- .../src/consumption/main/recent_executions.rs | 3 +- .../src/consumption/tests/helpers.rs | 24 ++++ .../tests/test_recent_executions.rs | 100 +++++++-------- .../src/consumption/tests/test_types.rs | 8 ++ fensu.toml | 1 + .../publication/_test_types.py | 8 ++ .../compiled_project/publication/helpers.py | 54 ++++++++ .../publication/test_plan_preview.py | 117 +++++------------- 11 files changed, 197 insertions(+), 149 deletions(-) diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs index bcbccb4abf..74df786f88 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs @@ -435,8 +435,7 @@ pub struct WarehouseSyncDue { pub struct PreviewCopyRead { pub session_id: Uuid, pub generation: i64, - /// Whether the copy carries consumption history, which only `--changed` plans read; sent only - /// when set, so other plans still reach servers that predate it. + /// Whether the copy carries consumption history; sent only when set, as older servers reject it. #[serde(default, skip_serializing_if = "std::ops::Not::not")] pub consumption: bool, } @@ -470,8 +469,7 @@ pub struct PreviewCopy { pub fingerprints: Vec, pub latest_runs: Vec, pub observations: Vec, - /// Every mirrored consumption row when the read asked for them; locations that never ask - /// never receive the field. + /// Every mirrored consumption row when asked for; omitted otherwise, as older locations reject it. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub consumption: Vec, } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/consumption_state.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/consumption_state.rs index 3542666348..191071123c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/consumption_state.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/consumption_state.rs @@ -456,7 +456,9 @@ impl ConsumptionConnection { ) -> PyResult> { if let Ok(copy) = self.adapter.bind(py).downcast::() { return Ok(recent_executions( - ©.get().consumption(location), + © + .get() + .consumption((location.database.as_deref(), &location.schema)), RECENT_EXECUTIONS, )); } @@ -470,7 +472,9 @@ impl ConsumptionConnection { execution_ids: &[&str], ) -> PyResult> { if let Ok(copy) = self.adapter.bind(py).downcast::() { - let mut rows: Vec = copy.get().consumption(location); + let mut rows: Vec = copy + .get() + .consumption((location.database.as_deref(), &location.schema)); rows.retain(|row| execution_ids.contains(&row.row.execution_id.as_str())); return Ok(rows); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/synced_copy.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/synced_copy.rs index afe951466b..93fc1dbe40 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/synced_copy.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/synced_copy.rs @@ -24,7 +24,6 @@ use sqlbuild_state::execution_history::constants::MEMORY_PATH; use sqlbuild_state::observations::constants::{CURSOR_MAX_KIND, SOURCE_COLUMNS_KIND}; use crate::bindings::_helpers::boundary::python_calls::{KeywordArguments, call_with}; -use crate::bindings::_helpers::native_host::consumption_state::Location; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::sqlite_connection::{SqliteConnection, SqliteCursor}; use crate::bindings::_helpers::records::fields::field as native_field; @@ -254,17 +253,17 @@ impl SyncedCopyAdapter { None } - /// Every copied consumption row of the table in `location`. - pub(crate) fn consumption(&self, location: &Location) -> Vec { + /// Every copied consumption row of the table in `(database, schema)`. + pub(crate) fn consumption( + &self, + (database, schema): (Option<&str>, &str), + ) -> Vec { let mut rows: Vec = Vec::new(); for record in &self.copy.consumption { - if !same(Some(&record.state_schema), Some(&location.schema)) - || !(location.database.is_none() + if !same(Some(&record.state_schema), Some(schema)) + || !(database.is_none() || record.state_database.is_none() - || same( - record.state_database.as_deref(), - location.database.as_deref(), - )) + || same(record.state_database.as_deref(), database)) { continue; } diff --git a/crates/sqlbuild-state/src/consumption/main/recent_executions.rs b/crates/sqlbuild-state/src/consumption/main/recent_executions.rs index 536bbf319c..caf764d07e 100644 --- a/crates/sqlbuild-state/src/consumption/main/recent_executions.rs +++ b/crates/sqlbuild-state/src/consumption/main/recent_executions.rs @@ -4,8 +4,7 @@ use std::collections::{BTreeMap, BTreeSet}; use crate::consumption::models::{ConsumptionRow, RelationKey}; -/// Rows of the newest `limit` executions per relation, timing each execution by its earliest row, -/// as `recent_executions_sql` selects them in the warehouse. +/// Rows of the newest `limit` executions per relation by earliest row, as `recent_executions_sql`. pub fn recent_executions(rows: &[ConsumptionRow], limit: usize) -> Vec { let mut started: BTreeMap<(&RelationKey, &str), i64> = BTreeMap::new(); for row in rows { diff --git a/crates/sqlbuild-state/src/consumption/tests/helpers.rs b/crates/sqlbuild-state/src/consumption/tests/helpers.rs index 349579917f..591aef47e1 100644 --- a/crates/sqlbuild-state/src/consumption/tests/helpers.rs +++ b/crates/sqlbuild-state/src/consumption/tests/helpers.rs @@ -761,3 +761,27 @@ pub(crate) fn two_jobs() -> Scenario { pub(crate) const JOB_A: (&str, &[&str]) = ("job_a", &["stg_orders", "int_orders"]); pub(crate) const JOB_B: (&str, &[&str]) = ("job_b", &["fct_revenue", "rpt_pnl"]); + +/// One succeeded row of `execution`, writing the `main` relation named `relation`. +pub(crate) fn recorded_row(relation: &str, execution: &str, recorded_at: i64) -> ConsumptionRow { + ConsumptionRow { + row: ConsumptionAppend { + execution_id: execution.into(), + phase: "succeeded".into(), + run_id: format!("run-{execution}"), + node: NodeKey { + kind: "model".into(), + name: relation.into(), + }, + relation: RelationKey { + database: None, + schema: "main".into(), + name: relation.into(), + }, + version_hash: "version".into(), + input_count: 0, + input: None, + }, + recorded_at, + } +} diff --git a/crates/sqlbuild-state/src/consumption/tests/test_recent_executions.rs b/crates/sqlbuild-state/src/consumption/tests/test_recent_executions.rs index b0fdb102c8..da17c10681 100644 --- a/crates/sqlbuild-state/src/consumption/tests/test_recent_executions.rs +++ b/crates/sqlbuild-state/src/consumption/tests/test_recent_executions.rs @@ -1,59 +1,61 @@ use crate::consumption::main::recent_executions::recent_executions; -use crate::consumption::models::{ConsumptionAppend, ConsumptionRow, NodeKey, RelationKey}; - -fn row(relation: &str, execution: &str, recorded_at: i64) -> ConsumptionRow { - ConsumptionRow { - row: ConsumptionAppend { - execution_id: execution.into(), - phase: "succeeded".into(), - run_id: format!("run-{execution}"), - node: NodeKey { - kind: "model".into(), - name: relation.into(), - }, - relation: RelationKey { - database: None, - schema: "main".into(), - name: relation.into(), - }, - version_hash: "version".into(), - input_count: 0, - input: None, - }, - recorded_at, - } -} +use crate::consumption::tests::helpers::recorded_row; +use crate::consumption::tests::test_types::RecentExecutionsTestCase; #[test] fn given_more_executions_than_the_limit_when_choosing_recent_then_the_newest_starts_win_per_relation() { - let mut rows: Vec = Vec::new(); - for execution in 1..=4 { - // `orders-1` started first but wrote its last row after every other execution. - let finished: i64 = if execution == 1 { - 50 - } else { - execution * 10 + 5 - }; - rows.push(row( + // `orders-1` started first but wrote its last row after every other execution. + let mut late_finisher = Vec::new(); + for (execution, started, finished) in [(1, 10, 50), (2, 20, 25), (3, 30, 35), (4, 40, 45)] { + late_finisher.push(recorded_row( + "orders", + &format!("orders-{execution}"), + started, + )); + late_finisher.push(recorded_row( "orders", &format!("orders-{execution}"), - execution * 10, + finished, )); - rows.push(row("orders", &format!("orders-{execution}"), finished)); } - rows.push(row("customers", "customers-1", 1)); - - let mut kept: Vec = recent_executions(&rows, 2) - .into_iter() - .map(|row| row.row.execution_id) - .collect(); - kept.dedup(); - - assert_eq!( - kept, - ["orders-3", "orders-4", "customers-1"], - "each relation keeps its newest two executions by earliest row, with every row of each" - ); - assert_eq!(recent_executions(&rows, 2).len(), 5); + late_finisher.push(recorded_row("customers", "customers-1", 1)); + let test_cases = [ + RecentExecutionsTestCase { + description: "an execution that started first but wrote last is the oldest; other relations keep theirs", + rows: late_finisher, + limit: 2, + expected_executions: vec!["orders-3", "orders-4", "customers-1"], + expected_rows: 5, + }, + RecentExecutionsTestCase { + description: "equal start times keep the larger execution id, as the warehouse query does", + rows: vec![ + recorded_row("orders", "orders-a", 10), + recorded_row("orders", "orders-b", 10), + ], + limit: 1, + expected_executions: vec!["orders-b"], + expected_rows: 1, + }, + ]; + for test_case in test_cases { + let kept = recent_executions(&test_case.rows, test_case.limit); + let mut executions: Vec<&str> = kept + .iter() + .map(|row| row.row.execution_id.as_str()) + .collect(); + executions.dedup(); + assert_eq!( + executions, test_case.expected_executions, + "{}", + test_case.description + ); + assert_eq!( + kept.len(), + test_case.expected_rows, + "{}", + test_case.description + ); + } } diff --git a/crates/sqlbuild-state/src/consumption/tests/test_types.rs b/crates/sqlbuild-state/src/consumption/tests/test_types.rs index 7ed8e9b34d..38e23c0c43 100644 --- a/crates/sqlbuild-state/src/consumption/tests/test_types.rs +++ b/crates/sqlbuild-state/src/consumption/tests/test_types.rs @@ -87,6 +87,14 @@ pub(crate) struct Scenario { pub(crate) clock: i64, } +pub(crate) struct RecentExecutionsTestCase { + pub(crate) description: &'static str, + pub(crate) rows: Vec, + pub(crate) limit: usize, + pub(crate) expected_executions: Vec<&'static str>, + pub(crate) expected_rows: usize, +} + pub(crate) struct DecideTestCase { pub(crate) description: &'static str, pub(crate) outcomes: Vec<(String, crate::consumption::models::ChangedOutcome)>, diff --git a/fensu.toml b/fensu.toml index 086077a723..e425f13b70 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1665,6 +1665,7 @@ allowed_paths = [ "crates/sqlbuild-cli/src/native_commands/main/_location_plan.rs=A location's requested plan names the synced copy for its sqb plan child.", "crates/sqlbuild-core/src/plan_display/constants.rs=Names the synced-copy environment variable.", "crates/sqlbuild-python/src/bindings/_helpers/freshness/planning.rs=Preview branch: a refused freshness observation leaves the source unknown.", + "crates/sqlbuild-python/src/bindings/_helpers/native_host/consumption_state.rs=Preview branch: --changed reads consumption history from the copy instead of the warehouse table.", "crates/sqlbuild-python/src/bindings/_helpers/native_host/registration.rs=Registers the synced copy's refusal error with Python.", "crates/sqlbuild-python/src/bindings/_helpers/native_host/synced_copy.rs=Defines the synced-copy adapter.", "crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs=Only sqb plan activates the synced copy and reports what it could not know.", diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py index b230cebb88..96888e85e6 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -408,6 +408,14 @@ class AdhocRunTestCase: expected_worker_log: str = "" +@dataclass(frozen=True) +class ChangedPlanPreviewTestCase: + description: str + edited_model: str + edit: tuple[str, str] + expected_changed: dict[str, Any] + + @dataclass(frozen=True) class PlanPreviewTestCase: description: str diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py index e39a4ba59d..ce40130425 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py @@ -68,6 +68,9 @@ WorkerDeployment, WorkerExecutionTestCase, ) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.catalog.helpers import ( + request_plan, +) from tests.integration.src.sqlbuild.compiler.compiled_project.publication.docker.helpers import ( docker_deployment, ) @@ -1601,3 +1604,54 @@ def create_raw_orders(warehouse: Path) -> None: _ = connection.execute( "CREATE TABLE main.raw_orders AS SELECT CAST(1 AS BIGINT) AS order_id" ) + + +def plan_from_copy( + tmp_path: Path, project: Path, selection: list[str], settings: dict[str, object] +) -> tuple[subprocess.CompletedProcess[str], dict[str, Any]]: + """Sync `project` to a location, lock its warehouse, then plan `selection` from the copy. + + Returns the refused live plan run while the lock was held and the location's plan document. + """ + control: Path = tmp_path / "control/.sqlbuild/control.sqlite3" + log: Path = tmp_path / "location.log" + started: datetime = datetime.now(UTC) + with ( + snapshot_api(tmp_path / "control") as api, + location_process( + [ + str(Path(sys.executable).with_name("sqb")), + "--project-dir", + str(project), + "location", + "serve", + "--server", + api.url, + "--location-id", + api.location_id, + "--target", + "development", + ], + log, + tmp_path / "location.stdout", + env=os.environ | {"SQLBUILD_LOCATION_TOKEN": api.bearer}, + ) as location, + ): + _ = wait_for_generation(api, "development", 1, location, log) + _ = wait_for_warehouse_sync_health( + api, + {"failure": None, "tables": ["catalog", "consumption", "fingerprints", "observations"]}, + location, + log, + after=started, + ) + lock: duckdb.DuckDBPyConnection = hold_write_lock(project / "warehouse.db") + try: + refused: subprocess.CompletedProcess[str] = live_plan(project, selection) + request_id: str = request_plan(control, selection, settings) + result: dict[str, Any] = wait_for_location_work(api, request_id, location, log) + finally: + lock.close() + outcome: dict[str, Any] = result["result_json"]["outcome"] + assert outcome["kind"] == "plan", outcome + return refused, outcome["document"] diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_plan_preview.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_plan_preview.py index e371d55655..e79a6f48f2 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_plan_preview.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_plan_preview.py @@ -1,10 +1,7 @@ """A location plans from the server's synced copy and never opens the warehouse to do it.""" import json -import os import subprocess -import sys -from datetime import UTC, datetime from pathlib import Path from textwrap import dedent from typing import Any @@ -13,20 +10,13 @@ import pytest from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + ChangedPlanPreviewTestCase, PlanPreviewTestCase, ) -from tests.integration.src.sqlbuild.compiler.compiled_project.publication.catalog.helpers import ( - request_plan, -) from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( - hold_write_lock, live_plan, - location_process, + plan_from_copy, run_sqb, - snapshot_api, - wait_for_generation, - wait_for_location_work, - wait_for_warehouse_sync_health, ) from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files @@ -190,7 +180,7 @@ def test_given_a_synced_location_when_a_plan_is_requested_then_it_plans_from_the assert live_run.returncode == 0, live_run.stderr live: dict[str, Any] = json.loads(live_run.stdout) log: Path = tmp_path / "location.log" - refused, preview = _plan_from_copy(tmp_path, project, {}) + refused, preview = plan_from_copy(tmp_path, project, _SELECTION, {}) assert refused.returncode != 0 assert "synced_copy" in preview, log.read_text() @@ -231,85 +221,46 @@ def test_given_a_synced_location_when_a_plan_is_requested_then_it_plans_from_the assert preview == live +@pytest.mark.parametrize( + "test_case", + [ + ChangedPlanPreviewTestCase( + description="an edited model is the only change the copy names", + edited_model="models/order_count.sql", + edit=("AS order_count", "AS order_count, 1 AS batch"), + expected_changed={ + "counts": ["definition"], + "reach": "job", + "runs": [ + { + "node": "model:order_count", + "reason": "planner change (query_changed)", + "downstream_of": None, + } + ], + "held_back": [], + "unchanged": ["model:daily_orders", "model:orders", "model:refunds"], + }, + ), + ], + ids=lambda case: case.description, +) def test_given_an_edited_model_when_a_changed_plan_is_requested_then_the_copy_names_what_changed( - tmp_path: Path, + test_case: ChangedPlanPreviewTestCase, tmp_path: Path ) -> None: project: Path = tmp_path / "project" write_files(project_dir=project, files=_PROJECT_FILES) built: subprocess.CompletedProcess[str] = run_sqb(project, "build", "--target", "development") assert built.returncode == 0, built.stderr - edited: str = _PROJECT_FILES["models/order_count.sql"].replace( - "AS order_count", "AS order_count, 1 AS batch" - ) - write_files(project_dir=project, files={"models/order_count.sql": edited}) + edited: str = _PROJECT_FILES[test_case.edited_model].replace(*test_case.edit) + write_files(project_dir=project, files={test_case.edited_model: edited}) live_run: subprocess.CompletedProcess[str] = live_plan(project, _SELECTION, "--changed") assert live_run.returncode == 0, live_run.stderr - refused, preview = _plan_from_copy(tmp_path, project, {"changed": True}) + refused, preview = plan_from_copy(tmp_path, project, _SELECTION, {"changed": True}) assert refused.returncode != 0 - assert preview.get("changed") == { - "counts": ["definition"], - "reach": "job", - "runs": [ - { - "node": "model:order_count", - "reason": "planner change (query_changed)", - "downstream_of": None, - } - ], - "held_back": [], - "unchanged": ["model:daily_orders", "model:orders", "model:refunds"], - }, (tmp_path / "location.log").read_text() + assert preview.get("changed") == test_case.expected_changed, ( + tmp_path / "location.log" + ).read_text() assert preview["changed"] == json.loads(live_run.stdout)["changed"] - - -def _plan_from_copy( - tmp_path: Path, project: Path, settings: dict[str, object] -) -> tuple[subprocess.CompletedProcess[str], dict[str, Any]]: - """Sync `project` to a location, lock its warehouse, then plan `_SELECTION` from the copy. - - Returns the refused live plan run while the lock was held and the location's plan document. - """ - control: Path = tmp_path / "control/.sqlbuild/control.sqlite3" - log: Path = tmp_path / "location.log" - started: datetime = datetime.now(UTC) - with ( - snapshot_api(tmp_path / "control") as api, - location_process( - [ - str(Path(sys.executable).with_name("sqb")), - "--project-dir", - str(project), - "location", - "serve", - "--server", - api.url, - "--location-id", - api.location_id, - "--target", - "development", - ], - log, - tmp_path / "location.stdout", - env=os.environ | {"SQLBUILD_LOCATION_TOKEN": api.bearer}, - ) as location, - ): - _ = wait_for_generation(api, "development", 1, location, log) - _ = wait_for_warehouse_sync_health( - api, - {"failure": None, "tables": ["catalog", "consumption", "fingerprints", "observations"]}, - location, - log, - after=started, - ) - lock: duckdb.DuckDBPyConnection = hold_write_lock(project / "warehouse.db") - try: - refused: subprocess.CompletedProcess[str] = live_plan(project, _SELECTION) - request_id: str = request_plan(control, _SELECTION, settings) - result: dict[str, Any] = wait_for_location_work(api, request_id, location, log) - finally: - lock.close() - outcome: dict[str, Any] = result["result_json"]["outcome"] - assert outcome["kind"] == "plan", outcome - return refused, outcome["document"] From de344de6db5710aafbaf008c5df79f7ffa094848 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 01:29:04 +0000 Subject: [PATCH 5/5] chore(client): regenerate the control client for the preview-copy fields --- clients/typescript/src/schema.d.ts | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index 5c5d8583a7..23f2ec18a7 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -2398,10 +2398,7 @@ export interface components { }; /** @description What the mirror knows as of `synced_at`; `None` when a required table never synced cleanly. */ PreviewCopy: { - /** - * @description Every mirrored consumption row when the read asked for them; locations that never ask - * never receive the field. - */ + /** @description Every mirrored consumption row when asked for; omitted otherwise, as older locations reject it. */ consumption?: components["schemas"]["WarehouseConsumptionRecord"][]; fingerprints: components["schemas"]["WarehouseFingerprintRecord"][]; latest_runs: components["schemas"]["PreviewLatestRun"][]; @@ -2412,10 +2409,7 @@ export interface components { }; /** @description A location asks for the mirrored copy a plan preview plans from, as its fenced reader. */ PreviewCopyRead: { - /** - * @description Whether the copy carries consumption history, which only `--changed` plans read; sent only - * when set, so other plans still reach servers that predate it. - */ + /** @description Whether the copy carries consumption history; sent only when set, as older servers reject it. */ consumption?: boolean; /** Format: int64 */ generation: number;