From c21806717c9989c4260ede410e7fd7cc8ab97d36 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 18 Nov 2026 19:31:36 +0000 Subject: [PATCH 1/2] fix(plan): read deferred upstreams from the deferred target and trust synced records from their last scan When deferring, unselected model and seed upstreams must exist in the deferred target, matched under their deferred relation names in every database the deferred locations use. S301 names the target each missing upstream was looked for in. A synced plan treats a mirrored record as current from the start of the last catalog scan that found it unchanged, timed by the server's clock, keeping the clock-skew allowance. --- .../deferred_reads/models/order_report.sql | 3 + .../deferred_reads/models/orders.sql | 3 + .../_helpers/missing_lines.rs | 36 ++++++++ .../src/planner_identity/_helpers/mod.rs | 1 + .../main/missing_upstream_message.rs | 65 ++++++-------- .../main/missing_upstreams.rs | 19 ++-- .../src/planner_identity/models.rs | 2 + .../tests/test_planner_identity.rs | 68 +++++++++++++++ .../_helpers/location/execution.rs | 2 +- .../src/native_commands/models.rs | 19 +++- .../tests/test_location_metadata.rs | 47 +++++++++- .../src/native_commands/tests/test_types.rs | 9 ++ .../src/warehouse_metadata/constants.rs | 2 +- .../location_reads/read_synced_catalog.rs | 32 +++++-- .../_helpers/planner_records/constants.rs | 1 + .../_helpers/stages/deferral.rs | 57 ++++++------ .../stages/resolve_model_deferral_inputs.rs | 1 + .../_helpers/outputs/buildability.rs | 86 ++++++++++++++++--- .../_helpers/compile_driver/plan_result.rs | 2 +- src/sqlbuild/_native.pyi | 8 ++ .../cli/commands/main/clone/_test_types.py | 12 +++ .../cli/commands/main/clone/test_clone.py | 73 ++++++++++++++++ .../compiler/planner/main/_test_types.py | 19 +++- .../sqlbuild/compiler/planner/main/helpers.py | 42 +++++++-- .../main/test_deferred_migration_origins.py | 68 +++++++++------ .../main/test_plan_from_synced_state.py | 80 ++++++++++++++++- .../compiler/pipeline/_helpers/_test_types.py | 10 +++ .../compiler/pipeline/_helpers/helpers.py | 36 ++++++++ .../_helpers/test_deferred_locations.py | 60 ++++++++++++- 29 files changed, 733 insertions(+), 130 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/deferred_reads/models/order_report.sql create mode 100644 benchmarks/duckshop/edge_cases/deferred_reads/models/orders.sql create mode 100644 crates/sqlbuild-analysis/src/planner_identity/_helpers/missing_lines.rs diff --git a/benchmarks/duckshop/edge_cases/deferred_reads/models/order_report.sql b/benchmarks/duckshop/edge_cases/deferred_reads/models/order_report.sql new file mode 100644 index 0000000000..15583d3c4e --- /dev/null +++ b/benchmarks/duckshop/edge_cases/deferred_reads/models/order_report.sql @@ -0,0 +1,3 @@ +MODEL (description "A report a deferring build selects alone, reading the aliased orders.", materialized table); + +SELECT order_id FROM __ref("orders") diff --git a/benchmarks/duckshop/edge_cases/deferred_reads/models/orders.sql b/benchmarks/duckshop/edge_cases/deferred_reads/models/orders.sql new file mode 100644 index 0000000000..a75e0ce500 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/deferred_reads/models/orders.sql @@ -0,0 +1,3 @@ +MODEL (description "Orders stored under an alias, so the deferred target lists them as orders_v2.", materialized table, alias "orders_v2"); + +SELECT 1 AS order_id diff --git a/crates/sqlbuild-analysis/src/planner_identity/_helpers/missing_lines.rs b/crates/sqlbuild-analysis/src/planner_identity/_helpers/missing_lines.rs new file mode 100644 index 0000000000..dea184b12e --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_identity/_helpers/missing_lines.rs @@ -0,0 +1,36 @@ +//! The lines naming upstreams of a missing-upstream message that one target lacks. + +use crate::planner_identity::constants::MISSING_MESSAGE_ITEM_LIMIT; + +/// How many upstreams of these kinds are missing from `target`, a few by name. +pub(crate) fn missing_lines(missing: &[(&str, &str)], target: Option<&String>) -> Vec { + let one: bool = missing.len() == 1; + let kinds: std::collections::BTreeSet<&str> = missing.iter().map(|(_, kind)| *kind).collect(); + let noun: String = match kinds.iter().collect::>().as_slice() { + [kind] if one => (*kind).to_string(), + [kind] => format!("{kind}s"), + _ => "models and seeds".to_owned(), + }; + let place: String = target.map_or_else( + || "the warehouse".to_owned(), + |target| format!("target '{target}'"), + ); + let mut lines: Vec = vec![format!( + "{} upstream {noun} this build reads {} exist in {place}", + missing.len(), + if one { "doesn't" } else { "don't" } + )]; + lines.extend( + missing + .iter() + .take(MISSING_MESSAGE_ITEM_LIMIT) + .map(|(name, _)| format!(" {name}")), + ); + if missing.len() > MISSING_MESSAGE_ITEM_LIMIT { + lines.push(format!( + " ... {} more (sqb plan --json lists all)", + missing.len() - MISSING_MESSAGE_ITEM_LIMIT + )); + } + lines +} diff --git a/crates/sqlbuild-analysis/src/planner_identity/_helpers/mod.rs b/crates/sqlbuild-analysis/src/planner_identity/_helpers/mod.rs index 7cf4c87d38..96dcbe3d63 100644 --- a/crates/sqlbuild-analysis/src/planner_identity/_helpers/mod.rs +++ b/crates/sqlbuild-analysis/src/planner_identity/_helpers/mod.rs @@ -2,4 +2,5 @@ pub(crate) mod compose_graph_identity; pub(crate) mod cycle_message; pub(crate) mod dependency_counts; pub(crate) mod identity_resolver; +pub(crate) mod missing_lines; pub(crate) mod staleness_walk; diff --git a/crates/sqlbuild-analysis/src/planner_identity/main/missing_upstream_message.rs b/crates/sqlbuild-analysis/src/planner_identity/main/missing_upstream_message.rs index c392668519..9409109229 100644 --- a/crates/sqlbuild-analysis/src/planner_identity/main/missing_upstream_message.rs +++ b/crates/sqlbuild-analysis/src/planner_identity/main/missing_upstream_message.rs @@ -2,7 +2,11 @@ use std::collections::HashMap; -use crate::planner_identity::constants::MISSING_MESSAGE_ITEM_LIMIT; +use crate::planner_identity::_helpers::missing_lines::missing_lines; + +use crate::planner_identity::constants::{ + MISSING_MESSAGE_ITEM_LIMIT, MODEL_RESOURCE, SEED_RESOURCE, +}; use crate::planner_identity::models::{GraphKey, MissingUpstream, MissingUpstreamFixes}; /// Missing lineage upstreams with how to fix them and, when only an audit or hook read requires one, that read's origin. @@ -11,8 +15,7 @@ pub fn missing_upstream_message( edge_origins: &HashMap<(GraphKey, GraphKey), String>, fixes: &MissingUpstreamFixes, ) -> String { - let mut lineage_missing: Vec<&str> = Vec::new(); - let mut kinds: std::collections::BTreeSet<&str> = std::collections::BTreeSet::new(); + let mut lineage_missing: Vec<(&str, &str)> = Vec::new(); let mut read_names: Vec<&str> = Vec::new(); for entry in missing { let origins: Vec<&str> = entry @@ -25,8 +28,7 @@ pub fn missing_upstream_message( }) .collect(); if origins.len() < entry.required_by.len() { - lineage_missing.push(entry.key.name.as_str()); - kinds.insert(entry.key.kind.as_str()); + lineage_missing.push((entry.key.name.as_str(), entry.key.kind.as_str())); continue; } for origin in origins { @@ -39,46 +41,35 @@ pub fn missing_upstream_message( if lineage_missing.is_empty() { return format!("cannot build selected scope: {}", reads.unwrap_or_default()); } - let mut text: String = lineage_text(&lineage_missing, &kinds, fixes); + let mut text: String = lineage_text(&lineage_missing, fixes); if let Some(reads) = reads { text.push_str(&format!("\n\n Also, {reads}")); } text } -/// The missing lineage upstreams, a few by name, then how to read, clone or build them. -fn lineage_text( - names: &[&str], - kinds: &std::collections::BTreeSet<&str>, - fixes: &MissingUpstreamFixes, -) -> String { +/// Missing lineage upstreams per target looked in (deferred for models and seeds when deferring), then the fixes. +fn lineage_text(missing: &[(&str, &str)], fixes: &MissingUpstreamFixes) -> String { + let names: Vec<&str> = missing.iter().map(|(name, _)| *name).collect(); let one: bool = names.len() == 1; - let noun: String = match kinds.iter().collect::>().as_slice() { - [kind] if one => (*kind).to_string(), - [kind] => format!("{kind}s"), - _ => "models and seeds".to_owned(), + let groups: Vec> = match fixes.deferred_to.as_ref() { + Some(deferred_to) => { + let (deferred, active): (Vec<_>, Vec<_>) = missing + .iter() + .copied() + .partition(|(_, kind)| [MODEL_RESOURCE, SEED_RESOURCE].contains(kind)); + [ + (deferred, Some(deferred_to)), + (active, fixes.target.as_ref()), + ] + .into_iter() + .filter(|(group, _)| !group.is_empty()) + .map(|(group, target)| missing_lines(&group, target)) + .collect() + } + None => vec![missing_lines(missing, fixes.target.as_ref())], }; - let place: String = fixes.target.as_ref().map_or_else( - || "the warehouse".to_owned(), - |target| format!("target '{target}'"), - ); - let mut lines: Vec = vec![format!( - "{} upstream {noun} this build reads {} exist in {place}", - names.len(), - if one { "doesn't" } else { "don't" } - )]; - lines.extend( - names - .iter() - .take(MISSING_MESSAGE_ITEM_LIMIT) - .map(|name| format!(" {name}")), - ); - if names.len() > MISSING_MESSAGE_ITEM_LIMIT { - lines.push(format!( - " ... {} more (sqb plan --json lists all)", - names.len() - MISSING_MESSAGE_ITEM_LIMIT - )); - } + let mut lines: Vec = groups.join(&String::new()); lines.push(String::new()); let (exists, them): (&str, &str) = if one { ("it exists", "it") diff --git a/crates/sqlbuild-analysis/src/planner_identity/main/missing_upstreams.rs b/crates/sqlbuild-analysis/src/planner_identity/main/missing_upstreams.rs index 59cb1c90a8..18f60d3d81 100644 --- a/crates/sqlbuild-analysis/src/planner_identity/main/missing_upstreams.rs +++ b/crates/sqlbuild-analysis/src/planner_identity/main/missing_upstreams.rs @@ -2,10 +2,12 @@ use std::collections::{HashMap, HashSet}; -use crate::planner_identity::constants::{DBT_REF_RESOURCE, SOURCE_RESOURCE, SQL_TEST_RESOURCE}; +use crate::planner_identity::constants::{ + DBT_REF_RESOURCE, MODEL_RESOURCE, SEED_RESOURCE, SOURCE_RESOURCE, SQL_TEST_RESOURCE, +}; use crate::planner_identity::models::{AvailableUpstreams, GraphKey, MissingUpstream}; -/// Missing upstreams, most dependents first, then by `(kind, name)`; dependents sorted by `(kind, name)`. +/// Missing upstreams, most dependents first; a deferring plan checks unselected models and seeds in the deferred target. pub fn missing_upstreams( selected_keys: &[GraphKey], upstream_deps: &HashMap>, @@ -19,11 +21,14 @@ pub fn missing_upstreams( .contains(&dep_key.kind.as_str()) || selected.contains(dep_key) || available.satisfied_keys.contains(dep_key) - || available.existing_relations.contains(&dep_key.name) - || available - .deferred_relations - .as_ref() - .is_some_and(|deferred| deferred.contains(&dep_key.name)) + || match &available.deferred_relations { + Some(deferred) + if [MODEL_RESOURCE, SEED_RESOURCE].contains(&dep_key.kind.as_str()) => + { + deferred.contains(&dep_key.name) + } + _ => available.existing_relations.contains(&dep_key.name), + } { continue; } diff --git a/crates/sqlbuild-analysis/src/planner_identity/models.rs b/crates/sqlbuild-analysis/src/planner_identity/models.rs index e0cd6716a0..0388d75735 100644 --- a/crates/sqlbuild-analysis/src/planner_identity/models.rs +++ b/crates/sqlbuild-analysis/src/planner_identity/models.rs @@ -92,6 +92,8 @@ pub struct AvailableUpstreams { pub struct MissingUpstreamFixes { /// The active target, named as where the upstreams are missing. pub target: Option, + /// The target a deferring plan reads unselected models and seeds from, named as where those are missing. + pub deferred_to: Option, /// Targets the active target may clone from, to defer to or clone the upstreams from. pub origins: Vec, /// The origins `--defer-to` accepts: project targets, offered only when this plan does not defer already. diff --git a/crates/sqlbuild-analysis/src/planner_identity/tests/test_planner_identity.rs b/crates/sqlbuild-analysis/src/planner_identity/tests/test_planner_identity.rs index 5c124eb943..da55a9535b 100644 --- a/crates/sqlbuild-analysis/src/planner_identity/tests/test_planner_identity.rs +++ b/crates/sqlbuild-analysis/src/planner_identity/tests/test_planner_identity.rs @@ -436,6 +436,7 @@ fn given_selected_keys_when_checking_buildability_then_reports_missing_upstreams origins: vec!["prod".to_owned(), "staging".to_owned()], deferrable: vec!["prod".to_owned()], clone_to: Some("dev".to_owned()), + deferred_to: None, }, expected_missing: ["a", "b", "c", "d", "e", "f"] .map(model) @@ -459,6 +460,73 @@ fn given_selected_keys_when_checking_buildability_then_reports_missing_upstreams " Or build them too: add + before your selection", ), }, + BuildabilityTestCase { + description: "deferring reads models and seeds only from the deferred target", + selected: vec![model("report")], + upstream: HashMap::from([( + model("report"), + vec![model("dev_only"), model("prod_only"), key("seed", "codes")], + )]), + available: AvailableUpstreams { + existing_relations: names(&["dev_only", "codes"]), + deferred_relations: Some(names(&["prod_only", "codes"])), + ..AvailableUpstreams::default() + }, + origins: HashMap::new(), + fixes: MissingUpstreamFixes { + target: Some("dev".to_owned()), + deferred_to: Some("prod".to_owned()), + ..MissingUpstreamFixes::default() + }, + expected_missing: vec![MissingUpstream { + key: model("dev_only"), + required_by: vec![model("report")], + }], + expected_message: concat!( + "1 upstream model this build reads doesn't exist in target 'prod'\n", + " dev_only\n", + "\n", + " Build it too: add + before your selection", + ), + }, + BuildabilityTestCase { + description: "deferring names the deferred target for models and the active one for functions", + selected: vec![model("report")], + upstream: HashMap::from([( + model("report"), + vec![model("dev_only"), key("udf", "parse_sku")], + )]), + available: AvailableUpstreams { + existing_relations: names(&["dev_only"]), + deferred_relations: Some(names(&["parse_sku"])), + ..AvailableUpstreams::default() + }, + origins: HashMap::new(), + fixes: MissingUpstreamFixes { + target: Some("dev".to_owned()), + deferred_to: Some("prod".to_owned()), + ..MissingUpstreamFixes::default() + }, + expected_missing: vec![ + MissingUpstream { + key: model("dev_only"), + required_by: vec![model("report")], + }, + MissingUpstream { + key: key("udf", "parse_sku"), + required_by: vec![model("report")], + }, + ], + expected_message: concat!( + "1 upstream model this build reads doesn't exist in target 'prod'\n", + " dev_only\n", + "\n", + "1 upstream udf this build reads doesn't exist in target 'dev'\n", + " parse_sku\n", + "\n", + " Build them too: add + before your selection", + ), + }, BuildabilityTestCase { description: "only an audit read requires the upstream", selected: vec![model("a")], diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/location/execution.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/location/execution.rs index f41fb37c34..1e64fe1398 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/location/execution.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/location/execution.rs @@ -745,7 +745,7 @@ pub(crate) fn next_scan(mut state: LocationMetadataSync) -> LocationMetadataSync state.progress.scan_unchanged = false; state.progress.scan_started = Instant::now(); state.index += 1; - state.scan_id = Uuid::now_v7(); + state.scan_id = state.server_scan_id(); state.observed_since = None; state.watermark_loaded = false; state.limit = MAX_HISTORY_PAGE_RECORDS; diff --git a/crates/sqlbuild-cli/src/native_commands/models.rs b/crates/sqlbuild-cli/src/native_commands/models.rs index 65c69f86f1..3967c069de 100644 --- a/crates/sqlbuild-cli/src/native_commands/models.rs +++ b/crates/sqlbuild-cli/src/native_commands/models.rs @@ -14,7 +14,7 @@ use sqlbuild_orchestration::warehouse_metadata::models::{ MetadataBatch, StatisticsRead, WarehouseHistoryFailure, WarehouseHistoryKind, WarehouseHistoryPage, WarehouseStateLocation, }; -use uuid::Uuid; +use uuid::{NoContext, Timestamp, Uuid}; use crate::argv::models::Namespace; use crate::native_commands::constants::{METADATA_FAILURE_LIMIT, METADATA_POLL_SECONDS}; @@ -425,12 +425,24 @@ impl LocationMetadataSync { self.reads_wake_compute && !self.safety_explicit } + /// A scan id timed by the server's clock: a synced plan trusts a record as of its scan's start, never after a read. + pub(crate) fn server_scan_id(&self) -> Uuid { + let now: DateTime = Utc::now() + self.server_offset; + let seconds: u64 = u64::try_from(now.timestamp()).unwrap_or_default(); + Uuid::new_v7(Timestamp::from_unix( + NoContext, + seconds, + now.timestamp_subsec_nanos(), + )) + } + /// Start the sync `sync_due` asked for at server time `now`; a stopped sync restarts its pass. pub(crate) fn begin_sync(&mut self, now: DateTime) { + self.server_offset = now - Utc::now(); + self.scan_id = self.server_scan_id(); if std::mem::take(&mut self.halted) { self.index = 0; self.source_after = None; - self.scan_id = Uuid::now_v7(); self.observed_since = None; self.watermark_loaded = false; self.statistics = None; @@ -441,7 +453,6 @@ impl LocationMetadataSync { self.applied = false; } self.idle = false; - self.server_offset = now - Utc::now(); self.progress = SyncProgress::new(); self.cycle_started_at = Some(now); self.next_at = tokio::time::Instant::now(); @@ -470,7 +481,7 @@ impl LocationMetadataSync { if statistics { self.index -= 1; self.source_after = None; - self.scan_id = Uuid::now_v7(); + self.scan_id = self.server_scan_id(); self.observed_since = None; self.watermark_loaded = false; } diff --git a/crates/sqlbuild-cli/src/native_commands/tests/test_location_metadata.rs b/crates/sqlbuild-cli/src/native_commands/tests/test_location_metadata.rs index f1a6e68c04..71c88d2577 100644 --- a/crates/sqlbuild-cli/src/native_commands/tests/test_location_metadata.rs +++ b/crates/sqlbuild-cli/src/native_commands/tests/test_location_metadata.rs @@ -14,6 +14,7 @@ use crate::native_commands::tests::helpers::{ acknowledge_complete_pages, one_location, published_location, }; use crate::native_commands::tests::test_types::BilledDeferralTestCase; +use crate::native_commands::tests::test_types::FastClockScanTestCase; use crate::native_commands::tests::test_types::{ FailedReadHealthTestCase, NewestVersionsTestCase, ReadFailureTestCase, }; @@ -24,7 +25,7 @@ use crate::native_commands::tests::test_types::{ use crate::native_commands::tests::test_types::{ RowsTextTestCase, ScanLineTestCase, SyncProgressTestCase, WakingIdleTestCase, }; -use chrono::Utc; +use chrono::{DateTime, Utc}; use sqlbuild_config::project::models::ControlConfig; use sqlbuild_orchestration::platform::models::{ LocationHealth, WarehouseReadFailure, WarehouseSyncFailure, @@ -1184,3 +1185,47 @@ fn given_a_published_location_when_its_sync_stops_before_any_page_succeeds_then_ ); } } + +#[test] +fn given_a_fast_location_clock_when_scanning_then_a_ddl_after_the_page_read_is_still_described_live() + { + let test_cases = [ + FastClockScanTestCase { + description: "the first scan of a sync", + scans_completed: 0, + expected_described_live: true, + }, + FastClockScanTestCase { + description: "a later scan of the same pass", + scans_completed: 1, + expected_described_live: true, + }, + ]; + // A synced plan reuses mirrored columns only when the last DDL ran this long before the scan began. + let ddl_clock_skew = chrono::Duration::minutes(5); + for test_case in test_cases { + let mut state = metadata_sync((one_location(), 1), (3600, 0)); + state.begin_sync(Utc::now() - chrono::Duration::minutes(10)); + state = acknowledge_complete_pages(state, test_case.scans_completed); + let page_read_at: DateTime = Utc::now() + state.server_offset; + let ddl_at: DateTime = page_read_at + chrono::Duration::seconds(1); + let (seconds, nanos) = state + .scan_id + .get_timestamp() + .expect("a v7 scan id") + .to_unix(); + let scan_started: DateTime = + DateTime::from_timestamp(i64::try_from(seconds).expect("seconds"), nanos) + .expect("scan start"); + + assert_eq!( + ( + scan_started <= page_read_at, + ddl_at > scan_started - ddl_clock_skew + ), + (true, test_case.expected_described_live), + "{}: the scan started at {scan_started}, the page was read at {page_read_at}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-cli/src/native_commands/tests/test_types.rs b/crates/sqlbuild-cli/src/native_commands/tests/test_types.rs index 83ce6e7039..076eb5dfaa 100644 --- a/crates/sqlbuild-cli/src/native_commands/tests/test_types.rs +++ b/crates/sqlbuild-cli/src/native_commands/tests/test_types.rs @@ -306,3 +306,12 @@ pub(super) struct ReplanTestCase { pub(super) syncs: Vec<(bool, bool)>, pub(super) expected_replans: Vec, } + +/// A scan whose location clock runs ten minutes ahead of the server's. +pub(super) struct FastClockScanTestCase { + pub(super) description: &'static str, + /// Scans of the pass completed before the one whose id is checked. + pub(super) scans_completed: usize, + /// Whether a table altered a second after the page read is still described live. + pub(super) expected_described_live: bool, +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/constants.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/constants.rs index 1f753dee9c..05ff648200 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/constants.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/constants.rs @@ -54,7 +54,7 @@ pub(crate) const INGESTION_CHUNK_RECORDS: usize = 200; /// The audit result metadata field marking a failure its build let through. pub(crate) const IGNORED_KEY: &str = "ignored"; /// The mirrored catalog of one location's target, with when each relation was observed. -pub(crate) const SYNCED_CATALOG_SQL: &str = "SELECT record_json, observed_at FROM sqb_warehouse_catalog \ +pub(crate) const SYNCED_CATALOG_SQL: &str = "SELECT record_json, observed_at, scan_id FROM sqb_warehouse_catalog \ WHERE location_id=$1 AND target=$2 ORDER BY state_database, state_schema, event_id"; /// The development locations of one target, by project path. pub(crate) const DEVELOPMENT_LOCATIONS_SQL: &str = diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/read_synced_catalog.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/read_synced_catalog.rs index c3104ec656..a669b187fa 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/read_synced_catalog.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/read_synced_catalog.rs @@ -1,5 +1,6 @@ //! A plan from synced state reads the target's mirrored catalog and when each relation was seen. +use chrono::{DateTime, Utc}; use uuid::Uuid; use crate::store::errors::ControlError; @@ -10,7 +11,7 @@ use crate::store::models::{ControlStore, Parameter, TransactionMode}; use crate::warehouse_metadata::constants::SYNCED_CATALOG_SQL; use crate::warehouse_metadata::models::{SyncedCatalogRelation, WarehouseCatalogRecord}; -/// Every mirrored relation of the location's target, with the time its catalog scan observed it. +/// Every mirrored relation of the location's target, current as of its last scan's start or later. pub async fn read_synced_catalog( store: &ControlStore, (location_id, target): (Uuid, &str), @@ -28,10 +29,18 @@ pub async fn read_synced_catalog( serde_json::from_value(row.remove("record_json").unwrap_or_default()).map_err( |error| ControlError::Invalid(format!("invalid mirrored relation: {error}")), )?; - let observed_at = serde_json::from_value(row.remove("observed_at").unwrap_or_default()) - .map_err(|error| { - ControlError::Invalid(format!("invalid mirrored observation time: {error}")) - })?; + let observed_at: DateTime = serde_json::from_value( + row.remove("observed_at").unwrap_or_default(), + ) + .map_err(|error| { + ControlError::Invalid(format!("invalid mirrored observation time: {error}")) + })?; + let scan_id: Option = serde_json::from_value( + row.remove("scan_id").unwrap_or_default(), + ) + .map_err(|error| ControlError::Invalid(format!("invalid mirrored scan: {error}")))?; + let observed_at = + scan_started(scan_id)?.map_or(observed_at, |started| started.max(observed_at)); Ok(SyncedCatalogRelation { record, observed_at, @@ -39,3 +48,16 @@ pub async fn read_synced_catalog( }) .collect() } + +/// When the scan that last delivered a record started: a later page of it re-read the record unchanged. +fn scan_started(scan_id: Option) -> Result>, ControlError> { + let Some((seconds, nanos)) = scan_id + .and_then(|scan_id| scan_id.get_timestamp()) + .map(|started| started.to_unix()) + else { + return Ok(None); + }; + let seconds: i64 = i64::try_from(seconds) + .map_err(|error| ControlError::Invalid(format!("invalid mirrored scan time: {error}")))?; + Ok(DateTime::from_timestamp(seconds, nanos)) +} diff --git a/crates/sqlbuild-python-base/src/records/_helpers/planner_records/constants.rs b/crates/sqlbuild-python-base/src/records/_helpers/planner_records/constants.rs index c094aa7ba3..bd568fa583 100644 --- a/crates/sqlbuild-python-base/src/records/_helpers/planner_records/constants.rs +++ b/crates/sqlbuild-python-base/src/records/_helpers/planner_records/constants.rs @@ -98,6 +98,7 @@ const R_DEFERRAL_INPUTS: RecordSpec = RecordSpec { fields: &[ FieldSpec::new("deferred_locations", FieldDefault::None), FieldSpec::new("deferred_relations", FieldDefault::None), + FieldSpec::new("defer_to", FieldDefault::None), FieldSpec::new("defer_sources_to", FieldDefault::None), FieldSpec::new("source_deferral_enabled", FieldDefault::True), ], diff --git a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/deferral.rs b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/deferral.rs index 572a536448..db20684f66 100644 --- a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/deferral.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/deferral.rs @@ -1,5 +1,6 @@ //! Deferred target relation locations, and the warehouse relations they name. +use pyo3::IntoPyObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyString, PyStringMethods, PyTuple}; use sqlbuild_python_base::records::main::construct::new_plain_error::new_plain_error; @@ -164,45 +165,47 @@ pub(crate) fn deferred_target_config<'py>( targets.get_item(defer_to) } -/// `gather_deferred_relations` with its arguments. +/// `gather_deferred_relations` with its arguments: one listing per database the locations live in. pub(crate) fn deferred_relations<'py>( adapter: &Bound<'py, PyAny>, connection: &Bound<'py, PyAny>, locations: &Bound<'py, PyAny>, ) -> PyResult> { let py: Python<'py> = adapter.py(); - let mut schemas: Vec = Vec::new(); - let mut database: Bound<'py, PyAny> = py.None().into_bound(py); + let mut databases: Vec<(Option, Vec)> = Vec::new(); for location in mapping_values(locations)?.try_iter()? { let location: Bound<'py, PyAny> = location?; - let schema: Bound<'py, PyAny> = native_field(&location, "schema")?; - if !schema.is_none() { - let schema: String = schema.extract()?; - if !schemas.contains(&schema) { - schemas.push(schema); + let schema: Option = native_field(&location, "schema")?.extract()?; + let database: Option = native_field(&location, "database")?.extract()?; + let Some(schema) = schema else { + continue; + }; + let index: usize = match databases.iter().position(|(listed, _)| *listed == database) { + Some(index) => index, + None => { + databases.push((database, Vec::new())); + databases.len() - 1 } - } - let location_database: Bound<'py, PyAny> = native_field(&location, "database")?; - if !location_database.is_none() && database.is_none() { - database = location_database; + }; + if !databases[index].1.contains(&schema) { + databases[index].1.push(schema); } } let relations_by_name: Bound<'py, PyDict> = PyDict::new(py); - if schemas.is_empty() { - return Ok(relations_by_name); - } - schemas.sort(); - let relations: Bound<'py, PyAny> = call_with( - &native_field(adapter, "list_relations")?, - &[ - ("connection", connection.clone()), - ("database", database), - ("schemas", PyTuple::new(py, schemas)?.into_any()), - ], - )?; - for relation in relations.try_iter()? { - let relation: Bound<'py, PyAny> = relation?; - relations_by_name.set_item(native_field(&relation, "name")?, relation)?; + for (database, mut schemas) in databases { + schemas.sort(); + let relations: Bound<'py, PyAny> = call_with( + &native_field(adapter, "list_relations")?, + &[ + ("connection", connection.clone()), + ("database", database.into_pyobject(py)?.into_any()), + ("schemas", PyTuple::new(py, schemas)?.into_any()), + ], + )?; + for relation in relations.try_iter()? { + let relation: Bound<'py, PyAny> = relation?; + relations_by_name.set_item(native_field(&relation, "name")?, relation)?; + } } Ok(relations_by_name) } diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/stages/resolve_model_deferral_inputs.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/stages/resolve_model_deferral_inputs.rs index 5795ff7349..5ca7b458fd 100644 --- a/crates/sqlbuild-python-compile/src/compile_driver/main/stages/resolve_model_deferral_inputs.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/stages/resolve_model_deferral_inputs.rs @@ -67,6 +67,7 @@ pub fn resolve_model_deferral_inputs<'py>( &[ ("deferred_locations", locations.into_any()), ("deferred_relations", relations.into_any()), + ("defer_to", defer_to), ], ) } diff --git a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/buildability.rs b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/buildability.rs index eb7288ec73..e478f7f2c0 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/buildability.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/buildability.rs @@ -8,7 +8,7 @@ use crate::planner_phases::main::inputs::runtime_dialect::runtime_dialect; use crate::planner_phases::main::inputs::runtime_reads::runtime_reads; use crate::planner_warehouse::main::sources::check_selected_source_tables_exist::check_selected_source_tables_exist; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::{PyString, PyTuple}; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; use sqlbuild_analysis::planner_identity::main::missing_upstream_message::missing_upstream_message; use sqlbuild_analysis::planner_identity::main::missing_upstreams::missing_upstreams; use sqlbuild_analysis::planner_identity::models::{ @@ -48,15 +48,10 @@ pub(crate) fn check_selected_scope_buildability<'py>( let names = |value: Bound<'py, PyAny>| -> PyResult> { value.try_iter()?.map(|value| value?.extract()).collect() }; - let deferred = native_field(deferral, "deferred_relations")?; let available = AvailableUpstreams { satisfied_keys, existing_relations: names(native_field(&snapshot, "existing_relations")?)?, - deferred_relations: if deferred.is_none() { - None - } else { - Some(names(deferred)?) - }, + deferred_relations: deferred_model_names(deferral)?, }; let selected: Vec<_> = scope.selected.iter().map(GraphKey::from).collect(); let deps: std::collections::HashMap> = scope @@ -79,7 +74,7 @@ pub(crate) fn check_selected_scope_buildability<'py>( let message = missing_upstream_message( &missing, &origins, - &missing_upstream_fixes(runtime, &project, available.deferred_relations.is_some())?, + &missing_upstream_fixes(runtime, &project, deferred_to(deferral, &available)?)?, ); let names = missing.iter().map(|entry| entry.key.name.as_str()); return Err(new_error_with_fields( @@ -97,14 +92,82 @@ pub(crate) fn check_selected_scope_buildability<'py>( ) } +/// The models and seeds whose deferred location, under an alias or a `migrate_from` origin, the deferred target lists. +fn deferred_model_names( + deferral: &Bound<'_, PyAny>, +) -> PyResult>> { + let relations = native_field(deferral, "deferred_relations")?; + let locations = native_field(deferral, "deferred_locations")?; + if relations.is_none() || locations.is_none() { + return Ok(None); + } + let mut listed: Vec = Vec::new(); + for relation in relations.downcast::()?.values() { + listed.push(relation_place(&relation)?); + } + let mut names: std::collections::HashSet = std::collections::HashSet::new(); + for (name, location) in locations.downcast::()? { + let place: RelationPlace = relation_place(&location)?; + if listed.iter().any(|relation| same_place(relation, &place)) { + names.insert(name.extract()?); + } + } + Ok(Some(names)) +} + +/// A relation's lowercased `(database, schema, name)`. +type RelationPlace = (Option, Option, String); + +/// Where a listed relation or a deferred location is, lowercased. +fn relation_place(relation: &Bound<'_, PyAny>) -> PyResult { + let lower = |field: &str| -> PyResult> { + Ok(native_field(relation, field)? + .extract::>()? + .map(|value| value.to_lowercase())) + }; + Ok(( + lower("database")?, + lower("schema")?, + native_field(relation, "name")? + .extract::()? + .to_lowercase(), + )) +} + +/// The same name, schema and database, where a part either side leaves to the connection's default matches any. +fn same_place(listed: &RelationPlace, location: &RelationPlace) -> bool { + let part = |left: &Option, right: &Option| { + left.is_none() || right.is_none() || left == right + }; + listed.2 == location.2 && part(&listed.1, &location.1) && part(&listed.0, &location.0) +} + +/// The target this plan reads unselected models and seeds from, when it defers. +fn deferred_to( + deferral: &Bound<'_, PyAny>, + available: &AvailableUpstreams, +) -> PyResult> { + if available.deferred_relations.is_none() { + return Ok(None); + } + native_field(deferral, "defer_to")?.extract() +} + /// The active target, its clone or deferral origins, and the clone `--to` it needs (also when the default target is unknown). fn missing_upstream_fixes( runtime: &Bound<'_, PyAny>, project: &Bound<'_, PyAny>, - deferring: bool, + deferred_to: Option, ) -> PyResult { - let py: Python<'_> = runtime.py(); let target: Option = native_field(project, "effective_target_name")?.extract()?; + if deferred_to.is_some() { + return Ok(MissingUpstreamFixes { + target, + deferred_to, + ..MissingUpstreamFixes::default() + }); + } + let py: Python<'_> = runtime.py(); let configs = ( runtime_field(runtime, |runtime| &runtime.project_config)?, runtime_field(runtime, |runtime| &runtime.local_config)?, @@ -129,7 +192,7 @@ fn missing_upstream_fixes( for name in native_field(config, "targets")?.try_iter()? { let name: String = name?.extract()?; if name != active && allows(&name, "allow_as_clone_origin")? { - if project_targets && !deferring { + if project_targets { deferrable.push(name.clone()); } origins.insert(name); @@ -143,6 +206,7 @@ fn missing_upstream_fixes( Err(_) => None, }; Ok(MissingUpstreamFixes { + deferred_to: None, clone_to: (default.as_deref() != Some(active.as_str())).then(|| active.clone()), target, origins: origins.into_iter().collect(), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs index 3f1c5c11c3..c9b9c1fcc7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs @@ -175,7 +175,7 @@ pub(crate) fn pipeline_result<'py>( ]; deferral_fields.extend(option_fields( options, - &["defer_sources_to", "source_deferral_enabled"], + &["defer_to", "defer_sources_to", "source_deferral_enabled"], )?); let deferral: Bound<'py, PyAny> = new_record_with(py, RecordId::DeferralInputs, &deferral_fields)?; diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 7b61bd1b3a..fe94cec490 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -3708,6 +3708,12 @@ def build_deferred_locations( default_database: str | None, render_qualified_name: Callable[..., str | None], ) -> dict[str, CompiledRelationLocation]: ... +def gather_deferred_relations( + *, + adapter: BaseAdapter, + connection: object, + deferred_locations: Mapping[str, CompiledRelationLocation], +) -> dict[str, RelationInfo]: ... def build_python_relation_targets( *, adapter: BaseAdapter, @@ -10515,12 +10521,14 @@ class DeferralInputs(NativeRecord): deferred_locations: dict[str, CompiledRelationLocation] | None deferred_relations: dict[str, RelationInfo] | None + defer_to: str | None defer_sources_to: str | None source_deferral_enabled: bool def __init__( self, deferred_locations: dict[str, CompiledRelationLocation] | None = ..., deferred_relations: dict[str, RelationInfo] | None = ..., + defer_to: str | None = ..., defer_sources_to: str | None = ..., source_deferral_enabled: bool = ..., ) -> None: ... diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clone/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/clone/_test_types.py index ba5cc3ee07..8ae7bbdce8 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/clone/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clone/_test_types.py @@ -123,3 +123,15 @@ class UnwritableConsumptionDeferCloneE2ETestCase: dropped_origin_table: str before_clone: tuple[str, ...] expected_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class DeferToMissingUpstreamE2ETestCase: + """Where an unselected upstream was built, and how a build deferring to prod plans it.""" + + description: str + upstream_sql: str + expected_exit_code: int + expected_fragments: tuple[str, ...] + report_sql: str + expected_report_rows: list[tuple[int, ...]] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clone/test_clone.py b/tests/e2e/src/sqlbuild/cli/commands/main/clone/test_clone.py index 2cf2258ba1..fb26169975 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/clone/test_clone.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clone/test_clone.py @@ -18,6 +18,7 @@ CloneMissingUpstreamE2ETestCase, ClonePolicyErrorTestCase, ClonePythonFunctionE2ETestCase, + DeferToMissingUpstreamE2ETestCase, MissingUpstreamFixesE2ETestCase, PartialDeferCloneE2ETestCase, UnwritableConsumptionDeferCloneE2ETestCase, @@ -29,6 +30,7 @@ ) from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( REPO_ROOT, + execute_duckdb, prepare_inline_project, query_duckdb, run_sqb, @@ -43,6 +45,20 @@ "returns", "shipments", ) +_DEFER_DATABASE: str = "warehouse.duckdb" +_DEFER_FIXTURE: Path = REPO_ROOT / "benchmarks/duckshop/edge_cases/deferred_reads" +_DEFER_PROJECT_FILES: dict[str, str] = { + "sqlbuild_project.toml": ( + 'name = "deferred_reads"\nadapter = "duckdb"\ndefault_target = "dev"\n\n' + f'[connection]\ndatabase = "{_DEFER_DATABASE}"\n\n' + '[targets.dev]\nschema = "dev"\n\n' + '[targets.prod]\nschema = "prod"\n' + ), + **{ + str(path.relative_to(_DEFER_FIXTURE)): path.read_text(encoding="utf-8") + for path in sorted(_DEFER_FIXTURE.glob("models/*.sql")) + }, +} @pytest.mark.parametrize( @@ -1147,3 +1163,60 @@ def test_given_unwritable_consumption_table_when_defer_clone_partly_fails_then_c output: str = cloned.stdout + cloned.stderr assert cloned.returncode == 1, output assert all(fragment in output for fragment in test_case.expected_fragments), output + + +@pytest.mark.parametrize( + "test_case", + [ + DeferToMissingUpstreamE2ETestCase( + description="an aliased upstream built only in the active target is missing from prod", + upstream_sql="CREATE TABLE dev.orders_v2 AS SELECT 5 AS order_id;", + expected_exit_code=1, + expected_fragments=( + "error[S301]: 1 upstream model this build reads doesn't exist in target 'prod'\n" + " orders\n\n Build it too: add + before your selection", + ), + report_sql=( + "SELECT count(*) FROM information_schema.tables " + "WHERE table_schema = 'dev' AND table_name = 'order_report'" + ), + expected_report_rows=[(0,)], + ), + DeferToMissingUpstreamE2ETestCase( + description="an aliased upstream built in prod is read from prod under its alias", + upstream_sql=( + "CREATE TABLE dev.orders_v2 AS SELECT 5 AS order_id;" + "CREATE TABLE prod.orders_v2 AS SELECT 7 AS order_id;" + ), + expected_exit_code=0, + expected_fragments=(), + report_sql="SELECT order_id FROM dev.order_report", + expected_report_rows=[(7,)], + ), + ], + ids=lambda case: case.description, +) +def test_given_defer_to_when_an_upstream_is_missing_from_the_deferred_target_then_s301( + test_case: DeferToMissingUpstreamE2ETestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, project_name="deferred_reads", repo_files=_DEFER_PROJECT_FILES + ) + database: Path = project_dir / _DEFER_DATABASE + execute_duckdb( + db_path=database, sql="CREATE SCHEMA dev; CREATE SCHEMA prod;" + test_case.upstream_sql + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build", "--select", "order_report", "--defer-to", "prod"), + project_dir=project_dir, + ) + + output: str = result.stdout + result.stderr + assert result.returncode == test_case.expected_exit_code, output + assert all(fragment in output for fragment in test_case.expected_fragments), output + assert "sqb clone" not in output, output + assert query_duckdb(db_path=database, sql=test_case.report_sql) == ( + test_case.expected_report_rows + ) diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py b/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py index c98ad03871..637eea6319 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py @@ -1,4 +1,5 @@ from dataclasses import dataclass, field +from datetime import timedelta from sqlbuild._native import ( BackfillAction, @@ -172,7 +173,14 @@ class SourceTableProbeBudgetTestCase: class UnlistedMigrationOriginTestCase: description: str select: tuple[str, ...] - expected_warnings: tuple[str, ...] + expected_unlisted_reads: int + + +@dataclass(frozen=True) +class DeferredMissingOriginTestCase: + description: str + select: tuple[str, ...] + expected_missing: tuple[str, ...] expected_unlisted_reads: int @@ -187,3 +195,12 @@ class PlanFromTestCase: expected_report: str expected_absent: tuple[str, ...] expected_live_listing: bool + + +@dataclass(frozen=True) +class SyncedScanTestCase: + """When a table's last DDL ran relative to the scan that last found its record unchanged.""" + + description: str + last_ddl_after_scan_start: timedelta + expected_live_describe: bool diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py index eeb122945f..08d271e41e 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py @@ -15,7 +15,7 @@ from tempfile import gettempdir from types import MappingProxyType from typing import Any, cast -from uuid import uuid4 +from uuid import UUID, uuid4 from sqlbuild._native import ( CompiledFunction, @@ -645,6 +645,17 @@ def _refuse_listing(database: str | None) -> None: ) +def deferring_unlisted_adapter( + *, connection: Any, unlisted_database: str +) -> UnlistedDatabaseDuckDbAdapter: + """An adapter over dev and prod schemas where only dev holds `order_ledger`.""" + + connection.execute("CREATE SCHEMA dev") + connection.execute("CREATE SCHEMA prod") + connection.execute("CREATE TABLE dev.order_ledger AS SELECT 1 AS order_id") + return UnlistedDatabaseDuckDbAdapter(connection=connection, unlisted_database=unlisted_database) + + def plan_deferred_project( *, project_dir: Path, @@ -725,8 +736,10 @@ def write_synced_control_store( target: str, warehouse: RecordingSnowflakeWarehouse, schemas: tuple[str, ...], + observed_at: datetime | None = None, + scan_id: str | None = None, ) -> None: - """A control store whose development location mirrored `schemas` of the warehouse just now.""" + """A control store whose location mirrored `schemas` at `observed_at` (default: now).""" catalog: InspectionCatalog = live_inspection_catalog(warehouse) relations: tuple[RelationInfo, ...] = catalog.list_relations( @@ -736,7 +749,7 @@ def write_synced_control_store( catalog.get_columns_for_relations(relations=relations) ) location: str = str(uuid4()) - observed: str = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S.%fZ") + observed: str = (observed_at or datetime.now(UTC)).strftime("%Y-%m-%dT%H:%M:%S.%fZ") (project_dir / ".sqlbuild").mkdir(parents=True) with closing(sqlite3.connect(project_dir / ".sqlbuild/control.sqlite3")) as store: _ = store.execute( @@ -745,29 +758,46 @@ def write_synced_control_store( ) _ = store.execute( "CREATE TABLE sqb_warehouse_catalog (event_id TEXT, location_id TEXT, target TEXT, " - "state_database TEXT, state_schema TEXT, observed_at TEXT, record_json JSON)" + "state_database TEXT, state_schema TEXT, observed_at TEXT, record_json JSON, " + "scan_id TEXT)" ) _ = store.execute( "INSERT INTO sqb_development_locations VALUES (?, ?, ?)", (str(project_dir), target, location), ) _ = store.executemany( - "INSERT INTO sqb_warehouse_catalog VALUES (?, ?, ?, ?, ?, ?, ?)", + "INSERT INTO sqb_warehouse_catalog VALUES (?, ?, ?, ?, ?, ?, ?, ?)", [ _mirrored_row( relation=relation, columns=columns.get(relation.identity, ()), scope=(location, target, observed), ) + + (scan_id,) for relation in relations ], ) store.commit() +def uuid7_at(started: datetime) -> str: + """A UUIDv7, as a catalog scan's id, whose timestamp is `started`.""" + + milliseconds: int = int(started.timestamp() * 1000) + random: int = uuid4().int + value: int = ( + milliseconds << 80 + | 0x7 << 76 + | (random & 0xFFF) << 64 + | 0b10 << 62 + | (random >> 12) & ((1 << 62) - 1) + ) + return str(UUID(int=value)) + + def _mirrored_row( *, relation: RelationInfo, columns: tuple[ColumnInfo, ...], scope: tuple[str, str, str] -) -> tuple[str, ...]: +) -> tuple[str | None, ...]: location, target, observed = scope record: dict[str, object] = { "state_database": "ANALYTICS", diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/test_deferred_migration_origins.py b/tests/integration/src/sqlbuild/compiler/planner/main/test_deferred_migration_origins.py index c00902ab45..8e3ed8f858 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/test_deferred_migration_origins.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/test_deferred_migration_origins.py @@ -1,4 +1,4 @@ -"""Deferred migrate_from lookups skip selected models and survive unlistable origins.""" +"""Deferred migrate_from lookups skip selected models; an unlistable origin falls back to the current name.""" from __future__ import annotations @@ -7,12 +7,14 @@ import pytest -from sqlbuild._native import CompilePipelineResult +from sqlbuild._native import CompilePipelineResult, PlannerInputError from tests.integration.src.sqlbuild.compiler.planner.main._test_types import ( + DeferredMissingOriginTestCase, UnlistedMigrationOriginTestCase, ) from tests.integration.src.sqlbuild.compiler.planner.main.helpers import ( UnlistedDatabaseDuckDbAdapter, + deferring_unlisted_adapter, plan_deferred_project, ) @@ -37,34 +39,53 @@ @pytest.mark.parametrize( "test_case", [ - UnlistedMigrationOriginTestCase( - description="unlistable origin of an unselected upstream falls back with a warning", + DeferredMissingOriginTestCase( + description="unlistable origin falls back to a current name the deferred target lacks", select=("order_report",), - expected_warnings=( - "could not list archive_db.prod.order_history in the deferred target to look " - "for a pre-migration relation, so model 'order_ledger' is deferred under its " - "current name: cannot list database archive_db", - ), + expected_missing=("order_ledger",), expected_unlisted_reads=1, ), + ], + ids=lambda case: case.description, +) +def test_given_unlistable_origin_of_an_unselected_upstream_when_deferring_then_s301_names_it( + test_case: DeferredMissingOriginTestCase, + tmp_path: Path, + connection: Any, +) -> None: + adapter: UnlistedDatabaseDuckDbAdapter = deferring_unlisted_adapter( + connection=connection, unlisted_database=_UNLISTED_DATABASE + ) + + with pytest.raises(PlannerInputError) as raised: + _ = plan_deferred_project( + project_dir=tmp_path, + adapter=adapter, + project_files=_PROJECT_FILES, + select=test_case.select, + ) + + assert raised.value.missing == test_case.expected_missing + assert adapter.listed_databases.count(_UNLISTED_DATABASE) == test_case.expected_unlisted_reads + + +@pytest.mark.parametrize( + "test_case", + [ UnlistedMigrationOriginTestCase( description="selected models are never looked up in the deferred target", select=("order_ledger", "order_report"), - expected_warnings=(), expected_unlisted_reads=0, ), ], ids=lambda case: case.description, ) -def test_given_unlistable_migration_origin_when_planning_with_defer_then_planning_continues( +def test_given_selected_models_when_deferring_then_their_origins_are_never_looked_up( test_case: UnlistedMigrationOriginTestCase, tmp_path: Path, connection: Any, ) -> None: - connection.execute("CREATE SCHEMA dev") - connection.execute("CREATE SCHEMA prod") - connection.execute("CREATE TABLE dev.order_ledger AS SELECT 1 AS order_id") - adapter: UnlistedDatabaseDuckDbAdapter = UnlistedDatabaseDuckDbAdapter( + adapter: UnlistedDatabaseDuckDbAdapter = deferring_unlisted_adapter( connection=connection, unlisted_database=_UNLISTED_DATABASE ) @@ -75,18 +96,13 @@ def test_given_unlistable_migration_origin_when_planning_with_defer_then_plannin select=test_case.select, ) - assert ( - tuple( - filter( - lambda message: "deferred target" in message, - (warning.message for warning in result.plan_output.warnings), - ) + assert not tuple( + filter( + lambda message: "deferred target" in message, + (warning.message for warning in result.plan_output.warnings), ) - == test_case.expected_warnings - ), test_case.description - assert ( - adapter.listed_databases.count(_UNLISTED_DATABASE) == test_case.expected_unlisted_reads - ), test_case.description + ) + assert adapter.listed_databases.count(_UNLISTED_DATABASE) == test_case.expected_unlisted_reads if __name__ == "__main__": diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/test_plan_from_synced_state.py b/tests/integration/src/sqlbuild/compiler/planner/main/test_plan_from_synced_state.py index f92caf9741..f10d08ae43 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/test_plan_from_synced_state.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/test_plan_from_synced_state.py @@ -2,19 +2,27 @@ from __future__ import annotations +import re +from dataclasses import replace +from datetime import UTC, datetime, timedelta from pathlib import Path import pytest from sqlbuild._native import CompilePipelineResult -from tests.integration.src.sqlbuild.compiler.planner.main._test_types import PlanFromTestCase +from tests.integration.src.sqlbuild.compiler.planner.main._test_types import ( + PlanFromTestCase, + SyncedScanTestCase, +) from tests.integration.src.sqlbuild.compiler.planner.main.helpers import ( plan_offline_snowflake_project, planned_entries, + uuid7_at, write_synced_control_store, write_targeted_snowflake_project, ) from tests.unit.src.sqlbuild.adapters.snowflake.inspection.helpers import ( + FakeRelation, RecordingSnowflakeWarehouse, SyntheticSnowflakeProject, ) @@ -24,6 +32,8 @@ _SYNCED: str = "Planned from synced state: checked in " _NO_STORE: str = "Planning live: the project has no control store" _LIVE: str = "Planning live:" +_FIRST_OBSERVED: datetime = datetime(2026, 2, 1, tzinfo=UTC) +_SCAN_STARTED: datetime = datetime(2026, 3, 1, tzinfo=UTC) @pytest.mark.parametrize( @@ -121,3 +131,71 @@ def test_given_plan_from_when_planning_then_reads_the_synced_copy_only_when_aske ) assert test_case.expected_report in err assert not any(text in err for text in test_case.expected_absent) + + +@pytest.mark.parametrize( + "test_case", + [ + SyncedScanTestCase( + description="a table replaced with the same columns before a later scan is not described", + last_ddl_after_scan_start=timedelta(hours=-1), + expected_live_describe=False, + ), + SyncedScanTestCase( + description="a table replaced within the clock skew before the scan started is described", + last_ddl_after_scan_start=timedelta(minutes=-1), + expected_live_describe=True, + ), + SyncedScanTestCase( + description="a table replaced after the scan started but before its page was read is described", + last_ddl_after_scan_start=timedelta(seconds=1), + expected_live_describe=True, + ), + SyncedScanTestCase( + description="a table replaced after the scan is described", + last_ddl_after_scan_start=timedelta(hours=1), + expected_live_describe=True, + ), + ], + ids=lambda case: case.description, +) +def test_given_a_later_scan_found_a_record_unchanged_when_planning_from_synced_state_then_describes_only_tables_altered_since_it_started( + test_case: SyncedScanTestCase, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("SQLBUILD_CONTROL_DATABASE_URL", raising=False) + project: SyntheticSnowflakeProject = write_targeted_snowflake_project( + project_dir=tmp_path / "orders_platform", target=_TARGET, plan_from="synced_state" + ) + name: str = project.incremental_model_names[0].upper() + original: FakeRelation = next(filter(lambda relation: relation.name == name, project.relations)) + others: tuple[FakeRelation, ...] = tuple( + filter(lambda relation: relation.name != name, project.relations) + ) + warehouse: RecordingSnowflakeWarehouse = RecordingSnowflakeWarehouse( + relations=( + *others, + replace(original, last_ddl=_SCAN_STARTED + test_case.last_ddl_after_scan_start), + ) + ) + write_synced_control_store( + project_dir=project.project_dir, + target=_TARGET, + warehouse=warehouse, + schemas=_MODEL_SCHEMAS, + observed_at=_FIRST_OBSERVED, + scan_id=uuid7_at(_SCAN_STARTED), + ) + warehouse.queries.clear() + + _ = plan_offline_snowflake_project(project=project, warehouse=warehouse) + + assert ( + any( + query.kind in ("show_columns", "columns") + and re.search(rf"\b{name}\b", f"{query.sql} {query.params}", re.IGNORECASE) + for query in warehouse.queries + ) + == test_case.expected_live_describe + ) diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/_test_types.py index 057679bf76..d9cced9b4c 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/_test_types.py @@ -43,3 +43,13 @@ class CloneTargetSchemaTestCase: class CloneConnectionInjectionTestCase: description: str expected_connection: dict[str, object] + + +@dataclass(frozen=True) +class DeferredListingTestCase: + """Deferred locations across databases, and what the deferred target is asked to list.""" + + description: str + locations: tuple[tuple[str | None, str, str], ...] + expected_listings: list[tuple[str | None, tuple[str, ...]]] + expected_names: list[str] diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/helpers.py index 2bdbe2429c..1a6f59cf26 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/helpers.py @@ -2,6 +2,7 @@ from __future__ import annotations +from dataclasses import dataclass, field from datetime import datetime from pathlib import Path @@ -14,6 +15,7 @@ CompileModelConfig, Fingerprint, ) +from sqlbuild.adapter.contract.models import RelationInfo from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, @@ -83,3 +85,37 @@ def build_previous_python_identity_map( ) } return ({}, identity_map)[previous_version_hash is not None] + + +@dataclass +class ListingAdapter: + """An adapter double that lists its relations per database and records each listing.""" + + relations: tuple[RelationInfo, ...] + listed: list[tuple[str | None, tuple[str, ...]]] = field(default_factory=list) + + def list_relations( + self, *, connection: object, database: str | None, schemas: tuple[str, ...] + ) -> tuple[RelationInfo, ...]: + """The relations of `schemas` in `database`.""" + + self.listed.append((database, schemas)) + return tuple( + filter( + lambda relation: relation.database == database and relation.schema in schemas, + self.relations, + ) + ) + + +def deferred_location(*, database: str | None, schema: str, name: str) -> CompiledRelationLocation: + """One deferred relation location.""" + + return CompiledRelationLocation( + database=database, + schema=schema, + name=name, + qualified_name=f"{database}.{schema}.{name}", + logical_schema=schema, + logical_database=database, + ) diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/test_deferred_locations.py b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/test_deferred_locations.py index ed79bf8e16..f0fda01220 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/test_deferred_locations.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/test_deferred_locations.py @@ -2,17 +2,35 @@ from __future__ import annotations +from typing import cast + import pytest -from sqlbuild._native import CompiledRelationLocation, TargetConfig, build_deferred_locations +from sqlbuild._native import ( + CompiledRelationLocation, + TargetConfig, + build_deferred_locations, + gather_deferred_relations, +) +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.adapter.contract.models import RelationInfo from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledProjectView from tests.unit.src.sqlbuild.compiler.pipeline._helpers._test_types import ( + DeferredListingTestCase, DeferredTargetTestCase, ) from tests.unit.src.sqlbuild.compiler.pipeline._helpers.helpers import ( + ListingAdapter, build_single_model_project, + deferred_location, +) + +_DEFERRED_RELATIONS: tuple[RelationInfo, ...] = ( + RelationInfo(database="analytics", schema="prod", name="orders", relation_type="table"), + RelationInfo(database="archive", schema="prod", name="order_history", relation_type="table"), + RelationInfo(database="archive", schema="audit", name="refunds", relation_type="table"), ) @@ -147,3 +165,43 @@ def test_given_deferred_target_config_when_building_targets_then_resolves_expect assert result.schema == test_case.expected_schema assert result.database == test_case.expected_database assert result.qualified_name == test_case.expected_qualified_name + + +@pytest.mark.parametrize( + "test_case", + [ + DeferredListingTestCase( + description="locations in two databases are listed in each database", + locations=( + ("analytics", "prod", "orders"), + ("archive", "prod", "order_history"), + ("archive", "audit", "refunds"), + ), + expected_listings=[("analytics", ("prod",)), ("archive", ("audit", "prod"))], + expected_names=["order_history", "orders", "refunds"], + ), + DeferredListingTestCase( + description="locations in one database are listed once", + locations=(("archive", "prod", "order_history"), ("archive", "audit", "refunds")), + expected_listings=[("archive", ("audit", "prod"))], + expected_names=["order_history", "refunds"], + ), + ], + ids=lambda case: case.description, +) +def test_given_deferred_locations_when_gathering_relations_then_lists_every_database( + test_case: DeferredListingTestCase, +) -> None: + adapter: ListingAdapter = ListingAdapter(relations=_DEFERRED_RELATIONS) + + relations: dict[str, RelationInfo] = gather_deferred_relations( + adapter=cast(BaseAdapter, adapter), + connection=None, + deferred_locations={ + name: deferred_location(database=database, schema=schema, name=name) + for database, schema, name in test_case.locations + }, + ) + + assert adapter.listed == test_case.expected_listings + assert sorted(relations) == test_case.expected_names From 22a8733c4456459aa362f7a31fdb938334e1576e Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 18 Nov 2026 20:18:32 +0000 Subject: [PATCH 2/2] fix(plan): bound a synced record's scan start by when the store received it A synced plan trusted a mirrored record from the start time in its scan id, which the location's clock sets. A location clock running ahead could push that past a later change and skip describing a table that changed. Each catalog scan now stamps the records it re-delivers unchanged with the store's receipt time, and the plan trusts a record from the earlier of its scan's start and that receipt, keeping the clock-skew allowance. --- .../warehouse_metadata/_helpers/ingestion.rs | 17 ++++++++++---- .../src/warehouse_metadata/constants.rs | 2 +- .../location_reads/read_synced_catalog.rs | 13 ++++++++--- .../api/warehouse_metadata/catalog/helpers.rs | 22 +++++++++++++++++++ .../compiler/planner/main/_test_types.py | 1 + .../sqlbuild/compiler/planner/main/helpers.py | 16 +++++++++----- .../main/test_plan_from_synced_state.py | 22 ++++++++++++++++++- 7 files changed, 79 insertions(+), 14 deletions(-) diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/ingestion.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/ingestion.rs index d8fde4e13f..55a7aa1b29 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/ingestion.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/ingestion.rs @@ -8,7 +8,7 @@ use crate::store::errors::ControlError; use crate::store::main::{begin::begin, commit::commit, query::query}; use crate::store::models::{Change, ControlStore, ControlTransaction, Parameter, TransactionMode}; use crate::warehouse_metadata::constants::{ - IGNORED_KEY, MAX_HISTORY_PAGE_RECORDS, MAX_HISTORY_RECORDS, + CATALOG_MIRROR, IGNORED_KEY, MAX_HISTORY_PAGE_RECORDS, MAX_HISTORY_RECORDS, }; use crate::warehouse_metadata::models::{ MetadataBatch, MetadataScan, WarehouseHistoryKind, WarehouseSyncResponse, @@ -80,7 +80,7 @@ pub(crate) async fn next_ingestion_chunk( .await } -/// Delivered rows take the scan's id; a complete scan deletes every row it did not deliver. +/// Delivered rows take the scan's id, and catalog rows the time they were received; a complete scan deletes every row it did not deliver. pub(crate) async fn commit_ingestion( mut transaction: ControlTransaction, (table, location_id, target): (&str, Uuid, &str), @@ -90,11 +90,20 @@ pub(crate) async fn commit_ingestion( if let Some(scan) = scan { if !keys.is_empty() { let mut parameters = vec![Parameter::Id(scan.scan_id)]; + let received: &str = if table == CATALOG_MIRROR { + parameters.push(Parameter::Time(transaction.now)); + ",received_at=$2" + } else { + "" + }; + let first_key: usize = parameters.len() + 1; parameters.extend(keys.iter().copied().map(Parameter::Id)); - let placeholders = Parameter::placeholders(2..=parameters.len()); + let placeholders = Parameter::placeholders(first_key..=parameters.len()); (transaction, _) = query( transaction, - &format!("UPDATE {table} SET scan_id=$1 WHERE event_id IN ({placeholders})"), + &format!( + "UPDATE {table} SET scan_id=$1{received} WHERE event_id IN ({placeholders})" + ), parameters, ) .await?; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/constants.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/constants.rs index 05ff648200..a495a62b01 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/constants.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/constants.rs @@ -54,7 +54,7 @@ pub(crate) const INGESTION_CHUNK_RECORDS: usize = 200; /// The audit result metadata field marking a failure its build let through. pub(crate) const IGNORED_KEY: &str = "ignored"; /// The mirrored catalog of one location's target, with when each relation was observed. -pub(crate) const SYNCED_CATALOG_SQL: &str = "SELECT record_json, observed_at, scan_id FROM sqb_warehouse_catalog \ +pub(crate) const SYNCED_CATALOG_SQL: &str = "SELECT record_json, observed_at, scan_id, received_at FROM sqb_warehouse_catalog \ WHERE location_id=$1 AND target=$2 ORDER BY state_database, state_schema, event_id"; /// The development locations of one target, by project path. pub(crate) const DEVELOPMENT_LOCATIONS_SQL: &str = diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/read_synced_catalog.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/read_synced_catalog.rs index a669b187fa..520587a9fd 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/read_synced_catalog.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/read_synced_catalog.rs @@ -11,7 +11,7 @@ use crate::store::models::{ControlStore, Parameter, TransactionMode}; use crate::warehouse_metadata::constants::SYNCED_CATALOG_SQL; use crate::warehouse_metadata::models::{SyncedCatalogRelation, WarehouseCatalogRecord}; -/// Every mirrored relation of the location's target, current as of its last scan's start or later. +/// Every mirrored relation of the location's target, current as of its last scan's start (no later than its receipt) or later. pub async fn read_synced_catalog( store: &ControlStore, (location_id, target): (Uuid, &str), @@ -39,8 +39,15 @@ pub async fn read_synced_catalog( row.remove("scan_id").unwrap_or_default(), ) .map_err(|error| ControlError::Invalid(format!("invalid mirrored scan: {error}")))?; - let observed_at = - scan_started(scan_id)?.map_or(observed_at, |started| started.max(observed_at)); + let received_at: DateTime = serde_json::from_value( + row.remove("received_at").unwrap_or_default(), + ) + .map_err(|error| { + ControlError::Invalid(format!("invalid mirrored receipt time: {error}")) + })?; + let observed_at = scan_started(scan_id)?.map_or(observed_at, |started| { + started.min(received_at).max(observed_at) + }); Ok(SyncedCatalogRelation { record, observed_at, diff --git a/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/catalog/helpers.rs b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/catalog/helpers.rs index 01aa5d9916..8b4b618b54 100644 --- a/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/catalog/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/catalog/helpers.rs @@ -73,6 +73,7 @@ pub(crate) async fn catalog_history( ) .await; batch["scan"]["scan_id"] = json!(Uuid::now_v7()); + let before = times(store).await; assert_eq!( post((client, url, bearer), &batch, StatusCode::OK).await["applied"], false @@ -82,6 +83,15 @@ pub(crate) async fn catalog_history( initial, "an unchanged scan changes nothing" ); + let after = times(store).await; + assert_eq!( + after["observed_at"], before["observed_at"], + "an unchanged record keeps the time it last changed" + ); + assert_ne!( + after["received_at"], before["received_at"], + "an unchanged record takes the time it was received again" + ); batch["scan"]["scan_id"] = json!(Uuid::now_v7()); batch["records"][0]["columns"][1]["data_type"] = json!("INTEGER"); let fault = TransactionFault { @@ -214,3 +224,15 @@ async fn current(store: &ControlStore) -> Value { commit(transaction).await.expect("close read"); json!(current) } + +async fn times(store: &ControlStore) -> Value { + let (transaction, mut rows) = query( + begin(store, TransactionMode::Read).await.expect("read"), + "SELECT observed_at,received_at FROM sqb_warehouse_catalog", + vec![], + ) + .await + .expect("record times"); + commit(transaction).await.expect("close read"); + json!(rows.remove(0)) +} diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py b/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py index 637eea6319..7c19d1c40c 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py @@ -204,3 +204,4 @@ class SyncedScanTestCase: description: str last_ddl_after_scan_start: timedelta expected_live_describe: bool + location_clock_ahead: timedelta = timedelta() diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py index 08d271e41e..9c63c10938 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py @@ -738,8 +738,12 @@ def write_synced_control_store( schemas: tuple[str, ...], observed_at: datetime | None = None, scan_id: str | None = None, + received_at: datetime | None = None, ) -> None: - """A control store whose location mirrored `schemas` at `observed_at` (default: now).""" + """A control store whose location mirrored `schemas` at `observed_at` (default: now). + + `received_at` is when the store last received each record (default: `observed_at`). + """ catalog: InspectionCatalog = live_inspection_catalog(warehouse) relations: tuple[RelationInfo, ...] = catalog.list_relations( @@ -749,7 +753,9 @@ def write_synced_control_store( catalog.get_columns_for_relations(relations=relations) ) location: str = str(uuid4()) - observed: str = (observed_at or datetime.now(UTC)).strftime("%Y-%m-%dT%H:%M:%S.%fZ") + observed_time: datetime = observed_at or datetime.now(UTC) + observed: str = observed_time.strftime("%Y-%m-%dT%H:%M:%S.%fZ") + received: str = (received_at or observed_time).strftime("%Y-%m-%dT%H:%M:%S.%fZ") (project_dir / ".sqlbuild").mkdir(parents=True) with closing(sqlite3.connect(project_dir / ".sqlbuild/control.sqlite3")) as store: _ = store.execute( @@ -759,21 +765,21 @@ def write_synced_control_store( _ = store.execute( "CREATE TABLE sqb_warehouse_catalog (event_id TEXT, location_id TEXT, target TEXT, " "state_database TEXT, state_schema TEXT, observed_at TEXT, record_json JSON, " - "scan_id TEXT)" + "scan_id TEXT, received_at TEXT)" ) _ = store.execute( "INSERT INTO sqb_development_locations VALUES (?, ?, ?)", (str(project_dir), target, location), ) _ = store.executemany( - "INSERT INTO sqb_warehouse_catalog VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + "INSERT INTO sqb_warehouse_catalog VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", [ _mirrored_row( relation=relation, columns=columns.get(relation.identity, ()), scope=(location, target, observed), ) - + (scan_id,) + + (scan_id, received) for relation in relations ], ) diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/test_plan_from_synced_state.py b/tests/integration/src/sqlbuild/compiler/planner/main/test_plan_from_synced_state.py index f10d08ae43..00897c2bf7 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/test_plan_from_synced_state.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/test_plan_from_synced_state.py @@ -34,6 +34,8 @@ _LIVE: str = "Planning live:" _FIRST_OBSERVED: datetime = datetime(2026, 2, 1, tzinfo=UTC) _SCAN_STARTED: datetime = datetime(2026, 3, 1, tzinfo=UTC) +# The scan read the page two seconds after it started, and the store received it a second later. +_PAGE_RECEIVED: datetime = _SCAN_STARTED + timedelta(seconds=3) @pytest.mark.parametrize( @@ -156,6 +158,23 @@ def test_given_plan_from_when_planning_then_reads_the_synced_copy_only_when_aske last_ddl_after_scan_start=timedelta(hours=1), expected_live_describe=True, ), + SyncedScanTestCase( + description="a table replaced a second after its page was read is described", + last_ddl_after_scan_start=timedelta(seconds=3), + expected_live_describe=True, + ), + SyncedScanTestCase( + description="a table replaced a second after its page was read by a location clock 10 minutes fast is described", + last_ddl_after_scan_start=timedelta(seconds=3), + expected_live_describe=True, + location_clock_ahead=timedelta(minutes=10), + ), + SyncedScanTestCase( + description="a table replaced with the same columns before a scan by a location clock 10 minutes fast is not described", + last_ddl_after_scan_start=timedelta(hours=-1), + expected_live_describe=False, + location_clock_ahead=timedelta(minutes=10), + ), ], ids=lambda case: case.description, ) @@ -185,7 +204,8 @@ def test_given_a_later_scan_found_a_record_unchanged_when_planning_from_synced_s warehouse=warehouse, schemas=_MODEL_SCHEMAS, observed_at=_FIRST_OBSERVED, - scan_id=uuid7_at(_SCAN_STARTED), + scan_id=uuid7_at(_SCAN_STARTED + test_case.location_clock_ahead), + received_at=_PAGE_RECEIVED, ) warehouse.queries.clear()