diff --git a/benchmarks/duckshop/edge_cases/deferred_reads/models/order_report.sql b/benchmarks/duckshop/edge_cases/deferred_reads/models/order_report.sql new file mode 100644 index 0000000000..15583d3c4e --- /dev/null +++ b/benchmarks/duckshop/edge_cases/deferred_reads/models/order_report.sql @@ -0,0 +1,3 @@ +MODEL (description "A report a deferring build selects alone, reading the aliased orders.", materialized table); + +SELECT order_id FROM __ref("orders") diff --git a/benchmarks/duckshop/edge_cases/deferred_reads/models/orders.sql b/benchmarks/duckshop/edge_cases/deferred_reads/models/orders.sql new file mode 100644 index 0000000000..a75e0ce500 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/deferred_reads/models/orders.sql @@ -0,0 +1,3 @@ +MODEL (description "Orders stored under an alias, so the deferred target lists them as orders_v2.", materialized table, alias "orders_v2"); + +SELECT 1 AS order_id diff --git a/crates/sqlbuild-analysis/src/planner_identity/_helpers/missing_lines.rs b/crates/sqlbuild-analysis/src/planner_identity/_helpers/missing_lines.rs new file mode 100644 index 0000000000..dea184b12e --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_identity/_helpers/missing_lines.rs @@ -0,0 +1,36 @@ +//! The lines naming upstreams of a missing-upstream message that one target lacks. + +use crate::planner_identity::constants::MISSING_MESSAGE_ITEM_LIMIT; + +/// How many upstreams of these kinds are missing from `target`, a few by name. +pub(crate) fn missing_lines(missing: &[(&str, &str)], target: Option<&String>) -> Vec { + let one: bool = missing.len() == 1; + let kinds: std::collections::BTreeSet<&str> = missing.iter().map(|(_, kind)| *kind).collect(); + let noun: String = match kinds.iter().collect::>().as_slice() { + [kind] if one => (*kind).to_string(), + [kind] => format!("{kind}s"), + _ => "models and seeds".to_owned(), + }; + let place: String = target.map_or_else( + || "the warehouse".to_owned(), + |target| format!("target '{target}'"), + ); + let mut lines: Vec = vec![format!( + "{} upstream {noun} this build reads {} exist in {place}", + missing.len(), + if one { "doesn't" } else { "don't" } + )]; + lines.extend( + missing + .iter() + .take(MISSING_MESSAGE_ITEM_LIMIT) + .map(|(name, _)| format!(" {name}")), + ); + if missing.len() > MISSING_MESSAGE_ITEM_LIMIT { + lines.push(format!( + " ... {} more (sqb plan --json lists all)", + missing.len() - MISSING_MESSAGE_ITEM_LIMIT + )); + } + lines +} diff --git a/crates/sqlbuild-analysis/src/planner_identity/_helpers/mod.rs b/crates/sqlbuild-analysis/src/planner_identity/_helpers/mod.rs index 7cf4c87d38..96dcbe3d63 100644 --- a/crates/sqlbuild-analysis/src/planner_identity/_helpers/mod.rs +++ b/crates/sqlbuild-analysis/src/planner_identity/_helpers/mod.rs @@ -2,4 +2,5 @@ pub(crate) mod compose_graph_identity; pub(crate) mod cycle_message; pub(crate) mod dependency_counts; pub(crate) mod identity_resolver; +pub(crate) mod missing_lines; pub(crate) mod staleness_walk; diff --git a/crates/sqlbuild-analysis/src/planner_identity/main/missing_upstream_message.rs b/crates/sqlbuild-analysis/src/planner_identity/main/missing_upstream_message.rs index c392668519..9409109229 100644 --- a/crates/sqlbuild-analysis/src/planner_identity/main/missing_upstream_message.rs +++ b/crates/sqlbuild-analysis/src/planner_identity/main/missing_upstream_message.rs @@ -2,7 +2,11 @@ use std::collections::HashMap; -use crate::planner_identity::constants::MISSING_MESSAGE_ITEM_LIMIT; +use crate::planner_identity::_helpers::missing_lines::missing_lines; + +use crate::planner_identity::constants::{ + MISSING_MESSAGE_ITEM_LIMIT, MODEL_RESOURCE, SEED_RESOURCE, +}; use crate::planner_identity::models::{GraphKey, MissingUpstream, MissingUpstreamFixes}; /// Missing lineage upstreams with how to fix them and, when only an audit or hook read requires one, that read's origin. @@ -11,8 +15,7 @@ pub fn missing_upstream_message( edge_origins: &HashMap<(GraphKey, GraphKey), String>, fixes: &MissingUpstreamFixes, ) -> String { - let mut lineage_missing: Vec<&str> = Vec::new(); - let mut kinds: std::collections::BTreeSet<&str> = std::collections::BTreeSet::new(); + let mut lineage_missing: Vec<(&str, &str)> = Vec::new(); let mut read_names: Vec<&str> = Vec::new(); for entry in missing { let origins: Vec<&str> = entry @@ -25,8 +28,7 @@ pub fn missing_upstream_message( }) .collect(); if origins.len() < entry.required_by.len() { - lineage_missing.push(entry.key.name.as_str()); - kinds.insert(entry.key.kind.as_str()); + lineage_missing.push((entry.key.name.as_str(), entry.key.kind.as_str())); continue; } for origin in origins { @@ -39,46 +41,35 @@ pub fn missing_upstream_message( if lineage_missing.is_empty() { return format!("cannot build selected scope: {}", reads.unwrap_or_default()); } - let mut text: String = lineage_text(&lineage_missing, &kinds, fixes); + let mut text: String = lineage_text(&lineage_missing, fixes); if let Some(reads) = reads { text.push_str(&format!("\n\n Also, {reads}")); } text } -/// The missing lineage upstreams, a few by name, then how to read, clone or build them. -fn lineage_text( - names: &[&str], - kinds: &std::collections::BTreeSet<&str>, - fixes: &MissingUpstreamFixes, -) -> String { +/// Missing lineage upstreams per target looked in (deferred for models and seeds when deferring), then the fixes. +fn lineage_text(missing: &[(&str, &str)], fixes: &MissingUpstreamFixes) -> String { + let names: Vec<&str> = missing.iter().map(|(name, _)| *name).collect(); let one: bool = names.len() == 1; - let noun: String = match kinds.iter().collect::>().as_slice() { - [kind] if one => (*kind).to_string(), - [kind] => format!("{kind}s"), - _ => "models and seeds".to_owned(), + let groups: Vec> = match fixes.deferred_to.as_ref() { + Some(deferred_to) => { + let (deferred, active): (Vec<_>, Vec<_>) = missing + .iter() + .copied() + .partition(|(_, kind)| [MODEL_RESOURCE, SEED_RESOURCE].contains(kind)); + [ + (deferred, Some(deferred_to)), + (active, fixes.target.as_ref()), + ] + .into_iter() + .filter(|(group, _)| !group.is_empty()) + .map(|(group, target)| missing_lines(&group, target)) + .collect() + } + None => vec![missing_lines(missing, fixes.target.as_ref())], }; - let place: String = fixes.target.as_ref().map_or_else( - || "the warehouse".to_owned(), - |target| format!("target '{target}'"), - ); - let mut lines: Vec = vec![format!( - "{} upstream {noun} this build reads {} exist in {place}", - names.len(), - if one { "doesn't" } else { "don't" } - )]; - lines.extend( - names - .iter() - .take(MISSING_MESSAGE_ITEM_LIMIT) - .map(|name| format!(" {name}")), - ); - if names.len() > MISSING_MESSAGE_ITEM_LIMIT { - lines.push(format!( - " ... {} more (sqb plan --json lists all)", - names.len() - MISSING_MESSAGE_ITEM_LIMIT - )); - } + let mut lines: Vec = groups.join(&String::new()); lines.push(String::new()); let (exists, them): (&str, &str) = if one { ("it exists", "it") diff --git a/crates/sqlbuild-analysis/src/planner_identity/main/missing_upstreams.rs b/crates/sqlbuild-analysis/src/planner_identity/main/missing_upstreams.rs index 59cb1c90a8..18f60d3d81 100644 --- a/crates/sqlbuild-analysis/src/planner_identity/main/missing_upstreams.rs +++ b/crates/sqlbuild-analysis/src/planner_identity/main/missing_upstreams.rs @@ -2,10 +2,12 @@ use std::collections::{HashMap, HashSet}; -use crate::planner_identity::constants::{DBT_REF_RESOURCE, SOURCE_RESOURCE, SQL_TEST_RESOURCE}; +use crate::planner_identity::constants::{ + DBT_REF_RESOURCE, MODEL_RESOURCE, SEED_RESOURCE, SOURCE_RESOURCE, SQL_TEST_RESOURCE, +}; use crate::planner_identity::models::{AvailableUpstreams, GraphKey, MissingUpstream}; -/// Missing upstreams, most dependents first, then by `(kind, name)`; dependents sorted by `(kind, name)`. +/// Missing upstreams, most dependents first; a deferring plan checks unselected models and seeds in the deferred target. pub fn missing_upstreams( selected_keys: &[GraphKey], upstream_deps: &HashMap>, @@ -19,11 +21,14 @@ pub fn missing_upstreams( .contains(&dep_key.kind.as_str()) || selected.contains(dep_key) || available.satisfied_keys.contains(dep_key) - || available.existing_relations.contains(&dep_key.name) - || available - .deferred_relations - .as_ref() - .is_some_and(|deferred| deferred.contains(&dep_key.name)) + || match &available.deferred_relations { + Some(deferred) + if [MODEL_RESOURCE, SEED_RESOURCE].contains(&dep_key.kind.as_str()) => + { + deferred.contains(&dep_key.name) + } + _ => available.existing_relations.contains(&dep_key.name), + } { continue; } diff --git a/crates/sqlbuild-analysis/src/planner_identity/models.rs b/crates/sqlbuild-analysis/src/planner_identity/models.rs index e0cd6716a0..0388d75735 100644 --- a/crates/sqlbuild-analysis/src/planner_identity/models.rs +++ b/crates/sqlbuild-analysis/src/planner_identity/models.rs @@ -92,6 +92,8 @@ pub struct AvailableUpstreams { pub struct MissingUpstreamFixes { /// The active target, named as where the upstreams are missing. pub target: Option, + /// The target a deferring plan reads unselected models and seeds from, named as where those are missing. + pub deferred_to: Option, /// Targets the active target may clone from, to defer to or clone the upstreams from. pub origins: Vec, /// The origins `--defer-to` accepts: project targets, offered only when this plan does not defer already. diff --git a/crates/sqlbuild-analysis/src/planner_identity/tests/test_planner_identity.rs b/crates/sqlbuild-analysis/src/planner_identity/tests/test_planner_identity.rs index 5c124eb943..da55a9535b 100644 --- a/crates/sqlbuild-analysis/src/planner_identity/tests/test_planner_identity.rs +++ b/crates/sqlbuild-analysis/src/planner_identity/tests/test_planner_identity.rs @@ -436,6 +436,7 @@ fn given_selected_keys_when_checking_buildability_then_reports_missing_upstreams origins: vec!["prod".to_owned(), "staging".to_owned()], deferrable: vec!["prod".to_owned()], clone_to: Some("dev".to_owned()), + deferred_to: None, }, expected_missing: ["a", "b", "c", "d", "e", "f"] .map(model) @@ -459,6 +460,73 @@ fn given_selected_keys_when_checking_buildability_then_reports_missing_upstreams " Or build them too: add + before your selection", ), }, + BuildabilityTestCase { + description: "deferring reads models and seeds only from the deferred target", + selected: vec![model("report")], + upstream: HashMap::from([( + model("report"), + vec![model("dev_only"), model("prod_only"), key("seed", "codes")], + )]), + available: AvailableUpstreams { + existing_relations: names(&["dev_only", "codes"]), + deferred_relations: Some(names(&["prod_only", "codes"])), + ..AvailableUpstreams::default() + }, + origins: HashMap::new(), + fixes: MissingUpstreamFixes { + target: Some("dev".to_owned()), + deferred_to: Some("prod".to_owned()), + ..MissingUpstreamFixes::default() + }, + expected_missing: vec![MissingUpstream { + key: model("dev_only"), + required_by: vec![model("report")], + }], + expected_message: concat!( + "1 upstream model this build reads doesn't exist in target 'prod'\n", + " dev_only\n", + "\n", + " Build it too: add + before your selection", + ), + }, + BuildabilityTestCase { + description: "deferring names the deferred target for models and the active one for functions", + selected: vec![model("report")], + upstream: HashMap::from([( + model("report"), + vec![model("dev_only"), key("udf", "parse_sku")], + )]), + available: AvailableUpstreams { + existing_relations: names(&["dev_only"]), + deferred_relations: Some(names(&["parse_sku"])), + ..AvailableUpstreams::default() + }, + origins: HashMap::new(), + fixes: MissingUpstreamFixes { + target: Some("dev".to_owned()), + deferred_to: Some("prod".to_owned()), + ..MissingUpstreamFixes::default() + }, + expected_missing: vec![ + MissingUpstream { + key: model("dev_only"), + required_by: vec![model("report")], + }, + MissingUpstream { + key: key("udf", "parse_sku"), + required_by: vec![model("report")], + }, + ], + expected_message: concat!( + "1 upstream model this build reads doesn't exist in target 'prod'\n", + " dev_only\n", + "\n", + "1 upstream udf this build reads doesn't exist in target 'dev'\n", + " parse_sku\n", + "\n", + " Build them too: add + before your selection", + ), + }, BuildabilityTestCase { description: "only an audit read requires the upstream", selected: vec![model("a")], diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/location/execution.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/location/execution.rs index f41fb37c34..1e64fe1398 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/location/execution.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/location/execution.rs @@ -745,7 +745,7 @@ pub(crate) fn next_scan(mut state: LocationMetadataSync) -> LocationMetadataSync state.progress.scan_unchanged = false; state.progress.scan_started = Instant::now(); state.index += 1; - state.scan_id = Uuid::now_v7(); + state.scan_id = state.server_scan_id(); state.observed_since = None; state.watermark_loaded = false; state.limit = MAX_HISTORY_PAGE_RECORDS; diff --git a/crates/sqlbuild-cli/src/native_commands/models.rs b/crates/sqlbuild-cli/src/native_commands/models.rs index 65c69f86f1..3967c069de 100644 --- a/crates/sqlbuild-cli/src/native_commands/models.rs +++ b/crates/sqlbuild-cli/src/native_commands/models.rs @@ -14,7 +14,7 @@ use sqlbuild_orchestration::warehouse_metadata::models::{ MetadataBatch, StatisticsRead, WarehouseHistoryFailure, WarehouseHistoryKind, WarehouseHistoryPage, WarehouseStateLocation, }; -use uuid::Uuid; +use uuid::{NoContext, Timestamp, Uuid}; use crate::argv::models::Namespace; use crate::native_commands::constants::{METADATA_FAILURE_LIMIT, METADATA_POLL_SECONDS}; @@ -425,12 +425,24 @@ impl LocationMetadataSync { self.reads_wake_compute && !self.safety_explicit } + /// A scan id timed by the server's clock: a synced plan trusts a record as of its scan's start, never after a read. + pub(crate) fn server_scan_id(&self) -> Uuid { + let now: DateTime = Utc::now() + self.server_offset; + let seconds: u64 = u64::try_from(now.timestamp()).unwrap_or_default(); + Uuid::new_v7(Timestamp::from_unix( + NoContext, + seconds, + now.timestamp_subsec_nanos(), + )) + } + /// Start the sync `sync_due` asked for at server time `now`; a stopped sync restarts its pass. pub(crate) fn begin_sync(&mut self, now: DateTime) { + self.server_offset = now - Utc::now(); + self.scan_id = self.server_scan_id(); if std::mem::take(&mut self.halted) { self.index = 0; self.source_after = None; - self.scan_id = Uuid::now_v7(); self.observed_since = None; self.watermark_loaded = false; self.statistics = None; @@ -441,7 +453,6 @@ impl LocationMetadataSync { self.applied = false; } self.idle = false; - self.server_offset = now - Utc::now(); self.progress = SyncProgress::new(); self.cycle_started_at = Some(now); self.next_at = tokio::time::Instant::now(); @@ -470,7 +481,7 @@ impl LocationMetadataSync { if statistics { self.index -= 1; self.source_after = None; - self.scan_id = Uuid::now_v7(); + self.scan_id = self.server_scan_id(); self.observed_since = None; self.watermark_loaded = false; } diff --git a/crates/sqlbuild-cli/src/native_commands/tests/test_location_metadata.rs b/crates/sqlbuild-cli/src/native_commands/tests/test_location_metadata.rs index f1a6e68c04..71c88d2577 100644 --- a/crates/sqlbuild-cli/src/native_commands/tests/test_location_metadata.rs +++ b/crates/sqlbuild-cli/src/native_commands/tests/test_location_metadata.rs @@ -14,6 +14,7 @@ use crate::native_commands::tests::helpers::{ acknowledge_complete_pages, one_location, published_location, }; use crate::native_commands::tests::test_types::BilledDeferralTestCase; +use crate::native_commands::tests::test_types::FastClockScanTestCase; use crate::native_commands::tests::test_types::{ FailedReadHealthTestCase, NewestVersionsTestCase, ReadFailureTestCase, }; @@ -24,7 +25,7 @@ use crate::native_commands::tests::test_types::{ use crate::native_commands::tests::test_types::{ RowsTextTestCase, ScanLineTestCase, SyncProgressTestCase, WakingIdleTestCase, }; -use chrono::Utc; +use chrono::{DateTime, Utc}; use sqlbuild_config::project::models::ControlConfig; use sqlbuild_orchestration::platform::models::{ LocationHealth, WarehouseReadFailure, WarehouseSyncFailure, @@ -1184,3 +1185,47 @@ fn given_a_published_location_when_its_sync_stops_before_any_page_succeeds_then_ ); } } + +#[test] +fn given_a_fast_location_clock_when_scanning_then_a_ddl_after_the_page_read_is_still_described_live() + { + let test_cases = [ + FastClockScanTestCase { + description: "the first scan of a sync", + scans_completed: 0, + expected_described_live: true, + }, + FastClockScanTestCase { + description: "a later scan of the same pass", + scans_completed: 1, + expected_described_live: true, + }, + ]; + // A synced plan reuses mirrored columns only when the last DDL ran this long before the scan began. + let ddl_clock_skew = chrono::Duration::minutes(5); + for test_case in test_cases { + let mut state = metadata_sync((one_location(), 1), (3600, 0)); + state.begin_sync(Utc::now() - chrono::Duration::minutes(10)); + state = acknowledge_complete_pages(state, test_case.scans_completed); + let page_read_at: DateTime = Utc::now() + state.server_offset; + let ddl_at: DateTime = page_read_at + chrono::Duration::seconds(1); + let (seconds, nanos) = state + .scan_id + .get_timestamp() + .expect("a v7 scan id") + .to_unix(); + let scan_started: DateTime = + DateTime::from_timestamp(i64::try_from(seconds).expect("seconds"), nanos) + .expect("scan start"); + + assert_eq!( + ( + scan_started <= page_read_at, + ddl_at > scan_started - ddl_clock_skew + ), + (true, test_case.expected_described_live), + "{}: the scan started at {scan_started}, the page was read at {page_read_at}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-cli/src/native_commands/tests/test_types.rs b/crates/sqlbuild-cli/src/native_commands/tests/test_types.rs index 83ce6e7039..076eb5dfaa 100644 --- a/crates/sqlbuild-cli/src/native_commands/tests/test_types.rs +++ b/crates/sqlbuild-cli/src/native_commands/tests/test_types.rs @@ -306,3 +306,12 @@ pub(super) struct ReplanTestCase { pub(super) syncs: Vec<(bool, bool)>, pub(super) expected_replans: Vec, } + +/// A scan whose location clock runs ten minutes ahead of the server's. +pub(super) struct FastClockScanTestCase { + pub(super) description: &'static str, + /// Scans of the pass completed before the one whose id is checked. + pub(super) scans_completed: usize, + /// Whether a table altered a second after the page read is still described live. + pub(super) expected_described_live: bool, +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/ingestion.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/ingestion.rs index d8fde4e13f..55a7aa1b29 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/ingestion.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/ingestion.rs @@ -8,7 +8,7 @@ use crate::store::errors::ControlError; use crate::store::main::{begin::begin, commit::commit, query::query}; use crate::store::models::{Change, ControlStore, ControlTransaction, Parameter, TransactionMode}; use crate::warehouse_metadata::constants::{ - IGNORED_KEY, MAX_HISTORY_PAGE_RECORDS, MAX_HISTORY_RECORDS, + CATALOG_MIRROR, IGNORED_KEY, MAX_HISTORY_PAGE_RECORDS, MAX_HISTORY_RECORDS, }; use crate::warehouse_metadata::models::{ MetadataBatch, MetadataScan, WarehouseHistoryKind, WarehouseSyncResponse, @@ -80,7 +80,7 @@ pub(crate) async fn next_ingestion_chunk( .await } -/// Delivered rows take the scan's id; a complete scan deletes every row it did not deliver. +/// Delivered rows take the scan's id, and catalog rows the time they were received; a complete scan deletes every row it did not deliver. pub(crate) async fn commit_ingestion( mut transaction: ControlTransaction, (table, location_id, target): (&str, Uuid, &str), @@ -90,11 +90,20 @@ pub(crate) async fn commit_ingestion( if let Some(scan) = scan { if !keys.is_empty() { let mut parameters = vec![Parameter::Id(scan.scan_id)]; + let received: &str = if table == CATALOG_MIRROR { + parameters.push(Parameter::Time(transaction.now)); + ",received_at=$2" + } else { + "" + }; + let first_key: usize = parameters.len() + 1; parameters.extend(keys.iter().copied().map(Parameter::Id)); - let placeholders = Parameter::placeholders(2..=parameters.len()); + let placeholders = Parameter::placeholders(first_key..=parameters.len()); (transaction, _) = query( transaction, - &format!("UPDATE {table} SET scan_id=$1 WHERE event_id IN ({placeholders})"), + &format!( + "UPDATE {table} SET scan_id=$1{received} WHERE event_id IN ({placeholders})" + ), parameters, ) .await?; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/constants.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/constants.rs index 1f753dee9c..a495a62b01 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/constants.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/constants.rs @@ -54,7 +54,7 @@ pub(crate) const INGESTION_CHUNK_RECORDS: usize = 200; /// The audit result metadata field marking a failure its build let through. pub(crate) const IGNORED_KEY: &str = "ignored"; /// The mirrored catalog of one location's target, with when each relation was observed. -pub(crate) const SYNCED_CATALOG_SQL: &str = "SELECT record_json, observed_at FROM sqb_warehouse_catalog \ +pub(crate) const SYNCED_CATALOG_SQL: &str = "SELECT record_json, observed_at, scan_id, received_at FROM sqb_warehouse_catalog \ WHERE location_id=$1 AND target=$2 ORDER BY state_database, state_schema, event_id"; /// The development locations of one target, by project path. pub(crate) const DEVELOPMENT_LOCATIONS_SQL: &str = diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/read_synced_catalog.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/read_synced_catalog.rs index c3104ec656..520587a9fd 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/read_synced_catalog.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/read_synced_catalog.rs @@ -1,5 +1,6 @@ //! A plan from synced state reads the target's mirrored catalog and when each relation was seen. +use chrono::{DateTime, Utc}; use uuid::Uuid; use crate::store::errors::ControlError; @@ -10,7 +11,7 @@ use crate::store::models::{ControlStore, Parameter, TransactionMode}; use crate::warehouse_metadata::constants::SYNCED_CATALOG_SQL; use crate::warehouse_metadata::models::{SyncedCatalogRelation, WarehouseCatalogRecord}; -/// Every mirrored relation of the location's target, with the time its catalog scan observed it. +/// Every mirrored relation of the location's target, current as of its last scan's start (no later than its receipt) or later. pub async fn read_synced_catalog( store: &ControlStore, (location_id, target): (Uuid, &str), @@ -28,10 +29,25 @@ pub async fn read_synced_catalog( serde_json::from_value(row.remove("record_json").unwrap_or_default()).map_err( |error| ControlError::Invalid(format!("invalid mirrored relation: {error}")), )?; - let observed_at = serde_json::from_value(row.remove("observed_at").unwrap_or_default()) - .map_err(|error| { - ControlError::Invalid(format!("invalid mirrored observation time: {error}")) - })?; + let observed_at: DateTime = serde_json::from_value( + row.remove("observed_at").unwrap_or_default(), + ) + .map_err(|error| { + ControlError::Invalid(format!("invalid mirrored observation time: {error}")) + })?; + let scan_id: Option = serde_json::from_value( + row.remove("scan_id").unwrap_or_default(), + ) + .map_err(|error| ControlError::Invalid(format!("invalid mirrored scan: {error}")))?; + let received_at: DateTime = serde_json::from_value( + row.remove("received_at").unwrap_or_default(), + ) + .map_err(|error| { + ControlError::Invalid(format!("invalid mirrored receipt time: {error}")) + })?; + let observed_at = scan_started(scan_id)?.map_or(observed_at, |started| { + started.min(received_at).max(observed_at) + }); Ok(SyncedCatalogRelation { record, observed_at, @@ -39,3 +55,16 @@ pub async fn read_synced_catalog( }) .collect() } + +/// When the scan that last delivered a record started: a later page of it re-read the record unchanged. +fn scan_started(scan_id: Option) -> Result>, ControlError> { + let Some((seconds, nanos)) = scan_id + .and_then(|scan_id| scan_id.get_timestamp()) + .map(|started| started.to_unix()) + else { + return Ok(None); + }; + let seconds: i64 = i64::try_from(seconds) + .map_err(|error| ControlError::Invalid(format!("invalid mirrored scan time: {error}")))?; + Ok(DateTime::from_timestamp(seconds, nanos)) +} diff --git a/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/catalog/helpers.rs b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/catalog/helpers.rs index 01aa5d9916..8b4b618b54 100644 --- a/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/catalog/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/catalog/helpers.rs @@ -73,6 +73,7 @@ pub(crate) async fn catalog_history( ) .await; batch["scan"]["scan_id"] = json!(Uuid::now_v7()); + let before = times(store).await; assert_eq!( post((client, url, bearer), &batch, StatusCode::OK).await["applied"], false @@ -82,6 +83,15 @@ pub(crate) async fn catalog_history( initial, "an unchanged scan changes nothing" ); + let after = times(store).await; + assert_eq!( + after["observed_at"], before["observed_at"], + "an unchanged record keeps the time it last changed" + ); + assert_ne!( + after["received_at"], before["received_at"], + "an unchanged record takes the time it was received again" + ); batch["scan"]["scan_id"] = json!(Uuid::now_v7()); batch["records"][0]["columns"][1]["data_type"] = json!("INTEGER"); let fault = TransactionFault { @@ -214,3 +224,15 @@ async fn current(store: &ControlStore) -> Value { commit(transaction).await.expect("close read"); json!(current) } + +async fn times(store: &ControlStore) -> Value { + let (transaction, mut rows) = query( + begin(store, TransactionMode::Read).await.expect("read"), + "SELECT observed_at,received_at FROM sqb_warehouse_catalog", + vec![], + ) + .await + .expect("record times"); + commit(transaction).await.expect("close read"); + json!(rows.remove(0)) +} diff --git a/crates/sqlbuild-python-base/src/records/_helpers/planner_records/constants.rs b/crates/sqlbuild-python-base/src/records/_helpers/planner_records/constants.rs index c094aa7ba3..bd568fa583 100644 --- a/crates/sqlbuild-python-base/src/records/_helpers/planner_records/constants.rs +++ b/crates/sqlbuild-python-base/src/records/_helpers/planner_records/constants.rs @@ -98,6 +98,7 @@ const R_DEFERRAL_INPUTS: RecordSpec = RecordSpec { fields: &[ FieldSpec::new("deferred_locations", FieldDefault::None), FieldSpec::new("deferred_relations", FieldDefault::None), + FieldSpec::new("defer_to", FieldDefault::None), FieldSpec::new("defer_sources_to", FieldDefault::None), FieldSpec::new("source_deferral_enabled", FieldDefault::True), ], diff --git a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/deferral.rs b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/deferral.rs index 572a536448..db20684f66 100644 --- a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/deferral.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/deferral.rs @@ -1,5 +1,6 @@ //! Deferred target relation locations, and the warehouse relations they name. +use pyo3::IntoPyObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyString, PyStringMethods, PyTuple}; use sqlbuild_python_base::records::main::construct::new_plain_error::new_plain_error; @@ -164,45 +165,47 @@ pub(crate) fn deferred_target_config<'py>( targets.get_item(defer_to) } -/// `gather_deferred_relations` with its arguments. +/// `gather_deferred_relations` with its arguments: one listing per database the locations live in. pub(crate) fn deferred_relations<'py>( adapter: &Bound<'py, PyAny>, connection: &Bound<'py, PyAny>, locations: &Bound<'py, PyAny>, ) -> PyResult> { let py: Python<'py> = adapter.py(); - let mut schemas: Vec = Vec::new(); - let mut database: Bound<'py, PyAny> = py.None().into_bound(py); + let mut databases: Vec<(Option, Vec)> = Vec::new(); for location in mapping_values(locations)?.try_iter()? { let location: Bound<'py, PyAny> = location?; - let schema: Bound<'py, PyAny> = native_field(&location, "schema")?; - if !schema.is_none() { - let schema: String = schema.extract()?; - if !schemas.contains(&schema) { - schemas.push(schema); + let schema: Option = native_field(&location, "schema")?.extract()?; + let database: Option = native_field(&location, "database")?.extract()?; + let Some(schema) = schema else { + continue; + }; + let index: usize = match databases.iter().position(|(listed, _)| *listed == database) { + Some(index) => index, + None => { + databases.push((database, Vec::new())); + databases.len() - 1 } - } - let location_database: Bound<'py, PyAny> = native_field(&location, "database")?; - if !location_database.is_none() && database.is_none() { - database = location_database; + }; + if !databases[index].1.contains(&schema) { + databases[index].1.push(schema); } } let relations_by_name: Bound<'py, PyDict> = PyDict::new(py); - if schemas.is_empty() { - return Ok(relations_by_name); - } - schemas.sort(); - let relations: Bound<'py, PyAny> = call_with( - &native_field(adapter, "list_relations")?, - &[ - ("connection", connection.clone()), - ("database", database), - ("schemas", PyTuple::new(py, schemas)?.into_any()), - ], - )?; - for relation in relations.try_iter()? { - let relation: Bound<'py, PyAny> = relation?; - relations_by_name.set_item(native_field(&relation, "name")?, relation)?; + for (database, mut schemas) in databases { + schemas.sort(); + let relations: Bound<'py, PyAny> = call_with( + &native_field(adapter, "list_relations")?, + &[ + ("connection", connection.clone()), + ("database", database.into_pyobject(py)?.into_any()), + ("schemas", PyTuple::new(py, schemas)?.into_any()), + ], + )?; + for relation in relations.try_iter()? { + let relation: Bound<'py, PyAny> = relation?; + relations_by_name.set_item(native_field(&relation, "name")?, relation)?; + } } Ok(relations_by_name) } diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/stages/resolve_model_deferral_inputs.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/stages/resolve_model_deferral_inputs.rs index 5795ff7349..5ca7b458fd 100644 --- a/crates/sqlbuild-python-compile/src/compile_driver/main/stages/resolve_model_deferral_inputs.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/stages/resolve_model_deferral_inputs.rs @@ -67,6 +67,7 @@ pub fn resolve_model_deferral_inputs<'py>( &[ ("deferred_locations", locations.into_any()), ("deferred_relations", relations.into_any()), + ("defer_to", defer_to), ], ) } diff --git a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/buildability.rs b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/buildability.rs index eb7288ec73..e478f7f2c0 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/buildability.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/buildability.rs @@ -8,7 +8,7 @@ use crate::planner_phases::main::inputs::runtime_dialect::runtime_dialect; use crate::planner_phases::main::inputs::runtime_reads::runtime_reads; use crate::planner_warehouse::main::sources::check_selected_source_tables_exist::check_selected_source_tables_exist; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::{PyString, PyTuple}; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; use sqlbuild_analysis::planner_identity::main::missing_upstream_message::missing_upstream_message; use sqlbuild_analysis::planner_identity::main::missing_upstreams::missing_upstreams; use sqlbuild_analysis::planner_identity::models::{ @@ -48,15 +48,10 @@ pub(crate) fn check_selected_scope_buildability<'py>( let names = |value: Bound<'py, PyAny>| -> PyResult> { value.try_iter()?.map(|value| value?.extract()).collect() }; - let deferred = native_field(deferral, "deferred_relations")?; let available = AvailableUpstreams { satisfied_keys, existing_relations: names(native_field(&snapshot, "existing_relations")?)?, - deferred_relations: if deferred.is_none() { - None - } else { - Some(names(deferred)?) - }, + deferred_relations: deferred_model_names(deferral)?, }; let selected: Vec<_> = scope.selected.iter().map(GraphKey::from).collect(); let deps: std::collections::HashMap> = scope @@ -79,7 +74,7 @@ pub(crate) fn check_selected_scope_buildability<'py>( let message = missing_upstream_message( &missing, &origins, - &missing_upstream_fixes(runtime, &project, available.deferred_relations.is_some())?, + &missing_upstream_fixes(runtime, &project, deferred_to(deferral, &available)?)?, ); let names = missing.iter().map(|entry| entry.key.name.as_str()); return Err(new_error_with_fields( @@ -97,14 +92,82 @@ pub(crate) fn check_selected_scope_buildability<'py>( ) } +/// The models and seeds whose deferred location, under an alias or a `migrate_from` origin, the deferred target lists. +fn deferred_model_names( + deferral: &Bound<'_, PyAny>, +) -> PyResult>> { + let relations = native_field(deferral, "deferred_relations")?; + let locations = native_field(deferral, "deferred_locations")?; + if relations.is_none() || locations.is_none() { + return Ok(None); + } + let mut listed: Vec = Vec::new(); + for relation in relations.downcast::()?.values() { + listed.push(relation_place(&relation)?); + } + let mut names: std::collections::HashSet = std::collections::HashSet::new(); + for (name, location) in locations.downcast::()? { + let place: RelationPlace = relation_place(&location)?; + if listed.iter().any(|relation| same_place(relation, &place)) { + names.insert(name.extract()?); + } + } + Ok(Some(names)) +} + +/// A relation's lowercased `(database, schema, name)`. +type RelationPlace = (Option, Option, String); + +/// Where a listed relation or a deferred location is, lowercased. +fn relation_place(relation: &Bound<'_, PyAny>) -> PyResult { + let lower = |field: &str| -> PyResult> { + Ok(native_field(relation, field)? + .extract::>()? + .map(|value| value.to_lowercase())) + }; + Ok(( + lower("database")?, + lower("schema")?, + native_field(relation, "name")? + .extract::()? + .to_lowercase(), + )) +} + +/// The same name, schema and database, where a part either side leaves to the connection's default matches any. +fn same_place(listed: &RelationPlace, location: &RelationPlace) -> bool { + let part = |left: &Option, right: &Option| { + left.is_none() || right.is_none() || left == right + }; + listed.2 == location.2 && part(&listed.1, &location.1) && part(&listed.0, &location.0) +} + +/// The target this plan reads unselected models and seeds from, when it defers. +fn deferred_to( + deferral: &Bound<'_, PyAny>, + available: &AvailableUpstreams, +) -> PyResult> { + if available.deferred_relations.is_none() { + return Ok(None); + } + native_field(deferral, "defer_to")?.extract() +} + /// The active target, its clone or deferral origins, and the clone `--to` it needs (also when the default target is unknown). fn missing_upstream_fixes( runtime: &Bound<'_, PyAny>, project: &Bound<'_, PyAny>, - deferring: bool, + deferred_to: Option, ) -> PyResult { - let py: Python<'_> = runtime.py(); let target: Option = native_field(project, "effective_target_name")?.extract()?; + if deferred_to.is_some() { + return Ok(MissingUpstreamFixes { + target, + deferred_to, + ..MissingUpstreamFixes::default() + }); + } + let py: Python<'_> = runtime.py(); let configs = ( runtime_field(runtime, |runtime| &runtime.project_config)?, runtime_field(runtime, |runtime| &runtime.local_config)?, @@ -129,7 +192,7 @@ fn missing_upstream_fixes( for name in native_field(config, "targets")?.try_iter()? { let name: String = name?.extract()?; if name != active && allows(&name, "allow_as_clone_origin")? { - if project_targets && !deferring { + if project_targets { deferrable.push(name.clone()); } origins.insert(name); @@ -143,6 +206,7 @@ fn missing_upstream_fixes( Err(_) => None, }; Ok(MissingUpstreamFixes { + deferred_to: None, clone_to: (default.as_deref() != Some(active.as_str())).then(|| active.clone()), target, origins: origins.into_iter().collect(), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs index 3f1c5c11c3..c9b9c1fcc7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs @@ -175,7 +175,7 @@ pub(crate) fn pipeline_result<'py>( ]; deferral_fields.extend(option_fields( options, - &["defer_sources_to", "source_deferral_enabled"], + &["defer_to", "defer_sources_to", "source_deferral_enabled"], )?); let deferral: Bound<'py, PyAny> = new_record_with(py, RecordId::DeferralInputs, &deferral_fields)?; diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 7b61bd1b3a..fe94cec490 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -3708,6 +3708,12 @@ def build_deferred_locations( default_database: str | None, render_qualified_name: Callable[..., str | None], ) -> dict[str, CompiledRelationLocation]: ... +def gather_deferred_relations( + *, + adapter: BaseAdapter, + connection: object, + deferred_locations: Mapping[str, CompiledRelationLocation], +) -> dict[str, RelationInfo]: ... def build_python_relation_targets( *, adapter: BaseAdapter, @@ -10515,12 +10521,14 @@ class DeferralInputs(NativeRecord): deferred_locations: dict[str, CompiledRelationLocation] | None deferred_relations: dict[str, RelationInfo] | None + defer_to: str | None defer_sources_to: str | None source_deferral_enabled: bool def __init__( self, deferred_locations: dict[str, CompiledRelationLocation] | None = ..., deferred_relations: dict[str, RelationInfo] | None = ..., + defer_to: str | None = ..., defer_sources_to: str | None = ..., source_deferral_enabled: bool = ..., ) -> None: ... diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clone/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/clone/_test_types.py index ba5cc3ee07..8ae7bbdce8 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/clone/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clone/_test_types.py @@ -123,3 +123,15 @@ class UnwritableConsumptionDeferCloneE2ETestCase: dropped_origin_table: str before_clone: tuple[str, ...] expected_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class DeferToMissingUpstreamE2ETestCase: + """Where an unselected upstream was built, and how a build deferring to prod plans it.""" + + description: str + upstream_sql: str + expected_exit_code: int + expected_fragments: tuple[str, ...] + report_sql: str + expected_report_rows: list[tuple[int, ...]] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clone/test_clone.py b/tests/e2e/src/sqlbuild/cli/commands/main/clone/test_clone.py index 2cf2258ba1..fb26169975 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/clone/test_clone.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clone/test_clone.py @@ -18,6 +18,7 @@ CloneMissingUpstreamE2ETestCase, ClonePolicyErrorTestCase, ClonePythonFunctionE2ETestCase, + DeferToMissingUpstreamE2ETestCase, MissingUpstreamFixesE2ETestCase, PartialDeferCloneE2ETestCase, UnwritableConsumptionDeferCloneE2ETestCase, @@ -29,6 +30,7 @@ ) from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( REPO_ROOT, + execute_duckdb, prepare_inline_project, query_duckdb, run_sqb, @@ -43,6 +45,20 @@ "returns", "shipments", ) +_DEFER_DATABASE: str = "warehouse.duckdb" +_DEFER_FIXTURE: Path = REPO_ROOT / "benchmarks/duckshop/edge_cases/deferred_reads" +_DEFER_PROJECT_FILES: dict[str, str] = { + "sqlbuild_project.toml": ( + 'name = "deferred_reads"\nadapter = "duckdb"\ndefault_target = "dev"\n\n' + f'[connection]\ndatabase = "{_DEFER_DATABASE}"\n\n' + '[targets.dev]\nschema = "dev"\n\n' + '[targets.prod]\nschema = "prod"\n' + ), + **{ + str(path.relative_to(_DEFER_FIXTURE)): path.read_text(encoding="utf-8") + for path in sorted(_DEFER_FIXTURE.glob("models/*.sql")) + }, +} @pytest.mark.parametrize( @@ -1147,3 +1163,60 @@ def test_given_unwritable_consumption_table_when_defer_clone_partly_fails_then_c output: str = cloned.stdout + cloned.stderr assert cloned.returncode == 1, output assert all(fragment in output for fragment in test_case.expected_fragments), output + + +@pytest.mark.parametrize( + "test_case", + [ + DeferToMissingUpstreamE2ETestCase( + description="an aliased upstream built only in the active target is missing from prod", + upstream_sql="CREATE TABLE dev.orders_v2 AS SELECT 5 AS order_id;", + expected_exit_code=1, + expected_fragments=( + "error[S301]: 1 upstream model this build reads doesn't exist in target 'prod'\n" + " orders\n\n Build it too: add + before your selection", + ), + report_sql=( + "SELECT count(*) FROM information_schema.tables " + "WHERE table_schema = 'dev' AND table_name = 'order_report'" + ), + expected_report_rows=[(0,)], + ), + DeferToMissingUpstreamE2ETestCase( + description="an aliased upstream built in prod is read from prod under its alias", + upstream_sql=( + "CREATE TABLE dev.orders_v2 AS SELECT 5 AS order_id;" + "CREATE TABLE prod.orders_v2 AS SELECT 7 AS order_id;" + ), + expected_exit_code=0, + expected_fragments=(), + report_sql="SELECT order_id FROM dev.order_report", + expected_report_rows=[(7,)], + ), + ], + ids=lambda case: case.description, +) +def test_given_defer_to_when_an_upstream_is_missing_from_the_deferred_target_then_s301( + test_case: DeferToMissingUpstreamE2ETestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, project_name="deferred_reads", repo_files=_DEFER_PROJECT_FILES + ) + database: Path = project_dir / _DEFER_DATABASE + execute_duckdb( + db_path=database, sql="CREATE SCHEMA dev; CREATE SCHEMA prod;" + test_case.upstream_sql + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build", "--select", "order_report", "--defer-to", "prod"), + project_dir=project_dir, + ) + + output: str = result.stdout + result.stderr + assert result.returncode == test_case.expected_exit_code, output + assert all(fragment in output for fragment in test_case.expected_fragments), output + assert "sqb clone" not in output, output + assert query_duckdb(db_path=database, sql=test_case.report_sql) == ( + test_case.expected_report_rows + ) diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py b/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py index c98ad03871..7c19d1c40c 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py @@ -1,4 +1,5 @@ from dataclasses import dataclass, field +from datetime import timedelta from sqlbuild._native import ( BackfillAction, @@ -172,7 +173,14 @@ class SourceTableProbeBudgetTestCase: class UnlistedMigrationOriginTestCase: description: str select: tuple[str, ...] - expected_warnings: tuple[str, ...] + expected_unlisted_reads: int + + +@dataclass(frozen=True) +class DeferredMissingOriginTestCase: + description: str + select: tuple[str, ...] + expected_missing: tuple[str, ...] expected_unlisted_reads: int @@ -187,3 +195,13 @@ class PlanFromTestCase: expected_report: str expected_absent: tuple[str, ...] expected_live_listing: bool + + +@dataclass(frozen=True) +class SyncedScanTestCase: + """When a table's last DDL ran relative to the scan that last found its record unchanged.""" + + description: str + last_ddl_after_scan_start: timedelta + expected_live_describe: bool + location_clock_ahead: timedelta = timedelta() diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py index eeb122945f..9c63c10938 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py @@ -15,7 +15,7 @@ from tempfile import gettempdir from types import MappingProxyType from typing import Any, cast -from uuid import uuid4 +from uuid import UUID, uuid4 from sqlbuild._native import ( CompiledFunction, @@ -645,6 +645,17 @@ def _refuse_listing(database: str | None) -> None: ) +def deferring_unlisted_adapter( + *, connection: Any, unlisted_database: str +) -> UnlistedDatabaseDuckDbAdapter: + """An adapter over dev and prod schemas where only dev holds `order_ledger`.""" + + connection.execute("CREATE SCHEMA dev") + connection.execute("CREATE SCHEMA prod") + connection.execute("CREATE TABLE dev.order_ledger AS SELECT 1 AS order_id") + return UnlistedDatabaseDuckDbAdapter(connection=connection, unlisted_database=unlisted_database) + + def plan_deferred_project( *, project_dir: Path, @@ -725,8 +736,14 @@ def write_synced_control_store( target: str, warehouse: RecordingSnowflakeWarehouse, schemas: tuple[str, ...], + observed_at: datetime | None = None, + scan_id: str | None = None, + received_at: datetime | None = None, ) -> None: - """A control store whose development location mirrored `schemas` of the warehouse just now.""" + """A control store whose location mirrored `schemas` at `observed_at` (default: now). + + `received_at` is when the store last received each record (default: `observed_at`). + """ catalog: InspectionCatalog = live_inspection_catalog(warehouse) relations: tuple[RelationInfo, ...] = catalog.list_relations( @@ -736,7 +753,9 @@ def write_synced_control_store( catalog.get_columns_for_relations(relations=relations) ) location: str = str(uuid4()) - observed: str = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S.%fZ") + observed_time: datetime = observed_at or datetime.now(UTC) + observed: str = observed_time.strftime("%Y-%m-%dT%H:%M:%S.%fZ") + received: str = (received_at or observed_time).strftime("%Y-%m-%dT%H:%M:%S.%fZ") (project_dir / ".sqlbuild").mkdir(parents=True) with closing(sqlite3.connect(project_dir / ".sqlbuild/control.sqlite3")) as store: _ = store.execute( @@ -745,29 +764,46 @@ def write_synced_control_store( ) _ = store.execute( "CREATE TABLE sqb_warehouse_catalog (event_id TEXT, location_id TEXT, target TEXT, " - "state_database TEXT, state_schema TEXT, observed_at TEXT, record_json JSON)" + "state_database TEXT, state_schema TEXT, observed_at TEXT, record_json JSON, " + "scan_id TEXT, received_at TEXT)" ) _ = store.execute( "INSERT INTO sqb_development_locations VALUES (?, ?, ?)", (str(project_dir), target, location), ) _ = store.executemany( - "INSERT INTO sqb_warehouse_catalog VALUES (?, ?, ?, ?, ?, ?, ?)", + "INSERT INTO sqb_warehouse_catalog VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", [ _mirrored_row( relation=relation, columns=columns.get(relation.identity, ()), scope=(location, target, observed), ) + + (scan_id, received) for relation in relations ], ) store.commit() +def uuid7_at(started: datetime) -> str: + """A UUIDv7, as a catalog scan's id, whose timestamp is `started`.""" + + milliseconds: int = int(started.timestamp() * 1000) + random: int = uuid4().int + value: int = ( + milliseconds << 80 + | 0x7 << 76 + | (random & 0xFFF) << 64 + | 0b10 << 62 + | (random >> 12) & ((1 << 62) - 1) + ) + return str(UUID(int=value)) + + def _mirrored_row( *, relation: RelationInfo, columns: tuple[ColumnInfo, ...], scope: tuple[str, str, str] -) -> tuple[str, ...]: +) -> tuple[str | None, ...]: location, target, observed = scope record: dict[str, object] = { "state_database": "ANALYTICS", diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/test_deferred_migration_origins.py b/tests/integration/src/sqlbuild/compiler/planner/main/test_deferred_migration_origins.py index c00902ab45..8e3ed8f858 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/test_deferred_migration_origins.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/test_deferred_migration_origins.py @@ -1,4 +1,4 @@ -"""Deferred migrate_from lookups skip selected models and survive unlistable origins.""" +"""Deferred migrate_from lookups skip selected models; an unlistable origin falls back to the current name.""" from __future__ import annotations @@ -7,12 +7,14 @@ import pytest -from sqlbuild._native import CompilePipelineResult +from sqlbuild._native import CompilePipelineResult, PlannerInputError from tests.integration.src.sqlbuild.compiler.planner.main._test_types import ( + DeferredMissingOriginTestCase, UnlistedMigrationOriginTestCase, ) from tests.integration.src.sqlbuild.compiler.planner.main.helpers import ( UnlistedDatabaseDuckDbAdapter, + deferring_unlisted_adapter, plan_deferred_project, ) @@ -37,34 +39,53 @@ @pytest.mark.parametrize( "test_case", [ - UnlistedMigrationOriginTestCase( - description="unlistable origin of an unselected upstream falls back with a warning", + DeferredMissingOriginTestCase( + description="unlistable origin falls back to a current name the deferred target lacks", select=("order_report",), - expected_warnings=( - "could not list archive_db.prod.order_history in the deferred target to look " - "for a pre-migration relation, so model 'order_ledger' is deferred under its " - "current name: cannot list database archive_db", - ), + expected_missing=("order_ledger",), expected_unlisted_reads=1, ), + ], + ids=lambda case: case.description, +) +def test_given_unlistable_origin_of_an_unselected_upstream_when_deferring_then_s301_names_it( + test_case: DeferredMissingOriginTestCase, + tmp_path: Path, + connection: Any, +) -> None: + adapter: UnlistedDatabaseDuckDbAdapter = deferring_unlisted_adapter( + connection=connection, unlisted_database=_UNLISTED_DATABASE + ) + + with pytest.raises(PlannerInputError) as raised: + _ = plan_deferred_project( + project_dir=tmp_path, + adapter=adapter, + project_files=_PROJECT_FILES, + select=test_case.select, + ) + + assert raised.value.missing == test_case.expected_missing + assert adapter.listed_databases.count(_UNLISTED_DATABASE) == test_case.expected_unlisted_reads + + +@pytest.mark.parametrize( + "test_case", + [ UnlistedMigrationOriginTestCase( description="selected models are never looked up in the deferred target", select=("order_ledger", "order_report"), - expected_warnings=(), expected_unlisted_reads=0, ), ], ids=lambda case: case.description, ) -def test_given_unlistable_migration_origin_when_planning_with_defer_then_planning_continues( +def test_given_selected_models_when_deferring_then_their_origins_are_never_looked_up( test_case: UnlistedMigrationOriginTestCase, tmp_path: Path, connection: Any, ) -> None: - connection.execute("CREATE SCHEMA dev") - connection.execute("CREATE SCHEMA prod") - connection.execute("CREATE TABLE dev.order_ledger AS SELECT 1 AS order_id") - adapter: UnlistedDatabaseDuckDbAdapter = UnlistedDatabaseDuckDbAdapter( + adapter: UnlistedDatabaseDuckDbAdapter = deferring_unlisted_adapter( connection=connection, unlisted_database=_UNLISTED_DATABASE ) @@ -75,18 +96,13 @@ def test_given_unlistable_migration_origin_when_planning_with_defer_then_plannin select=test_case.select, ) - assert ( - tuple( - filter( - lambda message: "deferred target" in message, - (warning.message for warning in result.plan_output.warnings), - ) + assert not tuple( + filter( + lambda message: "deferred target" in message, + (warning.message for warning in result.plan_output.warnings), ) - == test_case.expected_warnings - ), test_case.description - assert ( - adapter.listed_databases.count(_UNLISTED_DATABASE) == test_case.expected_unlisted_reads - ), test_case.description + ) + assert adapter.listed_databases.count(_UNLISTED_DATABASE) == test_case.expected_unlisted_reads if __name__ == "__main__": diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/test_plan_from_synced_state.py b/tests/integration/src/sqlbuild/compiler/planner/main/test_plan_from_synced_state.py index f92caf9741..00897c2bf7 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/test_plan_from_synced_state.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/test_plan_from_synced_state.py @@ -2,19 +2,27 @@ from __future__ import annotations +import re +from dataclasses import replace +from datetime import UTC, datetime, timedelta from pathlib import Path import pytest from sqlbuild._native import CompilePipelineResult -from tests.integration.src.sqlbuild.compiler.planner.main._test_types import PlanFromTestCase +from tests.integration.src.sqlbuild.compiler.planner.main._test_types import ( + PlanFromTestCase, + SyncedScanTestCase, +) from tests.integration.src.sqlbuild.compiler.planner.main.helpers import ( plan_offline_snowflake_project, planned_entries, + uuid7_at, write_synced_control_store, write_targeted_snowflake_project, ) from tests.unit.src.sqlbuild.adapters.snowflake.inspection.helpers import ( + FakeRelation, RecordingSnowflakeWarehouse, SyntheticSnowflakeProject, ) @@ -24,6 +32,10 @@ _SYNCED: str = "Planned from synced state: checked in " _NO_STORE: str = "Planning live: the project has no control store" _LIVE: str = "Planning live:" +_FIRST_OBSERVED: datetime = datetime(2026, 2, 1, tzinfo=UTC) +_SCAN_STARTED: datetime = datetime(2026, 3, 1, tzinfo=UTC) +# The scan read the page two seconds after it started, and the store received it a second later. +_PAGE_RECEIVED: datetime = _SCAN_STARTED + timedelta(seconds=3) @pytest.mark.parametrize( @@ -121,3 +133,89 @@ def test_given_plan_from_when_planning_then_reads_the_synced_copy_only_when_aske ) assert test_case.expected_report in err assert not any(text in err for text in test_case.expected_absent) + + +@pytest.mark.parametrize( + "test_case", + [ + SyncedScanTestCase( + description="a table replaced with the same columns before a later scan is not described", + last_ddl_after_scan_start=timedelta(hours=-1), + expected_live_describe=False, + ), + SyncedScanTestCase( + description="a table replaced within the clock skew before the scan started is described", + last_ddl_after_scan_start=timedelta(minutes=-1), + expected_live_describe=True, + ), + SyncedScanTestCase( + description="a table replaced after the scan started but before its page was read is described", + last_ddl_after_scan_start=timedelta(seconds=1), + expected_live_describe=True, + ), + SyncedScanTestCase( + description="a table replaced after the scan is described", + last_ddl_after_scan_start=timedelta(hours=1), + expected_live_describe=True, + ), + SyncedScanTestCase( + description="a table replaced a second after its page was read is described", + last_ddl_after_scan_start=timedelta(seconds=3), + expected_live_describe=True, + ), + SyncedScanTestCase( + description="a table replaced a second after its page was read by a location clock 10 minutes fast is described", + last_ddl_after_scan_start=timedelta(seconds=3), + expected_live_describe=True, + location_clock_ahead=timedelta(minutes=10), + ), + SyncedScanTestCase( + description="a table replaced with the same columns before a scan by a location clock 10 minutes fast is not described", + last_ddl_after_scan_start=timedelta(hours=-1), + expected_live_describe=False, + location_clock_ahead=timedelta(minutes=10), + ), + ], + ids=lambda case: case.description, +) +def test_given_a_later_scan_found_a_record_unchanged_when_planning_from_synced_state_then_describes_only_tables_altered_since_it_started( + test_case: SyncedScanTestCase, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("SQLBUILD_CONTROL_DATABASE_URL", raising=False) + project: SyntheticSnowflakeProject = write_targeted_snowflake_project( + project_dir=tmp_path / "orders_platform", target=_TARGET, plan_from="synced_state" + ) + name: str = project.incremental_model_names[0].upper() + original: FakeRelation = next(filter(lambda relation: relation.name == name, project.relations)) + others: tuple[FakeRelation, ...] = tuple( + filter(lambda relation: relation.name != name, project.relations) + ) + warehouse: RecordingSnowflakeWarehouse = RecordingSnowflakeWarehouse( + relations=( + *others, + replace(original, last_ddl=_SCAN_STARTED + test_case.last_ddl_after_scan_start), + ) + ) + write_synced_control_store( + project_dir=project.project_dir, + target=_TARGET, + warehouse=warehouse, + schemas=_MODEL_SCHEMAS, + observed_at=_FIRST_OBSERVED, + scan_id=uuid7_at(_SCAN_STARTED + test_case.location_clock_ahead), + received_at=_PAGE_RECEIVED, + ) + warehouse.queries.clear() + + _ = plan_offline_snowflake_project(project=project, warehouse=warehouse) + + assert ( + any( + query.kind in ("show_columns", "columns") + and re.search(rf"\b{name}\b", f"{query.sql} {query.params}", re.IGNORECASE) + for query in warehouse.queries + ) + == test_case.expected_live_describe + ) diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/_test_types.py index 057679bf76..d9cced9b4c 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/_test_types.py @@ -43,3 +43,13 @@ class CloneTargetSchemaTestCase: class CloneConnectionInjectionTestCase: description: str expected_connection: dict[str, object] + + +@dataclass(frozen=True) +class DeferredListingTestCase: + """Deferred locations across databases, and what the deferred target is asked to list.""" + + description: str + locations: tuple[tuple[str | None, str, str], ...] + expected_listings: list[tuple[str | None, tuple[str, ...]]] + expected_names: list[str] diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/helpers.py index 2bdbe2429c..1a6f59cf26 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/helpers.py @@ -2,6 +2,7 @@ from __future__ import annotations +from dataclasses import dataclass, field from datetime import datetime from pathlib import Path @@ -14,6 +15,7 @@ CompileModelConfig, Fingerprint, ) +from sqlbuild.adapter.contract.models import RelationInfo from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, @@ -83,3 +85,37 @@ def build_previous_python_identity_map( ) } return ({}, identity_map)[previous_version_hash is not None] + + +@dataclass +class ListingAdapter: + """An adapter double that lists its relations per database and records each listing.""" + + relations: tuple[RelationInfo, ...] + listed: list[tuple[str | None, tuple[str, ...]]] = field(default_factory=list) + + def list_relations( + self, *, connection: object, database: str | None, schemas: tuple[str, ...] + ) -> tuple[RelationInfo, ...]: + """The relations of `schemas` in `database`.""" + + self.listed.append((database, schemas)) + return tuple( + filter( + lambda relation: relation.database == database and relation.schema in schemas, + self.relations, + ) + ) + + +def deferred_location(*, database: str | None, schema: str, name: str) -> CompiledRelationLocation: + """One deferred relation location.""" + + return CompiledRelationLocation( + database=database, + schema=schema, + name=name, + qualified_name=f"{database}.{schema}.{name}", + logical_schema=schema, + logical_database=database, + ) diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/test_deferred_locations.py b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/test_deferred_locations.py index ed79bf8e16..f0fda01220 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/test_deferred_locations.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/test_deferred_locations.py @@ -2,17 +2,35 @@ from __future__ import annotations +from typing import cast + import pytest -from sqlbuild._native import CompiledRelationLocation, TargetConfig, build_deferred_locations +from sqlbuild._native import ( + CompiledRelationLocation, + TargetConfig, + build_deferred_locations, + gather_deferred_relations, +) +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.adapter.contract.models import RelationInfo from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledProjectView from tests.unit.src.sqlbuild.compiler.pipeline._helpers._test_types import ( + DeferredListingTestCase, DeferredTargetTestCase, ) from tests.unit.src.sqlbuild.compiler.pipeline._helpers.helpers import ( + ListingAdapter, build_single_model_project, + deferred_location, +) + +_DEFERRED_RELATIONS: tuple[RelationInfo, ...] = ( + RelationInfo(database="analytics", schema="prod", name="orders", relation_type="table"), + RelationInfo(database="archive", schema="prod", name="order_history", relation_type="table"), + RelationInfo(database="archive", schema="audit", name="refunds", relation_type="table"), ) @@ -147,3 +165,43 @@ def test_given_deferred_target_config_when_building_targets_then_resolves_expect assert result.schema == test_case.expected_schema assert result.database == test_case.expected_database assert result.qualified_name == test_case.expected_qualified_name + + +@pytest.mark.parametrize( + "test_case", + [ + DeferredListingTestCase( + description="locations in two databases are listed in each database", + locations=( + ("analytics", "prod", "orders"), + ("archive", "prod", "order_history"), + ("archive", "audit", "refunds"), + ), + expected_listings=[("analytics", ("prod",)), ("archive", ("audit", "prod"))], + expected_names=["order_history", "orders", "refunds"], + ), + DeferredListingTestCase( + description="locations in one database are listed once", + locations=(("archive", "prod", "order_history"), ("archive", "audit", "refunds")), + expected_listings=[("archive", ("audit", "prod"))], + expected_names=["order_history", "refunds"], + ), + ], + ids=lambda case: case.description, +) +def test_given_deferred_locations_when_gathering_relations_then_lists_every_database( + test_case: DeferredListingTestCase, +) -> None: + adapter: ListingAdapter = ListingAdapter(relations=_DEFERRED_RELATIONS) + + relations: dict[str, RelationInfo] = gather_deferred_relations( + adapter=cast(BaseAdapter, adapter), + connection=None, + deferred_locations={ + name: deferred_location(database=database, schema=schema, name=name) + for database, schema, name in test_case.locations + }, + ) + + assert adapter.listed == test_case.expected_listings + assert sorted(relations) == test_case.expected_names