diff --git a/benchmarks/duckshop/edge_cases/changed_builds_clone/models/dim_customers.sql b/benchmarks/duckshop/edge_cases/changed_builds_clone/models/dim_customers.sql new file mode 100644 index 0000000000..bfa81972e9 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/changed_builds_clone/models/dim_customers.sql @@ -0,0 +1,3 @@ +MODEL (description "Customers no clone touches, so --changed keeps skipping them.", materialized table); + +SELECT 101 AS customer_id diff --git a/benchmarks/duckshop/edge_cases/changed_builds_clone/models/fact_orders.sql b/benchmarks/duckshop/edge_cases/changed_builds_clone/models/fact_orders.sql new file mode 100644 index 0000000000..ba931f2eb7 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/changed_builds_clone/models/fact_orders.sql @@ -0,0 +1,3 @@ +MODEL (description "Orders a clone replaces in the destination target.", materialized table); + +SELECT 1 AS order_id diff --git a/benchmarks/duckshop/edge_cases/changed_builds_clone/models/orders_report.sql b/benchmarks/duckshop/edge_cases/changed_builds_clone/models/orders_report.sql new file mode 100644 index 0000000000..e85166ebe0 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/changed_builds_clone/models/orders_report.sql @@ -0,0 +1,3 @@ +MODEL (description "Orders read downstream of the cloned table.", materialized table); + +SELECT order_id FROM __ref("fact_orders") diff --git a/crates/sqlbuild-analysis/src/planner_identity/main/missing_upstream_message.rs b/crates/sqlbuild-analysis/src/planner_identity/main/missing_upstream_message.rs index 5afe8280f0..c392668519 100644 --- a/crates/sqlbuild-analysis/src/planner_identity/main/missing_upstream_message.rs +++ b/crates/sqlbuild-analysis/src/planner_identity/main/missing_upstream_message.rs @@ -3,14 +3,16 @@ use std::collections::HashMap; use crate::planner_identity::constants::MISSING_MESSAGE_ITEM_LIMIT; -use crate::planner_identity::models::{GraphKey, MissingUpstream}; +use crate::planner_identity::models::{GraphKey, MissingUpstream, MissingUpstreamFixes}; -/// Missing lineage upstreams by name and, when only an audit or hook read requires one, that read's origin. +/// Missing lineage upstreams with how to fix them and, when only an audit or hook read requires one, that read's origin. pub fn missing_upstream_message( missing: &[MissingUpstream], edge_origins: &HashMap<(GraphKey, GraphKey), String>, + fixes: &MissingUpstreamFixes, ) -> String { - let mut lineage_missing: Vec<&MissingUpstream> = Vec::new(); + let mut lineage_missing: Vec<&str> = Vec::new(); + let mut kinds: std::collections::BTreeSet<&str> = std::collections::BTreeSet::new(); let mut read_names: Vec<&str> = Vec::new(); for entry in missing { let origins: Vec<&str> = entry @@ -23,7 +25,8 @@ pub fn missing_upstream_message( }) .collect(); if origins.len() < entry.required_by.len() { - lineage_missing.push(entry); + lineage_missing.push(entry.key.name.as_str()); + kinds.insert(entry.key.kind.as_str()); continue; } for origin in origins { @@ -32,39 +35,121 @@ pub fn missing_upstream_message( } } } - let mut parts: Vec = Vec::new(); - if !lineage_missing.is_empty() { - let names: Vec<&str> = lineage_missing + let reads: Option = audit_reads_text(&read_names); + if lineage_missing.is_empty() { + return format!("cannot build selected scope: {}", reads.unwrap_or_default()); + } + let mut text: String = lineage_text(&lineage_missing, &kinds, fixes); + if let Some(reads) = reads { + text.push_str(&format!("\n\n Also, {reads}")); + } + text +} + +/// The missing lineage upstreams, a few by name, then how to read, clone or build them. +fn lineage_text( + names: &[&str], + kinds: &std::collections::BTreeSet<&str>, + fixes: &MissingUpstreamFixes, +) -> String { + let one: bool = names.len() == 1; + let noun: String = match kinds.iter().collect::>().as_slice() { + [kind] if one => (*kind).to_string(), + [kind] => format!("{kind}s"), + _ => "models and seeds".to_owned(), + }; + let place: String = fixes.target.as_ref().map_or_else( + || "the warehouse".to_owned(), + |target| format!("target '{target}'"), + ); + let mut lines: Vec = vec![format!( + "{} upstream {noun} this build reads {} exist in {place}", + names.len(), + if one { "doesn't" } else { "don't" } + )]; + lines.extend( + names .iter() .take(MISSING_MESSAGE_ITEM_LIMIT) - .map(|entry| entry.key.name.as_str()) - .collect(); - parts.push(format!( - "{} missing upstream dependencies ({})", - lineage_missing.len(), - names.join(", ") + .map(|name| format!(" {name}")), + ); + if names.len() > MISSING_MESSAGE_ITEM_LIMIT { + lines.push(format!( + " ... {} more (sqb plan --json lists all)", + names.len() - MISSING_MESSAGE_ITEM_LIMIT )); } - if let [name] = read_names.as_slice() { - parts.push(format!( - "{name}, which is not selected and does not exist in the warehouse; select it or build it first" - )); - } else if !read_names.is_empty() { - let shown: String = read_names - .iter() - .take(MISSING_MESSAGE_ITEM_LIMIT) - .copied() - .collect::>() - .join("; "); - let more: &str = if read_names.len() > MISSING_MESSAGE_ITEM_LIMIT { - "; ..." + lines.push(String::new()); + let (exists, them): (&str, &str) = if one { + ("it exists", "it") + } else { + ("they exist", "them") + }; + let read_label: String = format!("read {them} from it for this run:"); + let copy_label: String = format!("copy {them} into this target:"); + let build_label: String = format!( + "{} {them} too:", + if fixes.origins.is_empty() { + "Build" } else { - "" - }; - parts.push(format!( - "{} audit reads are not selected and do not exist in the warehouse ({shown}{more}); select them or build them first", - read_names.len() + "Or build" + } + ); + let width: usize = if fixes.origins.is_empty() { + build_label.len() + 2 + } else { + (read_label.len() + 4).max(copy_label.len() + 4) + } + .max(build_label.len() + 2); + let to: String = fixes + .clone_to + .as_ref() + .map(|target| format!(" --to {target}")) + .unwrap_or_default(); + for origin in &fixes.origins { + lines.push(format!(" If {exists} in {origin}:")); + if fixes.deferrable.contains(origin) { + lines.push(format!( + " {read_label:<0$} --defer-to {origin}", + width - 4 + )); + } + lines.push(format!( + " {copy_label:<0$} sqb clone --from {origin}{to} -s {1}", + width - 4, + names.join(" ") )); } - format!("cannot build selected scope: {}", parts.join("; ")) + lines.push(format!( + " {build_label:<0$} add + before your selection", + width - 2 + )); + lines.join("\n") +} + +/// The audit and hook reads that need a missing upstream, a few by name. +fn audit_reads_text(read_names: &[&str]) -> Option { + match read_names { + [] => None, + [name] => Some(format!( + "{name}, which is not selected and does not exist in the warehouse; select it or build it first" + )), + _ => { + let shown: String = read_names + .iter() + .take(MISSING_MESSAGE_ITEM_LIMIT) + .copied() + .collect::>() + .join("; "); + let more: &str = if read_names.len() > MISSING_MESSAGE_ITEM_LIMIT { + "; ..." + } else { + "" + }; + Some(format!( + "{} audit reads are not selected and do not exist in the warehouse ({shown}{more}); select them or build them first", + read_names.len() + )) + } + } } diff --git a/crates/sqlbuild-analysis/src/planner_identity/models.rs b/crates/sqlbuild-analysis/src/planner_identity/models.rs index f11d73b77b..e0cd6716a0 100644 --- a/crates/sqlbuild-analysis/src/planner_identity/models.rs +++ b/crates/sqlbuild-analysis/src/planner_identity/models.rs @@ -87,6 +87,19 @@ pub struct AvailableUpstreams { pub deferred_relations: Option>, } +/// What a missing-upstream message offers to fix the misses with, all from configuration. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct MissingUpstreamFixes { + /// The active target, named as where the upstreams are missing. + pub target: Option, + /// Targets the active target may clone from, to defer to or clone the upstreams from. + pub origins: Vec, + /// The origins `--defer-to` accepts: project targets, offered only when this plan does not defer already. + pub deferrable: Vec, + /// The clone command's `--to`, needed when the active target is not the default one. + pub clone_to: Option, +} + /// One upstream dependency missing from both scope and warehouse, with its dependents. #[derive(Clone, Debug, PartialEq, Eq)] pub struct MissingUpstream { diff --git a/crates/sqlbuild-analysis/src/planner_identity/tests/test_planner_identity.rs b/crates/sqlbuild-analysis/src/planner_identity/tests/test_planner_identity.rs index 6426f00d87..5c124eb943 100644 --- a/crates/sqlbuild-analysis/src/planner_identity/tests/test_planner_identity.rs +++ b/crates/sqlbuild-analysis/src/planner_identity/tests/test_planner_identity.rs @@ -10,8 +10,8 @@ use crate::planner_identity::main::missing_upstreams::missing_upstreams; use crate::planner_identity::main::stale_upstream_warning_message::stale_upstream_warning_message; use crate::planner_identity::main::topological_order::topological_order; use crate::planner_identity::models::{ - AvailableUpstreams, FunctionFingerprintFields, GraphKey, MissingUpstream, StaleWarning, - StalenessNames, + AvailableUpstreams, FunctionFingerprintFields, GraphKey, MissingUpstream, MissingUpstreamFixes, + StaleWarning, StalenessNames, }; use crate::planner_identity::tests::helpers::{ chain_head_hash, chain_identity_graph, chain_staleness_graph, cyclic_identity_graph, @@ -406,11 +406,58 @@ fn given_selected_keys_when_checking_buildability_then_reports_missing_upstreams ..AvailableUpstreams::default() }, origins: HashMap::new(), + fixes: MissingUpstreamFixes::default(), expected_missing: vec![MissingUpstream { key: model("raw"), required_by: vec![model("a"), model("b")], }], - expected_message: "cannot build selected scope: 1 missing upstream dependencies (raw)", + expected_message: concat!( + "1 upstream model this build reads doesn't exist in the warehouse\n", + " raw\n", + "\n", + " Build it too: add + before your selection", + ), + }, + BuildabilityTestCase { + description: "more misses than shown, with clone origins and a non-default target", + selected: vec![model("report")], + upstream: HashMap::from([( + model("report"), + ["f", "a", "e", "b", "d", "c"] + .map(model) + .into_iter() + .chain([key("seed", "codes")]) + .collect(), + )]), + available: AvailableUpstreams::default(), + origins: HashMap::new(), + fixes: MissingUpstreamFixes { + target: Some("dev".to_owned()), + origins: vec!["prod".to_owned(), "staging".to_owned()], + deferrable: vec!["prod".to_owned()], + clone_to: Some("dev".to_owned()), + }, + expected_missing: ["a", "b", "c", "d", "e", "f"] + .map(model) + .into_iter() + .chain([key("seed", "codes")]) + .map(|key| MissingUpstream { + key, + required_by: vec![model("report")], + }) + .collect(), + expected_message: concat!( + "7 upstream models and seeds this build reads don't exist in target 'dev'\n", + " a\n b\n c\n d\n e\n", + " ... 2 more (sqb plan --json lists all)\n", + "\n", + " If they exist in prod:\n", + " read them from it for this run: --defer-to prod\n", + " copy them into this target: sqb clone --from prod --to dev -s a b c d e f codes\n", + " If they exist in staging:\n", + " copy them into this target: sqb clone --from staging --to dev -s a b c d e f codes\n", + " Or build them too: add + before your selection", + ), }, BuildabilityTestCase { description: "only an audit read requires the upstream", @@ -421,6 +468,7 @@ fn given_selected_keys_when_checking_buildability_then_reports_missing_upstreams (model("a"), model("lookup")), "audit 'check' on 'a' reads 'lookup'".to_owned(), )]), + fixes: MissingUpstreamFixes::default(), expected_missing: vec![MissingUpstream { key: model("lookup"), required_by: vec![model("a")], @@ -436,7 +484,7 @@ fn given_selected_keys_when_checking_buildability_then_reports_missing_upstreams ); assert_eq!( ( - missing_upstream_message(&missing, &test_case.origins), + missing_upstream_message(&missing, &test_case.origins, &test_case.fixes), missing ), ( diff --git a/crates/sqlbuild-analysis/src/planner_identity/tests/test_types.rs b/crates/sqlbuild-analysis/src/planner_identity/tests/test_types.rs index 9c701080c4..3403315861 100644 --- a/crates/sqlbuild-analysis/src/planner_identity/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/planner_identity/tests/test_types.rs @@ -2,7 +2,7 @@ use std::collections::{HashMap, HashSet}; use crate::planner_identity::models::{ AvailableUpstreams, FunctionFingerprintFields, GraphKey, IdentityGraph, MissingUpstream, - StaleWarning, StalenessGraph, + MissingUpstreamFixes, StaleWarning, StalenessGraph, }; use crate::planner_identity::types::IdentityHashes; @@ -31,6 +31,7 @@ pub(super) struct BuildabilityTestCase { pub(super) upstream: HashMap>, pub(super) available: AvailableUpstreams, pub(super) origins: HashMap<(GraphKey, GraphKey), String>, + pub(super) fixes: MissingUpstreamFixes, pub(super) expected_missing: Vec, pub(super) expected_message: &'static str, } diff --git a/crates/sqlbuild-python-base/src/records/_helpers/compiler_errors/constants.rs b/crates/sqlbuild-python-base/src/records/_helpers/compiler_errors/constants.rs index 2a3bfbcd99..981e9522e6 100644 --- a/crates/sqlbuild-python-base/src/records/_helpers/compiler_errors/constants.rs +++ b/crates/sqlbuild-python-base/src/records/_helpers/compiler_errors/constants.rs @@ -398,7 +398,7 @@ pub(crate) const COMPILER_ERRORS: &[ErrorSpec] = &[ doc: "Raised when planner inputs cannot be resolved safely.", class_attributes: &[("code", ClassValue::Text("S000"))], required: &[], - optional: &[], + optional: &["missing"], fixed: &[], }, ErrorSpec { diff --git a/crates/sqlbuild-python-base/src/records/_helpers/native_classes/construct.rs b/crates/sqlbuild-python-base/src/records/_helpers/native_classes/construct.rs index d10b9621ac..1a1ced149a 100644 --- a/crates/sqlbuild-python-base/src/records/_helpers/native_classes/construct.rs +++ b/crates/sqlbuild-python-base/src/records/_helpers/native_classes/construct.rs @@ -60,7 +60,17 @@ pub(crate) fn new_error( pub(crate) fn exception_from( class: PyResult>, message: &str, + keywords: (Option<&str>, Option<&str>), +) -> PyErr { + exception_with_fields(class, message, keywords, &[]) +} + +/// The exception `class` with `message`, the given `code` and `help`, and its declared `fields`, or the error raising it. +pub(crate) fn exception_with_fields<'py>( + class: PyResult>, + message: &str, (code, help): (Option<&str>, Option<&str>), + fields: &[(&str, Bound<'py, PyAny>)], ) -> PyErr { class .and_then(|class| { @@ -71,6 +81,9 @@ pub(crate) fn exception_from( if let Some(help) = help { keywords.set_item("help", help)?; } + for (name, value) in fields { + keywords.set_item(*name, value)?; + } class.call((message,), Some(&keywords)) }) .map_or_else(|error| error, PyErr::from_value) diff --git a/crates/sqlbuild-python-base/src/records/main/construct/mod.rs b/crates/sqlbuild-python-base/src/records/main/construct/mod.rs index 393ad76d31..f18b59054c 100644 --- a/crates/sqlbuild-python-base/src/records/main/construct/mod.rs +++ b/crates/sqlbuild-python-base/src/records/main/construct/mod.rs @@ -4,6 +4,7 @@ pub mod error_class; pub mod exception_from; pub mod new_annotated_error; pub mod new_error; +pub mod new_error_with_fields; pub mod new_plain_error; pub mod new_record; pub mod new_record_with; diff --git a/crates/sqlbuild-python-base/src/records/main/construct/new_error_with_fields.rs b/crates/sqlbuild-python-base/src/records/main/construct/new_error_with_fields.rs new file mode 100644 index 0000000000..b7aae8a7d4 --- /dev/null +++ b/crates/sqlbuild-python-base/src/records/main/construct/new_error_with_fields.rs @@ -0,0 +1,21 @@ +//! The native `id` exception raised with `message`, its `code` and `help`, and declared keyword fields. + +use crate::records::_helpers::native_classes::construct; +use crate::records::types::ErrorId; +use pyo3::prelude::{Bound, PyAny, PyErr, Python}; + +/// The native `id` exception with `message`, the given `code` and `help`, and its declared `fields`, or the error raising it. +#[inline] +pub fn new_error_with_fields<'py>( + py: Python<'py>, + id: ErrorId, + (message, code, help): (&str, Option<&str>, Option<&str>), + fields: &[(&str, Bound<'py, PyAny>)], +) -> PyErr { + construct::exception_with_fields( + construct::error_class(py, id), + message, + (code, help), + fields, + ) +} diff --git a/crates/sqlbuild-python-front/src/native_host/_helpers/consumption_state.rs b/crates/sqlbuild-python-front/src/native_host/_helpers/consumption_state.rs index 4414633a9f..3c72bab1e6 100644 --- a/crates/sqlbuild-python-front/src/native_host/_helpers/consumption_state.rs +++ b/crates/sqlbuild-python-front/src/native_host/_helpers/consumption_state.rs @@ -125,7 +125,7 @@ pub(crate) fn ensure_table( } /// Whether the warehouse catalog lists the consumption table in `location`. -fn table_exists( +pub(crate) fn table_exists( adapter: &Bound<'_, PyAny>, connection: &Bound<'_, PyAny>, location: &Location, diff --git a/crates/sqlbuild-python-front/src/native_host/main/append_replaced_consumption.rs b/crates/sqlbuild-python-front/src/native_host/main/append_replaced_consumption.rs new file mode 100644 index 0000000000..075e9464b6 --- /dev/null +++ b/crates/sqlbuild-python-front/src/native_host/main/append_replaced_consumption.rs @@ -0,0 +1,32 @@ +//! Leave relations replaced outside a recorded build without a completed consumption record. + +use std::collections::BTreeMap; + +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_state::consumption::main::replaced_append::replaced_append; +use sqlbuild_state::consumption::models::{ConsumptionAppend, NodeKey, RelationKey}; +use uuid::Uuid; + +use crate::native_host::_helpers::consumption_state::{append, table_exists}; +use crate::native_host::models::Location; + +/// Append a never-completed execution for each of `replaced` where its schema has a consumption table. +pub fn append_replaced_consumption( + (adapter, connection): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), + run_id: &str, + replaced: Vec<(NodeKey, RelationKey)>, +) -> PyResult<()> { + let mut rows: BTreeMap> = BTreeMap::new(); + for (node, relation) in replaced { + let execution_id: String = Uuid::now_v7().to_string(); + rows.entry(Location::of(&relation)) + .or_default() + .push(replaced_append((node, relation), (&execution_id, run_id))); + } + for (location, rows) in rows { + if table_exists(adapter, connection, &location)? { + append(adapter, connection, &location, &rows)?; + } + } + Ok(()) +} diff --git a/crates/sqlbuild-python-front/src/native_host/main/mod.rs b/crates/sqlbuild-python-front/src/native_host/main/mod.rs index effd6e652f..4cb7f09303 100644 --- a/crates/sqlbuild-python-front/src/native_host/main/mod.rs +++ b/crates/sqlbuild-python-front/src/native_host/main/mod.rs @@ -1,4 +1,5 @@ pub mod append_observations; +pub mod append_replaced_consumption; pub mod execute; pub mod fixed_state_table_sql; pub mod integer_literal; diff --git a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/buildability.rs b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/buildability.rs index 76c81a61b9..d8e27b2193 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/buildability.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/buildability.rs @@ -5,15 +5,21 @@ use crate::planner_phases::_helpers::inputs::runtime::runtime_field; use crate::planner_phases::_helpers::inputs::runtime::warehouse_field; use crate::planner_phases::constants::MISSING_UPSTREAM_CODE; use crate::planner_warehouse::main::sources::check_selected_source_tables_exist::check_selected_source_tables_exist; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyString, PyTuple}; use sqlbuild_analysis::planner_identity::main::missing_upstream_message::missing_upstream_message; use sqlbuild_analysis::planner_identity::main::missing_upstreams::missing_upstreams; -use sqlbuild_analysis::planner_identity::models::{AvailableUpstreams, GraphKey}; +use sqlbuild_analysis::planner_identity::models::{ + AvailableUpstreams, GraphKey, MissingUpstreamFixes, +}; +use sqlbuild_python_base::records::main::construct::new_error_with_fields::new_error_with_fields; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::ErrorId; +use sqlbuild_python_front::contracts::main::resolve_target_config::resolve_target_config; +use sqlbuild_python_front::contracts::main::resolve_target_name::resolve_target_name; use sqlbuild_python_project::compiled_views::main::project::seeds::seeds; use sqlbuild_python_project::graph::main::nodes::native_key::native_key; use sqlbuild_python_project::planner_identity::main::execution_origins::execution_origins; -use sqlbuild_python_project::planner_phases::main::planner_error::planner_error; use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; use sqlbuild_python_project::planner_scope::models::PlannerScopeResolution; use sqlbuild_python_project::resource_views::main::seed_reads::external::external; @@ -68,11 +74,17 @@ pub(crate) fn check_selected_scope_buildability<'py>( .into_iter() .map(|((key, read), label)| ((GraphKey::from(&key), GraphKey::from(&read)), label)) .collect(); - let message = missing_upstream_message(&missing, &origins); - return Err(planner_error( + let message = missing_upstream_message( + &missing, + &origins, + &missing_upstream_fixes(runtime, &project, available.deferred_relations.is_some())?, + ); + let names = missing.iter().map(|entry| entry.key.name.as_str()); + return Err(new_error_with_fields( py, - &message, - (Some(MISSING_UPSTREAM_CODE), None), + ErrorId::PlannerInput, + (&message, Some(MISSING_UPSTREAM_CODE), None), + &[("missing", PyTuple::new(py, names)?.into_any())], )); } check_selected_source_tables_exist( @@ -85,3 +97,56 @@ pub(crate) fn check_selected_scope_buildability<'py>( &warehouse_field(warehouse, |warehouse| &warehouse.inspection_relations)?, ) } + +/// The active target, its clone or deferral origins, and the clone `--to` it needs (also when the default target is unknown). +fn missing_upstream_fixes( + runtime: &Bound<'_, PyAny>, + project: &Bound<'_, PyAny>, + deferring: bool, +) -> PyResult { + let py: Python<'_> = runtime.py(); + let target: Option = native_field(project, "effective_target_name")?.extract()?; + let configs = ( + runtime_field(runtime, |runtime| &runtime.project_config)?, + runtime_field(runtime, |runtime| &runtime.local_config)?, + ); + let Some(active) = target.clone() else { + return Ok(MissingUpstreamFixes::default()); + }; + if configs.0.is_none() || configs.1.is_none() { + return Ok(MissingUpstreamFixes { + target, + ..MissingUpstreamFixes::default() + }); + } + let allows = |name: &str, flag: &str| -> PyResult { + let config = resolve_target_config(py, &configs.0, &configs.1, &PyString::new(py, name))?; + native_field(&native_field(&config, "clone")?, flag)?.is_truthy() + }; + let mut origins: std::collections::BTreeSet = std::collections::BTreeSet::new(); + let mut deferrable: Vec = Vec::new(); + if allows(&active, "allow_as_clone_destination")? { + for (config, project_targets) in [(&configs.0, true), (&configs.1, false)] { + for name in native_field(config, "targets")?.try_iter()? { + let name: String = name?.extract()?; + if name != active && allows(&name, "allow_as_clone_origin")? { + if project_targets && !deferring { + deferrable.push(name.clone()); + } + origins.insert(name); + } + } + } + } + let default: Option = + match resolve_target_name(py, &configs.0, &configs.1, &py.None().into_bound(py)) { + Ok(name) => name.map(|name| name.to_string()), + Err(_) => None, + }; + Ok(MissingUpstreamFixes { + clone_to: (default.as_deref() != Some(active.as_str())).then(|| active.clone()), + target, + origins: origins.into_iter().collect(), + deferrable, + }) +} diff --git a/crates/sqlbuild-python-project/src/planner_identity/_helpers/graph.rs b/crates/sqlbuild-python-project/src/planner_identity/_helpers/graph.rs index aacdb803c1..2d63341beb 100644 --- a/crates/sqlbuild-python-project/src/planner_identity/_helpers/graph.rs +++ b/crates/sqlbuild-python-project/src/planner_identity/_helpers/graph.rs @@ -14,7 +14,9 @@ use sqlbuild_analysis::planner_identity::constants::MODEL_RESOURCE; use sqlbuild_analysis::planner_identity::main::missing_upstream_message::missing_upstream_message as message_of; use sqlbuild_analysis::planner_identity::main::missing_upstreams::missing_upstreams; use sqlbuild_analysis::planner_identity::main::topological_order::topological_order; -use sqlbuild_analysis::planner_identity::models::{AvailableUpstreams, GraphKey, MissingUpstream}; +use sqlbuild_analysis::planner_identity::models::{ + AvailableUpstreams, GraphKey, MissingUpstream, MissingUpstreamFixes, +}; use crate::graph::main::nodes::closure::closure; use crate::graph::main::nodes::inverted::inverted; @@ -270,5 +272,8 @@ pub(crate) fn missing_upstream_message<'py>( let rows: Vec = missing_rows(missing, &mut objects)?; let origins: HashMap<(GraphKey, GraphKey), String> = self::edge_origins(edge_origins, &mut objects)?; - Ok(PyString::new(missing.py(), &message_of(&rows, &origins))) + Ok(PyString::new( + missing.py(), + &message_of(&rows, &origins, &MissingUpstreamFixes::default()), + )) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs index 911e4772e6..3b23db5ffc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs @@ -19,6 +19,7 @@ use sqlbuild_host::entries::models::ExecutorHost; use crate::bindings::_helpers::build_command::invocation::BuildContext; use crate::bindings::_helpers::build_command::policies::build_command_error; use crate::bindings::_helpers::build_command::policies::enforce_model_execution_limit_binding; +use crate::bindings::_helpers::clone::clone_execution::forget_replaced_consumption; use crate::bindings::_helpers::clone::clone_pipeline::run_clone_pipeline_with_options; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; use sqlbuild_python_base::boundary::main::call_with::call_with; @@ -440,6 +441,8 @@ fn clone_boundaries<'py>( ), )?; let result: Bound<'py, PyAny> = executed_clone(context, prephase, (&pipeline, connection))?; + let marked: PyResult<()> = + forget_replaced_consumption((&prephase.adapter, connection), &pipeline, &result); let failed: Vec = failed_items(&result)?; if !failed.is_empty() { return Err(user_error( @@ -455,7 +458,7 @@ fn clone_boundaries<'py>( call_with( &host_entry(py, ExecutorHost::CopyCloneFingerprints)?, &[ - ("result", result), + ("result", result.clone()), ( "origin_model_entries", native_field(&pipeline, "origin_model_entries")?, @@ -481,7 +484,7 @@ fn clone_boundaries<'py>( ), ], )?; - Ok(()) + marked } /// Clone the selected boundary relations from the origin before build planning. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs index 1032360908..194c02db2e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs @@ -273,13 +273,18 @@ fn write_machine_error( } else { Some(help.str()?.to_string()) }; - let document = serde_json::json!({ - "error": { - "code": code.str()?.to_string(), - "message": message.str()?.to_string(), - "help": help, - } + let mut fields = serde_json::json!({ + "code": code.str()?.to_string(), + "message": message.str()?.to_string(), + "help": help, }); + if let Some(missing) = error + .getattr_opt("missing")? + .filter(|value| !value.is_none()) + { + fields["missing"] = serde_json::json!(missing.extract::>()?); + } + let document = serde_json::json!({ "error": fields }); print_object( &PyString::new(invocation.py(), &document.to_string()).into_any(), None, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_execution.rs index 5adac1f62d..da6ebcb25c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_execution.rs @@ -1,6 +1,6 @@ //! Execute the prepared clone plan and propagate fingerprints. -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::time::Instant; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; @@ -13,6 +13,7 @@ use crate::bindings::_helpers::clone::clone_command::Invocation; use crate::bindings::_helpers::clone::clone_progress::{ CloneFingerprintReporter, CloneItemPrinter, }; +use sqlbuild_analysis::fingerprint_storage::constants::{NODE_TYPE_MODEL, NODE_TYPE_SEED}; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::within::within; use sqlbuild_python_base::native_host::main::host_entry::host_entry; @@ -23,7 +24,10 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::contracts::main::resolve_target_config::resolve_target_config; +use sqlbuild_python_front::native_host::main::append_replaced_consumption::append_replaced_consumption; use sqlbuild_python_output::result_output::models::ExecutionEventWriter; +use sqlbuild_python_planner::consumption::main::relation::relation; +use sqlbuild_state::consumption::models::{NodeKey, RelationKey}; const DESTINATION_ENTRY_GROUPS: [&str; 4] = [ "destination_source_entries", @@ -37,6 +41,13 @@ const ORIGIN_ENTRY_GROUPS: [&str; 3] = [ "origin_seed_entries", ]; +/// A clone item that finished cleanly. +const SUCCESS_STATUS: &str = "success"; +/// Clone actions that replace the destination relation with the origin's data. +const REPLACING_ACTIONS: [&str; 2] = ["cloned", "copied"]; +/// The materialization of a model the clone recreates as a view rather than replacing. +const VIEW_MATERIALIZATION: &str = "view"; + /// The pipeline's plan entries of `groups`, in order. fn entries<'py>(pipeline: &Bound<'py, PyAny>, groups: &[&str]) -> PyResult>> { let mut found: Vec> = Vec::new(); @@ -310,6 +321,57 @@ fn copy_fingerprints<'py>( Ok(()) } +/// Mark each table and seed the clone replaced as having no completed consumption record, so `--changed` rebuilds it once. +pub(crate) fn forget_replaced_consumption( + (adapter, connection): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), + pipeline: &Bound<'_, PyAny>, + result: &Bound<'_, PyAny>, +) -> PyResult<()> { + let mut replaced_names: HashSet = HashSet::new(); + for item in native_field(result, "item_results")?.try_iter()? { + let item: Bound<'_, PyAny> = item?; + let status: String = native_field(&native_field(&item, "status")?, "value")?.extract()?; + let action: String = native_field(&native_field(&item, "action")?, "value")?.extract()?; + if status == SUCCESS_STATUS && REPLACING_ACTIONS.contains(&action.as_str()) { + replaced_names.insert(native_field(&item, "name")?.extract()?); + } + } + let mut replaced: Vec<(NodeKey, RelationKey)> = Vec::new(); + for (group, kind) in [ + ("destination_model_entries", NODE_TYPE_MODEL), + ("destination_seed_entries", NODE_TYPE_SEED), + ] { + for entry in native_field(pipeline, group)?.try_iter()? { + let entry: Bound<'_, PyAny> = entry?; + let name: String = native_field(&entry, "name")?.extract()?; + let destination: Bound<'_, PyAny> = native_field(&entry, "destination")?; + let view: bool = kind == NODE_TYPE_MODEL + && native_field(&native_field(&entry, "materialization_type")?, "value")? + .extract::()? + == VIEW_MATERIALIZATION; + if view + || !replaced_names.contains(&name) + || native_field(&destination, "schema")?.is_none() + { + continue; + } + replaced.push(( + NodeKey { + kind: kind.to_owned(), + name, + }, + relation(&destination)?, + )); + } + } + let project: Bound<'_, PyAny> = native_field(pipeline, "destination_project")?; + append_replaced_consumption( + (adapter, connection), + &native_field(&project, "run_id")?.extract::()?, + replaced, + ) +} + /// Execute the clone, streaming items, then copy fingerprints; returns the result and elapsed seconds. pub(crate) fn execute_clone_plan<'py>( request: &Bound<'py, PyAny>, @@ -351,6 +413,9 @@ pub(crate) fn execute_clone_plan<'py>( let closed: PyResult> = event_writer.call_method0("close"); let result: Bound<'py, PyAny> = executed?; closed?; + let marked: PyResult<()> = + forget_replaced_consumption((&invocation.adapter, connection), pipeline, &result); copy_fingerprints(invocation, (pipeline, connection), &result)?; + marked?; Ok((result, started.elapsed().as_secs_f64())) } diff --git a/crates/sqlbuild-state/src/consumption/_helpers/decision.rs b/crates/sqlbuild-state/src/consumption/_helpers/decision.rs index af9ed420a8..0c8abea530 100644 --- a/crates/sqlbuild-state/src/consumption/_helpers/decision.rs +++ b/crates/sqlbuild-state/src/consumption/_helpers/decision.rs @@ -235,7 +235,11 @@ fn execution_changes(node: &NodeFacts, context: &Context<'_>, window: &Window<'_ let mut changes: Vec = Vec::new(); for execution in window.executions { if !execution.complete { - return vec![always(RunReason::Unfinished)]; + return vec![always(if execution.replaced { + RunReason::NoRecord + } else { + RunReason::Unfinished + })]; } if execution.version_hash.as_deref() != Some(node.version_hash.as_str()) { changes.push(definition(RunReason::DefinitionChanged)); diff --git a/crates/sqlbuild-state/src/consumption/_helpers/history.rs b/crates/sqlbuild-state/src/consumption/_helpers/history.rs index e749ba91c3..c557c12cb8 100644 --- a/crates/sqlbuild-state/src/consumption/_helpers/history.rs +++ b/crates/sqlbuild-state/src/consumption/_helpers/history.rs @@ -2,7 +2,7 @@ use std::collections::{BTreeMap, BTreeSet, HashMap}; -use crate::consumption::constants::{ORDER_SKEW_MICROS, SUCCEEDED_PHASE}; +use crate::consumption::constants::{ORDER_SKEW_MICROS, REPLACED_PHASE, SUCCEEDED_PHASE}; use crate::consumption::models::{ ConsumedInput, ConsumptionHistory, ConsumptionRow, Execution, RelationHistory, RelationKey, }; @@ -18,7 +18,7 @@ struct ExecutionRows<'a> { /// A retried append repeats a row: its execution, phase and the input it records. type RowIdentity<'a> = (&'a str, &'a str, Option<(&'a str, &'a str, &'a str)>); -/// Recent executions per relation, from rows read newest first with at most `limit` per relation. +/// Recent executions per relation (rows newest first, at most `limit` each); a replacement marker counts only while newest. pub(crate) fn build_history(rows: &[ConsumptionRow], limit: usize) -> ConsumptionHistory { let mut relations: HashMap = HashMap::new(); for (relation, execution) in executions_of(rows) { @@ -33,6 +33,13 @@ pub(crate) fn build_history(rows: &[ConsumptionRow], limit: usize) -> Consumptio (right.started_at, &right.execution_id).cmp(&(left.started_at, &left.execution_id)) }); history.truncated = history.executions.len() >= limit; + let newest: Option = history + .executions + .first() + .map(|execution| execution.execution_id.clone()); + history.executions.retain(|execution| { + !execution.replaced || Some(&execution.execution_id) == newest.as_ref() + }); } ConsumptionHistory { relations, @@ -124,6 +131,7 @@ fn execution(execution_id: &str, rows: &ExecutionRows<'_>) -> Execution { execution_id: execution_id.to_owned(), started_at: rows.earliest, complete, + replaced: rows.rows.iter().all(|row| row.row.phase == REPLACED_PHASE), version_hash: (versions.len() == 1) .then(|| versions.first().map(|version| (*version).to_owned())) .flatten(), diff --git a/crates/sqlbuild-state/src/consumption/constants.rs b/crates/sqlbuild-state/src/consumption/constants.rs index 427d51b454..b511e5876d 100644 --- a/crates/sqlbuild-state/src/consumption/constants.rs +++ b/crates/sqlbuild-state/src/consumption/constants.rs @@ -34,6 +34,8 @@ pub const NULLABLE_COLUMNS: [&str; 4] = [ pub const STARTED_PHASE: &str = "started"; /// Phase of the rows appended after the write succeeded. pub const SUCCEEDED_PHASE: &str = "succeeded"; +/// Phase of the row appended when something other than a recorded build replaced the relation. +pub const REPLACED_PHASE: &str = "replaced"; /// Input kind of a source, versioned by its observed `data_version_hash`. pub const SOURCE_INPUT: &str = "source"; /// Input kind of an upstream model or seed, versioned by its `execution_id`. diff --git a/crates/sqlbuild-state/src/consumption/main/mod.rs b/crates/sqlbuild-state/src/consumption/main/mod.rs index 0fa927d55d..766e6e029c 100644 --- a/crates/sqlbuild-state/src/consumption/main/mod.rs +++ b/crates/sqlbuild-state/src/consumption/main/mod.rs @@ -6,5 +6,6 @@ pub mod prune_executions_sql; pub mod read_back_history; pub mod recent_executions; pub mod recent_executions_sql; +pub mod replaced_append; pub mod started_append; pub mod succeeded_appends; diff --git a/crates/sqlbuild-state/src/consumption/main/replaced_append.rs b/crates/sqlbuild-state/src/consumption/main/replaced_append.rs new file mode 100644 index 0000000000..a8fe49645a --- /dev/null +++ b/crates/sqlbuild-state/src/consumption/main/replaced_append.rs @@ -0,0 +1,21 @@ +//! The row appended for a relation replaced outside a recorded build. + +use crate::consumption::constants::{REPLACED_PHASE, UNKNOWN_VERSION}; +use crate::consumption::models::{ConsumptionAppend, NodeKey, RelationKey}; + +/// The `replaced` row of a new execution that never completes, so `relation` has no completed record. +pub fn replaced_append( + (node, relation): (NodeKey, RelationKey), + (execution_id, run_id): (&str, &str), +) -> ConsumptionAppend { + ConsumptionAppend { + execution_id: execution_id.to_owned(), + phase: REPLACED_PHASE.to_owned(), + run_id: run_id.to_owned(), + node, + relation, + version_hash: UNKNOWN_VERSION.to_owned(), + input_count: 0, + input: None, + } +} diff --git a/crates/sqlbuild-state/src/consumption/models.rs b/crates/sqlbuild-state/src/consumption/models.rs index a5b8ce8560..745afc921e 100644 --- a/crates/sqlbuild-state/src/consumption/models.rs +++ b/crates/sqlbuild-state/src/consumption/models.rs @@ -152,6 +152,8 @@ pub struct Execution { pub started_at: i64, /// Whether its `succeeded` rows are present, consistent and complete. pub complete: bool, + /// Whether its rows only mark the relation as replaced outside a recorded build. + pub replaced: bool, pub version_hash: Option, /// Input versions of a complete execution, keyed by `(kind, name)`. pub inputs: Vec, diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/plan.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/plan.md index e05e65aa7c..4885f90263 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/plan.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/plan.md @@ -152,10 +152,29 @@ Release note: earlier versions printed `Plan complete with errors` and exited `0 Planning a scoped selection whose upstream inputs were never built in the target fails with `S301` rather than producing a plan that would break at build time: ``` -error[S301]: cannot build selected scope: 3 missing upstream dependencies (stg_orders, stg_payments, waffle_types) +error[S301]: 3 upstream models and seeds this build reads don't exist in target 'dev' + stg_orders + stg_payments + waffle_types + + If they exist in prod: + read them from it for this run: --defer-to prod + copy them into this target: sqb clone --from prod -s stg_orders stg_payments waffle_types + Or build them too: add + before your selection ``` -Build the upstream chain first, or select it along with the model: `sqb build --select +fact_orders`. +The error names the first five missing resources; `sqb plan --json` lists all of them in +`error.missing`. Each fix fits a different case: + +| Fix | When to use it | +| --- | --- | +| `--defer-to ` | Read the missing resources from another target for this run only. | +| `sqb clone --from -s ...` | Copy them into this target once. The command lists every missing name, so you can run it as printed. | +| `+` before the selection, such as `sqb build --select +fact_orders` | Build the upstream chain along with the model. | + +The error offers an origin only for targets with `clone.allow_as_clone_origin = true`, and only +when the current target has `clone.allow_as_clone_destination = true`. It does not check that the +resources exist in the origin. The check also covers resources that an attached audit on a selected resource reads, even when the model SQL does not read them. Selecting `+fact_orders` follows data lineage only, so it does not @@ -167,7 +186,7 @@ error[S301]: cannot build selected scope: audit 'relationships' on 'stg_orders' ``` Load or build the resource first (for a seed, `sqb seed`), or add it to the selection. When both -kinds are missing, the error lists the missing upstream dependencies and then the audit reads. +kinds are missing, the error lists the missing upstream resources and then the audit reads. ## Missing source tables diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 8d683a5772..adc266cbe8 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -13535,8 +13535,14 @@ class PlannerInputError(ValueError): code: str message: str help: str | None + missing: tuple[str, ...] | None def __init__( - self, message: str, *, code: str | None = None, help: str | None = None + self, + message: str, + *, + code: str | None = None, + help: str | None = None, + missing: tuple[str, ...] | None = None, ) -> None: ... class FutureCursorSafetyError(PlannerInputError): diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_terminal_failure.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_terminal_failure.py index 759bbe8b4e..5e01acbc9c 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_terminal_failure.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_terminal_failure.py @@ -27,7 +27,7 @@ model_replacements=(), expected_screen_fragments=( "Inspected warehouse state.", - "error[S301]: cannot build selected scope: 2 missing upstream dependencies", + "error[S301]: 2 upstream models this build reads don't exist in the warehouse", ), ), TerminalBuildFailureE2ETestCase( diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clone/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/clone/_test_types.py index b1ca648fe1..ba5cc3ee07 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/clone/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clone/_test_types.py @@ -74,3 +74,52 @@ class CloneTimingsE2ETestCase: description: str expected_phases: tuple[str, ...] + + +@dataclass(frozen=True) +class CloneChangedPlanE2ETestCase: + """Commands run after building both targets, and what `plan --changed` says after them.""" + + description: str + commands: tuple[tuple[str, ...], ...] + expected_fragments: tuple[str, ...] + unexpected_fragments: tuple[str, ...] = () + + +@dataclass(frozen=True) +class CloneMissingUpstreamE2ETestCase: + """A target missing a selected model's upstreams, and the clone command S301 prints for it.""" + + description: str + default_target: str + target_args: tuple[str, ...] + expected_clone_command: str + + +@dataclass(frozen=True) +class MissingUpstreamFixesE2ETestCase: + """Whether the planned target may be cloned into, and what S301 then offers.""" + + description: str + destination_allowed: bool + expected_fragments: tuple[str, ...] + unexpected_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class PartialDeferCloneE2ETestCase: + """A defer clone with one boundary missing in the origin, and what `plan --changed` says.""" + + description: str + dropped_origin_table: str + expected_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class UnwritableConsumptionDeferCloneE2ETestCase: + """A partly failed defer clone whose destination consumption table rejects writes.""" + + description: str + dropped_origin_table: str + before_clone: tuple[str, ...] + expected_fragments: tuple[str, ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clone/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/clone/helpers.py index 7b810d14b1..7b653bc1ff 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/clone/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clone/helpers.py @@ -7,7 +7,11 @@ from pathlib import Path from textwrap import dedent -from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( + execute_duckdb, + prepare_inline_project, + run_sqb, +) PROJECT_FILES: dict[str, str] = { "sqlbuild_project.toml": dedent( @@ -189,3 +193,106 @@ def copy_project(*, origin_project: Path, tmp_path: Path) -> Path: project_dir: Path = tmp_path / origin_project.name shutil.copytree(origin_project, project_dir) return project_dir + + +MISSING_UPSTREAMS: tuple[str, ...] = ( + "customers", + "orders", + "payments", + "products", + "returns", + "shipments", +) + + +def plan_missing_upstreams( + *, tmp_path: Path, default_target: str, destination_allowed: bool, target_args: tuple[str, ...] +) -> tuple[Path, tuple[str, ...], subprocess.CompletedProcess[str]]: + """Build `MISSING_UPSTREAMS` in `prod`, then plan `report`, which reads them, in empty `dev`.""" + + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="missing_upstream_project", + repo_files={ + "sqlbuild_project.toml": ( + 'name = "missing_upstream_project"\nadapter = "duckdb"\n' + f'default_target = "{default_target}"\n\n' + '[connection]\ndatabase = "shop.duckdb"\n\n' + '[targets.prod]\nschema = "prod"\n\n' + "[targets.prod.clone]\nallow_as_clone_origin = true\n\n" + '[targets.dev]\nschema = "dev"\n\n' + "[targets.dev.clone]\n" + f"allow_as_clone_destination = {str(destination_allowed).lower()}\n" + ), + **{ + f"models/{name}.sql": ( + f"MODEL (description '{name}.', materialized table);\n\nSELECT 1 AS id\n" + ) + for name in MISSING_UPSTREAMS + }, + "models/report.sql": ( + "MODEL (description 'Report.', materialized table);\n\n" + 'SELECT customers.id FROM __ref("customers") AS customers\n' + + "".join( + f'JOIN __ref("{name}") AS {name} USING (id)\n' for name in MISSING_UPSTREAMS[1:] + ) + ), + }, + ) + built: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build", "--target", "prod"), project_dir=project_dir + ) + assert built.returncode == 0, built.stdout + built.stderr + plan_command: tuple[str, ...] = ("--no-color", "plan", "--select", "report", *target_args) + return ( + project_dir, + plan_command, + run_sqb(command=plan_command, project_dir=project_dir), + ) + + +def defer_clone_with_missing_origin( + *, tmp_path: Path, dropped_origin_table: str, before_clone: tuple[str, ...] = () +) -> tuple[subprocess.CompletedProcess[str], subprocess.CompletedProcess[str]]: + """Defer-clone `report` with one of its boundaries dropped in `prod`, then `plan --changed`.""" + + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="partial_defer_clone_project", + repo_files={ + "sqlbuild_project.toml": ( + 'name = "partial_defer_clone_project"\nadapter = "duckdb"\n' + 'default_target = "dev"\n\n[connection]\ndatabase = "shop.duckdb"\n\n' + '[targets.prod]\nschema = "prod"\n\n' + "[targets.prod.clone]\nallow_as_clone_origin = true\n\n" + '[targets.dev]\nschema = "dev"\n\n' + "[targets.dev.clone]\nallow_as_clone_destination = true\n" + ), + "models/orders.sql": ( + "MODEL (description 'Orders.', materialized table);\n\nSELECT 1 AS id\n" + ), + "models/customers.sql": ( + "MODEL (description 'Customers.', materialized table);\n\nSELECT 1 AS id\n" + ), + "models/report.sql": ( + "MODEL (description 'Report.', materialized table);\n\n" + 'SELECT o.id FROM __ref("orders") AS o JOIN __ref("customers") AS c USING (id)\n' + ), + }, + ) + for target in ("dev", "prod"): + built: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build", "--target", target), project_dir=project_dir + ) + assert built.returncode == 0, built.stdout + built.stderr + for sql in (f"DROP TABLE prod.{dropped_origin_table}", *before_clone): + execute_duckdb(db_path=project_dir / "shop.duckdb", sql=sql) + cloned: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build", "--select", "report", "--defer-clone-from", "prod"), + project_dir=project_dir, + ) + planned: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "plan", "--changed", "--select", "orders", "customers"), + project_dir=project_dir, + ) + return cloned, planned diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clone/test_clone.py b/tests/e2e/src/sqlbuild/cli/commands/main/clone/test_clone.py index 3d7dbd2ae1..2cf2258ba1 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/clone/test_clone.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clone/test_clone.py @@ -2,6 +2,8 @@ from __future__ import annotations +import json +import re import subprocess from pathlib import Path from textwrap import dedent @@ -9,18 +11,39 @@ import pytest from tests.e2e.src.sqlbuild.cli.commands.main.clone._test_types import ( + CloneChangedPlanE2ETestCase, CloneDeferredSourceFunctionE2ETestCase, CloneE2ETestCase, CloneFunctionGraphE2ETestCase, + CloneMissingUpstreamE2ETestCase, ClonePolicyErrorTestCase, ClonePythonFunctionE2ETestCase, + MissingUpstreamFixesE2ETestCase, + PartialDeferCloneE2ETestCase, + UnwritableConsumptionDeferCloneE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.clone.helpers import ( + MISSING_UPSTREAMS, + defer_clone_with_missing_origin, + plan_missing_upstreams, ) from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( + REPO_ROOT, prepare_inline_project, query_duckdb, run_sqb, ) +_CHANGED_CLONE_FIXTURE: Path = REPO_ROOT / "benchmarks/duckshop/edge_cases/changed_builds_clone" +_UPSTREAMS: tuple[str, ...] = ( + "customers", + "orders", + "payments", + "products", + "returns", + "shipments", +) + @pytest.mark.parametrize( "test_case", @@ -879,3 +902,248 @@ def test_given_denied_clone_policy_when_running_then_error_identifies_configurat assert result.returncode != 0 assert test_case.expected_error_code in combined_output assert test_case.expected_policy_key in combined_output + + +@pytest.mark.parametrize( + "test_case", + [ + CloneChangedPlanE2ETestCase( + description="a cloned table rebuilds and an untouched one stays skipped", + commands=(("clone", "--from", "prod", "--select", "fact_orders"),), + expected_fragments=("no completed record", "Unchanged (1): skipped by --changed"), + ), + CloneChangedPlanE2ETestCase( + description="a cloned table rebuilds only once", + commands=( + ("clone", "--from", "prod", "--select", "fact_orders"), + ("build", "--changed", "--select", "fact_orders", "dim_customers"), + ), + expected_fragments=("Unchanged (2): skipped by --changed",), + unexpected_fragments=("no completed record",), + ), + CloneChangedPlanE2ETestCase( + description="an upstream a build clones for its deferral rebuilds", + commands=(("build", "--select", "orders_report", "--defer-clone-from", "prod"),), + expected_fragments=("Prephase defer clone", "no completed record"), + ), + ], + ids=lambda case: case.description, +) +def test_given_cloned_table_when_planning_changed_then_it_rebuilds( + test_case: CloneChangedPlanE2ETestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="clone_changed_project", + repo_files={ + "sqlbuild_project.toml": dedent( + """ + name = "clone_changed_project" + adapter = "duckdb" + default_target = "dev" + + [connection] + database = "clone.duckdb" + + [targets.prod] + schema = "prod" + + [targets.prod.clone] + allow_as_clone_origin = true + + [targets.dev] + schema = "dev" + + [targets.dev.clone] + allow_as_clone_destination = true + """ + ).strip() + + "\n", + **{ + str(path.relative_to(_CHANGED_CLONE_FIXTURE)): path.read_text(encoding="utf-8") + for path in sorted(_CHANGED_CLONE_FIXTURE.glob("models/*.sql")) + }, + }, + ) + for command in ( + ("build", "--target", "dev"), + ("build", "--target", "prod"), + *test_case.commands, + ): + result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", *command), project_dir=project_dir + ) + assert result.returncode == 0, result.stdout + result.stderr + output: str = result.stdout + + plan: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "plan", "--changed", "--select", "fact_orders", "dim_customers"), + project_dir=project_dir, + ) + + assert plan.returncode == 0, plan.stdout + plan.stderr + for fragment in test_case.expected_fragments: + assert fragment in output + plan.stdout, output + plan.stdout + for fragment in test_case.unexpected_fragments: + assert fragment not in plan.stdout, plan.stdout + + +@pytest.mark.parametrize( + "test_case", + [ + MissingUpstreamFixesE2ETestCase( + description="a clone destination is offered deferring to and cloning from each origin", + destination_allowed=True, + expected_fragments=( + " If they exist in prod:\n" + " read them from it for this run: --defer-to prod\n" + " copy them into this target: sqb clone --from prod -s " + + " ".join(MISSING_UPSTREAMS) + + "\n Or build them too: add + before your selection", + ), + unexpected_fragments=(), + ), + MissingUpstreamFixesE2ETestCase( + description="a target that may not be cloned into is offered only building", + destination_allowed=False, + expected_fragments=(" Build them too: add + before your selection",), + unexpected_fragments=("sqb clone", "--defer-to"), + ), + ], + ids=lambda case: case.description, +) +def test_given_missing_upstreams_when_planning_then_s301_lists_them_with_fixes( + test_case: MissingUpstreamFixesE2ETestCase, + tmp_path: Path, +) -> None: + project_dir, plan_command, refused = plan_missing_upstreams( + tmp_path=tmp_path, + default_target="dev", + destination_allowed=test_case.destination_allowed, + target_args=(), + ) + + as_json: subprocess.CompletedProcess[str] = run_sqb( + command=(*plan_command, "--json"), project_dir=project_dir + ) + + output: str = refused.stdout + refused.stderr + assert refused.returncode == 1, output + assert "error[S301]: 6 upstream models this build reads don't exist in target 'dev'\n" in output + assert " returns\n ... 1 more (sqb plan --json lists all)\n" in output + assert all(fragment in output for fragment in test_case.expected_fragments), output + assert not any(fragment in output for fragment in test_case.unexpected_fragments), output + assert json.loads(as_json.stdout)["error"]["missing"] == list(MISSING_UPSTREAMS) + + +@pytest.mark.parametrize( + "test_case", + [ + CloneMissingUpstreamE2ETestCase( + description="the default target is cloned into without naming it", + default_target="dev", + target_args=(), + expected_clone_command="sqb clone --from prod -s " + " ".join(MISSING_UPSTREAMS), + ), + CloneMissingUpstreamE2ETestCase( + description="a target other than the default is named in the clone command", + default_target="prod", + target_args=("--target", "dev"), + expected_clone_command="sqb clone --from prod --to dev -s " + + " ".join(MISSING_UPSTREAMS), + ), + CloneMissingUpstreamE2ETestCase( + description="a default target that does not exist still yields a runnable clone", + default_target="retired", + target_args=("--target", "dev"), + expected_clone_command="sqb clone --from prod --to dev -s " + + " ".join(MISSING_UPSTREAMS), + ), + ], + ids=lambda case: case.description, +) +def test_given_missing_upstreams_when_running_printed_clone_then_plan_succeeds( + test_case: CloneMissingUpstreamE2ETestCase, + tmp_path: Path, +) -> None: + project_dir, plan_command, refused = plan_missing_upstreams( + tmp_path=tmp_path, + default_target=test_case.default_target, + destination_allowed=True, + target_args=test_case.target_args, + ) + printed: list[str] = re.findall( + r"^ +copy them into this target: +(sqb clone .+)$", refused.stdout + refused.stderr, re.M + ) + + cloned: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", *printed[0].split()[1:]), project_dir=project_dir + ) + planned: subprocess.CompletedProcess[str] = run_sqb( + command=plan_command, project_dir=project_dir + ) + + assert printed == [test_case.expected_clone_command], refused.stdout + refused.stderr + assert cloned.returncode == 0, cloned.stdout + cloned.stderr + assert planned.returncode == 0, planned.stdout + planned.stderr + + +@pytest.mark.parametrize( + "test_case", + [ + PartialDeferCloneE2ETestCase( + description="a boundary cloned before another failed still rebuilds", + dropped_origin_table="customers", + expected_fragments=( + "orders no completed record", + "Unchanged (1): skipped by --changed", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_partly_failed_defer_clone_when_planning_changed_then_cloned_boundary_rebuilds( + test_case: PartialDeferCloneE2ETestCase, + tmp_path: Path, +) -> None: + cloned, planned = defer_clone_with_missing_origin( + tmp_path=tmp_path, dropped_origin_table=test_case.dropped_origin_table + ) + + assert cloned.returncode == 1, cloned.stdout + cloned.stderr + assert "error[C411]" in cloned.stdout + cloned.stderr + assert planned.returncode == 0, planned.stdout + planned.stderr + assert all(fragment in planned.stdout for fragment in test_case.expected_fragments), ( + planned.stdout + ) + + +@pytest.mark.parametrize( + "test_case", + [ + UnwritableConsumptionDeferCloneE2ETestCase( + description="the failed boundary is reported, not the consumption write", + dropped_origin_table="customers", + before_clone=( + "DROP TABLE dev._sqlbuild_consumption", + "CREATE TABLE dev._sqlbuild_consumption (unexpected INTEGER)", + ), + expected_fragments=("error[C411]", "customers: missing in origin environment"), + ), + ], + ids=lambda case: case.description, +) +def test_given_unwritable_consumption_table_when_defer_clone_partly_fails_then_c411_is_reported( + test_case: UnwritableConsumptionDeferCloneE2ETestCase, + tmp_path: Path, +) -> None: + cloned, _ = defer_clone_with_missing_origin( + tmp_path=tmp_path, + dropped_origin_table=test_case.dropped_origin_table, + before_clone=test_case.before_clone, + ) + + output: str = cloned.stdout + cloned.stderr + assert cloned.returncode == 1, output + assert all(fragment in output for fragment in test_case.expected_fragments), output diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_buildability.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_buildability.py index 7c38e4b974..312e180d63 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_buildability.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_buildability.py @@ -173,14 +173,16 @@ def test_given_scope_and_snapshot_when_checking_buildability_then_returns_expect "test_case", [ MissingUpstreamMessageTestCase( - description="data-lineage misses keep the existing wording", + description="data-lineage misses are listed with how to build them", missing=( MissingUpstream(key=model_key("a"), required_by=(model_key("b"),)), MissingUpstream(key=model_key("c"), required_by=(model_key("b"),)), ), edge_origins={}, expected_message=( - "cannot build selected scope: 2 missing upstream dependencies (a, c)" + "2 upstream models this build reads don't exist in the warehouse\n" + " a\n c\n\n" + " Build them too: add + before your selection" ), ), MissingUpstreamMessageTestCase( @@ -211,7 +213,9 @@ def test_given_scope_and_snapshot_when_checking_buildability_then_returns_expect (model_key("stg_orders"), seed_key("waffle_types")): _RELATIONSHIPS_ORIGIN }, expected_message=( - "cannot build selected scope: 1 missing upstream dependencies (waffle_types)" + "1 upstream seed this build reads doesn't exist in the warehouse\n" + " waffle_types\n\n" + " Build it too: add + before your selection" ), ), MissingUpstreamMessageTestCase( @@ -228,7 +232,10 @@ def test_given_scope_and_snapshot_when_checking_buildability_then_returns_expect (model_key("orders"), seed_key("order_statuses")): _STATUS_ORIGIN, }, expected_message=( - "cannot build selected scope: 1 missing upstream dependencies (a); 2 audit reads " + "1 upstream model this build reads doesn't exist in the warehouse\n" + " a\n\n" + " Build it too: add + before your selection\n\n" + " Also, 2 audit reads " f"are not selected and do not exist in the warehouse ({_RELATIONSHIPS_ORIGIN}; " f"{_STATUS_ORIGIN}); select them or build them first" ), diff --git a/ui/src/lib/run/plan-text.test.ts b/ui/src/lib/run/plan-text.test.ts index c61eed4a05..6c48fed5eb 100644 --- a/ui/src/lib/run/plan-text.test.ts +++ b/ui/src/lib/run/plan-text.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest'; import type { PlanDocument, PlanModel } from '$lib/api'; -import { actionText, changedSections, jobRefreshText, looseWarnings, planSections, warningText } from './plan-text'; +import { actionText, changedSections, cloneCommands, jobRefreshText, looseWarnings, planSections, warningText } from './plan-text'; const model = (name: string, over: Partial = {}): PlanModel => ({ name, @@ -184,3 +184,25 @@ describe('plan under --changed', () => { }); }); + +describe('cloneCommands', () => { + it('returns each origin\'s clone command with every missing name, and nothing for other errors', () => { + const error = [ + "S301: 7 upstream models this build reads don't exist in target 'dev'", + ' a', + ' ... 2 more (sqb plan --json lists all)', + '', + ' If they exist in prod:', + ' read them from it for this run: --defer-to prod', + ' copy them into this target: sqb clone --from prod --to dev -s a b c d e f g', + ' If it exists in staging:', + ' copy it into this target: sqb clone --from staging -s a', + ' Or build them too: add + before your selection' + ].join('\n'); + expect(cloneCommands(error)).toEqual([ + { origin: 'prod', command: 'sqb clone --from prod --to dev -s a b c d e f g' }, + { origin: 'staging', command: 'sqb clone --from staging -s a' } + ]); + expect(cloneCommands('S007: unknown selector sqb clone --from prod')).toEqual([]); + }); +}); diff --git a/ui/src/lib/run/plan-text.ts b/ui/src/lib/run/plan-text.ts index 6b339e2cb4..a34d36e3ae 100644 --- a/ui/src/lib/run/plan-text.ts +++ b/ui/src/lib/run/plan-text.ts @@ -209,3 +209,8 @@ export function jobRefreshText(job: string, triggers: Pick t.type === 'sensor') ? `job ${job} · when its sensor fires` : `job ${job}`; } + +/** The `sqb clone` commands a plan error offers for missing upstreams (S301), each ready to copy as is. */ +export function cloneCommands(error: string): { origin: string; command: string }[] { + return [...error.matchAll(/^\s*copy .+?:\s+(sqb clone --from (\S+) .+)$/gm)].map((match) => ({ origin: match[2], command: match[1].trim() })); +} diff --git a/ui/src/routes/plan/+page.svelte b/ui/src/routes/plan/+page.svelte index 82c28f9e68..ffbf8b1f57 100644 --- a/ui/src/routes/plan/+page.svelte +++ b/ui/src/routes/plan/+page.svelte @@ -20,7 +20,8 @@ import PlanReport from '$lib/run/plan-report.svelte'; import StatusLabel from '$lib/components/list/status-label.svelte'; import { checkOutcomeLabel, checkOutcomeTone } from '$lib/status'; - import { builtModels, builtSeeds, changedModelKeys, jobRefreshText } from '$lib/run/plan-text'; + import { builtModels, builtSeeds, changedModelKeys, cloneCommands, jobRefreshText } from '$lib/run/plan-text'; + import CopyButton from '$lib/components/copy-button.svelte'; import { app } from '$lib/state/app.svelte'; import { createResource } from '$lib/state/resource.svelte'; @@ -323,7 +324,12 @@
Data (cmd = withFullRefresh(withWindow(cmd, c.window), c.fullRefresh))} />
{/if} - {#if problem}

{problem}

{/if} + {#if problem} +
+

{problem}

+ {#each cloneCommands(problem) as clone (clone.command)}{/each} +
+ {/if}
@@ -554,7 +560,10 @@ color: var(--sb-error); font-family: var(--font-mono); font-size: 12px; - white-space: pre-line; + } + .problem-bar p { + margin: 0; + white-space: pre-wrap; } .overlay { position: absolute; diff --git a/website/src/content/docs/docs/cli/plan.mdx b/website/src/content/docs/docs/cli/plan.mdx index a1a4d562c0..92b8fe5ebf 100644 --- a/website/src/content/docs/docs/cli/plan.mdx +++ b/website/src/content/docs/docs/cli/plan.mdx @@ -142,10 +142,29 @@ Release note: earlier versions printed `Plan complete with errors` and exited `0 Planning a scoped selection whose upstream inputs were never built in the target fails with `S301` rather than producing a plan that would break at build time: ``` -error[S301]: cannot build selected scope: 3 missing upstream dependencies (stg_orders, stg_payments, waffle_types) +error[S301]: 3 upstream models and seeds this build reads don't exist in target 'dev' + stg_orders + stg_payments + waffle_types + + If they exist in prod: + read them from it for this run: --defer-to prod + copy them into this target: sqb clone --from prod -s stg_orders stg_payments waffle_types + Or build them too: add + before your selection ``` -Build the upstream chain first, or select it along with the model: `sqb build --select +fact_orders`. +The error names the first five missing resources; `sqb plan --json` lists all of them in +`error.missing`. Each fix fits a different case: + +| Fix | When to use it | +| --- | --- | +| `--defer-to ` | Read the missing resources from another target for this run only. | +| `sqb clone --from -s ...` | Copy them into this target once. The command lists every missing name, so you can run it as printed. | +| `+` before the selection, such as `sqb build --select +fact_orders` | Build the upstream chain along with the model. | + +The error offers an origin only for targets with `clone.allow_as_clone_origin = true`, and only +when the current target has `clone.allow_as_clone_destination = true`. It does not check that the +resources exist in the origin. The check also covers resources that an attached audit on a selected resource reads, even when the model SQL does not read them. Selecting `+fact_orders` follows data lineage only, so it does not @@ -157,7 +176,7 @@ error[S301]: cannot build selected scope: audit 'relationships' on 'stg_orders' ``` Load or build the resource first (for a seed, `sqb seed`), or add it to the selection. When both -kinds are missing, the error lists the missing upstream dependencies and then the audit reads. +kinds are missing, the error lists the missing upstream resources and then the audit reads. ## Missing source tables