From feaf6230f641f67c3c722fbeef0afc1fa8079bf8 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 17 Nov 2026 19:26:42 +0000 Subject: [PATCH 1/3] fix(clone): leave replaced tables without a completed consumption record sqb clone (and a build's --defer-clone-from) replaced a destination table but kept its consumption rows, which describe the build the clone overwrote, so --changed could treat the cloned table as up to date with inputs it never read. A clone now appends a 'replaced' row for each table and seed it replaces; --changed reads that as no completed record and rebuilds the table once, and a later build supersedes the marker. --- .../models/dim_customers.sql | 3 + .../models/fact_orders.sql | 3 + .../models/orders_report.sql | 3 + .../native_host/_helpers/consumption_state.rs | 2 +- .../main/append_replaced_consumption.rs | 32 +++++++ .../src/native_host/main/mod.rs | 1 + .../_helpers/build_command/defer_clone.rs | 5 +- .../_helpers/clone/clone_execution.rs | 65 +++++++++++++- .../src/consumption/_helpers/decision.rs | 6 +- .../src/consumption/_helpers/history.rs | 12 ++- .../src/consumption/constants.rs | 2 + .../src/consumption/main/mod.rs | 1 + .../src/consumption/main/replaced_append.rs | 21 +++++ .../sqlbuild-state/src/consumption/models.rs | 2 + .../cli/commands/main/clone/_test_types.py | 10 +++ .../cli/commands/main/clone/test_clone.py | 89 +++++++++++++++++++ 16 files changed, 250 insertions(+), 7 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/changed_builds_clone/models/dim_customers.sql create mode 100644 benchmarks/duckshop/edge_cases/changed_builds_clone/models/fact_orders.sql create mode 100644 benchmarks/duckshop/edge_cases/changed_builds_clone/models/orders_report.sql create mode 100644 crates/sqlbuild-python-front/src/native_host/main/append_replaced_consumption.rs create mode 100644 crates/sqlbuild-state/src/consumption/main/replaced_append.rs diff --git a/benchmarks/duckshop/edge_cases/changed_builds_clone/models/dim_customers.sql b/benchmarks/duckshop/edge_cases/changed_builds_clone/models/dim_customers.sql new file mode 100644 index 0000000000..bfa81972e9 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/changed_builds_clone/models/dim_customers.sql @@ -0,0 +1,3 @@ +MODEL (description "Customers no clone touches, so --changed keeps skipping them.", materialized table); + +SELECT 101 AS customer_id diff --git a/benchmarks/duckshop/edge_cases/changed_builds_clone/models/fact_orders.sql b/benchmarks/duckshop/edge_cases/changed_builds_clone/models/fact_orders.sql new file mode 100644 index 0000000000..ba931f2eb7 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/changed_builds_clone/models/fact_orders.sql @@ -0,0 +1,3 @@ +MODEL (description "Orders a clone replaces in the destination target.", materialized table); + +SELECT 1 AS order_id diff --git a/benchmarks/duckshop/edge_cases/changed_builds_clone/models/orders_report.sql b/benchmarks/duckshop/edge_cases/changed_builds_clone/models/orders_report.sql new file mode 100644 index 0000000000..e85166ebe0 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/changed_builds_clone/models/orders_report.sql @@ -0,0 +1,3 @@ +MODEL (description "Orders read downstream of the cloned table.", materialized table); + +SELECT order_id FROM __ref("fact_orders") diff --git a/crates/sqlbuild-python-front/src/native_host/_helpers/consumption_state.rs b/crates/sqlbuild-python-front/src/native_host/_helpers/consumption_state.rs index 4414633a9f..3c72bab1e6 100644 --- a/crates/sqlbuild-python-front/src/native_host/_helpers/consumption_state.rs +++ b/crates/sqlbuild-python-front/src/native_host/_helpers/consumption_state.rs @@ -125,7 +125,7 @@ pub(crate) fn ensure_table( } /// Whether the warehouse catalog lists the consumption table in `location`. -fn table_exists( +pub(crate) fn table_exists( adapter: &Bound<'_, PyAny>, connection: &Bound<'_, PyAny>, location: &Location, diff --git a/crates/sqlbuild-python-front/src/native_host/main/append_replaced_consumption.rs b/crates/sqlbuild-python-front/src/native_host/main/append_replaced_consumption.rs new file mode 100644 index 0000000000..075e9464b6 --- /dev/null +++ b/crates/sqlbuild-python-front/src/native_host/main/append_replaced_consumption.rs @@ -0,0 +1,32 @@ +//! Leave relations replaced outside a recorded build without a completed consumption record. + +use std::collections::BTreeMap; + +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_state::consumption::main::replaced_append::replaced_append; +use sqlbuild_state::consumption::models::{ConsumptionAppend, NodeKey, RelationKey}; +use uuid::Uuid; + +use crate::native_host::_helpers::consumption_state::{append, table_exists}; +use crate::native_host::models::Location; + +/// Append a never-completed execution for each of `replaced` where its schema has a consumption table. +pub fn append_replaced_consumption( + (adapter, connection): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), + run_id: &str, + replaced: Vec<(NodeKey, RelationKey)>, +) -> PyResult<()> { + let mut rows: BTreeMap> = BTreeMap::new(); + for (node, relation) in replaced { + let execution_id: String = Uuid::now_v7().to_string(); + rows.entry(Location::of(&relation)) + .or_default() + .push(replaced_append((node, relation), (&execution_id, run_id))); + } + for (location, rows) in rows { + if table_exists(adapter, connection, &location)? { + append(adapter, connection, &location, &rows)?; + } + } + Ok(()) +} diff --git a/crates/sqlbuild-python-front/src/native_host/main/mod.rs b/crates/sqlbuild-python-front/src/native_host/main/mod.rs index effd6e652f..4cb7f09303 100644 --- a/crates/sqlbuild-python-front/src/native_host/main/mod.rs +++ b/crates/sqlbuild-python-front/src/native_host/main/mod.rs @@ -1,4 +1,5 @@ pub mod append_observations; +pub mod append_replaced_consumption; pub mod execute; pub mod fixed_state_table_sql; pub mod integer_literal; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs index 911e4772e6..5265b247bb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs @@ -19,6 +19,7 @@ use sqlbuild_host::entries::models::ExecutorHost; use crate::bindings::_helpers::build_command::invocation::BuildContext; use crate::bindings::_helpers::build_command::policies::build_command_error; use crate::bindings::_helpers::build_command::policies::enforce_model_execution_limit_binding; +use crate::bindings::_helpers::clone::clone_execution::forget_replaced_consumption; use crate::bindings::_helpers::clone::clone_pipeline::run_clone_pipeline_with_options; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; use sqlbuild_python_base::boundary::main::call_with::call_with; @@ -455,7 +456,7 @@ fn clone_boundaries<'py>( call_with( &host_entry(py, ExecutorHost::CopyCloneFingerprints)?, &[ - ("result", result), + ("result", result.clone()), ( "origin_model_entries", native_field(&pipeline, "origin_model_entries")?, @@ -481,7 +482,7 @@ fn clone_boundaries<'py>( ), ], )?; - Ok(()) + forget_replaced_consumption((&prephase.adapter, connection), &pipeline, &result) } /// Clone the selected boundary relations from the origin before build planning. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_execution.rs index 5adac1f62d..02087f26f2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_execution.rs @@ -1,6 +1,6 @@ //! Execute the prepared clone plan and propagate fingerprints. -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::time::Instant; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; @@ -13,6 +13,7 @@ use crate::bindings::_helpers::clone::clone_command::Invocation; use crate::bindings::_helpers::clone::clone_progress::{ CloneFingerprintReporter, CloneItemPrinter, }; +use sqlbuild_analysis::fingerprint_storage::constants::{NODE_TYPE_MODEL, NODE_TYPE_SEED}; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::within::within; use sqlbuild_python_base::native_host::main::host_entry::host_entry; @@ -23,7 +24,10 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::contracts::main::resolve_target_config::resolve_target_config; +use sqlbuild_python_front::native_host::main::append_replaced_consumption::append_replaced_consumption; use sqlbuild_python_output::result_output::models::ExecutionEventWriter; +use sqlbuild_python_planner::consumption::main::relation::relation; +use sqlbuild_state::consumption::models::{NodeKey, RelationKey}; const DESTINATION_ENTRY_GROUPS: [&str; 4] = [ "destination_source_entries", @@ -37,6 +41,13 @@ const ORIGIN_ENTRY_GROUPS: [&str; 3] = [ "origin_seed_entries", ]; +/// A clone item that finished cleanly. +const SUCCESS_STATUS: &str = "success"; +/// Clone actions that replace the destination relation with the origin's data. +const REPLACING_ACTIONS: [&str; 2] = ["cloned", "copied"]; +/// The materialization of a model the clone recreates as a view rather than replacing. +const VIEW_MATERIALIZATION: &str = "view"; + /// The pipeline's plan entries of `groups`, in order. fn entries<'py>(pipeline: &Bound<'py, PyAny>, groups: &[&str]) -> PyResult>> { let mut found: Vec> = Vec::new(); @@ -310,6 +321,57 @@ fn copy_fingerprints<'py>( Ok(()) } +/// Mark each table and seed the clone replaced as having no completed consumption record, so `--changed` rebuilds it once. +pub(crate) fn forget_replaced_consumption( + (adapter, connection): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), + pipeline: &Bound<'_, PyAny>, + result: &Bound<'_, PyAny>, +) -> PyResult<()> { + let mut replaced_names: HashSet = HashSet::new(); + for item in native_field(result, "item_results")?.try_iter()? { + let item: Bound<'_, PyAny> = item?; + let status: String = native_field(&native_field(&item, "status")?, "value")?.extract()?; + let action: String = native_field(&native_field(&item, "action")?, "value")?.extract()?; + if status == SUCCESS_STATUS && REPLACING_ACTIONS.contains(&action.as_str()) { + replaced_names.insert(native_field(&item, "name")?.extract()?); + } + } + let mut replaced: Vec<(NodeKey, RelationKey)> = Vec::new(); + for (group, kind) in [ + ("destination_model_entries", NODE_TYPE_MODEL), + ("destination_seed_entries", NODE_TYPE_SEED), + ] { + for entry in native_field(pipeline, group)?.try_iter()? { + let entry: Bound<'_, PyAny> = entry?; + let name: String = native_field(&entry, "name")?.extract()?; + let destination: Bound<'_, PyAny> = native_field(&entry, "destination")?; + let view: bool = kind == NODE_TYPE_MODEL + && native_field(&native_field(&entry, "materialization_type")?, "value")? + .extract::()? + == VIEW_MATERIALIZATION; + if view + || !replaced_names.contains(&name) + || native_field(&destination, "schema")?.is_none() + { + continue; + } + replaced.push(( + NodeKey { + kind: kind.to_owned(), + name, + }, + relation(&destination)?, + )); + } + } + let project: Bound<'_, PyAny> = native_field(pipeline, "destination_project")?; + append_replaced_consumption( + (adapter, connection), + &native_field(&project, "run_id")?.extract::()?, + replaced, + ) +} + /// Execute the clone, streaming items, then copy fingerprints; returns the result and elapsed seconds. pub(crate) fn execute_clone_plan<'py>( request: &Bound<'py, PyAny>, @@ -352,5 +414,6 @@ pub(crate) fn execute_clone_plan<'py>( let result: Bound<'py, PyAny> = executed?; closed?; copy_fingerprints(invocation, (pipeline, connection), &result)?; + forget_replaced_consumption((&invocation.adapter, connection), pipeline, &result)?; Ok((result, started.elapsed().as_secs_f64())) } diff --git a/crates/sqlbuild-state/src/consumption/_helpers/decision.rs b/crates/sqlbuild-state/src/consumption/_helpers/decision.rs index af9ed420a8..0c8abea530 100644 --- a/crates/sqlbuild-state/src/consumption/_helpers/decision.rs +++ b/crates/sqlbuild-state/src/consumption/_helpers/decision.rs @@ -235,7 +235,11 @@ fn execution_changes(node: &NodeFacts, context: &Context<'_>, window: &Window<'_ let mut changes: Vec = Vec::new(); for execution in window.executions { if !execution.complete { - return vec![always(RunReason::Unfinished)]; + return vec![always(if execution.replaced { + RunReason::NoRecord + } else { + RunReason::Unfinished + })]; } if execution.version_hash.as_deref() != Some(node.version_hash.as_str()) { changes.push(definition(RunReason::DefinitionChanged)); diff --git a/crates/sqlbuild-state/src/consumption/_helpers/history.rs b/crates/sqlbuild-state/src/consumption/_helpers/history.rs index e749ba91c3..c557c12cb8 100644 --- a/crates/sqlbuild-state/src/consumption/_helpers/history.rs +++ b/crates/sqlbuild-state/src/consumption/_helpers/history.rs @@ -2,7 +2,7 @@ use std::collections::{BTreeMap, BTreeSet, HashMap}; -use crate::consumption::constants::{ORDER_SKEW_MICROS, SUCCEEDED_PHASE}; +use crate::consumption::constants::{ORDER_SKEW_MICROS, REPLACED_PHASE, SUCCEEDED_PHASE}; use crate::consumption::models::{ ConsumedInput, ConsumptionHistory, ConsumptionRow, Execution, RelationHistory, RelationKey, }; @@ -18,7 +18,7 @@ struct ExecutionRows<'a> { /// A retried append repeats a row: its execution, phase and the input it records. type RowIdentity<'a> = (&'a str, &'a str, Option<(&'a str, &'a str, &'a str)>); -/// Recent executions per relation, from rows read newest first with at most `limit` per relation. +/// Recent executions per relation (rows newest first, at most `limit` each); a replacement marker counts only while newest. pub(crate) fn build_history(rows: &[ConsumptionRow], limit: usize) -> ConsumptionHistory { let mut relations: HashMap = HashMap::new(); for (relation, execution) in executions_of(rows) { @@ -33,6 +33,13 @@ pub(crate) fn build_history(rows: &[ConsumptionRow], limit: usize) -> Consumptio (right.started_at, &right.execution_id).cmp(&(left.started_at, &left.execution_id)) }); history.truncated = history.executions.len() >= limit; + let newest: Option = history + .executions + .first() + .map(|execution| execution.execution_id.clone()); + history.executions.retain(|execution| { + !execution.replaced || Some(&execution.execution_id) == newest.as_ref() + }); } ConsumptionHistory { relations, @@ -124,6 +131,7 @@ fn execution(execution_id: &str, rows: &ExecutionRows<'_>) -> Execution { execution_id: execution_id.to_owned(), started_at: rows.earliest, complete, + replaced: rows.rows.iter().all(|row| row.row.phase == REPLACED_PHASE), version_hash: (versions.len() == 1) .then(|| versions.first().map(|version| (*version).to_owned())) .flatten(), diff --git a/crates/sqlbuild-state/src/consumption/constants.rs b/crates/sqlbuild-state/src/consumption/constants.rs index 427d51b454..b511e5876d 100644 --- a/crates/sqlbuild-state/src/consumption/constants.rs +++ b/crates/sqlbuild-state/src/consumption/constants.rs @@ -34,6 +34,8 @@ pub const NULLABLE_COLUMNS: [&str; 4] = [ pub const STARTED_PHASE: &str = "started"; /// Phase of the rows appended after the write succeeded. pub const SUCCEEDED_PHASE: &str = "succeeded"; +/// Phase of the row appended when something other than a recorded build replaced the relation. +pub const REPLACED_PHASE: &str = "replaced"; /// Input kind of a source, versioned by its observed `data_version_hash`. pub const SOURCE_INPUT: &str = "source"; /// Input kind of an upstream model or seed, versioned by its `execution_id`. diff --git a/crates/sqlbuild-state/src/consumption/main/mod.rs b/crates/sqlbuild-state/src/consumption/main/mod.rs index 0fa927d55d..766e6e029c 100644 --- a/crates/sqlbuild-state/src/consumption/main/mod.rs +++ b/crates/sqlbuild-state/src/consumption/main/mod.rs @@ -6,5 +6,6 @@ pub mod prune_executions_sql; pub mod read_back_history; pub mod recent_executions; pub mod recent_executions_sql; +pub mod replaced_append; pub mod started_append; pub mod succeeded_appends; diff --git a/crates/sqlbuild-state/src/consumption/main/replaced_append.rs b/crates/sqlbuild-state/src/consumption/main/replaced_append.rs new file mode 100644 index 0000000000..a8fe49645a --- /dev/null +++ b/crates/sqlbuild-state/src/consumption/main/replaced_append.rs @@ -0,0 +1,21 @@ +//! The row appended for a relation replaced outside a recorded build. + +use crate::consumption::constants::{REPLACED_PHASE, UNKNOWN_VERSION}; +use crate::consumption::models::{ConsumptionAppend, NodeKey, RelationKey}; + +/// The `replaced` row of a new execution that never completes, so `relation` has no completed record. +pub fn replaced_append( + (node, relation): (NodeKey, RelationKey), + (execution_id, run_id): (&str, &str), +) -> ConsumptionAppend { + ConsumptionAppend { + execution_id: execution_id.to_owned(), + phase: REPLACED_PHASE.to_owned(), + run_id: run_id.to_owned(), + node, + relation, + version_hash: UNKNOWN_VERSION.to_owned(), + input_count: 0, + input: None, + } +} diff --git a/crates/sqlbuild-state/src/consumption/models.rs b/crates/sqlbuild-state/src/consumption/models.rs index a5b8ce8560..745afc921e 100644 --- a/crates/sqlbuild-state/src/consumption/models.rs +++ b/crates/sqlbuild-state/src/consumption/models.rs @@ -152,6 +152,8 @@ pub struct Execution { pub started_at: i64, /// Whether its `succeeded` rows are present, consistent and complete. pub complete: bool, + /// Whether its rows only mark the relation as replaced outside a recorded build. + pub replaced: bool, pub version_hash: Option, /// Input versions of a complete execution, keyed by `(kind, name)`. pub inputs: Vec, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clone/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/clone/_test_types.py index b1ca648fe1..461de7cb85 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/clone/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clone/_test_types.py @@ -74,3 +74,13 @@ class CloneTimingsE2ETestCase: description: str expected_phases: tuple[str, ...] + + +@dataclass(frozen=True) +class CloneChangedPlanE2ETestCase: + """Commands run after building both targets, and what `plan --changed` says after them.""" + + description: str + commands: tuple[tuple[str, ...], ...] + expected_fragments: tuple[str, ...] + unexpected_fragments: tuple[str, ...] = () diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clone/test_clone.py b/tests/e2e/src/sqlbuild/cli/commands/main/clone/test_clone.py index 3d7dbd2ae1..4c4c7a0a76 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/clone/test_clone.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clone/test_clone.py @@ -9,6 +9,7 @@ import pytest from tests.e2e.src.sqlbuild.cli.commands.main.clone._test_types import ( + CloneChangedPlanE2ETestCase, CloneDeferredSourceFunctionE2ETestCase, CloneE2ETestCase, CloneFunctionGraphE2ETestCase, @@ -16,11 +17,14 @@ ClonePythonFunctionE2ETestCase, ) from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( + REPO_ROOT, prepare_inline_project, query_duckdb, run_sqb, ) +_CHANGED_CLONE_FIXTURE: Path = REPO_ROOT / "benchmarks/duckshop/edge_cases/changed_builds_clone" + @pytest.mark.parametrize( "test_case", @@ -879,3 +883,88 @@ def test_given_denied_clone_policy_when_running_then_error_identifies_configurat assert result.returncode != 0 assert test_case.expected_error_code in combined_output assert test_case.expected_policy_key in combined_output + + +@pytest.mark.parametrize( + "test_case", + [ + CloneChangedPlanE2ETestCase( + description="a cloned table rebuilds and an untouched one stays skipped", + commands=(("clone", "--from", "prod", "--select", "fact_orders"),), + expected_fragments=("no completed record", "Unchanged (1): skipped by --changed"), + ), + CloneChangedPlanE2ETestCase( + description="a cloned table rebuilds only once", + commands=( + ("clone", "--from", "prod", "--select", "fact_orders"), + ("build", "--changed", "--select", "fact_orders", "dim_customers"), + ), + expected_fragments=("Unchanged (2): skipped by --changed",), + unexpected_fragments=("no completed record",), + ), + CloneChangedPlanE2ETestCase( + description="an upstream a build clones for its deferral rebuilds", + commands=(("build", "--select", "orders_report", "--defer-clone-from", "prod"),), + expected_fragments=("Prephase defer clone", "no completed record"), + ), + ], + ids=lambda case: case.description, +) +def test_given_cloned_table_when_planning_changed_then_it_rebuilds( + test_case: CloneChangedPlanE2ETestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="clone_changed_project", + repo_files={ + "sqlbuild_project.toml": dedent( + """ + name = "clone_changed_project" + adapter = "duckdb" + default_target = "dev" + + [connection] + database = "clone.duckdb" + + [targets.prod] + schema = "prod" + + [targets.prod.clone] + allow_as_clone_origin = true + + [targets.dev] + schema = "dev" + + [targets.dev.clone] + allow_as_clone_destination = true + """ + ).strip() + + "\n", + **{ + str(path.relative_to(_CHANGED_CLONE_FIXTURE)): path.read_text(encoding="utf-8") + for path in sorted(_CHANGED_CLONE_FIXTURE.glob("models/*.sql")) + }, + }, + ) + for command in ( + ("build", "--target", "dev"), + ("build", "--target", "prod"), + *test_case.commands, + ): + result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", *command), project_dir=project_dir + ) + assert result.returncode == 0, result.stdout + result.stderr + output: str = result.stdout + + plan: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "plan", "--changed", "--select", "fact_orders", "dim_customers"), + project_dir=project_dir, + ) + + assert plan.returncode == 0, plan.stdout + plan.stderr + for fragment in test_case.expected_fragments: + assert fragment in output + plan.stdout, output + plan.stdout + for fragment in test_case.unexpected_fragments: + assert fragment not in plan.stdout, plan.stdout From 834cbb35726aab9cbd34d4ca7f73cf36a540c1df Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 17 Nov 2026 19:34:11 +0000 Subject: [PATCH 2/3] feat(plan): say how to fix missing upstreams in S301 and list them all S301 named at most five missing upstreams and stopped. It now lists the first five, says how many more there are, and offers each fix: defer to or clone from every target configured as a clone origin for this one (with a copyable `sqb clone` command naming every missing resource), or build them with `+`. Origins come from configuration only. `sqb plan --json` lists every missing name in `error.missing`, and the plan page shows the message as printed with a copy button per clone command. --- .../main/missing_upstream_message.rs | 147 ++++++++++++++---- .../src/planner_identity/models.rs | 11 ++ .../tests/test_planner_identity.rs | 56 ++++++- .../src/planner_identity/tests/test_types.rs | 3 +- .../_helpers/compiler_errors/constants.rs | 2 +- .../_helpers/native_classes/construct.rs | 13 ++ .../src/records/main/construct/mod.rs | 1 + .../main/construct/new_error_with_fields.rs | 21 +++ .../_helpers/outputs/buildability.rs | 71 ++++++++- .../src/planner_identity/_helpers/graph.rs | 9 +- .../src/bindings/_helpers/cli/entry.rs | 17 +- .../sqlbuild/references/docs/cli/plan.md | 25 ++- src/sqlbuild/_native.pyi | 8 +- .../main/build/test_terminal_failure.py | 2 +- .../cli/commands/main/clone/_test_types.py | 20 +++ .../cli/commands/main/clone/helpers.py | 56 +++++++ .../cli/commands/main/clone/test_clone.py | 109 +++++++++++++ .../planner/_helpers/test_buildability.py | 15 +- ui/src/lib/run/plan-text.test.ts | 24 ++- ui/src/lib/run/plan-text.ts | 5 + ui/src/routes/plan/+page.svelte | 15 +- website/src/content/docs/docs/cli/plan.mdx | 25 ++- 22 files changed, 586 insertions(+), 69 deletions(-) create mode 100644 crates/sqlbuild-python-base/src/records/main/construct/new_error_with_fields.rs diff --git a/crates/sqlbuild-analysis/src/planner_identity/main/missing_upstream_message.rs b/crates/sqlbuild-analysis/src/planner_identity/main/missing_upstream_message.rs index 5afe8280f0..61ca107a5e 100644 --- a/crates/sqlbuild-analysis/src/planner_identity/main/missing_upstream_message.rs +++ b/crates/sqlbuild-analysis/src/planner_identity/main/missing_upstream_message.rs @@ -3,14 +3,16 @@ use std::collections::HashMap; use crate::planner_identity::constants::MISSING_MESSAGE_ITEM_LIMIT; -use crate::planner_identity::models::{GraphKey, MissingUpstream}; +use crate::planner_identity::models::{GraphKey, MissingUpstream, MissingUpstreamFixes}; -/// Missing lineage upstreams by name and, when only an audit or hook read requires one, that read's origin. +/// Missing lineage upstreams with how to fix them and, when only an audit or hook read requires one, that read's origin. pub fn missing_upstream_message( missing: &[MissingUpstream], edge_origins: &HashMap<(GraphKey, GraphKey), String>, + fixes: &MissingUpstreamFixes, ) -> String { - let mut lineage_missing: Vec<&MissingUpstream> = Vec::new(); + let mut lineage_missing: Vec<&str> = Vec::new(); + let mut kinds: std::collections::BTreeSet<&str> = std::collections::BTreeSet::new(); let mut read_names: Vec<&str> = Vec::new(); for entry in missing { let origins: Vec<&str> = entry @@ -23,7 +25,8 @@ pub fn missing_upstream_message( }) .collect(); if origins.len() < entry.required_by.len() { - lineage_missing.push(entry); + lineage_missing.push(entry.key.name.as_str()); + kinds.insert(entry.key.kind.as_str()); continue; } for origin in origins { @@ -32,39 +35,119 @@ pub fn missing_upstream_message( } } } - let mut parts: Vec = Vec::new(); - if !lineage_missing.is_empty() { - let names: Vec<&str> = lineage_missing + let reads: Option = audit_reads_text(&read_names); + if lineage_missing.is_empty() { + return format!("cannot build selected scope: {}", reads.unwrap_or_default()); + } + let mut text: String = lineage_text(&lineage_missing, &kinds, fixes); + if let Some(reads) = reads { + text.push_str(&format!("\n\n Also, {reads}")); + } + text +} + +/// The missing lineage upstreams, a few by name, then how to read, clone or build them. +fn lineage_text( + names: &[&str], + kinds: &std::collections::BTreeSet<&str>, + fixes: &MissingUpstreamFixes, +) -> String { + let one: bool = names.len() == 1; + let noun: String = match kinds.iter().collect::>().as_slice() { + [kind] if one => (*kind).to_string(), + [kind] => format!("{kind}s"), + _ => "models and seeds".to_owned(), + }; + let place: String = fixes.target.as_ref().map_or_else( + || "the warehouse".to_owned(), + |target| format!("target '{target}'"), + ); + let mut lines: Vec = vec![format!( + "{} upstream {noun} this build reads {} exist in {place}", + names.len(), + if one { "doesn't" } else { "don't" } + )]; + lines.extend( + names .iter() .take(MISSING_MESSAGE_ITEM_LIMIT) - .map(|entry| entry.key.name.as_str()) - .collect(); - parts.push(format!( - "{} missing upstream dependencies ({})", - lineage_missing.len(), - names.join(", ") + .map(|name| format!(" {name}")), + ); + if names.len() > MISSING_MESSAGE_ITEM_LIMIT { + lines.push(format!( + " ... {} more (sqb plan --json lists all)", + names.len() - MISSING_MESSAGE_ITEM_LIMIT )); } - if let [name] = read_names.as_slice() { - parts.push(format!( - "{name}, which is not selected and does not exist in the warehouse; select it or build it first" - )); - } else if !read_names.is_empty() { - let shown: String = read_names - .iter() - .take(MISSING_MESSAGE_ITEM_LIMIT) - .copied() - .collect::>() - .join("; "); - let more: &str = if read_names.len() > MISSING_MESSAGE_ITEM_LIMIT { - "; ..." + lines.push(String::new()); + let (exists, them): (&str, &str) = if one { + ("it exists", "it") + } else { + ("they exist", "them") + }; + let read_label: String = format!("read {them} from it for this run:"); + let copy_label: String = format!("copy {them} into this target:"); + let build_label: String = format!( + "{} {them} too:", + if fixes.origins.is_empty() { + "Build" } else { - "" - }; - parts.push(format!( - "{} audit reads are not selected and do not exist in the warehouse ({shown}{more}); select them or build them first", - read_names.len() + "Or build" + } + ); + let width: usize = if fixes.origins.is_empty() { + build_label.len() + 2 + } else { + (read_label.len() + 4).max(copy_label.len() + 4) + } + .max(build_label.len() + 2); + let to: String = fixes + .clone_to + .as_ref() + .map(|target| format!(" --to {target}")) + .unwrap_or_default(); + for origin in &fixes.origins { + lines.push(format!(" If {exists} in {origin}:")); + lines.push(format!( + " {read_label:<0$} --defer-to {origin}", + width - 4 )); + lines.push(format!( + " {copy_label:<0$} sqb clone --from {origin}{to} -s {1}", + width - 4, + names.join(" ") + )); + } + lines.push(format!( + " {build_label:<0$} add + before your selection", + width - 2 + )); + lines.join("\n") +} + +/// The audit and hook reads that need a missing upstream, a few by name. +fn audit_reads_text(read_names: &[&str]) -> Option { + match read_names { + [] => None, + [name] => Some(format!( + "{name}, which is not selected and does not exist in the warehouse; select it or build it first" + )), + _ => { + let shown: String = read_names + .iter() + .take(MISSING_MESSAGE_ITEM_LIMIT) + .copied() + .collect::>() + .join("; "); + let more: &str = if read_names.len() > MISSING_MESSAGE_ITEM_LIMIT { + "; ..." + } else { + "" + }; + Some(format!( + "{} audit reads are not selected and do not exist in the warehouse ({shown}{more}); select them or build them first", + read_names.len() + )) + } } - format!("cannot build selected scope: {}", parts.join("; ")) } diff --git a/crates/sqlbuild-analysis/src/planner_identity/models.rs b/crates/sqlbuild-analysis/src/planner_identity/models.rs index f11d73b77b..d4b39ac024 100644 --- a/crates/sqlbuild-analysis/src/planner_identity/models.rs +++ b/crates/sqlbuild-analysis/src/planner_identity/models.rs @@ -87,6 +87,17 @@ pub struct AvailableUpstreams { pub deferred_relations: Option>, } +/// What a missing-upstream message offers to fix the misses with, all from configuration. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct MissingUpstreamFixes { + /// The active target, named as where the upstreams are missing. + pub target: Option, + /// Targets the active target may clone from, to defer to or clone the upstreams from. + pub origins: Vec, + /// The clone command's `--to`, needed when the active target is not the default one. + pub clone_to: Option, +} + /// One upstream dependency missing from both scope and warehouse, with its dependents. #[derive(Clone, Debug, PartialEq, Eq)] pub struct MissingUpstream { diff --git a/crates/sqlbuild-analysis/src/planner_identity/tests/test_planner_identity.rs b/crates/sqlbuild-analysis/src/planner_identity/tests/test_planner_identity.rs index 6426f00d87..7bfc536891 100644 --- a/crates/sqlbuild-analysis/src/planner_identity/tests/test_planner_identity.rs +++ b/crates/sqlbuild-analysis/src/planner_identity/tests/test_planner_identity.rs @@ -10,8 +10,8 @@ use crate::planner_identity::main::missing_upstreams::missing_upstreams; use crate::planner_identity::main::stale_upstream_warning_message::stale_upstream_warning_message; use crate::planner_identity::main::topological_order::topological_order; use crate::planner_identity::models::{ - AvailableUpstreams, FunctionFingerprintFields, GraphKey, MissingUpstream, StaleWarning, - StalenessNames, + AvailableUpstreams, FunctionFingerprintFields, GraphKey, MissingUpstream, MissingUpstreamFixes, + StaleWarning, StalenessNames, }; use crate::planner_identity::tests::helpers::{ chain_head_hash, chain_identity_graph, chain_staleness_graph, cyclic_identity_graph, @@ -406,11 +406,58 @@ fn given_selected_keys_when_checking_buildability_then_reports_missing_upstreams ..AvailableUpstreams::default() }, origins: HashMap::new(), + fixes: MissingUpstreamFixes::default(), expected_missing: vec![MissingUpstream { key: model("raw"), required_by: vec![model("a"), model("b")], }], - expected_message: "cannot build selected scope: 1 missing upstream dependencies (raw)", + expected_message: concat!( + "1 upstream model this build reads doesn't exist in the warehouse\n", + " raw\n", + "\n", + " Build it too: add + before your selection", + ), + }, + BuildabilityTestCase { + description: "more misses than shown, with clone origins and a non-default target", + selected: vec![model("report")], + upstream: HashMap::from([( + model("report"), + ["f", "a", "e", "b", "d", "c"] + .map(model) + .into_iter() + .chain([key("seed", "codes")]) + .collect(), + )]), + available: AvailableUpstreams::default(), + origins: HashMap::new(), + fixes: MissingUpstreamFixes { + target: Some("dev".to_owned()), + origins: vec!["prod".to_owned(), "staging".to_owned()], + clone_to: Some("dev".to_owned()), + }, + expected_missing: ["a", "b", "c", "d", "e", "f"] + .map(model) + .into_iter() + .chain([key("seed", "codes")]) + .map(|key| MissingUpstream { + key, + required_by: vec![model("report")], + }) + .collect(), + expected_message: concat!( + "7 upstream models and seeds this build reads don't exist in target 'dev'\n", + " a\n b\n c\n d\n e\n", + " ... 2 more (sqb plan --json lists all)\n", + "\n", + " If they exist in prod:\n", + " read them from it for this run: --defer-to prod\n", + " copy them into this target: sqb clone --from prod --to dev -s a b c d e f codes\n", + " If they exist in staging:\n", + " read them from it for this run: --defer-to staging\n", + " copy them into this target: sqb clone --from staging --to dev -s a b c d e f codes\n", + " Or build them too: add + before your selection", + ), }, BuildabilityTestCase { description: "only an audit read requires the upstream", @@ -421,6 +468,7 @@ fn given_selected_keys_when_checking_buildability_then_reports_missing_upstreams (model("a"), model("lookup")), "audit 'check' on 'a' reads 'lookup'".to_owned(), )]), + fixes: MissingUpstreamFixes::default(), expected_missing: vec![MissingUpstream { key: model("lookup"), required_by: vec![model("a")], @@ -436,7 +484,7 @@ fn given_selected_keys_when_checking_buildability_then_reports_missing_upstreams ); assert_eq!( ( - missing_upstream_message(&missing, &test_case.origins), + missing_upstream_message(&missing, &test_case.origins, &test_case.fixes), missing ), ( diff --git a/crates/sqlbuild-analysis/src/planner_identity/tests/test_types.rs b/crates/sqlbuild-analysis/src/planner_identity/tests/test_types.rs index 9c701080c4..3403315861 100644 --- a/crates/sqlbuild-analysis/src/planner_identity/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/planner_identity/tests/test_types.rs @@ -2,7 +2,7 @@ use std::collections::{HashMap, HashSet}; use crate::planner_identity::models::{ AvailableUpstreams, FunctionFingerprintFields, GraphKey, IdentityGraph, MissingUpstream, - StaleWarning, StalenessGraph, + MissingUpstreamFixes, StaleWarning, StalenessGraph, }; use crate::planner_identity::types::IdentityHashes; @@ -31,6 +31,7 @@ pub(super) struct BuildabilityTestCase { pub(super) upstream: HashMap>, pub(super) available: AvailableUpstreams, pub(super) origins: HashMap<(GraphKey, GraphKey), String>, + pub(super) fixes: MissingUpstreamFixes, pub(super) expected_missing: Vec, pub(super) expected_message: &'static str, } diff --git a/crates/sqlbuild-python-base/src/records/_helpers/compiler_errors/constants.rs b/crates/sqlbuild-python-base/src/records/_helpers/compiler_errors/constants.rs index 2a3bfbcd99..981e9522e6 100644 --- a/crates/sqlbuild-python-base/src/records/_helpers/compiler_errors/constants.rs +++ b/crates/sqlbuild-python-base/src/records/_helpers/compiler_errors/constants.rs @@ -398,7 +398,7 @@ pub(crate) const COMPILER_ERRORS: &[ErrorSpec] = &[ doc: "Raised when planner inputs cannot be resolved safely.", class_attributes: &[("code", ClassValue::Text("S000"))], required: &[], - optional: &[], + optional: &["missing"], fixed: &[], }, ErrorSpec { diff --git a/crates/sqlbuild-python-base/src/records/_helpers/native_classes/construct.rs b/crates/sqlbuild-python-base/src/records/_helpers/native_classes/construct.rs index d10b9621ac..1a1ced149a 100644 --- a/crates/sqlbuild-python-base/src/records/_helpers/native_classes/construct.rs +++ b/crates/sqlbuild-python-base/src/records/_helpers/native_classes/construct.rs @@ -60,7 +60,17 @@ pub(crate) fn new_error( pub(crate) fn exception_from( class: PyResult>, message: &str, + keywords: (Option<&str>, Option<&str>), +) -> PyErr { + exception_with_fields(class, message, keywords, &[]) +} + +/// The exception `class` with `message`, the given `code` and `help`, and its declared `fields`, or the error raising it. +pub(crate) fn exception_with_fields<'py>( + class: PyResult>, + message: &str, (code, help): (Option<&str>, Option<&str>), + fields: &[(&str, Bound<'py, PyAny>)], ) -> PyErr { class .and_then(|class| { @@ -71,6 +81,9 @@ pub(crate) fn exception_from( if let Some(help) = help { keywords.set_item("help", help)?; } + for (name, value) in fields { + keywords.set_item(*name, value)?; + } class.call((message,), Some(&keywords)) }) .map_or_else(|error| error, PyErr::from_value) diff --git a/crates/sqlbuild-python-base/src/records/main/construct/mod.rs b/crates/sqlbuild-python-base/src/records/main/construct/mod.rs index 393ad76d31..f18b59054c 100644 --- a/crates/sqlbuild-python-base/src/records/main/construct/mod.rs +++ b/crates/sqlbuild-python-base/src/records/main/construct/mod.rs @@ -4,6 +4,7 @@ pub mod error_class; pub mod exception_from; pub mod new_annotated_error; pub mod new_error; +pub mod new_error_with_fields; pub mod new_plain_error; pub mod new_record; pub mod new_record_with; diff --git a/crates/sqlbuild-python-base/src/records/main/construct/new_error_with_fields.rs b/crates/sqlbuild-python-base/src/records/main/construct/new_error_with_fields.rs new file mode 100644 index 0000000000..b7aae8a7d4 --- /dev/null +++ b/crates/sqlbuild-python-base/src/records/main/construct/new_error_with_fields.rs @@ -0,0 +1,21 @@ +//! The native `id` exception raised with `message`, its `code` and `help`, and declared keyword fields. + +use crate::records::_helpers::native_classes::construct; +use crate::records::types::ErrorId; +use pyo3::prelude::{Bound, PyAny, PyErr, Python}; + +/// The native `id` exception with `message`, the given `code` and `help`, and its declared `fields`, or the error raising it. +#[inline] +pub fn new_error_with_fields<'py>( + py: Python<'py>, + id: ErrorId, + (message, code, help): (&str, Option<&str>, Option<&str>), + fields: &[(&str, Bound<'py, PyAny>)], +) -> PyErr { + construct::exception_with_fields( + construct::error_class(py, id), + message, + (code, help), + fields, + ) +} diff --git a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/buildability.rs b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/buildability.rs index 76c81a61b9..3aecf62cdd 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/buildability.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/buildability.rs @@ -5,15 +5,21 @@ use crate::planner_phases::_helpers::inputs::runtime::runtime_field; use crate::planner_phases::_helpers::inputs::runtime::warehouse_field; use crate::planner_phases::constants::MISSING_UPSTREAM_CODE; use crate::planner_warehouse::main::sources::check_selected_source_tables_exist::check_selected_source_tables_exist; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyString, PyTuple}; use sqlbuild_analysis::planner_identity::main::missing_upstream_message::missing_upstream_message; use sqlbuild_analysis::planner_identity::main::missing_upstreams::missing_upstreams; -use sqlbuild_analysis::planner_identity::models::{AvailableUpstreams, GraphKey}; +use sqlbuild_analysis::planner_identity::models::{ + AvailableUpstreams, GraphKey, MissingUpstreamFixes, +}; +use sqlbuild_python_base::records::main::construct::new_error_with_fields::new_error_with_fields; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::ErrorId; +use sqlbuild_python_front::contracts::main::resolve_target_config::resolve_target_config; +use sqlbuild_python_front::contracts::main::resolve_target_name::resolve_target_name; use sqlbuild_python_project::compiled_views::main::project::seeds::seeds; use sqlbuild_python_project::graph::main::nodes::native_key::native_key; use sqlbuild_python_project::planner_identity::main::execution_origins::execution_origins; -use sqlbuild_python_project::planner_phases::main::planner_error::planner_error; use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; use sqlbuild_python_project::planner_scope::models::PlannerScopeResolution; use sqlbuild_python_project::resource_views::main::seed_reads::external::external; @@ -68,11 +74,17 @@ pub(crate) fn check_selected_scope_buildability<'py>( .into_iter() .map(|((key, read), label)| ((GraphKey::from(&key), GraphKey::from(&read)), label)) .collect(); - let message = missing_upstream_message(&missing, &origins); - return Err(planner_error( + let message = missing_upstream_message( + &missing, + &origins, + &missing_upstream_fixes(runtime, &project)?, + ); + let names = missing.iter().map(|entry| entry.key.name.as_str()); + return Err(new_error_with_fields( py, - &message, - (Some(MISSING_UPSTREAM_CODE), None), + ErrorId::PlannerInput, + (&message, Some(MISSING_UPSTREAM_CODE), None), + &[("missing", PyTuple::new(py, names)?.into_any())], )); } check_selected_source_tables_exist( @@ -85,3 +97,48 @@ pub(crate) fn check_selected_scope_buildability<'py>( &warehouse_field(warehouse, |warehouse| &warehouse.inspection_relations)?, ) } + +/// The active target, the targets it may clone from, and the clone `--to` it needs, from configuration only. +fn missing_upstream_fixes( + runtime: &Bound<'_, PyAny>, + project: &Bound<'_, PyAny>, +) -> PyResult { + let py: Python<'_> = runtime.py(); + let target: Option = native_field(project, "effective_target_name")?.extract()?; + let configs = ( + runtime_field(runtime, |runtime| &runtime.project_config)?, + runtime_field(runtime, |runtime| &runtime.local_config)?, + ); + let Some(active) = target.clone() else { + return Ok(MissingUpstreamFixes::default()); + }; + if configs.0.is_none() || configs.1.is_none() { + return Ok(MissingUpstreamFixes { + target, + ..MissingUpstreamFixes::default() + }); + } + let allows = |name: &str, flag: &str| -> PyResult { + let config = resolve_target_config(py, &configs.0, &configs.1, &PyString::new(py, name))?; + native_field(&native_field(&config, "clone")?, flag)?.is_truthy() + }; + let mut origins: std::collections::BTreeSet = std::collections::BTreeSet::new(); + if allows(&active, "allow_as_clone_destination")? { + for config in [&configs.0, &configs.1] { + for name in native_field(config, "targets")?.try_iter()? { + let name: String = name?.extract()?; + if name != active && allows(&name, "allow_as_clone_origin")? { + origins.insert(name); + } + } + } + } + let default: Option = + resolve_target_name(py, &configs.0, &configs.1, &py.None().into_bound(py))? + .map(|name| name.to_string()); + Ok(MissingUpstreamFixes { + clone_to: (default.as_deref() != Some(active.as_str())).then(|| active.clone()), + target, + origins: origins.into_iter().collect(), + }) +} diff --git a/crates/sqlbuild-python-project/src/planner_identity/_helpers/graph.rs b/crates/sqlbuild-python-project/src/planner_identity/_helpers/graph.rs index aacdb803c1..2d63341beb 100644 --- a/crates/sqlbuild-python-project/src/planner_identity/_helpers/graph.rs +++ b/crates/sqlbuild-python-project/src/planner_identity/_helpers/graph.rs @@ -14,7 +14,9 @@ use sqlbuild_analysis::planner_identity::constants::MODEL_RESOURCE; use sqlbuild_analysis::planner_identity::main::missing_upstream_message::missing_upstream_message as message_of; use sqlbuild_analysis::planner_identity::main::missing_upstreams::missing_upstreams; use sqlbuild_analysis::planner_identity::main::topological_order::topological_order; -use sqlbuild_analysis::planner_identity::models::{AvailableUpstreams, GraphKey, MissingUpstream}; +use sqlbuild_analysis::planner_identity::models::{ + AvailableUpstreams, GraphKey, MissingUpstream, MissingUpstreamFixes, +}; use crate::graph::main::nodes::closure::closure; use crate::graph::main::nodes::inverted::inverted; @@ -270,5 +272,8 @@ pub(crate) fn missing_upstream_message<'py>( let rows: Vec = missing_rows(missing, &mut objects)?; let origins: HashMap<(GraphKey, GraphKey), String> = self::edge_origins(edge_origins, &mut objects)?; - Ok(PyString::new(missing.py(), &message_of(&rows, &origins))) + Ok(PyString::new( + missing.py(), + &message_of(&rows, &origins, &MissingUpstreamFixes::default()), + )) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs index 1032360908..194c02db2e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs @@ -273,13 +273,18 @@ fn write_machine_error( } else { Some(help.str()?.to_string()) }; - let document = serde_json::json!({ - "error": { - "code": code.str()?.to_string(), - "message": message.str()?.to_string(), - "help": help, - } + let mut fields = serde_json::json!({ + "code": code.str()?.to_string(), + "message": message.str()?.to_string(), + "help": help, }); + if let Some(missing) = error + .getattr_opt("missing")? + .filter(|value| !value.is_none()) + { + fields["missing"] = serde_json::json!(missing.extract::>()?); + } + let document = serde_json::json!({ "error": fields }); print_object( &PyString::new(invocation.py(), &document.to_string()).into_any(), None, diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/plan.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/plan.md index e05e65aa7c..4885f90263 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/plan.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/plan.md @@ -152,10 +152,29 @@ Release note: earlier versions printed `Plan complete with errors` and exited `0 Planning a scoped selection whose upstream inputs were never built in the target fails with `S301` rather than producing a plan that would break at build time: ``` -error[S301]: cannot build selected scope: 3 missing upstream dependencies (stg_orders, stg_payments, waffle_types) +error[S301]: 3 upstream models and seeds this build reads don't exist in target 'dev' + stg_orders + stg_payments + waffle_types + + If they exist in prod: + read them from it for this run: --defer-to prod + copy them into this target: sqb clone --from prod -s stg_orders stg_payments waffle_types + Or build them too: add + before your selection ``` -Build the upstream chain first, or select it along with the model: `sqb build --select +fact_orders`. +The error names the first five missing resources; `sqb plan --json` lists all of them in +`error.missing`. Each fix fits a different case: + +| Fix | When to use it | +| --- | --- | +| `--defer-to ` | Read the missing resources from another target for this run only. | +| `sqb clone --from -s ...` | Copy them into this target once. The command lists every missing name, so you can run it as printed. | +| `+` before the selection, such as `sqb build --select +fact_orders` | Build the upstream chain along with the model. | + +The error offers an origin only for targets with `clone.allow_as_clone_origin = true`, and only +when the current target has `clone.allow_as_clone_destination = true`. It does not check that the +resources exist in the origin. The check also covers resources that an attached audit on a selected resource reads, even when the model SQL does not read them. Selecting `+fact_orders` follows data lineage only, so it does not @@ -167,7 +186,7 @@ error[S301]: cannot build selected scope: audit 'relationships' on 'stg_orders' ``` Load or build the resource first (for a seed, `sqb seed`), or add it to the selection. When both -kinds are missing, the error lists the missing upstream dependencies and then the audit reads. +kinds are missing, the error lists the missing upstream resources and then the audit reads. ## Missing source tables diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 8d683a5772..adc266cbe8 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -13535,8 +13535,14 @@ class PlannerInputError(ValueError): code: str message: str help: str | None + missing: tuple[str, ...] | None def __init__( - self, message: str, *, code: str | None = None, help: str | None = None + self, + message: str, + *, + code: str | None = None, + help: str | None = None, + missing: tuple[str, ...] | None = None, ) -> None: ... class FutureCursorSafetyError(PlannerInputError): diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_terminal_failure.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_terminal_failure.py index 759bbe8b4e..5e01acbc9c 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_terminal_failure.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_terminal_failure.py @@ -27,7 +27,7 @@ model_replacements=(), expected_screen_fragments=( "Inspected warehouse state.", - "error[S301]: cannot build selected scope: 2 missing upstream dependencies", + "error[S301]: 2 upstream models this build reads don't exist in the warehouse", ), ), TerminalBuildFailureE2ETestCase( diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clone/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/clone/_test_types.py index 461de7cb85..68f4b77748 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/clone/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clone/_test_types.py @@ -84,3 +84,23 @@ class CloneChangedPlanE2ETestCase: commands: tuple[tuple[str, ...], ...] expected_fragments: tuple[str, ...] unexpected_fragments: tuple[str, ...] = () + + +@dataclass(frozen=True) +class CloneMissingUpstreamE2ETestCase: + """A target missing a selected model's upstreams, and the clone command S301 prints for it.""" + + description: str + default_target: str + target_args: tuple[str, ...] + expected_clone_command: str + + +@dataclass(frozen=True) +class MissingUpstreamFixesE2ETestCase: + """Whether the planned target may be cloned into, and what S301 then offers.""" + + description: str + destination_allowed: bool + expected_fragments: tuple[str, ...] + unexpected_fragments: tuple[str, ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clone/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/clone/helpers.py index 7b810d14b1..b1b18e261d 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/clone/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clone/helpers.py @@ -189,3 +189,59 @@ def copy_project(*, origin_project: Path, tmp_path: Path) -> Path: project_dir: Path = tmp_path / origin_project.name shutil.copytree(origin_project, project_dir) return project_dir + + +MISSING_UPSTREAMS: tuple[str, ...] = ( + "customers", + "orders", + "payments", + "products", + "returns", + "shipments", +) + + +def plan_missing_upstreams( + *, tmp_path: Path, default_target: str, destination_allowed: bool, target_args: tuple[str, ...] +) -> tuple[Path, tuple[str, ...], subprocess.CompletedProcess[str]]: + """Build `MISSING_UPSTREAMS` in `prod`, then plan `report`, which reads them, in empty `dev`.""" + + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="missing_upstream_project", + repo_files={ + "sqlbuild_project.toml": ( + 'name = "missing_upstream_project"\nadapter = "duckdb"\n' + f'default_target = "{default_target}"\n\n' + '[connection]\ndatabase = "shop.duckdb"\n\n' + '[targets.prod]\nschema = "prod"\n\n' + "[targets.prod.clone]\nallow_as_clone_origin = true\n\n" + '[targets.dev]\nschema = "dev"\n\n' + "[targets.dev.clone]\n" + f"allow_as_clone_destination = {str(destination_allowed).lower()}\n" + ), + **{ + f"models/{name}.sql": ( + f"MODEL (description '{name}.', materialized table);\n\nSELECT 1 AS id\n" + ) + for name in MISSING_UPSTREAMS + }, + "models/report.sql": ( + "MODEL (description 'Report.', materialized table);\n\n" + 'SELECT customers.id FROM __ref("customers") AS customers\n' + + "".join( + f'JOIN __ref("{name}") AS {name} USING (id)\n' for name in MISSING_UPSTREAMS[1:] + ) + ), + }, + ) + built: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build", "--target", "prod"), project_dir=project_dir + ) + assert built.returncode == 0, built.stdout + built.stderr + plan_command: tuple[str, ...] = ("--no-color", "plan", "--select", "report", *target_args) + return ( + project_dir, + plan_command, + run_sqb(command=plan_command, project_dir=project_dir), + ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clone/test_clone.py b/tests/e2e/src/sqlbuild/cli/commands/main/clone/test_clone.py index 4c4c7a0a76..88f9f9f6ed 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/clone/test_clone.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clone/test_clone.py @@ -2,6 +2,8 @@ from __future__ import annotations +import json +import re import subprocess from pathlib import Path from textwrap import dedent @@ -13,8 +15,14 @@ CloneDeferredSourceFunctionE2ETestCase, CloneE2ETestCase, CloneFunctionGraphE2ETestCase, + CloneMissingUpstreamE2ETestCase, ClonePolicyErrorTestCase, ClonePythonFunctionE2ETestCase, + MissingUpstreamFixesE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.clone.helpers import ( + MISSING_UPSTREAMS, + plan_missing_upstreams, ) from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( REPO_ROOT, @@ -24,6 +32,14 @@ ) _CHANGED_CLONE_FIXTURE: Path = REPO_ROOT / "benchmarks/duckshop/edge_cases/changed_builds_clone" +_UPSTREAMS: tuple[str, ...] = ( + "customers", + "orders", + "payments", + "products", + "returns", + "shipments", +) @pytest.mark.parametrize( @@ -968,3 +984,96 @@ def test_given_cloned_table_when_planning_changed_then_it_rebuilds( assert fragment in output + plan.stdout, output + plan.stdout for fragment in test_case.unexpected_fragments: assert fragment not in plan.stdout, plan.stdout + + +@pytest.mark.parametrize( + "test_case", + [ + MissingUpstreamFixesE2ETestCase( + description="a clone destination is offered deferring to and cloning from each origin", + destination_allowed=True, + expected_fragments=( + " If they exist in prod:\n" + " read them from it for this run: --defer-to prod\n" + " copy them into this target: sqb clone --from prod -s " + + " ".join(MISSING_UPSTREAMS) + + "\n Or build them too: add + before your selection", + ), + unexpected_fragments=(), + ), + MissingUpstreamFixesE2ETestCase( + description="a target that may not be cloned into is offered only building", + destination_allowed=False, + expected_fragments=(" Build them too: add + before your selection",), + unexpected_fragments=("sqb clone", "--defer-to"), + ), + ], + ids=lambda case: case.description, +) +def test_given_missing_upstreams_when_planning_then_s301_lists_them_with_fixes( + test_case: MissingUpstreamFixesE2ETestCase, + tmp_path: Path, +) -> None: + project_dir, plan_command, refused = plan_missing_upstreams( + tmp_path=tmp_path, + default_target="dev", + destination_allowed=test_case.destination_allowed, + target_args=(), + ) + + as_json: subprocess.CompletedProcess[str] = run_sqb( + command=(*plan_command, "--json"), project_dir=project_dir + ) + + output: str = refused.stdout + refused.stderr + assert refused.returncode == 1, output + assert "error[S301]: 6 upstream models this build reads don't exist in target 'dev'\n" in output + assert " returns\n ... 1 more (sqb plan --json lists all)\n" in output + assert all(fragment in output for fragment in test_case.expected_fragments), output + assert not any(fragment in output for fragment in test_case.unexpected_fragments), output + assert json.loads(as_json.stdout)["error"]["missing"] == list(MISSING_UPSTREAMS) + + +@pytest.mark.parametrize( + "test_case", + [ + CloneMissingUpstreamE2ETestCase( + description="the default target is cloned into without naming it", + default_target="dev", + target_args=(), + expected_clone_command="sqb clone --from prod -s " + " ".join(MISSING_UPSTREAMS), + ), + CloneMissingUpstreamE2ETestCase( + description="a target other than the default is named in the clone command", + default_target="prod", + target_args=("--target", "dev"), + expected_clone_command="sqb clone --from prod --to dev -s " + + " ".join(MISSING_UPSTREAMS), + ), + ], + ids=lambda case: case.description, +) +def test_given_missing_upstreams_when_running_printed_clone_then_plan_succeeds( + test_case: CloneMissingUpstreamE2ETestCase, + tmp_path: Path, +) -> None: + project_dir, plan_command, refused = plan_missing_upstreams( + tmp_path=tmp_path, + default_target=test_case.default_target, + destination_allowed=True, + target_args=test_case.target_args, + ) + printed: list[str] = re.findall( + r"^ +copy them into this target: +(sqb clone .+)$", refused.stdout + refused.stderr, re.M + ) + + cloned: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", *printed[0].split()[1:]), project_dir=project_dir + ) + planned: subprocess.CompletedProcess[str] = run_sqb( + command=plan_command, project_dir=project_dir + ) + + assert printed == [test_case.expected_clone_command], refused.stdout + refused.stderr + assert cloned.returncode == 0, cloned.stdout + cloned.stderr + assert planned.returncode == 0, planned.stdout + planned.stderr diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_buildability.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_buildability.py index 7c38e4b974..312e180d63 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_buildability.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_buildability.py @@ -173,14 +173,16 @@ def test_given_scope_and_snapshot_when_checking_buildability_then_returns_expect "test_case", [ MissingUpstreamMessageTestCase( - description="data-lineage misses keep the existing wording", + description="data-lineage misses are listed with how to build them", missing=( MissingUpstream(key=model_key("a"), required_by=(model_key("b"),)), MissingUpstream(key=model_key("c"), required_by=(model_key("b"),)), ), edge_origins={}, expected_message=( - "cannot build selected scope: 2 missing upstream dependencies (a, c)" + "2 upstream models this build reads don't exist in the warehouse\n" + " a\n c\n\n" + " Build them too: add + before your selection" ), ), MissingUpstreamMessageTestCase( @@ -211,7 +213,9 @@ def test_given_scope_and_snapshot_when_checking_buildability_then_returns_expect (model_key("stg_orders"), seed_key("waffle_types")): _RELATIONSHIPS_ORIGIN }, expected_message=( - "cannot build selected scope: 1 missing upstream dependencies (waffle_types)" + "1 upstream seed this build reads doesn't exist in the warehouse\n" + " waffle_types\n\n" + " Build it too: add + before your selection" ), ), MissingUpstreamMessageTestCase( @@ -228,7 +232,10 @@ def test_given_scope_and_snapshot_when_checking_buildability_then_returns_expect (model_key("orders"), seed_key("order_statuses")): _STATUS_ORIGIN, }, expected_message=( - "cannot build selected scope: 1 missing upstream dependencies (a); 2 audit reads " + "1 upstream model this build reads doesn't exist in the warehouse\n" + " a\n\n" + " Build it too: add + before your selection\n\n" + " Also, 2 audit reads " f"are not selected and do not exist in the warehouse ({_RELATIONSHIPS_ORIGIN}; " f"{_STATUS_ORIGIN}); select them or build them first" ), diff --git a/ui/src/lib/run/plan-text.test.ts b/ui/src/lib/run/plan-text.test.ts index c61eed4a05..6c48fed5eb 100644 --- a/ui/src/lib/run/plan-text.test.ts +++ b/ui/src/lib/run/plan-text.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest'; import type { PlanDocument, PlanModel } from '$lib/api'; -import { actionText, changedSections, jobRefreshText, looseWarnings, planSections, warningText } from './plan-text'; +import { actionText, changedSections, cloneCommands, jobRefreshText, looseWarnings, planSections, warningText } from './plan-text'; const model = (name: string, over: Partial = {}): PlanModel => ({ name, @@ -184,3 +184,25 @@ describe('plan under --changed', () => { }); }); + +describe('cloneCommands', () => { + it('returns each origin\'s clone command with every missing name, and nothing for other errors', () => { + const error = [ + "S301: 7 upstream models this build reads don't exist in target 'dev'", + ' a', + ' ... 2 more (sqb plan --json lists all)', + '', + ' If they exist in prod:', + ' read them from it for this run: --defer-to prod', + ' copy them into this target: sqb clone --from prod --to dev -s a b c d e f g', + ' If it exists in staging:', + ' copy it into this target: sqb clone --from staging -s a', + ' Or build them too: add + before your selection' + ].join('\n'); + expect(cloneCommands(error)).toEqual([ + { origin: 'prod', command: 'sqb clone --from prod --to dev -s a b c d e f g' }, + { origin: 'staging', command: 'sqb clone --from staging -s a' } + ]); + expect(cloneCommands('S007: unknown selector sqb clone --from prod')).toEqual([]); + }); +}); diff --git a/ui/src/lib/run/plan-text.ts b/ui/src/lib/run/plan-text.ts index 6b339e2cb4..a34d36e3ae 100644 --- a/ui/src/lib/run/plan-text.ts +++ b/ui/src/lib/run/plan-text.ts @@ -209,3 +209,8 @@ export function jobRefreshText(job: string, triggers: Pick t.type === 'sensor') ? `job ${job} · when its sensor fires` : `job ${job}`; } + +/** The `sqb clone` commands a plan error offers for missing upstreams (S301), each ready to copy as is. */ +export function cloneCommands(error: string): { origin: string; command: string }[] { + return [...error.matchAll(/^\s*copy .+?:\s+(sqb clone --from (\S+) .+)$/gm)].map((match) => ({ origin: match[2], command: match[1].trim() })); +} diff --git a/ui/src/routes/plan/+page.svelte b/ui/src/routes/plan/+page.svelte index 82c28f9e68..ffbf8b1f57 100644 --- a/ui/src/routes/plan/+page.svelte +++ b/ui/src/routes/plan/+page.svelte @@ -20,7 +20,8 @@ import PlanReport from '$lib/run/plan-report.svelte'; import StatusLabel from '$lib/components/list/status-label.svelte'; import { checkOutcomeLabel, checkOutcomeTone } from '$lib/status'; - import { builtModels, builtSeeds, changedModelKeys, jobRefreshText } from '$lib/run/plan-text'; + import { builtModels, builtSeeds, changedModelKeys, cloneCommands, jobRefreshText } from '$lib/run/plan-text'; + import CopyButton from '$lib/components/copy-button.svelte'; import { app } from '$lib/state/app.svelte'; import { createResource } from '$lib/state/resource.svelte'; @@ -323,7 +324,12 @@
Data (cmd = withFullRefresh(withWindow(cmd, c.window), c.fullRefresh))} />
{/if} - {#if problem}

{problem}

{/if} + {#if problem} +
+

{problem}

+ {#each cloneCommands(problem) as clone (clone.command)}{/each} +
+ {/if}
@@ -554,7 +560,10 @@ color: var(--sb-error); font-family: var(--font-mono); font-size: 12px; - white-space: pre-line; + } + .problem-bar p { + margin: 0; + white-space: pre-wrap; } .overlay { position: absolute; diff --git a/website/src/content/docs/docs/cli/plan.mdx b/website/src/content/docs/docs/cli/plan.mdx index a1a4d562c0..92b8fe5ebf 100644 --- a/website/src/content/docs/docs/cli/plan.mdx +++ b/website/src/content/docs/docs/cli/plan.mdx @@ -142,10 +142,29 @@ Release note: earlier versions printed `Plan complete with errors` and exited `0 Planning a scoped selection whose upstream inputs were never built in the target fails with `S301` rather than producing a plan that would break at build time: ``` -error[S301]: cannot build selected scope: 3 missing upstream dependencies (stg_orders, stg_payments, waffle_types) +error[S301]: 3 upstream models and seeds this build reads don't exist in target 'dev' + stg_orders + stg_payments + waffle_types + + If they exist in prod: + read them from it for this run: --defer-to prod + copy them into this target: sqb clone --from prod -s stg_orders stg_payments waffle_types + Or build them too: add + before your selection ``` -Build the upstream chain first, or select it along with the model: `sqb build --select +fact_orders`. +The error names the first five missing resources; `sqb plan --json` lists all of them in +`error.missing`. Each fix fits a different case: + +| Fix | When to use it | +| --- | --- | +| `--defer-to ` | Read the missing resources from another target for this run only. | +| `sqb clone --from -s ...` | Copy them into this target once. The command lists every missing name, so you can run it as printed. | +| `+` before the selection, such as `sqb build --select +fact_orders` | Build the upstream chain along with the model. | + +The error offers an origin only for targets with `clone.allow_as_clone_origin = true`, and only +when the current target has `clone.allow_as_clone_destination = true`. It does not check that the +resources exist in the origin. The check also covers resources that an attached audit on a selected resource reads, even when the model SQL does not read them. Selecting `+fact_orders` follows data lineage only, so it does not @@ -157,7 +176,7 @@ error[S301]: cannot build selected scope: audit 'relationships' on 'stg_orders' ``` Load or build the resource first (for a seed, `sqb seed`), or add it to the selection. When both -kinds are missing, the error lists the missing upstream dependencies and then the audit reads. +kinds are missing, the error lists the missing upstream resources and then the audit reads. ## Missing source tables From 6ea74e6b2644987906186f1e308c63a4e94bfaba Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 17 Nov 2026 20:54:49 +0000 Subject: [PATCH 3/3] fix: keep S301 when its hints fail and mark partly cloned boundaries as replaced A default target that does not resolve no longer turns S301 into an unknown-target error; the clone command then names its destination. --defer-to is offered only for project targets and not when the plan already defers. A defer clone that fails for one boundary still marks the boundaries it replaced. A failed marker write still fails the command, but only after fingerprints are copied, and a defer clone's C411 is reported first. --- .../main/missing_upstream_message.rs | 10 +-- .../src/planner_identity/models.rs | 2 + .../tests/test_planner_identity.rs | 2 +- .../_helpers/outputs/buildability.rs | 18 +++-- .../_helpers/build_command/defer_clone.rs | 4 +- .../_helpers/clone/clone_execution.rs | 4 +- .../cli/commands/main/clone/_test_types.py | 19 +++++ .../cli/commands/main/clone/helpers.py | 53 +++++++++++++- .../cli/commands/main/clone/test_clone.py | 70 +++++++++++++++++++ 9 files changed, 169 insertions(+), 13 deletions(-) diff --git a/crates/sqlbuild-analysis/src/planner_identity/main/missing_upstream_message.rs b/crates/sqlbuild-analysis/src/planner_identity/main/missing_upstream_message.rs index 61ca107a5e..c392668519 100644 --- a/crates/sqlbuild-analysis/src/planner_identity/main/missing_upstream_message.rs +++ b/crates/sqlbuild-analysis/src/planner_identity/main/missing_upstream_message.rs @@ -108,10 +108,12 @@ fn lineage_text( .unwrap_or_default(); for origin in &fixes.origins { lines.push(format!(" If {exists} in {origin}:")); - lines.push(format!( - " {read_label:<0$} --defer-to {origin}", - width - 4 - )); + if fixes.deferrable.contains(origin) { + lines.push(format!( + " {read_label:<0$} --defer-to {origin}", + width - 4 + )); + } lines.push(format!( " {copy_label:<0$} sqb clone --from {origin}{to} -s {1}", width - 4, diff --git a/crates/sqlbuild-analysis/src/planner_identity/models.rs b/crates/sqlbuild-analysis/src/planner_identity/models.rs index d4b39ac024..e0cd6716a0 100644 --- a/crates/sqlbuild-analysis/src/planner_identity/models.rs +++ b/crates/sqlbuild-analysis/src/planner_identity/models.rs @@ -94,6 +94,8 @@ pub struct MissingUpstreamFixes { pub target: Option, /// Targets the active target may clone from, to defer to or clone the upstreams from. pub origins: Vec, + /// The origins `--defer-to` accepts: project targets, offered only when this plan does not defer already. + pub deferrable: Vec, /// The clone command's `--to`, needed when the active target is not the default one. pub clone_to: Option, } diff --git a/crates/sqlbuild-analysis/src/planner_identity/tests/test_planner_identity.rs b/crates/sqlbuild-analysis/src/planner_identity/tests/test_planner_identity.rs index 7bfc536891..5c124eb943 100644 --- a/crates/sqlbuild-analysis/src/planner_identity/tests/test_planner_identity.rs +++ b/crates/sqlbuild-analysis/src/planner_identity/tests/test_planner_identity.rs @@ -434,6 +434,7 @@ fn given_selected_keys_when_checking_buildability_then_reports_missing_upstreams fixes: MissingUpstreamFixes { target: Some("dev".to_owned()), origins: vec!["prod".to_owned(), "staging".to_owned()], + deferrable: vec!["prod".to_owned()], clone_to: Some("dev".to_owned()), }, expected_missing: ["a", "b", "c", "d", "e", "f"] @@ -454,7 +455,6 @@ fn given_selected_keys_when_checking_buildability_then_reports_missing_upstreams " read them from it for this run: --defer-to prod\n", " copy them into this target: sqb clone --from prod --to dev -s a b c d e f codes\n", " If they exist in staging:\n", - " read them from it for this run: --defer-to staging\n", " copy them into this target: sqb clone --from staging --to dev -s a b c d e f codes\n", " Or build them too: add + before your selection", ), diff --git a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/buildability.rs b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/buildability.rs index 3aecf62cdd..d8e27b2193 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/buildability.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/buildability.rs @@ -77,7 +77,7 @@ pub(crate) fn check_selected_scope_buildability<'py>( let message = missing_upstream_message( &missing, &origins, - &missing_upstream_fixes(runtime, &project)?, + &missing_upstream_fixes(runtime, &project, available.deferred_relations.is_some())?, ); let names = missing.iter().map(|entry| entry.key.name.as_str()); return Err(new_error_with_fields( @@ -98,10 +98,11 @@ pub(crate) fn check_selected_scope_buildability<'py>( ) } -/// The active target, the targets it may clone from, and the clone `--to` it needs, from configuration only. +/// The active target, its clone or deferral origins, and the clone `--to` it needs (also when the default target is unknown). fn missing_upstream_fixes( runtime: &Bound<'_, PyAny>, project: &Bound<'_, PyAny>, + deferring: bool, ) -> PyResult { let py: Python<'_> = runtime.py(); let target: Option = native_field(project, "effective_target_name")?.extract()?; @@ -123,22 +124,29 @@ fn missing_upstream_fixes( native_field(&native_field(&config, "clone")?, flag)?.is_truthy() }; let mut origins: std::collections::BTreeSet = std::collections::BTreeSet::new(); + let mut deferrable: Vec = Vec::new(); if allows(&active, "allow_as_clone_destination")? { - for config in [&configs.0, &configs.1] { + for (config, project_targets) in [(&configs.0, true), (&configs.1, false)] { for name in native_field(config, "targets")?.try_iter()? { let name: String = name?.extract()?; if name != active && allows(&name, "allow_as_clone_origin")? { + if project_targets && !deferring { + deferrable.push(name.clone()); + } origins.insert(name); } } } } let default: Option = - resolve_target_name(py, &configs.0, &configs.1, &py.None().into_bound(py))? - .map(|name| name.to_string()); + match resolve_target_name(py, &configs.0, &configs.1, &py.None().into_bound(py)) { + Ok(name) => name.map(|name| name.to_string()), + Err(_) => None, + }; Ok(MissingUpstreamFixes { clone_to: (default.as_deref() != Some(active.as_str())).then(|| active.clone()), target, origins: origins.into_iter().collect(), + deferrable, }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs index 5265b247bb..3b23db5ffc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs @@ -441,6 +441,8 @@ fn clone_boundaries<'py>( ), )?; let result: Bound<'py, PyAny> = executed_clone(context, prephase, (&pipeline, connection))?; + let marked: PyResult<()> = + forget_replaced_consumption((&prephase.adapter, connection), &pipeline, &result); let failed: Vec = failed_items(&result)?; if !failed.is_empty() { return Err(user_error( @@ -482,7 +484,7 @@ fn clone_boundaries<'py>( ), ], )?; - forget_replaced_consumption((&prephase.adapter, connection), &pipeline, &result) + marked } /// Clone the selected boundary relations from the origin before build planning. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_execution.rs index 02087f26f2..da6ebcb25c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_execution.rs @@ -413,7 +413,9 @@ pub(crate) fn execute_clone_plan<'py>( let closed: PyResult> = event_writer.call_method0("close"); let result: Bound<'py, PyAny> = executed?; closed?; + let marked: PyResult<()> = + forget_replaced_consumption((&invocation.adapter, connection), pipeline, &result); copy_fingerprints(invocation, (pipeline, connection), &result)?; - forget_replaced_consumption((&invocation.adapter, connection), pipeline, &result)?; + marked?; Ok((result, started.elapsed().as_secs_f64())) } diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clone/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/clone/_test_types.py index 68f4b77748..ba5cc3ee07 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/clone/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clone/_test_types.py @@ -104,3 +104,22 @@ class MissingUpstreamFixesE2ETestCase: destination_allowed: bool expected_fragments: tuple[str, ...] unexpected_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class PartialDeferCloneE2ETestCase: + """A defer clone with one boundary missing in the origin, and what `plan --changed` says.""" + + description: str + dropped_origin_table: str + expected_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class UnwritableConsumptionDeferCloneE2ETestCase: + """A partly failed defer clone whose destination consumption table rejects writes.""" + + description: str + dropped_origin_table: str + before_clone: tuple[str, ...] + expected_fragments: tuple[str, ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clone/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/clone/helpers.py index b1b18e261d..7b653bc1ff 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/clone/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clone/helpers.py @@ -7,7 +7,11 @@ from pathlib import Path from textwrap import dedent -from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( + execute_duckdb, + prepare_inline_project, + run_sqb, +) PROJECT_FILES: dict[str, str] = { "sqlbuild_project.toml": dedent( @@ -245,3 +249,50 @@ def plan_missing_upstreams( plan_command, run_sqb(command=plan_command, project_dir=project_dir), ) + + +def defer_clone_with_missing_origin( + *, tmp_path: Path, dropped_origin_table: str, before_clone: tuple[str, ...] = () +) -> tuple[subprocess.CompletedProcess[str], subprocess.CompletedProcess[str]]: + """Defer-clone `report` with one of its boundaries dropped in `prod`, then `plan --changed`.""" + + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="partial_defer_clone_project", + repo_files={ + "sqlbuild_project.toml": ( + 'name = "partial_defer_clone_project"\nadapter = "duckdb"\n' + 'default_target = "dev"\n\n[connection]\ndatabase = "shop.duckdb"\n\n' + '[targets.prod]\nschema = "prod"\n\n' + "[targets.prod.clone]\nallow_as_clone_origin = true\n\n" + '[targets.dev]\nschema = "dev"\n\n' + "[targets.dev.clone]\nallow_as_clone_destination = true\n" + ), + "models/orders.sql": ( + "MODEL (description 'Orders.', materialized table);\n\nSELECT 1 AS id\n" + ), + "models/customers.sql": ( + "MODEL (description 'Customers.', materialized table);\n\nSELECT 1 AS id\n" + ), + "models/report.sql": ( + "MODEL (description 'Report.', materialized table);\n\n" + 'SELECT o.id FROM __ref("orders") AS o JOIN __ref("customers") AS c USING (id)\n' + ), + }, + ) + for target in ("dev", "prod"): + built: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build", "--target", target), project_dir=project_dir + ) + assert built.returncode == 0, built.stdout + built.stderr + for sql in (f"DROP TABLE prod.{dropped_origin_table}", *before_clone): + execute_duckdb(db_path=project_dir / "shop.duckdb", sql=sql) + cloned: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build", "--select", "report", "--defer-clone-from", "prod"), + project_dir=project_dir, + ) + planned: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "plan", "--changed", "--select", "orders", "customers"), + project_dir=project_dir, + ) + return cloned, planned diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clone/test_clone.py b/tests/e2e/src/sqlbuild/cli/commands/main/clone/test_clone.py index 88f9f9f6ed..2cf2258ba1 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/clone/test_clone.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clone/test_clone.py @@ -19,9 +19,12 @@ ClonePolicyErrorTestCase, ClonePythonFunctionE2ETestCase, MissingUpstreamFixesE2ETestCase, + PartialDeferCloneE2ETestCase, + UnwritableConsumptionDeferCloneE2ETestCase, ) from tests.e2e.src.sqlbuild.cli.commands.main.clone.helpers import ( MISSING_UPSTREAMS, + defer_clone_with_missing_origin, plan_missing_upstreams, ) from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( @@ -1050,6 +1053,13 @@ def test_given_missing_upstreams_when_planning_then_s301_lists_them_with_fixes( expected_clone_command="sqb clone --from prod --to dev -s " + " ".join(MISSING_UPSTREAMS), ), + CloneMissingUpstreamE2ETestCase( + description="a default target that does not exist still yields a runnable clone", + default_target="retired", + target_args=("--target", "dev"), + expected_clone_command="sqb clone --from prod --to dev -s " + + " ".join(MISSING_UPSTREAMS), + ), ], ids=lambda case: case.description, ) @@ -1077,3 +1087,63 @@ def test_given_missing_upstreams_when_running_printed_clone_then_plan_succeeds( assert printed == [test_case.expected_clone_command], refused.stdout + refused.stderr assert cloned.returncode == 0, cloned.stdout + cloned.stderr assert planned.returncode == 0, planned.stdout + planned.stderr + + +@pytest.mark.parametrize( + "test_case", + [ + PartialDeferCloneE2ETestCase( + description="a boundary cloned before another failed still rebuilds", + dropped_origin_table="customers", + expected_fragments=( + "orders no completed record", + "Unchanged (1): skipped by --changed", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_partly_failed_defer_clone_when_planning_changed_then_cloned_boundary_rebuilds( + test_case: PartialDeferCloneE2ETestCase, + tmp_path: Path, +) -> None: + cloned, planned = defer_clone_with_missing_origin( + tmp_path=tmp_path, dropped_origin_table=test_case.dropped_origin_table + ) + + assert cloned.returncode == 1, cloned.stdout + cloned.stderr + assert "error[C411]" in cloned.stdout + cloned.stderr + assert planned.returncode == 0, planned.stdout + planned.stderr + assert all(fragment in planned.stdout for fragment in test_case.expected_fragments), ( + planned.stdout + ) + + +@pytest.mark.parametrize( + "test_case", + [ + UnwritableConsumptionDeferCloneE2ETestCase( + description="the failed boundary is reported, not the consumption write", + dropped_origin_table="customers", + before_clone=( + "DROP TABLE dev._sqlbuild_consumption", + "CREATE TABLE dev._sqlbuild_consumption (unexpected INTEGER)", + ), + expected_fragments=("error[C411]", "customers: missing in origin environment"), + ), + ], + ids=lambda case: case.description, +) +def test_given_unwritable_consumption_table_when_defer_clone_partly_fails_then_c411_is_reported( + test_case: UnwritableConsumptionDeferCloneE2ETestCase, + tmp_path: Path, +) -> None: + cloned, _ = defer_clone_with_missing_origin( + tmp_path=tmp_path, + dropped_origin_table=test_case.dropped_origin_table, + before_clone=test_case.before_clone, + ) + + output: str = cloned.stdout + cloned.stderr + assert cloned.returncode == 1, output + assert all(fragment in output for fragment in test_case.expected_fragments), output