From e6e9b13443df7f87c1f833486ac98905b0237563 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 17 Nov 2026 14:55:53 +0000 Subject: [PATCH 1/3] fix: check the resolved loader_schema for preserved node results The preserved-target node results check now expands loader_schema with the effective vars and applies the same usability test as the runtime location: non-empty, not preserve, and no context placeholder. A target whose loader_schema resolves to nothing usable is refused before the build instead of writing results to the wrong schema. --- .../_helpers/setup/model_free_locations.rs | 10 ++++- .../_helpers/setup/node_results.rs | 41 +++++++++++++++---- .../compile_driver/_helpers/setup/project.rs | 5 ++- .../main/setup/preserved_loader_schemas.rs | 6 ++- .../bindings/_helpers/load/load_selection.rs | 9 +++- 5 files changed, 56 insertions(+), 15 deletions(-) diff --git a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/model_free_locations.rs b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/model_free_locations.rs index e001269c72..30554a4044 100644 --- a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/model_free_locations.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/model_free_locations.rs @@ -6,14 +6,20 @@ use sqlbuild_analysis::assembly::target_checks::constants::PRESERVE_TARGET_VALUE use sqlbuild_model_config::templates::main::holds_context_placeholder::holds_context_placeholder; use sqlbuild_python_base::warehouse_access::main::adapter_default_location::adapter_default_location; +/// The resolved `loader_schema` a `preserve` schema keeps Python node results in, unless it is unusable. +pub(crate) fn kept_loader_schema(loader_schema: Option<&str>) -> Option<&str> { + loader_schema.filter(|name| { + !name.is_empty() && *name != PRESERVE_TARGET_VALUE && !holds_context_placeholder(name) + }) +} + /// Python nodes' `(database, schema)`: a `preserve` schema reads `loader_schema`; unusable values the adapter's. pub(crate) fn model_free_locations<'py>( (database, schema): (Bound<'py, PyAny>, Bound<'py, PyAny>), loader_schema: Option<&str>, adapter: &Bound<'py, PyAny>, ) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { - let kept_schema: Option<&str> = - loader_schema.filter(|name| !name.is_empty() && *name != PRESERVE_TARGET_VALUE); + let kept_schema: Option<&str> = kept_loader_schema(loader_schema); let location = |value: Bound<'py, PyAny>, method: &str, preserved: Option<&str>| match ( value.extract::>()?.as_deref(), preserved, diff --git a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/node_results.rs b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/node_results.rs index d302abe94b..f5065d8805 100644 --- a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/node_results.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/node_results.rs @@ -1,10 +1,12 @@ //! The schema a `preserve` target keeps Python node results in. +use crate::compile_driver::_helpers::setup::model_free_locations::kept_loader_schema; use crate::compile_driver::_helpers::setup::targets::planner_error; +use crate::compile_driver::main::setup::expanded_loader_namespace::expanded_loader_namespace; +use pyo3::exceptions::PyException; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; -use sqlbuild_analysis::assembly::target_checks::constants::{ - PRESERVE_TARGET_VALUE, PRESERVED_NODE_RESULTS_CODE, -}; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_analysis::assembly::target_checks::constants::PRESERVED_NODE_RESULTS_CODE; use sqlbuild_analysis::assembly::target_checks::errors::TargetError; use sqlbuild_analysis::assembly::target_checks::main::target_error_message::target_error_message; use sqlbuild_core::text::main::python_strip::python_strip; @@ -19,16 +21,16 @@ const PYTHON_NODE_FIELDS: [&str; 4] = [ "check_functions", ]; -/// Raise when a `preserve` target has Python nodes but no loader schema to keep their results in. +/// Raise when a `preserve` target has Python nodes but no usable loader schema, after `effective_vars`, to keep their results in. pub(crate) fn require_node_results_schema( discovered: &Bound<'_, PyAny>, (target_name, target_config): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), + effective_vars: &Bound<'_, PyAny>, ) -> PyResult<()> { - let loader_schema: Option = - optional_text(&native_field(target_config, "loader_schema")?)? - .map(|schema| python_strip(&schema).to_owned()) - .filter(|schema| !schema.is_empty() && schema != PRESERVE_TARGET_VALUE); - if loader_schema.is_some() || python_node_count(discovered)? == 0 { + let loader_schema: Option = resolved_loader_schema(target_config, effective_vars)?; + if kept_loader_schema(loader_schema.as_deref()).is_some() + || python_node_count(discovered)? == 0 + { return Ok(()); } let target: String = target_name.extract()?; @@ -44,6 +46,27 @@ pub(crate) fn require_node_results_schema( )) } +/// The target's `loader_schema` as runtime resolves it: vars expanded, `None` when it cannot resolve outside a model. +fn resolved_loader_schema( + target_config: &Bound<'_, PyAny>, + effective_vars: &Bound<'_, PyAny>, +) -> PyResult> { + let Some(schema) = optional_text(&native_field(target_config, "loader_schema")?)? else { + return Ok(None); + }; + let py = target_config.py(); + let namespace: Bound<'_, PyDict> = PyDict::new(py); + namespace.set_item("schema", python_strip(&schema))?; + match expanded_loader_namespace(namespace.as_any(), effective_vars) { + Ok(expanded) => { + let schema: Bound<'_, PyAny> = expanded.get_item("schema")?; + Ok((!schema.is_none()).then(|| schema.str()?.extract()).transpose()?) + } + Err(error) if error.is_instance_of::(py) => Ok(None), + Err(error) => Err(error), + } +} + /// How many loaders, tasks, assets and checks the project declares in Python. fn python_node_count(discovered: &Bound<'_, PyAny>) -> PyResult { PYTHON_NODE_FIELDS diff --git a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/project.rs b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/project.rs index 4190c20d49..8fcac2dff0 100644 --- a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/project.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/project.rs @@ -205,7 +205,10 @@ fn targeted_project<'py>( } preserved_loader_schemas( discovered, - request.selected_target.bind(py), + ( + request.selected_target.bind(py), + &native_field(project, "effective_vars")?, + ), &intermediate_loaders, )?; managed_write_schemas(adapter, project)?; diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/setup/preserved_loader_schemas.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/preserved_loader_schemas.rs index 9a0c3d649b..aaa643cf8b 100644 --- a/crates/sqlbuild-python-compile/src/compile_driver/main/setup/preserved_loader_schemas.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/preserved_loader_schemas.rs @@ -15,9 +15,11 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use std::collections::HashSet; /// Raise when a `preserve` target leaves one of `loaders`, or the Python node results, without a schema. +/// +/// `effective_vars` expand the target's `loader_schema` as runtime does. pub fn preserved_loader_schemas( discovered: &Bound<'_, PyAny>, - selected_target: &Bound<'_, PyAny>, + (selected_target, effective_vars): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), loaders: &HashSet, ) -> PyResult<()> { let py: Python<'_> = discovered.py(); @@ -48,5 +50,5 @@ pub fn preserved_loader_schemas( )), )); } - require_node_results_schema(discovered, (&target_name, &target_config)) + require_node_results_schema(discovered, (&target_name, &target_config), effective_vars) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs index 10315d480b..bd8e8b5ad6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs @@ -410,7 +410,14 @@ pub(crate) fn loader_write_schemas<'py>( intermediate_loaders.insert(native_field(entry, "name")?.extract()?); } } - preserved_loader_schemas(discovered, selected_target, &intermediate_loaders)?; + preserved_loader_schemas( + discovered, + ( + selected_target, + &target_vars(discovered, (selected_target, cli_vars))?, + ), + &intermediate_loaders, + )?; if connection_is_default_target(discovered)? { return Ok(()); } From 01dd426e16f6902fcd63e5cc2d9a622d5713670c Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 17 Nov 2026 18:06:07 +0000 Subject: [PATCH 2/3] test: refuse a preserved target whose loader schema resolves to nothing usable --- .../cli/commands/main/build/_test_types.py | 3 ++- .../main/build/test_node_result_location.py | 17 +++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py index 8728f33513..d3810a9189 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py @@ -1680,11 +1680,12 @@ class PreservedTargetNodeResultsE2ETestCase: @dataclass(frozen=True) class PreservedTargetRefusalE2ETestCase: - """A preserved target without a loader schema and the compile error it must print.""" + """A preserved target without a usable loader schema and the compile error it must print.""" description: str python_files: dict[str, str] expected_error: str + target_lines: str = "" @dataclass(frozen=True) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_node_result_location.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_node_result_location.py index 02de124e27..f421f7abbd 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_node_result_location.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_node_result_location.py @@ -158,6 +158,22 @@ def test_given_target_location_when_building_and_loading_then_node_results_stay_ }, expected_error="error[S105]: Target 'dev' keeps each model's own schema", ), + PreservedTargetRefusalE2ETestCase( + description="a loader schema whose var resolves to empty is no loader schema", + python_files={ + "python/tasks/announce.py": _TASK.replace('depends_on=source("raw_orders")', "") + }, + expected_error="error[S105]: Target 'dev' keeps each model's own schema", + target_lines='loader_schema = "${results_schema}"\n\n[vars]\nresults_schema = ""\n', + ), + PreservedTargetRefusalE2ETestCase( + description="a loader schema that resolves only inside a model is no loader schema", + python_files={ + "python/tasks/announce.py": _TASK.replace('depends_on=source("raw_orders")', "") + }, + expected_error="error[S105]: Target 'dev' keeps each model's own schema", + target_lines='loader_schema = "raw_${CTX:model.name}"\n', + ), ], ids=lambda case: case.description, ) @@ -172,6 +188,7 @@ def test_given_preserved_schema_without_loader_schema_when_compiling_then_python 'name = "orders"\nadapter = "duckdb"\ndefault_target = "dev"\n\n' '[connection]\ndatabase = "dev.duckdb"\n\n' '[defaults]\nschema = "mart"\n\n[targets.dev]\nschema = "preserve"\n' + + test_case.target_lines ), "models/orders.sql": "MODEL (description 'Orders.');\n\nSELECT 1 AS order_id\n", } From 92de5b4be0186c517360fb37b8eb48a04278b5ba Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 17 Nov 2026 18:32:08 +0000 Subject: [PATCH 3/3] fix: read Python node fingerprints from their results location when planning Under a preserved target schema, Python nodes record their results and fingerprints in the loader schema, which planning never read, so every plan showed loaders, tasks and assets as new. Planning now passes the node results location into the warehouse snapshot, which reads that schema's fingerprints table without listing its other relations. --- .../_helpers/setup/node_results.rs | 9 +- .../src/compile_driver/main/setup/mod.rs | 1 + .../main/setup/preserved_loader_schemas.rs | 4 +- .../main/setup/python_node_count.rs | 9 ++ .../main/plans/build_execution_plan.rs | 8 +- .../planner_phases/_helpers/inputs/getters.rs | 1 + .../planner_phases/_helpers/inputs/runtime.rs | 4 +- .../_helpers/inputs/warehouse_state.rs | 7 +- .../src/planner_phases/models.rs | 2 + .../_helpers/inspection/snapshot.rs | 122 +++++++++++++++++- .../src/planner_warehouse/models.rs | 2 + .../_helpers/compile_driver/plan_pipeline.rs | 22 +++- .../_helpers/compile_driver/plan_result.rs | 8 +- src/sqlbuild/_native.pyi | 3 + .../cli/commands/main/build/_test_types.py | 20 +++ .../main/build/test_node_result_location.py | 116 +++++++++++++++++ 16 files changed, 319 insertions(+), 19 deletions(-) create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/main/setup/python_node_count.rs diff --git a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/node_results.rs b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/node_results.rs index f5065d8805..a98fb8e915 100644 --- a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/node_results.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/node_results.rs @@ -28,8 +28,7 @@ pub(crate) fn require_node_results_schema( effective_vars: &Bound<'_, PyAny>, ) -> PyResult<()> { let loader_schema: Option = resolved_loader_schema(target_config, effective_vars)?; - if kept_loader_schema(loader_schema.as_deref()).is_some() - || python_node_count(discovered)? == 0 + if kept_loader_schema(loader_schema.as_deref()).is_some() || python_node_count(discovered)? == 0 { return Ok(()); } @@ -60,7 +59,9 @@ fn resolved_loader_schema( match expanded_loader_namespace(namespace.as_any(), effective_vars) { Ok(expanded) => { let schema: Bound<'_, PyAny> = expanded.get_item("schema")?; - Ok((!schema.is_none()).then(|| schema.str()?.extract()).transpose()?) + Ok((!schema.is_none()) + .then(|| schema.str()?.extract()) + .transpose()?) } Err(error) if error.is_instance_of::(py) => Ok(None), Err(error) => Err(error), @@ -68,7 +69,7 @@ fn resolved_loader_schema( } /// How many loaders, tasks, assets and checks the project declares in Python. -fn python_node_count(discovered: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn python_node_count(discovered: &Bound<'_, PyAny>) -> PyResult { PYTHON_NODE_FIELDS .iter() .map(|field| native_field(discovered, field)?.len()) diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/setup/mod.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/mod.rs index 7217f6eb02..3dc72339e8 100644 --- a/crates/sqlbuild-python-compile/src/compile_driver/main/setup/mod.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/mod.rs @@ -16,6 +16,7 @@ pub mod preserved_loader_schemas; pub mod progress; pub mod project_graph; pub mod project_target_settings; +pub mod python_node_count; pub mod python_read_source_names_for; pub mod register_inputs; pub mod register_pipeline; diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/setup/preserved_loader_schemas.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/preserved_loader_schemas.rs index aaa643cf8b..ec28da636e 100644 --- a/crates/sqlbuild-python-compile/src/compile_driver/main/setup/preserved_loader_schemas.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/preserved_loader_schemas.rs @@ -14,9 +14,7 @@ use sqlbuild_python_base::boundary::main::optional_text::optional_text; use sqlbuild_python_base::records::main::fields::field::field as native_field; use std::collections::HashSet; -/// Raise when a `preserve` target leaves one of `loaders`, or the Python node results, without a schema. -/// -/// `effective_vars` expand the target's `loader_schema` as runtime does. +/// Raise when a `preserve` target leaves `loaders` or Python node results without a schema, after `effective_vars`. pub fn preserved_loader_schemas( discovered: &Bound<'_, PyAny>, (selected_target, effective_vars): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/setup/python_node_count.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/python_node_count.rs new file mode 100644 index 0000000000..92f25771f4 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/python_node_count.rs @@ -0,0 +1,9 @@ +//! How many Python nodes a project declares. + +use crate::compile_driver::_helpers::setup::node_results; +use pyo3::prelude::{Bound, PyAny, PyResult}; + +/// How many loaders, tasks, assets and checks the project declares in Python. +pub fn python_node_count(discovered: &Bound<'_, PyAny>) -> PyResult { + node_results::python_node_count(discovered) +} diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_execution_plan.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_execution_plan.rs index a854ea1cda..3c15840c05 100644 --- a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_execution_plan.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_execution_plan.rs @@ -38,7 +38,12 @@ pub fn build_execution_plan<'py>( "deferral", "policies", ], - &["on_progress", "project_config", "local_config"], + &[ + "on_progress", + "project_config", + "local_config", + "node_results_location", + ], ), )?; let project = args.value("project")?; @@ -54,6 +59,7 @@ pub fn build_execution_plan<'py>( local_config: args.value("local_config")?.unbind(), on_progress: args.value("on_progress")?.unbind(), invocation_time: utc_now(py)?.unbind(), + node_results_location: args.value("node_results_location")?.extract()?, }, )? .into_any(); diff --git a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/getters.rs b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/getters.rs index 48d89d4596..f215690bac 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/getters.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/getters.rs @@ -56,6 +56,7 @@ pub(crate) fn read_planner_runtime<'py>( "local_config" => (&this.local_config).into_bound_py_any(py), "on_progress" => (&this.on_progress).into_bound_py_any(py), "invocation_time" => (&this.invocation_time).into_bound_py_any(py), + "node_results_location" => (&this.node_results_location).into_bound_py_any(py), _ => return None, }) } diff --git a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/runtime.rs b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/runtime.rs index 5b3535f9a9..f2f9136fc3 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/runtime.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/runtime.rs @@ -1,7 +1,7 @@ //! The planner runtime and warehouse state: built from keyword arguments and read by planner phases. use crate::planner_warehouse::models::MigrationFingerprintCache; -use pyo3::prelude::{Bound, Py, PyAny, PyResult, Python}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::pymethods; use pyo3::types::{PyDict, PyTuple}; @@ -25,6 +25,7 @@ impl PlannerRuntime { "local_config", "on_progress", "invocation_time", + "node_results_location", ], ), )?; @@ -41,6 +42,7 @@ impl PlannerRuntime { local_config: args.value("local_config")?.unbind(), on_progress: args.value("on_progress")?.unbind(), invocation_time: invocation_time.unbind(), + node_results_location: args.value("node_results_location")?.extract()?, }) } } diff --git a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/warehouse_state.rs b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/warehouse_state.rs index 422b8bdbc9..444a59987c 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/warehouse_state.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/warehouse_state.rs @@ -9,7 +9,7 @@ use crate::planner_migrations::models::{ColumnMigrationInputs, MigrationPhase}; use crate::planner_phases::_helpers::inputs::contexts::class_field; use crate::planner_phases::_helpers::inputs::runtime::runtime_field; use crate::planner_phases::main::rebuilds::effectively_full_refreshed_model_names::effectively_full_refreshed_model_names; -use crate::planner_phases::models::PlannerWarehouseState; +use crate::planner_phases::models::{PlannerRuntime, PlannerWarehouseState}; use crate::planner_warehouse::main::inspection::warehouse_snapshot::warehouse_snapshot; use crate::planner_warehouse::models::MigrationFingerprintCache; use crate::planner_warehouse::models::SnapshotRequest; @@ -104,6 +104,11 @@ pub(crate) fn gather_planner_warehouse_state<'py>( on_progress: runtime_field(runtime, |runtime| &runtime.on_progress)?, deferred_locations: native_field(deferral, "deferred_locations")?, cursor_scope, + node_results_location: runtime + .cast::()? + .get() + .node_results_location + .clone(), })?; let phase = MigrationPhase { runtime: runtime.clone(), diff --git a/crates/sqlbuild-python-planner/src/planner_phases/models.rs b/crates/sqlbuild-python-planner/src/planner_phases/models.rs index a7a3da7fd2..14219134aa 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/models.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/models.rs @@ -72,6 +72,8 @@ pub struct PlannerRuntime { pub local_config: Py, pub on_progress: Py, pub invocation_time: Py, + /// Python nodes' `(database, schema)`, where their results and fingerprints live; `None` when unknown. + pub node_results_location: Option<(Option, Option)>, } /// Warehouse snapshot, inspection relations and migration plans gathered once per plan. diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/snapshot.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/snapshot.rs index 5919658d1a..6fec23f4ec 100644 --- a/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/snapshot.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/snapshot.rs @@ -7,11 +7,11 @@ use crate::planner_warehouse::constants::{ RELATIONS_MAIN_COLUMNS, RELATIONS_MAIN_LIST, }; use crate::planner_warehouse::models::SnapshotRequest; -use pyo3::IntoPyObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::pyfunction; use pyo3::types::PyDict as KeywordDict; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyString, PyTuple}; +use pyo3::{IntoPyObject, IntoPyObjectExt}; use sqlbuild_analysis::fingerprint_storage::constants::NODE_TYPE_MODEL; use sqlbuild_analysis::fingerprint_storage::constants::NODE_TYPE_SEED; use sqlbuild_analysis::graph::main::reachable_keys::reachable_keys; @@ -511,6 +511,7 @@ fn gather_fingerprints<'py>( let Some(schemas) = schemas else { return native(py, RecordId::WarehouseFingerprints, &[]); }; + let location = node_results_location(request, database, schemas)?; let node_names: Option> = if relevant.is_none() { None } else { @@ -609,7 +610,11 @@ fn gather_fingerprints<'py>( } } let by_identity = native_field(&set, "fingerprints_by_identity")?; - if !by_identity.is_none() { + let holds_python_nodes = match &location { + None => true, + Some(location) => location.read_as.as_deref() == Some(schema_lower.as_str()), + }; + if holds_python_nodes && !by_identity.is_none() { for item in mapping_items(&by_identity)?.try_iter()? { let (key, fingerprint): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item?.extract()?; let node_type: String = native_field(&fingerprint, "node_type")?.extract()?; @@ -622,6 +627,9 @@ fn gather_fingerprints<'py>( } } } + if let Some(location) = location.filter(|location| location.read_as.is_none()) { + read_node_result_fingerprints(request, location, &python_nodes)?; + } native( py, RecordId::WarehouseFingerprints, @@ -636,6 +644,114 @@ fn gather_fingerprints<'py>( ) } +/// Where the Python nodes of a plan record their fingerprints. +struct NodeResultsLocation { + database: Option, + schema: String, + /// The lowercased schema when the snapshot reads it in its own database. + read_as: Option, +} + +/// The request's node results location, or `None` when it has none. +fn node_results_location( + request: &SnapshotRequest<'_>, + database: &Bound<'_, PyAny>, + schemas: &BTreeSet, +) -> PyResult> { + let py = request.project.py(); + let Some((node_database, Some(schema))) = request.node_results_location.clone() else { + return Ok(None); + }; + let schema_lower = lower(&text(py, &schema))?; + let same_database = match (&node_database, database.is_none()) { + (None, true) => true, + (Some(name), false) => lower(&text(py, name))? == lower(database)?, + _ => false, + }; + let mut read_as = None; + for name in schemas { + if same_database && lower(&text(py, name))? == schema_lower { + read_as = Some(schema_lower.clone()); + } + } + Ok(Some(NodeResultsLocation { + database: node_database, + schema, + read_as, + })) +} + +/// Add Python node fingerprints from an unread results location, listing only its fingerprints table. +fn read_node_result_fingerprints<'py>( + request: &SnapshotRequest<'py>, + NodeResultsLocation { + database, schema, .. + }: NodeResultsLocation, + python_nodes: &Bound<'py, PyDict>, +) -> PyResult<()> { + let py = request.project.py(); + let node_database: Bound<'py, PyAny> = database.into_bound_py_any(py)?; + let node_schema: Bound<'py, PyAny> = text(py, &schema); + let mut table_exists = false; + for relation in list_relations( + (&request.adapter, &request.connection), + &node_database, + tuple(py, vec![node_schema.clone()])?, + tuple(py, vec![text(py, FINGERPRINT_TABLE_NAME)])?, + )? + .try_iter()? + { + let name: String = native_field(&relation?, "name")?.extract()?; + table_exists |= name.eq_ignore_ascii_case(FINGERPRINT_TABLE_NAME); + } + let set = read_latest_fingerprints(Some(&keywords( + py, + &[ + ("connection", request.connection.clone()), + ("execute", request.execute.clone()), + ( + "table_exists", + table_exists.into_pyobject(py)?.to_owned().into_any(), + ), + ("database", node_database), + ("schema", node_schema), + ( + "render_qualified_name", + native_field(&request.adapter, "render_qualified_name")?, + ), + ( + "render_read_latest_sql", + native_field(&request.adapter, "render_read_latest_fingerprints_sql")?, + ), + ("node_names", tuple(py, Vec::new())?), + ( + "filtered_node_types", + tuple( + py, + [ + NODE_TYPE_MODEL, + FUNCTION_NODE_TYPES[0], + FUNCTION_NODE_TYPES[1], + NODE_TYPE_SEED, + ] + .iter() + .map(|name| text(py, name)) + .collect(), + )?, + ), + ], + )?))?; + let by_identity = native_field(&set, "fingerprints_by_identity")?; + if by_identity.is_none() { + return Ok(()); + } + for item in mapping_items(&by_identity)?.try_iter()? { + let (key, fingerprint): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item?.extract()?; + python_nodes.set_item(key, fingerprint)?; + } + Ok(()) +} + fn scope_field<'py>(scope: &Bound<'py, PyAny>, name: &str) -> PyResult> { if scope.is_none() { Ok(scope.clone()) @@ -782,6 +898,7 @@ pub(crate) fn gather_warehouse_snapshot<'py>( on_progress: on_progress.clone(), deferred_locations: deferred_locations.clone(), cursor_scope: cursor_scope.clone(), + node_results_location: None, }) } @@ -836,6 +953,7 @@ pub(crate) fn gather_redirected_cursor_snapshots<'py>( on_progress: args.value("on_progress")?, deferred_locations: deferred_locations.clone(), cursor_scope: cursor_scope.clone(), + node_results_location: None, }; let snapshots = cursor_snapshots( &request, diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/models.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/models.rs index fc106c2d09..76f09c8206 100644 --- a/crates/sqlbuild-python-planner/src/planner_warehouse/models.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/models.rs @@ -343,4 +343,6 @@ pub struct SnapshotRequest<'py> { pub on_progress: Bound<'py, PyAny>, pub deferred_locations: Bound<'py, PyAny>, pub cursor_scope: Bound<'py, PyAny>, + /// Python nodes' `(database, schema)`; their fingerprints are read there too. `None` when unknown. + pub node_results_location: Option<(Option, Option)>, } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_pipeline.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_pipeline.rs index b7eebb23d9..d2b3895b16 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_pipeline.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_pipeline.rs @@ -24,6 +24,8 @@ use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_compile::compile_driver::main::setup::compiled_phase::compiled_phase; use sqlbuild_python_compile::compile_driver::main::setup::progress::progress; use sqlbuild_python_compile::compile_driver::main::setup::project_graph::project_graph; +use sqlbuild_python_compile::compile_driver::main::setup::project_target_settings::project_target_settings; +use sqlbuild_python_compile::compile_driver::main::setup::python_node_count::python_node_count; use sqlbuild_python_compile::compile_driver::main::setup::timing_tracker::timing_tracker; use sqlbuild_python_compile::compile_driver::main::stages::apply_unit_test_selectors::apply_unit_test_selectors; use sqlbuild_python_compile::compile_driver::main::stages::monotonic::monotonic; @@ -204,19 +206,27 @@ fn planned<'py>( let started: f64 = monotonic(py)?; let mut connection: Option> = None; let result: PyResult> = (|| { - let opened: Bound<'py, PyAny> = connection_with_hooks( - adapter, - &native_field(compiled, "connection_config")?, - hooks, - )?; + let connection_config: Bound<'py, PyAny> = native_field(compiled, "connection_config")?; + let project: Bound<'py, PyAny> = native_field(compiled, "project")?; + let node_results_location: Option<(Option, Option)> = + if python_node_count(discovered)? == 0 { + None + } else { + let (database, schema) = + project_target_settings(discovered, &project, (&connection_config, adapter))? + .node_locations(adapter)?; + Some((database.extract()?, schema.extract()?)) + }; + let opened: Bound<'py, PyAny> = connection_with_hooks(adapter, &connection_config, hooks)?; connection = Some(opened.clone()); pipeline_result( &PlanningInputs { discovered_inputs: discovered, adapter, connection: &opened, + node_results_location, }, - &native_field(compiled, "project")?, + &project, options, &native_field(hooks, "on_progress")?, ) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs index 1c45c533c1..60f4c4bec3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs @@ -21,7 +21,7 @@ use std::collections::HashSet; use pyo3::exceptions::PyException; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet}; -use pyo3::{pyfunction, wrap_pyfunction}; +use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::native_host::main::connection_failure::connection_failure; @@ -41,6 +41,8 @@ pub(crate) struct PlanningInputs<'a, 'py> { pub(crate) discovered_inputs: &'a Bound<'py, PyAny>, pub(crate) adapter: &'a Bound<'py, PyAny>, pub(crate) connection: &'a Bound<'py, PyAny>, + /// Python nodes' `(database, schema)` under the active target. + pub(crate) node_results_location: Option<(Option, Option)>, } /// The selected Python nodes and SQL keys when loading is off: loaders out, their sources in. @@ -204,6 +206,10 @@ pub(crate) fn pipeline_result<'py>( native_field(discovered, "project_config")?, ), ("local_config", native_field(discovered, "local_config")?), + ( + "node_results_location", + inputs.node_results_location.clone().into_bound_py_any(py)?, + ), ], )?), )?; diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 8d683a5772..964216a912 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -1244,6 +1244,7 @@ class PlannerRuntime: local_config: LocalConfig | None on_progress: Callable[[str], None] | None invocation_time: datetime + node_results_location: tuple[str | None, str | None] | None def __init__( self, *, @@ -1254,6 +1255,7 @@ class PlannerRuntime: local_config: LocalConfig | None = ..., on_progress: Callable[[str], None] | None = ..., invocation_time: datetime = ..., + node_results_location: tuple[str | None, str | None] | None = ..., ) -> None: ... class PlannerScopeResolution: @@ -5736,6 +5738,7 @@ def build_execution_plan( on_progress: Callable[[str], None] | None = None, project_config: ProjectConfig | None = None, local_config: LocalConfig | None = None, + node_results_location: tuple[str | None, str | None] | None = None, ) -> PlanOutput: ... class SqlTestPlanning: diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py index d3810a9189..400d99277b 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py @@ -1688,6 +1688,26 @@ class PreservedTargetRefusalE2ETestCase: target_lines: str = "" +@dataclass(frozen=True) +class PlanAfterBuildsE2ETestCase: + """A target's location settings, and the line `sqb plan` prints for a task after two builds.""" + + description: str + target_lines: str + expected_plan_line: str + plan_args: tuple[str, ...] = () + + +@dataclass(frozen=True) +class PlanAfterTargetChangeE2ETestCase: + """Location settings before and after a change, and the plan line for the task.""" + + description: str + earlier_target_lines: str + target_lines: str + expected_plan_line: str + + @dataclass(frozen=True) class AdapterDefaultResultsFallbackE2ETestCase: """Results left in the adapter default by an earlier build, and the loader rebuilt after it.""" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_node_result_location.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_node_result_location.py index f421f7abbd..1a009b12f8 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_node_result_location.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_node_result_location.py @@ -11,6 +11,8 @@ from tests.e2e.src.sqlbuild.cli.commands.main.build._test_types import ( AdapterDefaultResultsFallbackE2ETestCase, + PlanAfterBuildsE2ETestCase, + PlanAfterTargetChangeE2ETestCase, PreservedTargetNodeResultsE2ETestCase, PreservedTargetRefusalE2ETestCase, ) @@ -66,6 +68,16 @@ expected_results_schema="raw", expected_runs_per_node=2, ), + PreservedTargetNodeResultsE2ETestCase( + description="a preserved schema keeps node results in a loader schema set by a var", + target_lines=( + 'schema = "preserve"\nloader_schema = "${results_schema}"\n\n' + '[vars]\nresults_schema = "raw"\n' + ), + source_location=" schema: raw\n", + expected_results_schema="raw", + expected_runs_per_node=2, + ), PreservedTargetNodeResultsE2ETestCase( description="a preserved database keeps node results in the connection database", target_lines='database = "preserve"\nschema = "analytics"\nloader_schema = "raw"\n', @@ -203,6 +215,110 @@ def test_given_preserved_schema_without_loader_schema_when_compiling_then_python assert test_case.expected_error in result.stderr +@pytest.mark.parametrize( + "test_case", + [ + PlanAfterBuildsE2ETestCase( + description="a preserved schema reads the task's fingerprint from the loader schema", + target_lines='schema = "preserve"\nloader_schema = "raw"\n', + expected_plan_line="announce_orders task (unchanged)", + ), + PlanAfterBuildsE2ETestCase( + description="a plan of the task alone reads its fingerprint from the loader schema", + target_lines='schema = "preserve"\nloader_schema = "raw"\n', + expected_plan_line="announce_orders task (unchanged)", + plan_args=("--select", "announce_orders"), + ), + PlanAfterBuildsE2ETestCase( + description="a named schema reads the task's fingerprint from the target schema", + target_lines='schema = "analytics"\n', + expected_plan_line="announce_orders task (unchanged)", + ), + ], + ids=lambda case: case.description, +) +def test_given_two_builds_when_planning_then_python_nodes_are_not_new( + test_case: PlanAfterBuildsE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = node_result_project( + tmp_path=tmp_path, + target_lines=test_case.target_lines, + files={ + "sources/raw.yml": node_result_source_yml(" schema: raw\n"), + "python/loaders/orders.py": _LOADER, + "python/tasks/announce.py": _TASK, + }, + ) + for _ in range(2): + build: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + assert build.returncode == 0, build.stdout + build.stderr + + plan: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "plan", *test_case.plan_args), project_dir=project_dir + ) + + assert plan.returncode == 0, plan.stdout + plan.stderr + assert test_case.expected_plan_line in plan.stdout, plan.stdout + + +@pytest.mark.parametrize( + "test_case", + [ + PlanAfterTargetChangeE2ETestCase( + description="older task results in a later-sorting model schema do not hide current ones", + earlier_target_lines='schema = "staging"\n', + target_lines='schema = "preserve"\nloader_schema = "raw"\n', + expected_plan_line="announce_orders task (unchanged)", + ), + ], + ids=lambda case: case.description, +) +def test_given_older_named_schema_results_when_planning_preserved_target_then_loader_schema_wins( + test_case: PlanAfterTargetChangeE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = node_result_project( + tmp_path=tmp_path, + target_lines=test_case.earlier_target_lines, + files={ + "sources/raw.yml": node_result_source_yml(" schema: raw\n"), + "python/loaders/orders.py": _LOADER, + "python/tasks/announce.py": _TASK, + "models/staged_orders.sql": ( + "MODEL (description 'Staged orders.', schema \"staging\");\n\n" + "SELECT 1 AS order_id\n" + ), + "models/raw_order_ids.sql": ( + "MODEL (description 'Order ids beside the raw data.', schema \"raw\");\n\n" + "SELECT 1 AS order_id\n" + ), + }, + ) + first: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + assert first.returncode == 0, first.stdout + first.stderr + config: Path = project_dir / "sqlbuild_project.toml" + config.write_text( + config.read_text().replace(test_case.earlier_target_lines, test_case.target_lines) + ) + task: Path = project_dir / "python/tasks/announce.py" + task.write_text(task.read_text().replace("'announced': True", "'announced': 1")) + for _ in range(2): + build: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + assert build.returncode == 0, build.stdout + build.stderr + + plan: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "plan"), project_dir=project_dir + ) + + assert plan.returncode == 0, plan.stdout + plan.stderr + assert test_case.expected_plan_line in plan.stdout, plan.stdout + + @pytest.mark.parametrize( "test_case", [