diff --git a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/model_free_locations.rs b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/model_free_locations.rs index e001269c72..30554a4044 100644 --- a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/model_free_locations.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/model_free_locations.rs @@ -6,14 +6,20 @@ use sqlbuild_analysis::assembly::target_checks::constants::PRESERVE_TARGET_VALUE use sqlbuild_model_config::templates::main::holds_context_placeholder::holds_context_placeholder; use sqlbuild_python_base::warehouse_access::main::adapter_default_location::adapter_default_location; +/// The resolved `loader_schema` a `preserve` schema keeps Python node results in, unless it is unusable. +pub(crate) fn kept_loader_schema(loader_schema: Option<&str>) -> Option<&str> { + loader_schema.filter(|name| { + !name.is_empty() && *name != PRESERVE_TARGET_VALUE && !holds_context_placeholder(name) + }) +} + /// Python nodes' `(database, schema)`: a `preserve` schema reads `loader_schema`; unusable values the adapter's. pub(crate) fn model_free_locations<'py>( (database, schema): (Bound<'py, PyAny>, Bound<'py, PyAny>), loader_schema: Option<&str>, adapter: &Bound<'py, PyAny>, ) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { - let kept_schema: Option<&str> = - loader_schema.filter(|name| !name.is_empty() && *name != PRESERVE_TARGET_VALUE); + let kept_schema: Option<&str> = kept_loader_schema(loader_schema); let location = |value: Bound<'py, PyAny>, method: &str, preserved: Option<&str>| match ( value.extract::>()?.as_deref(), preserved, diff --git a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/node_results.rs b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/node_results.rs index d302abe94b..a98fb8e915 100644 --- a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/node_results.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/node_results.rs @@ -1,10 +1,12 @@ //! The schema a `preserve` target keeps Python node results in. +use crate::compile_driver::_helpers::setup::model_free_locations::kept_loader_schema; use crate::compile_driver::_helpers::setup::targets::planner_error; +use crate::compile_driver::main::setup::expanded_loader_namespace::expanded_loader_namespace; +use pyo3::exceptions::PyException; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; -use sqlbuild_analysis::assembly::target_checks::constants::{ - PRESERVE_TARGET_VALUE, PRESERVED_NODE_RESULTS_CODE, -}; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_analysis::assembly::target_checks::constants::PRESERVED_NODE_RESULTS_CODE; use sqlbuild_analysis::assembly::target_checks::errors::TargetError; use sqlbuild_analysis::assembly::target_checks::main::target_error_message::target_error_message; use sqlbuild_core::text::main::python_strip::python_strip; @@ -19,16 +21,15 @@ const PYTHON_NODE_FIELDS: [&str; 4] = [ "check_functions", ]; -/// Raise when a `preserve` target has Python nodes but no loader schema to keep their results in. +/// Raise when a `preserve` target has Python nodes but no usable loader schema, after `effective_vars`, to keep their results in. pub(crate) fn require_node_results_schema( discovered: &Bound<'_, PyAny>, (target_name, target_config): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), + effective_vars: &Bound<'_, PyAny>, ) -> PyResult<()> { - let loader_schema: Option = - optional_text(&native_field(target_config, "loader_schema")?)? - .map(|schema| python_strip(&schema).to_owned()) - .filter(|schema| !schema.is_empty() && schema != PRESERVE_TARGET_VALUE); - if loader_schema.is_some() || python_node_count(discovered)? == 0 { + let loader_schema: Option = resolved_loader_schema(target_config, effective_vars)?; + if kept_loader_schema(loader_schema.as_deref()).is_some() || python_node_count(discovered)? == 0 + { return Ok(()); } let target: String = target_name.extract()?; @@ -44,8 +45,31 @@ pub(crate) fn require_node_results_schema( )) } +/// The target's `loader_schema` as runtime resolves it: vars expanded, `None` when it cannot resolve outside a model. +fn resolved_loader_schema( + target_config: &Bound<'_, PyAny>, + effective_vars: &Bound<'_, PyAny>, +) -> PyResult> { + let Some(schema) = optional_text(&native_field(target_config, "loader_schema")?)? else { + return Ok(None); + }; + let py = target_config.py(); + let namespace: Bound<'_, PyDict> = PyDict::new(py); + namespace.set_item("schema", python_strip(&schema))?; + match expanded_loader_namespace(namespace.as_any(), effective_vars) { + Ok(expanded) => { + let schema: Bound<'_, PyAny> = expanded.get_item("schema")?; + Ok((!schema.is_none()) + .then(|| schema.str()?.extract()) + .transpose()?) + } + Err(error) if error.is_instance_of::(py) => Ok(None), + Err(error) => Err(error), + } +} + /// How many loaders, tasks, assets and checks the project declares in Python. -fn python_node_count(discovered: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn python_node_count(discovered: &Bound<'_, PyAny>) -> PyResult { PYTHON_NODE_FIELDS .iter() .map(|field| native_field(discovered, field)?.len()) diff --git a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/project.rs b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/project.rs index 4190c20d49..8fcac2dff0 100644 --- a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/project.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/project.rs @@ -205,7 +205,10 @@ fn targeted_project<'py>( } preserved_loader_schemas( discovered, - request.selected_target.bind(py), + ( + request.selected_target.bind(py), + &native_field(project, "effective_vars")?, + ), &intermediate_loaders, )?; managed_write_schemas(adapter, project)?; diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/setup/mod.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/mod.rs index 7217f6eb02..3dc72339e8 100644 --- a/crates/sqlbuild-python-compile/src/compile_driver/main/setup/mod.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/mod.rs @@ -16,6 +16,7 @@ pub mod preserved_loader_schemas; pub mod progress; pub mod project_graph; pub mod project_target_settings; +pub mod python_node_count; pub mod python_read_source_names_for; pub mod register_inputs; pub mod register_pipeline; diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/setup/preserved_loader_schemas.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/preserved_loader_schemas.rs index 9a0c3d649b..ec28da636e 100644 --- a/crates/sqlbuild-python-compile/src/compile_driver/main/setup/preserved_loader_schemas.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/preserved_loader_schemas.rs @@ -14,10 +14,10 @@ use sqlbuild_python_base::boundary::main::optional_text::optional_text; use sqlbuild_python_base::records::main::fields::field::field as native_field; use std::collections::HashSet; -/// Raise when a `preserve` target leaves one of `loaders`, or the Python node results, without a schema. +/// Raise when a `preserve` target leaves `loaders` or Python node results without a schema, after `effective_vars`. pub fn preserved_loader_schemas( discovered: &Bound<'_, PyAny>, - selected_target: &Bound<'_, PyAny>, + (selected_target, effective_vars): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), loaders: &HashSet, ) -> PyResult<()> { let py: Python<'_> = discovered.py(); @@ -48,5 +48,5 @@ pub fn preserved_loader_schemas( )), )); } - require_node_results_schema(discovered, (&target_name, &target_config)) + require_node_results_schema(discovered, (&target_name, &target_config), effective_vars) } diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/setup/python_node_count.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/python_node_count.rs new file mode 100644 index 0000000000..92f25771f4 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/python_node_count.rs @@ -0,0 +1,9 @@ +//! How many Python nodes a project declares. + +use crate::compile_driver::_helpers::setup::node_results; +use pyo3::prelude::{Bound, PyAny, PyResult}; + +/// How many loaders, tasks, assets and checks the project declares in Python. +pub fn python_node_count(discovered: &Bound<'_, PyAny>) -> PyResult { + node_results::python_node_count(discovered) +} diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_execution_plan.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_execution_plan.rs index a854ea1cda..3c15840c05 100644 --- a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_execution_plan.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_execution_plan.rs @@ -38,7 +38,12 @@ pub fn build_execution_plan<'py>( "deferral", "policies", ], - &["on_progress", "project_config", "local_config"], + &[ + "on_progress", + "project_config", + "local_config", + "node_results_location", + ], ), )?; let project = args.value("project")?; @@ -54,6 +59,7 @@ pub fn build_execution_plan<'py>( local_config: args.value("local_config")?.unbind(), on_progress: args.value("on_progress")?.unbind(), invocation_time: utc_now(py)?.unbind(), + node_results_location: args.value("node_results_location")?.extract()?, }, )? .into_any(); diff --git a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/getters.rs b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/getters.rs index 48d89d4596..f215690bac 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/getters.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/getters.rs @@ -56,6 +56,7 @@ pub(crate) fn read_planner_runtime<'py>( "local_config" => (&this.local_config).into_bound_py_any(py), "on_progress" => (&this.on_progress).into_bound_py_any(py), "invocation_time" => (&this.invocation_time).into_bound_py_any(py), + "node_results_location" => (&this.node_results_location).into_bound_py_any(py), _ => return None, }) } diff --git a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/runtime.rs b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/runtime.rs index 5b3535f9a9..f2f9136fc3 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/runtime.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/runtime.rs @@ -1,7 +1,7 @@ //! The planner runtime and warehouse state: built from keyword arguments and read by planner phases. use crate::planner_warehouse::models::MigrationFingerprintCache; -use pyo3::prelude::{Bound, Py, PyAny, PyResult, Python}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::pymethods; use pyo3::types::{PyDict, PyTuple}; @@ -25,6 +25,7 @@ impl PlannerRuntime { "local_config", "on_progress", "invocation_time", + "node_results_location", ], ), )?; @@ -41,6 +42,7 @@ impl PlannerRuntime { local_config: args.value("local_config")?.unbind(), on_progress: args.value("on_progress")?.unbind(), invocation_time: invocation_time.unbind(), + node_results_location: args.value("node_results_location")?.extract()?, }) } } diff --git a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/warehouse_state.rs b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/warehouse_state.rs index 422b8bdbc9..444a59987c 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/warehouse_state.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/warehouse_state.rs @@ -9,7 +9,7 @@ use crate::planner_migrations::models::{ColumnMigrationInputs, MigrationPhase}; use crate::planner_phases::_helpers::inputs::contexts::class_field; use crate::planner_phases::_helpers::inputs::runtime::runtime_field; use crate::planner_phases::main::rebuilds::effectively_full_refreshed_model_names::effectively_full_refreshed_model_names; -use crate::planner_phases::models::PlannerWarehouseState; +use crate::planner_phases::models::{PlannerRuntime, PlannerWarehouseState}; use crate::planner_warehouse::main::inspection::warehouse_snapshot::warehouse_snapshot; use crate::planner_warehouse::models::MigrationFingerprintCache; use crate::planner_warehouse::models::SnapshotRequest; @@ -104,6 +104,11 @@ pub(crate) fn gather_planner_warehouse_state<'py>( on_progress: runtime_field(runtime, |runtime| &runtime.on_progress)?, deferred_locations: native_field(deferral, "deferred_locations")?, cursor_scope, + node_results_location: runtime + .cast::()? + .get() + .node_results_location + .clone(), })?; let phase = MigrationPhase { runtime: runtime.clone(), diff --git a/crates/sqlbuild-python-planner/src/planner_phases/models.rs b/crates/sqlbuild-python-planner/src/planner_phases/models.rs index a7a3da7fd2..14219134aa 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/models.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/models.rs @@ -72,6 +72,8 @@ pub struct PlannerRuntime { pub local_config: Py, pub on_progress: Py, pub invocation_time: Py, + /// Python nodes' `(database, schema)`, where their results and fingerprints live; `None` when unknown. + pub node_results_location: Option<(Option, Option)>, } /// Warehouse snapshot, inspection relations and migration plans gathered once per plan. diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/snapshot.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/snapshot.rs index 5919658d1a..6fec23f4ec 100644 --- a/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/snapshot.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/snapshot.rs @@ -7,11 +7,11 @@ use crate::planner_warehouse::constants::{ RELATIONS_MAIN_COLUMNS, RELATIONS_MAIN_LIST, }; use crate::planner_warehouse::models::SnapshotRequest; -use pyo3::IntoPyObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::pyfunction; use pyo3::types::PyDict as KeywordDict; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyString, PyTuple}; +use pyo3::{IntoPyObject, IntoPyObjectExt}; use sqlbuild_analysis::fingerprint_storage::constants::NODE_TYPE_MODEL; use sqlbuild_analysis::fingerprint_storage::constants::NODE_TYPE_SEED; use sqlbuild_analysis::graph::main::reachable_keys::reachable_keys; @@ -511,6 +511,7 @@ fn gather_fingerprints<'py>( let Some(schemas) = schemas else { return native(py, RecordId::WarehouseFingerprints, &[]); }; + let location = node_results_location(request, database, schemas)?; let node_names: Option> = if relevant.is_none() { None } else { @@ -609,7 +610,11 @@ fn gather_fingerprints<'py>( } } let by_identity = native_field(&set, "fingerprints_by_identity")?; - if !by_identity.is_none() { + let holds_python_nodes = match &location { + None => true, + Some(location) => location.read_as.as_deref() == Some(schema_lower.as_str()), + }; + if holds_python_nodes && !by_identity.is_none() { for item in mapping_items(&by_identity)?.try_iter()? { let (key, fingerprint): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item?.extract()?; let node_type: String = native_field(&fingerprint, "node_type")?.extract()?; @@ -622,6 +627,9 @@ fn gather_fingerprints<'py>( } } } + if let Some(location) = location.filter(|location| location.read_as.is_none()) { + read_node_result_fingerprints(request, location, &python_nodes)?; + } native( py, RecordId::WarehouseFingerprints, @@ -636,6 +644,114 @@ fn gather_fingerprints<'py>( ) } +/// Where the Python nodes of a plan record their fingerprints. +struct NodeResultsLocation { + database: Option, + schema: String, + /// The lowercased schema when the snapshot reads it in its own database. + read_as: Option, +} + +/// The request's node results location, or `None` when it has none. +fn node_results_location( + request: &SnapshotRequest<'_>, + database: &Bound<'_, PyAny>, + schemas: &BTreeSet, +) -> PyResult> { + let py = request.project.py(); + let Some((node_database, Some(schema))) = request.node_results_location.clone() else { + return Ok(None); + }; + let schema_lower = lower(&text(py, &schema))?; + let same_database = match (&node_database, database.is_none()) { + (None, true) => true, + (Some(name), false) => lower(&text(py, name))? == lower(database)?, + _ => false, + }; + let mut read_as = None; + for name in schemas { + if same_database && lower(&text(py, name))? == schema_lower { + read_as = Some(schema_lower.clone()); + } + } + Ok(Some(NodeResultsLocation { + database: node_database, + schema, + read_as, + })) +} + +/// Add Python node fingerprints from an unread results location, listing only its fingerprints table. +fn read_node_result_fingerprints<'py>( + request: &SnapshotRequest<'py>, + NodeResultsLocation { + database, schema, .. + }: NodeResultsLocation, + python_nodes: &Bound<'py, PyDict>, +) -> PyResult<()> { + let py = request.project.py(); + let node_database: Bound<'py, PyAny> = database.into_bound_py_any(py)?; + let node_schema: Bound<'py, PyAny> = text(py, &schema); + let mut table_exists = false; + for relation in list_relations( + (&request.adapter, &request.connection), + &node_database, + tuple(py, vec![node_schema.clone()])?, + tuple(py, vec![text(py, FINGERPRINT_TABLE_NAME)])?, + )? + .try_iter()? + { + let name: String = native_field(&relation?, "name")?.extract()?; + table_exists |= name.eq_ignore_ascii_case(FINGERPRINT_TABLE_NAME); + } + let set = read_latest_fingerprints(Some(&keywords( + py, + &[ + ("connection", request.connection.clone()), + ("execute", request.execute.clone()), + ( + "table_exists", + table_exists.into_pyobject(py)?.to_owned().into_any(), + ), + ("database", node_database), + ("schema", node_schema), + ( + "render_qualified_name", + native_field(&request.adapter, "render_qualified_name")?, + ), + ( + "render_read_latest_sql", + native_field(&request.adapter, "render_read_latest_fingerprints_sql")?, + ), + ("node_names", tuple(py, Vec::new())?), + ( + "filtered_node_types", + tuple( + py, + [ + NODE_TYPE_MODEL, + FUNCTION_NODE_TYPES[0], + FUNCTION_NODE_TYPES[1], + NODE_TYPE_SEED, + ] + .iter() + .map(|name| text(py, name)) + .collect(), + )?, + ), + ], + )?))?; + let by_identity = native_field(&set, "fingerprints_by_identity")?; + if by_identity.is_none() { + return Ok(()); + } + for item in mapping_items(&by_identity)?.try_iter()? { + let (key, fingerprint): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item?.extract()?; + python_nodes.set_item(key, fingerprint)?; + } + Ok(()) +} + fn scope_field<'py>(scope: &Bound<'py, PyAny>, name: &str) -> PyResult> { if scope.is_none() { Ok(scope.clone()) @@ -782,6 +898,7 @@ pub(crate) fn gather_warehouse_snapshot<'py>( on_progress: on_progress.clone(), deferred_locations: deferred_locations.clone(), cursor_scope: cursor_scope.clone(), + node_results_location: None, }) } @@ -836,6 +953,7 @@ pub(crate) fn gather_redirected_cursor_snapshots<'py>( on_progress: args.value("on_progress")?, deferred_locations: deferred_locations.clone(), cursor_scope: cursor_scope.clone(), + node_results_location: None, }; let snapshots = cursor_snapshots( &request, diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/models.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/models.rs index fc106c2d09..76f09c8206 100644 --- a/crates/sqlbuild-python-planner/src/planner_warehouse/models.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/models.rs @@ -343,4 +343,6 @@ pub struct SnapshotRequest<'py> { pub on_progress: Bound<'py, PyAny>, pub deferred_locations: Bound<'py, PyAny>, pub cursor_scope: Bound<'py, PyAny>, + /// Python nodes' `(database, schema)`; their fingerprints are read there too. `None` when unknown. + pub node_results_location: Option<(Option, Option)>, } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_pipeline.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_pipeline.rs index b7eebb23d9..d2b3895b16 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_pipeline.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_pipeline.rs @@ -24,6 +24,8 @@ use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_compile::compile_driver::main::setup::compiled_phase::compiled_phase; use sqlbuild_python_compile::compile_driver::main::setup::progress::progress; use sqlbuild_python_compile::compile_driver::main::setup::project_graph::project_graph; +use sqlbuild_python_compile::compile_driver::main::setup::project_target_settings::project_target_settings; +use sqlbuild_python_compile::compile_driver::main::setup::python_node_count::python_node_count; use sqlbuild_python_compile::compile_driver::main::setup::timing_tracker::timing_tracker; use sqlbuild_python_compile::compile_driver::main::stages::apply_unit_test_selectors::apply_unit_test_selectors; use sqlbuild_python_compile::compile_driver::main::stages::monotonic::monotonic; @@ -204,19 +206,27 @@ fn planned<'py>( let started: f64 = monotonic(py)?; let mut connection: Option> = None; let result: PyResult> = (|| { - let opened: Bound<'py, PyAny> = connection_with_hooks( - adapter, - &native_field(compiled, "connection_config")?, - hooks, - )?; + let connection_config: Bound<'py, PyAny> = native_field(compiled, "connection_config")?; + let project: Bound<'py, PyAny> = native_field(compiled, "project")?; + let node_results_location: Option<(Option, Option)> = + if python_node_count(discovered)? == 0 { + None + } else { + let (database, schema) = + project_target_settings(discovered, &project, (&connection_config, adapter))? + .node_locations(adapter)?; + Some((database.extract()?, schema.extract()?)) + }; + let opened: Bound<'py, PyAny> = connection_with_hooks(adapter, &connection_config, hooks)?; connection = Some(opened.clone()); pipeline_result( &PlanningInputs { discovered_inputs: discovered, adapter, connection: &opened, + node_results_location, }, - &native_field(compiled, "project")?, + &project, options, &native_field(hooks, "on_progress")?, ) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs index 1c45c533c1..60f4c4bec3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs @@ -21,7 +21,7 @@ use std::collections::HashSet; use pyo3::exceptions::PyException; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet}; -use pyo3::{pyfunction, wrap_pyfunction}; +use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::native_host::main::connection_failure::connection_failure; @@ -41,6 +41,8 @@ pub(crate) struct PlanningInputs<'a, 'py> { pub(crate) discovered_inputs: &'a Bound<'py, PyAny>, pub(crate) adapter: &'a Bound<'py, PyAny>, pub(crate) connection: &'a Bound<'py, PyAny>, + /// Python nodes' `(database, schema)` under the active target. + pub(crate) node_results_location: Option<(Option, Option)>, } /// The selected Python nodes and SQL keys when loading is off: loaders out, their sources in. @@ -204,6 +206,10 @@ pub(crate) fn pipeline_result<'py>( native_field(discovered, "project_config")?, ), ("local_config", native_field(discovered, "local_config")?), + ( + "node_results_location", + inputs.node_results_location.clone().into_bound_py_any(py)?, + ), ], )?), )?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs index 10315d480b..bd8e8b5ad6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs @@ -410,7 +410,14 @@ pub(crate) fn loader_write_schemas<'py>( intermediate_loaders.insert(native_field(entry, "name")?.extract()?); } } - preserved_loader_schemas(discovered, selected_target, &intermediate_loaders)?; + preserved_loader_schemas( + discovered, + ( + selected_target, + &target_vars(discovered, (selected_target, cli_vars))?, + ), + &intermediate_loaders, + )?; if connection_is_default_target(discovered)? { return Ok(()); } diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 8d683a5772..964216a912 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -1244,6 +1244,7 @@ class PlannerRuntime: local_config: LocalConfig | None on_progress: Callable[[str], None] | None invocation_time: datetime + node_results_location: tuple[str | None, str | None] | None def __init__( self, *, @@ -1254,6 +1255,7 @@ class PlannerRuntime: local_config: LocalConfig | None = ..., on_progress: Callable[[str], None] | None = ..., invocation_time: datetime = ..., + node_results_location: tuple[str | None, str | None] | None = ..., ) -> None: ... class PlannerScopeResolution: @@ -5736,6 +5738,7 @@ def build_execution_plan( on_progress: Callable[[str], None] | None = None, project_config: ProjectConfig | None = None, local_config: LocalConfig | None = None, + node_results_location: tuple[str | None, str | None] | None = None, ) -> PlanOutput: ... class SqlTestPlanning: diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py index 8728f33513..400d99277b 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py @@ -1680,11 +1680,32 @@ class PreservedTargetNodeResultsE2ETestCase: @dataclass(frozen=True) class PreservedTargetRefusalE2ETestCase: - """A preserved target without a loader schema and the compile error it must print.""" + """A preserved target without a usable loader schema and the compile error it must print.""" description: str python_files: dict[str, str] expected_error: str + target_lines: str = "" + + +@dataclass(frozen=True) +class PlanAfterBuildsE2ETestCase: + """A target's location settings, and the line `sqb plan` prints for a task after two builds.""" + + description: str + target_lines: str + expected_plan_line: str + plan_args: tuple[str, ...] = () + + +@dataclass(frozen=True) +class PlanAfterTargetChangeE2ETestCase: + """Location settings before and after a change, and the plan line for the task.""" + + description: str + earlier_target_lines: str + target_lines: str + expected_plan_line: str @dataclass(frozen=True) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_node_result_location.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_node_result_location.py index 02de124e27..1a009b12f8 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_node_result_location.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_node_result_location.py @@ -11,6 +11,8 @@ from tests.e2e.src.sqlbuild.cli.commands.main.build._test_types import ( AdapterDefaultResultsFallbackE2ETestCase, + PlanAfterBuildsE2ETestCase, + PlanAfterTargetChangeE2ETestCase, PreservedTargetNodeResultsE2ETestCase, PreservedTargetRefusalE2ETestCase, ) @@ -66,6 +68,16 @@ expected_results_schema="raw", expected_runs_per_node=2, ), + PreservedTargetNodeResultsE2ETestCase( + description="a preserved schema keeps node results in a loader schema set by a var", + target_lines=( + 'schema = "preserve"\nloader_schema = "${results_schema}"\n\n' + '[vars]\nresults_schema = "raw"\n' + ), + source_location=" schema: raw\n", + expected_results_schema="raw", + expected_runs_per_node=2, + ), PreservedTargetNodeResultsE2ETestCase( description="a preserved database keeps node results in the connection database", target_lines='database = "preserve"\nschema = "analytics"\nloader_schema = "raw"\n', @@ -158,6 +170,22 @@ def test_given_target_location_when_building_and_loading_then_node_results_stay_ }, expected_error="error[S105]: Target 'dev' keeps each model's own schema", ), + PreservedTargetRefusalE2ETestCase( + description="a loader schema whose var resolves to empty is no loader schema", + python_files={ + "python/tasks/announce.py": _TASK.replace('depends_on=source("raw_orders")', "") + }, + expected_error="error[S105]: Target 'dev' keeps each model's own schema", + target_lines='loader_schema = "${results_schema}"\n\n[vars]\nresults_schema = ""\n', + ), + PreservedTargetRefusalE2ETestCase( + description="a loader schema that resolves only inside a model is no loader schema", + python_files={ + "python/tasks/announce.py": _TASK.replace('depends_on=source("raw_orders")', "") + }, + expected_error="error[S105]: Target 'dev' keeps each model's own schema", + target_lines='loader_schema = "raw_${CTX:model.name}"\n', + ), ], ids=lambda case: case.description, ) @@ -172,6 +200,7 @@ def test_given_preserved_schema_without_loader_schema_when_compiling_then_python 'name = "orders"\nadapter = "duckdb"\ndefault_target = "dev"\n\n' '[connection]\ndatabase = "dev.duckdb"\n\n' '[defaults]\nschema = "mart"\n\n[targets.dev]\nschema = "preserve"\n' + + test_case.target_lines ), "models/orders.sql": "MODEL (description 'Orders.');\n\nSELECT 1 AS order_id\n", } @@ -186,6 +215,110 @@ def test_given_preserved_schema_without_loader_schema_when_compiling_then_python assert test_case.expected_error in result.stderr +@pytest.mark.parametrize( + "test_case", + [ + PlanAfterBuildsE2ETestCase( + description="a preserved schema reads the task's fingerprint from the loader schema", + target_lines='schema = "preserve"\nloader_schema = "raw"\n', + expected_plan_line="announce_orders task (unchanged)", + ), + PlanAfterBuildsE2ETestCase( + description="a plan of the task alone reads its fingerprint from the loader schema", + target_lines='schema = "preserve"\nloader_schema = "raw"\n', + expected_plan_line="announce_orders task (unchanged)", + plan_args=("--select", "announce_orders"), + ), + PlanAfterBuildsE2ETestCase( + description="a named schema reads the task's fingerprint from the target schema", + target_lines='schema = "analytics"\n', + expected_plan_line="announce_orders task (unchanged)", + ), + ], + ids=lambda case: case.description, +) +def test_given_two_builds_when_planning_then_python_nodes_are_not_new( + test_case: PlanAfterBuildsE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = node_result_project( + tmp_path=tmp_path, + target_lines=test_case.target_lines, + files={ + "sources/raw.yml": node_result_source_yml(" schema: raw\n"), + "python/loaders/orders.py": _LOADER, + "python/tasks/announce.py": _TASK, + }, + ) + for _ in range(2): + build: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + assert build.returncode == 0, build.stdout + build.stderr + + plan: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "plan", *test_case.plan_args), project_dir=project_dir + ) + + assert plan.returncode == 0, plan.stdout + plan.stderr + assert test_case.expected_plan_line in plan.stdout, plan.stdout + + +@pytest.mark.parametrize( + "test_case", + [ + PlanAfterTargetChangeE2ETestCase( + description="older task results in a later-sorting model schema do not hide current ones", + earlier_target_lines='schema = "staging"\n', + target_lines='schema = "preserve"\nloader_schema = "raw"\n', + expected_plan_line="announce_orders task (unchanged)", + ), + ], + ids=lambda case: case.description, +) +def test_given_older_named_schema_results_when_planning_preserved_target_then_loader_schema_wins( + test_case: PlanAfterTargetChangeE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = node_result_project( + tmp_path=tmp_path, + target_lines=test_case.earlier_target_lines, + files={ + "sources/raw.yml": node_result_source_yml(" schema: raw\n"), + "python/loaders/orders.py": _LOADER, + "python/tasks/announce.py": _TASK, + "models/staged_orders.sql": ( + "MODEL (description 'Staged orders.', schema \"staging\");\n\n" + "SELECT 1 AS order_id\n" + ), + "models/raw_order_ids.sql": ( + "MODEL (description 'Order ids beside the raw data.', schema \"raw\");\n\n" + "SELECT 1 AS order_id\n" + ), + }, + ) + first: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + assert first.returncode == 0, first.stdout + first.stderr + config: Path = project_dir / "sqlbuild_project.toml" + config.write_text( + config.read_text().replace(test_case.earlier_target_lines, test_case.target_lines) + ) + task: Path = project_dir / "python/tasks/announce.py" + task.write_text(task.read_text().replace("'announced': True", "'announced': 1")) + for _ in range(2): + build: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + assert build.returncode == 0, build.stdout + build.stderr + + plan: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "plan"), project_dir=project_dir + ) + + assert plan.returncode == 0, plan.stdout + plan.stderr + assert test_case.expected_plan_line in plan.stdout, plan.stdout + + @pytest.mark.parametrize( "test_case", [