From 0a6e3d3571a005bd742153a8c47895c9cd98edc7 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 17 Nov 2026 12:48:57 +0000 Subject: [PATCH 1/6] fix: keep Python node results in one location per target Loaders, tasks, assets and checks now write their results and loader fingerprints to one location for every target: the target's schema, the target's loader_schema when the schema is preserve, or the adapter's default schema when the target schema is unset or resolved per model. Scheduled loaders and build-time checks used the adapter default before, so adapters without one, such as Snowflake, skipped those writes. Compile stops with S105 when a preserve target without a loader_schema has Python nodes. For one release, a node whose result is missing from the new location is read from the adapter's default location instead. --- .../edge_cases/loaders/loader_results.py | 34 +++ .../edge_cases/loaders/loader_results.yml | 9 + .../src/assembly/target_checks/constants.rs | 2 + .../src/assembly/target_checks/errors.rs | 2 + .../main/target_error_message.rs | 5 + .../src/compile_driver/main/setup/mod.rs | 1 + .../main/setup/model_free_locations.rs | 38 ++- .../main/setup/preserved_loader_schemas.rs | 44 +++- .../setup/project_model_free_locations.rs | 22 ++ .../src/compile_driver/models.rs | 6 + .../_helpers/build_command/execution.rs | 45 ++-- .../_helpers/build_command/lifecycle.rs | 12 +- .../bindings/_helpers/check/check_command.rs | 6 +- .../src/bindings/_helpers/check/check_core.rs | 19 +- .../bindings/_helpers/load/load_command.rs | 13 + .../docs/concepts/project-configuration.md | 2 +- .../docs/concepts/python-nodes/overview.md | 14 +- .../executor/build/_helpers/source_node.py | 26 +- src/sqlbuild/executor/build/models.py | 2 + .../executor/load/_helpers/dag_runtime.py | 4 +- src/sqlbuild/executor/load/models.py | 2 + .../node_results/classes/direct_store.py | 26 +- .../node_results/main/_direct_store.py | 4 +- .../_helpers/ingress_execution.py | 6 +- .../cli/commands/main/build/_test_types.py | 20 ++ .../main/build/test_node_result_location.py | 229 ++++++++++++++++++ .../executor/build/_helpers/_test_types.py | 10 + .../build/_helpers/test_source_node.py | 73 ++++++ .../docs/concepts/project-configuration.mdx | 2 +- .../docs/concepts/python-nodes/overview.mdx | 14 +- 30 files changed, 607 insertions(+), 85 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/loaders/loader_results.py create mode 100644 benchmarks/duckshop/edge_cases/loaders/loader_results.yml create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/main/setup/project_model_free_locations.rs create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/build/test_node_result_location.py diff --git a/benchmarks/duckshop/edge_cases/loaders/loader_results.py b/benchmarks/duckshop/edge_cases/loaders/loader_results.py new file mode 100644 index 0000000000..4c000683c4 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/loaders/loader_results.py @@ -0,0 +1,34 @@ +"""Two loaders and a task that reads both of their results. + +A build that selects only one loader still reads the other loader's latest result, which may come +from an earlier build. On Snowflake the adapter has no default schema, so the results must go to +the target's schema for the task to find them. +""" + +from sqlbuild.executor.load.models import LoaderContext +from sqlbuild.loaders import loader +from sqlbuild.refs import source +from sqlbuild.tasks import task + + +@loader +def raw_order_events(ctx: LoaderContext) -> list[dict[str, object]]: + """Order events.""" + return [{"event_id": 1}] + + +@loader +def raw_customer_events(ctx: LoaderContext) -> list[dict[str, object]]: + """Customer events.""" + return [{"event_id": 1}] + + +@task(depends_on=[source("raw_order_events"), source("raw_customer_events")]) +def announce_events(ctx): + """Record the run each loader result came from.""" + return ctx.result( + payload={ + "orders": ctx.result_of(node_function=raw_order_events).run_id, + "customers": ctx.result_of(node_function=raw_customer_events).run_id, + } + ) diff --git a/benchmarks/duckshop/edge_cases/loaders/loader_results.yml b/benchmarks/duckshop/edge_cases/loaders/loader_results.yml new file mode 100644 index 0000000000..1caf61e111 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/loaders/loader_results.yml @@ -0,0 +1,9 @@ +sources: + - name: raw_order_events + description: Order events, read by announce_events. + managed: true + write_strategy: table + - name: raw_customer_events + description: Customer events, read by announce_events. + managed: true + write_strategy: table diff --git a/crates/sqlbuild-analysis/src/assembly/target_checks/constants.rs b/crates/sqlbuild-analysis/src/assembly/target_checks/constants.rs index 947493d5bb..01df1688b4 100644 --- a/crates/sqlbuild-analysis/src/assembly/target_checks/constants.rs +++ b/crates/sqlbuild-analysis/src/assembly/target_checks/constants.rs @@ -10,5 +10,7 @@ pub const LOADER_COLLISION_CODE: &str = "S102"; pub const MANAGED_WRITE_SCHEMA_CODE: &str = "S103"; /// A named target chooses no schema strategy. pub const NAMED_TARGET_SCHEMA_CODE: &str = "S104"; +/// A target that keeps each model's schema has no schema for Python node results. +pub const PRESERVED_NODE_RESULTS_CODE: &str = "S105"; pub(crate) const DATABASE_PART: &str = "database"; pub(crate) const SCHEMA_PART: &str = "schema"; diff --git a/crates/sqlbuild-analysis/src/assembly/target_checks/errors.rs b/crates/sqlbuild-analysis/src/assembly/target_checks/errors.rs index 9c1dcf2dc0..d65fe743e9 100644 --- a/crates/sqlbuild-analysis/src/assembly/target_checks/errors.rs +++ b/crates/sqlbuild-analysis/src/assembly/target_checks/errors.rs @@ -15,4 +15,6 @@ pub enum TargetError { NamedTargetSchema(String), /// An intermediate loader, by name, whose schema resolves to the target's `preserve`. PreservedLoaderSchema(String), + /// A target, by name, whose schema is `preserve` and that names no loader schema. + PreservedNodeResults(String), } diff --git a/crates/sqlbuild-analysis/src/assembly/target_checks/main/target_error_message.rs b/crates/sqlbuild-analysis/src/assembly/target_checks/main/target_error_message.rs index ba4b45c4a6..7cfb39622b 100644 --- a/crates/sqlbuild-analysis/src/assembly/target_checks/main/target_error_message.rs +++ b/crates/sqlbuild-analysis/src/assembly/target_checks/main/target_error_message.rs @@ -48,5 +48,10 @@ pub fn target_error_message(error: &TargetError) -> String { '{PRESERVE_TARGET_VALUE}'. The target's schema '{PRESERVE_TARGET_VALUE}' keeps each \ model's own schema, and a loader has none, so its destination must name a schema." ), + TargetError::PreservedNodeResults(target) => format!( + "Target '{target}' keeps each model's own schema, so it has no schema for the \ + results of loaders, tasks, assets and checks. Set targets.{target}.loader_schema; \ + Python node results are kept there." + ), } } diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/setup/mod.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/mod.rs index 2e7fb32797..7ecbc5aaa6 100644 --- a/crates/sqlbuild-python-compile/src/compile_driver/main/setup/mod.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/mod.rs @@ -16,6 +16,7 @@ pub mod plan_has_executable_work; pub mod preserved_loader_schemas; pub mod progress; pub mod project_graph; +pub mod project_model_free_locations; pub mod project_target_settings; pub mod python_read_source_names_for; pub mod register_inputs; diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/setup/model_free_locations.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/model_free_locations.rs index 38e17b630f..526836b3ce 100644 --- a/crates/sqlbuild-python-compile/src/compile_driver/main/setup/model_free_locations.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/model_free_locations.rs @@ -1,23 +1,39 @@ //! The target database and schema that Python nodes use outside any model. use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_analysis::assembly::target_checks::constants::PRESERVE_TARGET_VALUE; use sqlbuild_model_config::templates::main::holds_context_placeholder::holds_context_placeholder; -use sqlbuild_python_base::records::main::fields::field::field as native_field; -/// The compiled project's target database and schema, each from `default` with the adapter method's name when it is unset, empty, or resolves only inside a model, such as a `${CTX:...}` schema. +/// The database and schema of Python nodes, their results and loader fingerprints, from the +/// target's configured `(database, schema)`; a `preserve` schema reads `loader_schema`, and an +/// unset, empty, `preserve` or model-only value, such as a `${CTX:...}` schema, reads `default` +/// with the adapter method's name. pub fn model_free_locations<'py>( - project: &Bound<'py, PyAny>, + (database, schema): (Bound<'py, PyAny>, Bound<'py, PyAny>), + loader_schema: Option<&str>, default: impl Fn(&str) -> PyResult>, ) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { - let location = |field: &str, fallback: &str| { - let value = native_field(project, field)?; - match value.extract::>()? { - Some(text) if !text.is_empty() && !holds_context_placeholder(&text) => Ok(value), + let location = + |value: Bound<'py, PyAny>, fallback: &str| match value.extract::>()? { + Some(text) if text == PRESERVE_TARGET_VALUE && fallback == SCHEMA_DEFAULT => { + match loader_schema.filter(|schema| *schema != PRESERVE_TARGET_VALUE) { + Some(schema) => Ok(pyo3::types::PyString::new(value.py(), schema).into_any()), + None => default(fallback), + } + } + Some(text) + if !text.is_empty() + && text != PRESERVE_TARGET_VALUE + && !holds_context_placeholder(&text) => + { + Ok(value) + } _ => default(fallback), - } - }; + }; Ok(( - location("effective_target_database", "default_database")?, - location("effective_target_schema", "default_schema")?, + location(database, "default_database")?, + location(schema, SCHEMA_DEFAULT)?, )) } + +const SCHEMA_DEFAULT: &str = "default_schema"; diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/setup/preserved_loader_schemas.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/preserved_loader_schemas.rs index 54162399ed..e48bc19623 100644 --- a/crates/sqlbuild-python-compile/src/compile_driver/main/setup/preserved_loader_schemas.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/preserved_loader_schemas.rs @@ -1,10 +1,10 @@ -//! Raise when the selected target's schema is `preserve` and one of `loaders` names no schema. +//! Raise when the selected target's schema is `preserve` and a loader or Python node has no schema. use crate::compile_attachment::main::resolved_target::resolved_target; use crate::compile_driver::_helpers::setup::targets::planner_error; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use sqlbuild_analysis::assembly::target_checks::constants::{ - MANAGED_WRITE_SCHEMA_CODE, PRESERVE_TARGET_VALUE, + MANAGED_WRITE_SCHEMA_CODE, PRESERVE_TARGET_VALUE, PRESERVED_NODE_RESULTS_CODE, }; use sqlbuild_analysis::assembly::target_checks::errors::TargetError; use sqlbuild_analysis::assembly::target_checks::main::target_error_message::target_error_message; @@ -13,7 +13,16 @@ use sqlbuild_python_base::boundary::main::optional_text::optional_text; use sqlbuild_python_base::records::main::fields::field::field as native_field; use std::collections::HashSet; -/// Raise when the selected target's schema is `preserve` and one of `loaders` names no schema. +/// The discovered inputs' Python node functions, each kind of which keeps node results. +const PYTHON_NODE_FIELDS: [&str; 4] = [ + "loader_functions", + "task_functions", + "asset_functions", + "check_functions", +]; + +/// Raise when the selected target's schema is `preserve` and one of `loaders` names no schema, or +/// the project has Python nodes and the target names no loader schema to keep their results in. pub fn preserved_loader_schemas( discovered: &Bound<'_, PyAny>, selected_target: &Bound<'_, PyAny>, @@ -47,5 +56,32 @@ pub fn preserved_loader_schemas( )), )); } - Ok(()) + let loader_schema: Option = + optional_text(&native_field(&target_config, "loader_schema")?)? + .map(|schema| python_strip(&schema).to_owned()) + .filter(|schema| !schema.is_empty() && schema != PRESERVE_TARGET_VALUE); + if loader_schema.is_some() || !has_python_nodes(discovered)? { + return Ok(()); + } + let target: String = target_name.extract()?; + Err(planner_error( + py, + target_error_message(&TargetError::PreservedNodeResults(target.clone())), + PRESERVED_NODE_RESULTS_CODE, + Some(format!( + "to keep Python node results in a schema, set this in sqlbuild_project.toml:\n \ + [targets.{target}]\n \ + loader_schema = \"my_schema\"" + )), + )) +} + +/// Whether the project declares any loader, task, asset or check written in Python. +fn has_python_nodes(discovered: &Bound<'_, PyAny>) -> PyResult { + for field in PYTHON_NODE_FIELDS { + if native_field(discovered, field)?.len()? > 0 { + return Ok(true); + } + } + Ok(false) } diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/setup/project_model_free_locations.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/project_model_free_locations.rs new file mode 100644 index 0000000000..58d8ed401c --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/project_model_free_locations.rs @@ -0,0 +1,22 @@ +//! The compiled project's database and schema for Python nodes outside any model. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +use crate::compile_driver::main::setup::model_free_locations::model_free_locations; + +/// The compiled project's `model_free_locations`, falling back to the adapter's defaults. +pub fn project_model_free_locations<'py>( + project: &Bound<'py, PyAny>, + loader_schema: Option<&str>, + adapter: &Bound<'py, PyAny>, +) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { + model_free_locations( + ( + native_field(project, "effective_target_database")?, + native_field(project, "effective_target_schema")?, + ), + loader_schema, + |method| adapter.call_method0(method), + ) +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/models.rs b/crates/sqlbuild-python-compile/src/compile_driver/models.rs index aece8e4552..c400be8c3b 100644 --- a/crates/sqlbuild-python-compile/src/compile_driver/models.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/models.rs @@ -77,6 +77,12 @@ pub struct TargetSettingsValues { } impl TargetSettingsValues { + /// The resolved schema managed loader data goes to. + #[must_use] + pub fn loader_schema(&self) -> Option<&str> { + self.loader_schema.as_deref() + } + /// The `TargetSettings` record: a `preserve` location reads `None` with its flag set. pub fn record<'py>(&self, py: Python<'py>) -> PyResult> { let located = |value: &Option| -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs index c4bebd360d..ff98f45069 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs @@ -43,7 +43,9 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_compile::compile_driver::main::setup::build_project_relation_targets::build_project_relation_targets; +use sqlbuild_python_compile::compile_driver::main::setup::project_model_free_locations::project_model_free_locations; use sqlbuild_python_compile::compile_driver::main::setup::project_target_settings::project_target_settings; +use sqlbuild_python_compile::compile_driver::models::TargetSettingsValues; use sqlbuild_python_front::cost_context::models::CostContext; use sqlbuild_python_front::cursor::main::overrides::cursor_integer::cursor_integer; use sqlbuild_python_front::cursor::main::overrides::cursor_timestamp::cursor_timestamp; @@ -333,6 +335,13 @@ pub(crate) fn execute_build_plan<'py>( let lifecycle: &DirectPythonLifecycle = preparation.lifecycle.get(); let [start_ts, end_ts, start_int, end_int] = &preparation.cursors; let request = |name: &str| native_field(&context.request, name); + let target_config: TargetSettingsValues = project_target_settings( + &context.get("discovered_inputs")?, + &project, + (&connection_config, &adapter), + )?; + let (default_database, default_schema) = + project_model_free_locations(&project, target_config.loader_schema(), &adapter)?; let runtime: Bound<'py, PyAny> = call_with( &host_entry(py, ExecutorHost::BuildRuntimeParams)?, &[ @@ -360,15 +369,9 @@ pub(crate) fn execute_build_plan<'py>( preparation.concurrency.into_pyobject(py)?.into_any(), ), ("loader_is_reload", request("reload_sources")?), - ( - "target_config", - project_target_settings( - &context.get("discovered_inputs")?, - &project, - (&connection_config, &adapter), - )? - .record(py)?, - ), + ("target_config", target_config.record(py)?), + ("default_database", default_database), + ("default_schema", default_schema), ("start_cursor_ts", start_ts.clone()), ("end_cursor_ts", end_ts.clone()), ("start_cursor_int", start_int.clone()), @@ -633,6 +636,13 @@ fn check_runtime<'py>( .iter() .map(|check| native_field(check, "name")) .collect::>()?; + let target_config: TargetSettingsValues = project_target_settings( + &context.get("discovered_inputs")?, + &project, + (&context.get("connection_config")?, &adapter), + )?; + let (default_database, default_schema) = + project_model_free_locations(&project, target_config.loader_schema(), &adapter)?; call_with( &host_entry(py, ExecutorHost::PythonNodeRuntime)?, &[ @@ -646,20 +656,9 @@ fn check_runtime<'py>( "is_reload", native_field(&context.request, "reload_sources")?, ), - ( - "target_config", - project_target_settings( - &context.get("discovered_inputs")?, - &project, - (&context.get("connection_config")?, &adapter), - )? - .record(py)?, - ), - ( - "default_database", - adapter.call_method0("default_database")?, - ), - ("default_schema", adapter.call_method0("default_schema")?), + ("target_config", target_config.record(py)?), + ("default_database", default_database), + ("default_schema", default_schema), ( "relation_targets", build_check_relation_targets(&adapter, pipeline, graph, frozenset(py, &names)?)?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs index c0dbe28177..591968e64b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs @@ -28,7 +28,7 @@ use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_key use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_compile::compile_driver::main::setup::build_project_relation_targets::build_project_relation_targets; use sqlbuild_python_compile::compile_driver::main::setup::build_python_relation_targets::build_python_relation_targets; -use sqlbuild_python_compile::compile_driver::main::setup::model_free_locations::model_free_locations; +use sqlbuild_python_compile::compile_driver::main::setup::project_model_free_locations::project_model_free_locations; use sqlbuild_python_compile::compile_driver::main::setup::project_target_settings::project_target_settings; use sqlbuild_python_compile::compile_driver::models::TargetSettingsValues; use sqlbuild_python_project::graph::main::python_nodes::build_python_sql_run_lifecycle::build_python_sql_run_lifecycle; @@ -432,17 +432,15 @@ pub(crate) fn prepare_direct_python_lifecycle<'py>( )?), ) .and_then(|value| pyo3::IntoPyObjectExt::into_bound_py_any(value, py))?; + let target_config: TargetSettingsValues = + project_target_settings(&discovered, &project, (&connection_config, &adapter))?; let parts: RuntimeParts<'_, 'py> = RuntimeParts { adapter: &adapter, connection_config: &connection_config, project: &project, reload: context.request_flag("reload_sources")?, - defaults: model_free_locations(&project, |method| adapter.call_method0(method))?, - target_config: project_target_settings( - &discovered, - &project, - (&connection_config, &adapter), - )?, + defaults: project_model_free_locations(&project, target_config.loader_schema(), &adapter)?, + target_config, relations: (relation_targets, project_relations), inputs, }; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs index 7b544a29f9..f9c9b8ce21 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs @@ -25,7 +25,7 @@ use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_compile::compile_driver::main::setup::build_project_relation_targets::build_project_relation_targets; use sqlbuild_python_compile::compile_driver::main::setup::build_python_relation_targets::build_python_relation_targets; use sqlbuild_python_compile::compile_driver::main::setup::build_static_pipeline_result::build_static_pipeline_result; -use sqlbuild_python_compile::compile_driver::main::setup::model_free_locations::model_free_locations; +use sqlbuild_python_compile::compile_driver::main::setup::project_model_free_locations::project_model_free_locations; use sqlbuild_python_compile::compile_driver::main::setup::project_target_settings::project_target_settings; use sqlbuild_python_compile::compile_driver::models::TargetSettingsValues; use sqlbuild_python_front::discovery::main::sources::consumer_uses::consumer_uses; @@ -343,12 +343,12 @@ fn prepare<'py>( (&native_field(invocation, "connection_config")?, &adapter), )?; Ok(Prepared { - target_config, check_functions: PyTuple::new(py, check_functions)?.into_any(), lifecycle_plan, relation_targets, relation_refs, - defaults: model_free_locations(&project, |method| adapter.call_method0(method))?, + defaults: project_model_free_locations(&project, target_config.loader_schema(), &adapter)?, + target_config, project_relations: object( py, build_project_relation_targets( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/check_core.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/check_core.rs index 98b2b61bb4..bd31197b5e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/check/check_core.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/check_core.rs @@ -11,6 +11,7 @@ use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyFrozenSetMethods, PyTupl use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_compile::compile_driver::main::setup::build_python_relation_targets::build_python_relation_targets; +use sqlbuild_python_compile::compile_driver::main::setup::project_model_free_locations::project_model_free_locations; use sqlbuild_python_project::graph::main::python_nodes::resolve_python_check_excludes::resolve_python_check_excludes; use sqlbuild_python_project::planner_phases::main::object::object; @@ -446,6 +447,14 @@ fn read_side_references<'py>( fn node_runtime<'py>(inputs: &ReadSideInputs<'py>) -> PyResult> { let py: Python<'py> = inputs.adapter.py(); let project: Bound<'py, PyAny> = native_field(&inputs.pipeline_result, "project")?; + let (default_database, default_schema) = project_model_free_locations( + &project, + inputs + .target_config + .as_ref() + .and_then(TargetSettingsValues::loader_schema), + &inputs.adapter, + )?; call_with( &host_entry(py, ExecutorHost::PythonNodeRuntime)?, &[ @@ -463,14 +472,8 @@ fn node_runtime<'py>(inputs: &ReadSideInputs<'py>) -> PyResult None => py.None().into_bound(py), }, ), - ( - "default_database", - inputs.adapter.call_method0("default_database")?, - ), - ( - "default_schema", - inputs.adapter.call_method0("default_schema")?, - ), + ("default_database", default_database), + ("default_schema", default_schema), ("relation_targets", inputs.relation_targets.clone()), ("project_relations", inputs.project_relations.clone()), ("providers", inputs.providers.clone()), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs index 4ea2a1ffd1..2f8ae9a18f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs @@ -35,6 +35,7 @@ use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_compile::compile_driver::main::setup::model_free_locations::model_free_locations; use sqlbuild_python_compile::compile_driver::main::setup::selected_target_settings::selected_target_settings; use sqlbuild_python_compile::compile_driver::main::stages::discover_project_inputs::discover_project_inputs; use sqlbuild_python_compile::compile_driver::models::TargetSettingsValues; @@ -67,6 +68,7 @@ struct Invocation<'py> { selected: Vec>, references: Vec>, relation_sources: Vec>, + defaults: LoaderDefaults<'py>, style: CliStyle, stream: Bound<'py, PyAny>, } @@ -181,6 +183,7 @@ fn invocation<'py>(py: Python<'py>, request: &Bound<'py, PyAny>) -> PyResult( let py: Python<'py> = request.py(); let (target_name, effective_vars, run_id) = &preparation.runtime; let overrides: &Bound<'py, PyAny> = &preparation.cursor_overrides; + let (default_database, default_schema) = model_free_locations( + ( + invocation.defaults.database.clone(), + invocation.defaults.schema.clone(), + ), + preparation.target_config.loader_schema(), + |method| preparation.adapter.call_method0(method), + )?; call_with( &host_entry(py, ExecutorHost::LoadRuntimeParams)?, &[ @@ -408,6 +419,8 @@ fn runtime_params<'py>( ("vars", effective_vars.clone()), ("is_reload", native_field(request, "reload")?), ("target_config", preparation.target_config.record(py)?), + ("default_database", default_database), + ("default_schema", default_schema), ( "start_cursor_ts", cursor_timestamp(&native_field(overrides, "start_ts")?)?, diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md index 15338d5372..98b07f8260 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md @@ -177,7 +177,7 @@ A target is a named build context - the database and schema you build into, plus | Field | Description | |-------|-------------| | `schema` | Schema for all models in this target; required for named targets | -| `loader_schema` | Default write schema for managed source loaders; falls back to `schema` | +| `loader_schema` | Default write schema for managed source loaders; falls back to `schema`. Required when `schema` is `preserve` and the project has loaders, tasks, assets or checks, because their [results](python-nodes/overview.md#result-persistence) go there | | `database` | Database for all models in this target | | `connection` | Name from `[connections.]` used for endpoint, authentication, and compute; optional when exactly one named connection is defined | | `vars` | Target-specific project variables | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md index bd1768654e..bbac4abad1 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md @@ -172,7 +172,19 @@ Within one run, downstream nodes run only if at least one upstream succeeded. If ## Result persistence -Node results (payload, metadata, status, errors) are persisted after each execution. Results are stored in `_sqlbuild_node_results` in the warehouse alongside your data. Results persist across runs, so they are available for observability, debugging, and downstream consumption. +Node results (payload, metadata, status, errors) are persisted after each execution in a +`_sqlbuild_node_results` table. Results persist across runs, so they are available for +observability, debugging, and downstream consumption through `ctx.result_of`. + +Results of loaders, tasks, assets and checks all go to one schema, chosen by the target's `schema`: + +| Target `schema` | Results table schema | +|-----------------|----------------------| +| A schema name, such as `analytics` | That schema, in the target's database | +| `preserve` | The target's `loader_schema`; commands that compile the project stop with `S105` until it is set | +| Unset, or resolved per model, such as `${CTX:...}` | The adapter's default schema, such as `main` on DuckDB or `public` on PostgreSQL | + +A target `database` of `preserve` keeps results in the connection's database. ## Selection diff --git a/src/sqlbuild/executor/build/_helpers/source_node.py b/src/sqlbuild/executor/build/_helpers/source_node.py index 146df17c03..475fa9ec99 100644 --- a/src/sqlbuild/executor/build/_helpers/source_node.py +++ b/src/sqlbuild/executor/build/_helpers/source_node.py @@ -60,8 +60,8 @@ def execute_build_source_node( result_store: DirectNodeResultStore = build_direct_node_result_store( adapter=adapter, connection=connection, - database=adapter.default_database(), - schema=adapter.default_schema(), + database=runtime.default_database, + schema=runtime.default_schema, ) result: LoadExecutionResult = execute_source_load( source_entry=source_entry, @@ -73,6 +73,8 @@ def execute_build_source_node( run_id=runtime.run_id, target=runtime.target, target_config=runtime.target_config, + default_database=runtime.default_database, + default_schema=runtime.default_schema, vars=runtime.effective_vars or {}, is_reload=runtime.loader_is_reload, runtime_dir=runtime.runtime_dir, @@ -99,8 +101,6 @@ def execute_build_source_node( phase="finalize", ): _persist_loader_result( - adapter=adapter, - connection=connection, loader_name=loader_name, result=timed_result, run_id=runtime.run_id, @@ -111,27 +111,19 @@ def execute_build_source_node( def _persist_loader_result( *, - adapter: BaseAdapter, - connection: Any, loader_name: str, result: LoadExecutionResult, run_id: str, - result_store: DirectNodeResultStore | None = None, + result_store: DirectNodeResultStore, ) -> None: - if connection is None and (result_store is None or result_store.connection is None): + if result_store.connection is None: return - resolved_result_store: DirectNodeResultStore = result_store or build_direct_node_result_store( - adapter=adapter, - connection=connection, - database=adapter.default_database(), - schema=adapter.default_schema(), - ) - resolved_result_store.write( + result_store.write( NodeResultRecord( node_type="loader", node_name=loader_name, - target_database=resolved_result_store.database, - target_schema=resolved_result_store.schema, + target_database=result_store.database, + target_schema=result_store.schema, target_name=None, run_id=run_id, status=result.status.value, diff --git a/src/sqlbuild/executor/build/models.py b/src/sqlbuild/executor/build/models.py index 5b9521edf0..5f79313275 100644 --- a/src/sqlbuild/executor/build/models.py +++ b/src/sqlbuild/executor/build/models.py @@ -80,6 +80,8 @@ class BuildRuntimeParams: end_cursor_int: int | None = None target: str = "" target_config: TargetSettings | None = None + default_database: str | None = None + default_schema: str | None = None effective_vars: dict[str, object] | None = None use_color: bool = False providers: ProviderContainer | None = None diff --git a/src/sqlbuild/executor/load/_helpers/dag_runtime.py b/src/sqlbuild/executor/load/_helpers/dag_runtime.py index 0695bf42ec..a26bdfddcc 100644 --- a/src/sqlbuild/executor/load/_helpers/dag_runtime.py +++ b/src/sqlbuild/executor/load/_helpers/dag_runtime.py @@ -189,8 +189,8 @@ def _execute_ready_dag_source( resolved_result_store = build_direct_node_result_store( adapter=adapter, connection=connection, - database=adapter.default_database(), - schema=adapter.default_schema(), + database=runtime.default_database, + schema=runtime.default_schema, ) return execute_source_load( source_entry=source, diff --git a/src/sqlbuild/executor/load/models.py b/src/sqlbuild/executor/load/models.py index 38f6e970b0..74ed892674 100644 --- a/src/sqlbuild/executor/load/models.py +++ b/src/sqlbuild/executor/load/models.py @@ -283,6 +283,8 @@ class LoadRuntimeParams: vars: dict[str, object] is_reload: bool target_config: TargetSettings | None = None + default_database: str | None = None + default_schema: str | None = None runtime_dir: Path = Path("target") start_cursor_ts: datetime | None = None end_cursor_ts: datetime | None = None diff --git a/src/sqlbuild/executor/node_results/classes/direct_store.py b/src/sqlbuild/executor/node_results/classes/direct_store.py index 020a015a93..3e42718578 100644 --- a/src/sqlbuild/executor/node_results/classes/direct_store.py +++ b/src/sqlbuild/executor/node_results/classes/direct_store.py @@ -2,6 +2,7 @@ from __future__ import annotations +from dataclasses import replace from typing import Any from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter @@ -26,9 +27,11 @@ def __init__( connection: Any, database: str | None, schema: str | None, + previous_location: tuple[str | None, str | None] | None = None, ) -> None: self.adapter: BaseAdapter = adapter self.connection: Any = connection + self.previous_location: tuple[str | None, str | None] | None = previous_location super().__init__(database=database, schema=schema) def write(self, record: NodeResultRecord) -> None: @@ -54,14 +57,31 @@ def write(self, record: NodeResultRecord) -> None: self._cache_record(record=record) def _read_persisted(self, *, query: NodeResultQuery) -> tuple[NodeResultEnvelope, ...]: - if self.schema is None: + results: tuple[NodeResultEnvelope, ...] = self._read_at( + database=self.database, schema=self.schema, query=query + ) + if results or self.previous_location is None: + return results + # Results written before this release's location move: read them for one release only, + # and remove this fallback with the control-store migrations. + database, schema = self.previous_location + return self._read_at( + database=database, + schema=schema, + query=replace(query, target_database=database, target_schema=schema), + ) + + def _read_at( + self, *, database: str | None, schema: str | None, query: NodeResultQuery + ) -> tuple[NodeResultEnvelope, ...]: + if schema is None: return () return read_node_results( connection=self.connection, execute=self.adapter.execute, relation_exists=self.adapter.relation_exists, - database=self.database, - schema=self.schema, + database=database, + schema=schema, query=query, render_qualified_name=self.adapter.render_qualified_name, ) diff --git a/src/sqlbuild/executor/node_results/main/_direct_store.py b/src/sqlbuild/executor/node_results/main/_direct_store.py index 9c1b9edb8b..94a71d239c 100644 --- a/src/sqlbuild/executor/node_results/main/_direct_store.py +++ b/src/sqlbuild/executor/node_results/main/_direct_store.py @@ -15,11 +15,13 @@ def build_direct_node_result_store( database: str | None, schema: str | None, ) -> DirectNodeResultStore: - """Build a direct-mode runtime node result store.""" + """Build a direct-mode node result store that also reads the adapter's default location.""" + previous: tuple[str | None, str | None] = (adapter.default_database(), adapter.default_schema()) return DirectNodeResultStore( adapter=adapter, connection=connection, database=database, schema=schema, + previous_location=previous if previous != (database, schema) else None, ) diff --git a/src/sqlbuild/executor/python_nodes/_helpers/ingress_execution.py b/src/sqlbuild/executor/python_nodes/_helpers/ingress_execution.py index d364544c48..ab671bae2f 100644 --- a/src/sqlbuild/executor/python_nodes/_helpers/ingress_execution.py +++ b/src/sqlbuild/executor/python_nodes/_helpers/ingress_execution.py @@ -284,6 +284,8 @@ def _execute_ingress_loader( run_id=runtime.run_id, target=runtime.target, target_config=runtime.target_config, + default_database=runtime.default_database, + default_schema=runtime.default_schema, vars=runtime.vars, is_reload=runtime.is_reload, start_cursor_ts=runtime.start_cursor_ts, @@ -320,8 +322,8 @@ def _execute_ingress_loader( adapter=runtime.adapter, connection=runtime.connection, run_id=runtime.run_id, - database=runtime.adapter.default_database(), - schema=runtime.adapter.default_schema(), + database=runtime.default_database, + schema=runtime.default_schema, target_name=source_entry.name, ) if attempt is not None and result.status == ExecutionStatus.FAILED: diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py index a444f30373..165db667db 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py @@ -1664,3 +1664,23 @@ class CtxTargetSchemaE2ETestCase: target_schema: str expected_model_schema: str expected_loader_schema: str | None + + +@dataclass(frozen=True) +class PreservedTargetNodeResultsE2ETestCase: + """A target that keeps its database or schema, and where two builds' node results land.""" + + description: str + target_lines: str + source_location: str + expected_results_schema: str + expected_runs_per_node: int + + +@dataclass(frozen=True) +class PreservedTargetRefusalE2ETestCase: + """A preserved target without a loader schema and the compile error it must print.""" + + description: str + python_files: dict[str, str] + expected_error: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_node_result_location.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_node_result_location.py new file mode 100644 index 0000000000..fc68b2141d --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_node_result_location.py @@ -0,0 +1,229 @@ +"""CLI e2e coverage: where Python node results go, so later builds read them back.""" + +from __future__ import annotations + +import base64 +import json +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.build._test_types import ( + PreservedTargetNodeResultsE2ETestCase, + PreservedTargetRefusalE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( + prepare_inline_project, + query_duckdb, + run_sqb, +) + +_LOADER: str = ( + "from sqlbuild.loaders import loader\n\n\n@loader\n" + "def raw_orders(ctx: object) -> list[dict[str, object]]:\n" + ' """Raw orders."""\n _ = ctx\n return [{"order_id": 1}]\n' +) +_TASK: str = ( + "from sqlbuild.refs import source\nfrom sqlbuild.tasks import task\n\n\n" + '@task(depends_on=source("raw_orders"))\n' + "def announce_orders(ctx):\n" + ' """Announce the loaded orders."""\n' + " return ctx.result(payload={'announced': True})\n" +) + + +def _source_yml(location: str) -> str: + return ( + "sources:\n - name: raw_orders\n description: Raw orders.\n" + f" managed: true\n{location} write_strategy: table\n" + " columns:\n - name: order_id\n type: INTEGER\n" + ) + + +def _project(*, tmp_path: Path, target_lines: str, files: dict[str, str]) -> Path: + return prepare_inline_project( + tmp_path=tmp_path, + project_name="orders", + repo_files={ + "sqlbuild_project.toml": ( + 'name = "orders"\nadapter = "duckdb"\ndefault_target = "dev"\n\n' + '[connection]\ndatabase = "dev.duckdb"\n\n' + f'[defaults]\ndatabase = "dev"\nschema = "mart"\n\n[targets.dev]\n{target_lines}' + ), + "models/orders.sql": ( + "MODEL (description 'Orders.', materialized table);\n\n" + 'SELECT order_id FROM __source("raw_orders")\n' + ), + } + | files, + ) + + +@pytest.mark.parametrize( + "test_case", + [ + PreservedTargetNodeResultsE2ETestCase( + description="a preserved schema keeps node results in the loader schema", + target_lines='schema = "preserve"\nloader_schema = "raw"\n', + source_location=" schema: raw\n", + expected_results_schema="raw", + expected_runs_per_node=2, + ), + PreservedTargetNodeResultsE2ETestCase( + description="a preserved database keeps node results in the connection database", + target_lines='database = "preserve"\nschema = "analytics"\nloader_schema = "raw"\n', + source_location=" database: dev\n schema: raw\n", + expected_results_schema="analytics", + expected_runs_per_node=2, + ), + ], + ids=lambda case: case.description, +) +def test_given_preserved_target_when_building_twice_then_node_results_stay_in_one_real_location( + test_case: PreservedTargetNodeResultsE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = _project( + tmp_path=tmp_path, + target_lines=test_case.target_lines, + files={ + "sources/raw.yml": _source_yml(test_case.source_location), + "python/loaders/orders.py": _LOADER, + "python/tasks/announce.py": _TASK, + }, + ) + + results: list[subprocess.CompletedProcess[str]] = [ + run_sqb(command=("--no-color", "build"), project_dir=project_dir) for _ in range(2) + ] + + for result in results: + assert result.returncode == 0, result.stdout + result.stderr + db_path: Path = project_dir / "dev.duckdb" + assert ( + query_duckdb( + db_path=db_path, + sql=( + "SELECT catalog_name, schema_name FROM information_schema.schemata " + "WHERE lower(schema_name) = 'preserve' OR lower(catalog_name) = 'preserve'" + ), + ) + == [] + ) + assert query_duckdb( + db_path=db_path, + sql=( + "SELECT table_schema FROM information_schema.tables " + "WHERE table_name = '_sqlbuild_node_results'" + ), + ) == [(test_case.expected_results_schema,)] + assert query_duckdb( + db_path=db_path, + sql=( + "SELECT node_type, node_name, count(DISTINCT run_id) " + f"FROM {test_case.expected_results_schema}._sqlbuild_node_results " + "GROUP BY ALL ORDER BY ALL" + ), + ) == [ + ("loader", "raw_orders", test_case.expected_runs_per_node), + ("task", "announce_orders", test_case.expected_runs_per_node), + ] + + +@pytest.mark.parametrize( + "test_case", + [ + PreservedTargetRefusalE2ETestCase( + description="a task under a preserved schema needs a loader schema", + python_files={ + "python/tasks/announce.py": _TASK.replace('depends_on=source("raw_orders")', "") + }, + expected_error="error[S105]: Target 'dev' keeps each model's own schema", + ), + ], + ids=lambda case: case.description, +) +def test_given_preserved_schema_without_loader_schema_when_compiling_then_python_nodes_are_refused( + test_case: PreservedTargetRefusalE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="orders", + repo_files={ + "sqlbuild_project.toml": ( + 'name = "orders"\nadapter = "duckdb"\ndefault_target = "dev"\n\n' + '[connection]\ndatabase = "dev.duckdb"\n\n' + '[defaults]\nschema = "mart"\n\n[targets.dev]\nschema = "preserve"\n' + ), + "models/orders.sql": "MODEL (description 'Orders.');\n\nSELECT 1 AS order_id\n", + } + | test_case.python_files, + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "compile"), project_dir=project_dir + ) + + assert result.returncode == 1, result.stdout + result.stderr + assert test_case.expected_error in result.stderr + + +_EDGE_CASES: Path = Path("benchmarks/duckshop/edge_cases/loaders") + + +def _results(*, db_path: Path, schema: str, node_type: str) -> list[tuple[str, str, str]]: + return query_duckdb( + db_path=db_path, + sql=( + f"SELECT node_name, run_id, payload_json_b64 FROM {schema}._sqlbuild_node_results " + f"WHERE node_type = '{node_type}' ORDER BY node_name" + ), + ) + + +def test_given_results_in_the_adapter_default_when_one_loader_rebuilds_then_both_are_still_read( + tmp_path: Path, +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="orders", + repo_files={ + "sqlbuild_project.toml": ( + 'name = "orders"\nadapter = "duckdb"\ndefault_target = "dev"\n\n' + '[connection]\ndatabase = "dev.duckdb"\n\n[targets.dev]\n' + ), + "sources/events.yml": (_EDGE_CASES / "loader_results.yml").read_text(), + "python/loaders/events.py": (_EDGE_CASES / "loader_results.py").read_text(), + }, + ) + db_path: Path = project_dir / "dev.duckdb" + first: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + assert first.returncode == 0, first.stdout + first.stderr + old_runs: dict[str, str] = { + name: run for name, run, _ in _results(db_path=db_path, schema="main", node_type="loader") + } + toml: Path = project_dir / "sqlbuild_project.toml" + toml.write_text(toml.read_text(encoding="utf-8") + 'schema = "mart"\n', encoding="utf-8") + + second: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build", "--select", "raw_order_events", "announce_events"), + project_dir=project_dir, + ) + + assert second.returncode == 0, second.stdout + second.stderr + new_runs: dict[str, str] = { + name: run for name, run, _ in _results(db_path=db_path, schema="mart", node_type="loader") + } + assert set(new_runs) == {"raw_order_events"} + assert new_runs["raw_order_events"] != old_runs["raw_order_events"] + [(_, task_run, payload)] = _results(db_path=db_path, schema="mart", node_type="task") + assert task_run == new_runs["raw_order_events"] + assert json.loads(base64.b64decode(payload)) == { + "orders": new_runs["raw_order_events"], + "customers": old_runs["raw_customer_events"], + } + assert { + name: run for name, run, _ in _results(db_path=db_path, schema="main", node_type="loader") + } == old_runs diff --git a/tests/unit/src/sqlbuild/executor/build/_helpers/_test_types.py b/tests/unit/src/sqlbuild/executor/build/_helpers/_test_types.py index e30da1baae..22f4b0e984 100644 --- a/tests/unit/src/sqlbuild/executor/build/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/executor/build/_helpers/_test_types.py @@ -121,6 +121,16 @@ class BuildSourceNodeExecutionTestCase: expected_rows: tuple[tuple[object, ...], ...] +@dataclass(frozen=True) +class BuildSourceNodeResultLocationTestCase: + """Where a scheduled loader's result goes for an adapter default and a runtime location.""" + + description: str + adapter_default_schema: str | None + runtime_schema: str + expected_result_schemas: tuple[str, ...] + + @dataclass(frozen=True) class ExternalBuildSourceLoadTestCase: """One pre-connection build source-load behavior case.""" diff --git a/tests/unit/src/sqlbuild/executor/build/_helpers/test_source_node.py b/tests/unit/src/sqlbuild/executor/build/_helpers/test_source_node.py index 1d1d63143e..cfea281a4c 100644 --- a/tests/unit/src/sqlbuild/executor/build/_helpers/test_source_node.py +++ b/tests/unit/src/sqlbuild/executor/build/_helpers/test_source_node.py @@ -15,6 +15,7 @@ from sqlbuild.runtime.contracts.types import ExecutionResourceKind from tests.unit.src.sqlbuild.executor.build._helpers._test_types import ( BuildSourceNodeExecutionTestCase, + BuildSourceNodeResultLocationTestCase, ) from tests.unit.src.sqlbuild.executor.build._helpers.helpers import ( build_discovered_source_loader, @@ -86,3 +87,75 @@ def test_given_build_source_node_when_executing_then_runs_loader_runtime( assert rows == test_case.expected_rows assert progress_events == [test_case.expected_progress_event] assert start_events == [test_case.expected_start_event] + + +class _NoDefaultSchemaAdapter(DuckDbAdapter): + """A DuckDB adapter that, like Snowflake, has no default schema.""" + + def default_schema(self) -> str | None: + return None + + +@pytest.mark.parametrize( + "test_case", + [ + BuildSourceNodeResultLocationTestCase( + description="an adapter without a default schema still records the result", + adapter_default_schema=None, + runtime_schema="analytics", + expected_result_schemas=("analytics",), + ), + BuildSourceNodeResultLocationTestCase( + description="the runtime location wins over the adapter default schema", + adapter_default_schema="main", + runtime_schema="analytics", + expected_result_schemas=("analytics",), + ), + ], + ids=lambda case: case.description, +) +def test_given_runtime_location_when_executing_source_node_then_result_is_written_there( + test_case: BuildSourceNodeResultLocationTestCase, + tmp_path: Path, +) -> None: + adapter: DuckDbAdapter = ( + DuckDbAdapter() if test_case.adapter_default_schema else _NoDefaultSchemaAdapter() + ) + connection: object = adapter.connect({"database": str(tmp_path / "source_node.duckdb")}) + try: + result: LoadExecutionResult = execute_build_source_node( + key=CompiledObjectKey(CompiledResourceType.SOURCE, "raw_orders"), + plan=build_source_load_plan_output( + source_name="raw_orders", loader_name="raw_orders_loader" + ), + loader_functions_by_name={ + "raw_orders_loader": build_discovered_source_loader(loader_name="raw_orders_loader") + }, + loader_ref_entries={}, + adapter=adapter, + connection_config={"database": str(tmp_path / "source_node.duckdb")}, + connection=connection, + runtime=BuildRuntimeParams( + run_id="run-1", + target="dev", + default_schema=test_case.runtime_schema, + ), + callbacks=BuildCallbacks(), + ) + result_schemas: tuple[str, ...] = tuple( + row[0] + for row in connection.execute( + "SELECT table_schema FROM information_schema.tables " + "WHERE table_name = '_sqlbuild_node_results' ORDER BY ALL" + ).fetchall() + ) + recorded: list[tuple[object, ...]] = connection.execute( + f"SELECT node_type, target_schema, run_id " + f"FROM {test_case.runtime_schema}._sqlbuild_node_results" + ).fetchall() + finally: + adapter.close(connection) + + assert result.status == ExecutionStatus.SUCCESS + assert result_schemas == test_case.expected_result_schemas + assert recorded == [("loader", test_case.runtime_schema, "run-1")] diff --git a/website/src/content/docs/docs/concepts/project-configuration.mdx b/website/src/content/docs/docs/concepts/project-configuration.mdx index b43a5fbcc9..6c556efbf0 100644 --- a/website/src/content/docs/docs/concepts/project-configuration.mdx +++ b/website/src/content/docs/docs/concepts/project-configuration.mdx @@ -152,7 +152,7 @@ A target is a named build context - the database and schema you build into, plus | Field | Description | |-------|-------------| | `schema` | Schema for all models in this target; required for named targets | -| `loader_schema` | Default write schema for managed source loaders; falls back to `schema` | +| `loader_schema` | Default write schema for managed source loaders; falls back to `schema`. Required when `schema` is `preserve` and the project has loaders, tasks, assets or checks, because their [results](/docs/concepts/python-nodes/overview#result-persistence) go there | | `database` | Database for all models in this target | | `connection` | Name from `[connections.]` used for endpoint, authentication, and compute; optional when exactly one named connection is defined | | `vars` | Target-specific project variables | diff --git a/website/src/content/docs/docs/concepts/python-nodes/overview.mdx b/website/src/content/docs/docs/concepts/python-nodes/overview.mdx index 87c93836f3..5954224c00 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/overview.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/overview.mdx @@ -156,7 +156,19 @@ Within one run, downstream nodes run only if at least one upstream succeeded. If ## Result persistence -Node results (payload, metadata, status, errors) are persisted after each execution. Results are stored in `_sqlbuild_node_results` in the warehouse alongside your data. Results persist across runs, so they are available for observability, debugging, and downstream consumption. +Node results (payload, metadata, status, errors) are persisted after each execution in a +`_sqlbuild_node_results` table. Results persist across runs, so they are available for +observability, debugging, and downstream consumption through `ctx.result_of`. + +Results of loaders, tasks, assets and checks all go to one schema, chosen by the target's `schema`: + +| Target `schema` | Results table schema | +|-----------------|----------------------| +| A schema name, such as `analytics` | That schema, in the target's database | +| `preserve` | The target's `loader_schema`; commands that compile the project stop with `S105` until it is set | +| Unset, or resolved per model, such as `${CTX:...}` | The adapter's default schema, such as `main` on DuckDB or `public` on PostgreSQL | + +A target `database` of `preserve` keeps results in the connection's database. ## Selection From af54c84eeb25aca123f64c53ccb01133eccc20c5 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 17 Nov 2026 13:14:27 +0000 Subject: [PATCH 2/6] fix: read results left in a preserve schema and cover every runtime Under a preserve target, also read a node's result from the schema named preserve that earlier releases wrote to, after the adapter default. The location for `sqb load`, build-time checks and `sqb check` now comes from the target settings, so each runtime reads and writes the same place. --- .../main/adapter_default_location.rs | 11 ++ .../src/warehouse_access/main/mod.rs | 1 + .../src/compile_driver/_helpers/setup/mod.rs | 2 + .../_helpers/setup/model_free_locations.rs | 35 +++++ .../_helpers/setup/node_results.rs | 53 +++++++ .../src/compile_driver/main/setup/mod.rs | 2 - .../main/setup/model_free_locations.rs | 39 ----- .../main/setup/preserved_loader_schemas.rs | 43 +----- .../setup/project_model_free_locations.rs | 22 --- .../src/compile_driver/models.rs | 17 +++ .../_helpers/build_command/execution.rs | 7 +- .../_helpers/build_command/lifecycle.rs | 3 +- .../bindings/_helpers/check/check_command.rs | 3 +- .../src/bindings/_helpers/check/check_core.rs | 17 ++- .../bindings/_helpers/load/load_command.rs | 14 +- fensu.toml | 8 +- .../executor/build/_helpers/source_node.py | 1 + .../executor/load/_helpers/dag_runtime.py | 1 + .../node_results/classes/direct_store.py | 27 ++-- .../executor/node_results/constants.py | 1 + .../node_results/main/_direct_store.py | 15 +- .../python_nodes/_helpers/execution.py | 1 + .../_helpers/ingress_execution.py | 1 + .../python_nodes/_helpers/python_checks.py | 1 + .../read_side_python_execution_tracker.py | 1 + .../cli/commands/main/build/_test_types.py | 12 ++ .../cli/commands/main/build/helpers.py | 43 ++++++ .../main/build/test_node_result_location.py | 133 +++++++++++------- .../executor/node_results/main/_test_types.py | 11 ++ .../executor/node_results/main/helpers.py | 21 +++ .../node_results/main/test_read_write.py | 74 ++++++++++ .../executor/build/_helpers/_test_types.py | 3 +- .../executor/build/_helpers/helpers.py | 8 ++ .../build/_helpers/test_source_node.py | 16 +-- 34 files changed, 431 insertions(+), 216 deletions(-) create mode 100644 crates/sqlbuild-python-base/src/warehouse_access/main/adapter_default_location.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/model_free_locations.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/node_results.rs delete mode 100644 crates/sqlbuild-python-compile/src/compile_driver/main/setup/model_free_locations.rs delete mode 100644 crates/sqlbuild-python-compile/src/compile_driver/main/setup/project_model_free_locations.rs create mode 100644 tests/integration/src/sqlbuild/executor/node_results/main/helpers.py diff --git a/crates/sqlbuild-python-base/src/warehouse_access/main/adapter_default_location.rs b/crates/sqlbuild-python-base/src/warehouse_access/main/adapter_default_location.rs new file mode 100644 index 0000000000..69a56b834e --- /dev/null +++ b/crates/sqlbuild-python-base/src/warehouse_access/main/adapter_default_location.rs @@ -0,0 +1,11 @@ +//! The adapter's own default database or schema. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; + +/// The adapter's `default_database` or `default_schema`, named by `method`. +pub fn adapter_default_location<'py>( + adapter: &Bound<'py, PyAny>, + method: &str, +) -> PyResult> { + adapter.call_method0(method) +} diff --git a/crates/sqlbuild-python-base/src/warehouse_access/main/mod.rs b/crates/sqlbuild-python-base/src/warehouse_access/main/mod.rs index b0a2d9b65e..e9d3d3e481 100644 --- a/crates/sqlbuild-python-base/src/warehouse_access/main/mod.rs +++ b/crates/sqlbuild-python-base/src/warehouse_access/main/mod.rs @@ -1,3 +1,4 @@ +pub mod adapter_default_location; pub mod ensure_event_id_column; pub mod is_warehouse_access_refused; pub mod read_catalog; diff --git a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/mod.rs b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/mod.rs index ba9e864c07..a30b0e8ab7 100644 --- a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/mod.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/mod.rs @@ -1,5 +1,7 @@ pub(crate) mod inputs; pub(crate) mod integration_loaders; +pub(crate) mod model_free_locations; +pub(crate) mod node_results; pub(crate) mod pipeline; pub(crate) mod pipeline_entries; pub(crate) mod project; diff --git a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/model_free_locations.rs b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/model_free_locations.rs new file mode 100644 index 0000000000..e001269c72 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/model_free_locations.rs @@ -0,0 +1,35 @@ +//! The target database and schema that Python nodes use outside any model. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::PyString; +use sqlbuild_analysis::assembly::target_checks::constants::PRESERVE_TARGET_VALUE; +use sqlbuild_model_config::templates::main::holds_context_placeholder::holds_context_placeholder; +use sqlbuild_python_base::warehouse_access::main::adapter_default_location::adapter_default_location; + +/// Python nodes' `(database, schema)`: a `preserve` schema reads `loader_schema`; unusable values the adapter's. +pub(crate) fn model_free_locations<'py>( + (database, schema): (Bound<'py, PyAny>, Bound<'py, PyAny>), + loader_schema: Option<&str>, + adapter: &Bound<'py, PyAny>, +) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { + let kept_schema: Option<&str> = + loader_schema.filter(|name| !name.is_empty() && *name != PRESERVE_TARGET_VALUE); + let location = |value: Bound<'py, PyAny>, method: &str, preserved: Option<&str>| match ( + value.extract::>()?.as_deref(), + preserved, + ) { + (Some(PRESERVE_TARGET_VALUE), Some(name)) => Ok(PyString::new(value.py(), name).into_any()), + (Some(text), _) + if !text.is_empty() + && text != PRESERVE_TARGET_VALUE + && !holds_context_placeholder(text) => + { + Ok(value) + } + _ => adapter_default_location(adapter, method), + }; + Ok(( + location(database, "default_database", None)?, + location(schema, "default_schema", kept_schema)?, + )) +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/node_results.rs b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/node_results.rs new file mode 100644 index 0000000000..d302abe94b --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/node_results.rs @@ -0,0 +1,53 @@ +//! The schema a `preserve` target keeps Python node results in. + +use crate::compile_driver::_helpers::setup::targets::planner_error; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_analysis::assembly::target_checks::constants::{ + PRESERVE_TARGET_VALUE, PRESERVED_NODE_RESULTS_CODE, +}; +use sqlbuild_analysis::assembly::target_checks::errors::TargetError; +use sqlbuild_analysis::assembly::target_checks::main::target_error_message::target_error_message; +use sqlbuild_core::text::main::python_strip::python_strip; +use sqlbuild_python_base::boundary::main::optional_text::optional_text; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The discovered inputs' Python node functions, each kind of which keeps node results. +const PYTHON_NODE_FIELDS: [&str; 4] = [ + "loader_functions", + "task_functions", + "asset_functions", + "check_functions", +]; + +/// Raise when a `preserve` target has Python nodes but no loader schema to keep their results in. +pub(crate) fn require_node_results_schema( + discovered: &Bound<'_, PyAny>, + (target_name, target_config): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), +) -> PyResult<()> { + let loader_schema: Option = + optional_text(&native_field(target_config, "loader_schema")?)? + .map(|schema| python_strip(&schema).to_owned()) + .filter(|schema| !schema.is_empty() && schema != PRESERVE_TARGET_VALUE); + if loader_schema.is_some() || python_node_count(discovered)? == 0 { + return Ok(()); + } + let target: String = target_name.extract()?; + Err(planner_error( + discovered.py(), + target_error_message(&TargetError::PreservedNodeResults(target.clone())), + PRESERVED_NODE_RESULTS_CODE, + Some(format!( + "to keep Python node results in a schema, set this in sqlbuild_project.toml:\n \ + [targets.{target}]\n \ + loader_schema = \"my_schema\"" + )), + )) +} + +/// How many loaders, tasks, assets and checks the project declares in Python. +fn python_node_count(discovered: &Bound<'_, PyAny>) -> PyResult { + PYTHON_NODE_FIELDS + .iter() + .map(|field| native_field(discovered, field)?.len()) + .sum() +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/setup/mod.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/mod.rs index 7ecbc5aaa6..7217f6eb02 100644 --- a/crates/sqlbuild-python-compile/src/compile_driver/main/setup/mod.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/mod.rs @@ -11,12 +11,10 @@ pub mod compile_project_phase; pub mod compiled_phase; pub mod expanded_loader_namespace; pub mod managed_write_schema; -pub mod model_free_locations; pub mod plan_has_executable_work; pub mod preserved_loader_schemas; pub mod progress; pub mod project_graph; -pub mod project_model_free_locations; pub mod project_target_settings; pub mod python_read_source_names_for; pub mod register_inputs; diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/setup/model_free_locations.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/model_free_locations.rs deleted file mode 100644 index 526836b3ce..0000000000 --- a/crates/sqlbuild-python-compile/src/compile_driver/main/setup/model_free_locations.rs +++ /dev/null @@ -1,39 +0,0 @@ -//! The target database and schema that Python nodes use outside any model. - -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; -use sqlbuild_analysis::assembly::target_checks::constants::PRESERVE_TARGET_VALUE; -use sqlbuild_model_config::templates::main::holds_context_placeholder::holds_context_placeholder; - -/// The database and schema of Python nodes, their results and loader fingerprints, from the -/// target's configured `(database, schema)`; a `preserve` schema reads `loader_schema`, and an -/// unset, empty, `preserve` or model-only value, such as a `${CTX:...}` schema, reads `default` -/// with the adapter method's name. -pub fn model_free_locations<'py>( - (database, schema): (Bound<'py, PyAny>, Bound<'py, PyAny>), - loader_schema: Option<&str>, - default: impl Fn(&str) -> PyResult>, -) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { - let location = - |value: Bound<'py, PyAny>, fallback: &str| match value.extract::>()? { - Some(text) if text == PRESERVE_TARGET_VALUE && fallback == SCHEMA_DEFAULT => { - match loader_schema.filter(|schema| *schema != PRESERVE_TARGET_VALUE) { - Some(schema) => Ok(pyo3::types::PyString::new(value.py(), schema).into_any()), - None => default(fallback), - } - } - Some(text) - if !text.is_empty() - && text != PRESERVE_TARGET_VALUE - && !holds_context_placeholder(&text) => - { - Ok(value) - } - _ => default(fallback), - }; - Ok(( - location(database, "default_database")?, - location(schema, SCHEMA_DEFAULT)?, - )) -} - -const SCHEMA_DEFAULT: &str = "default_schema"; diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/setup/preserved_loader_schemas.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/preserved_loader_schemas.rs index e48bc19623..9a0c3d649b 100644 --- a/crates/sqlbuild-python-compile/src/compile_driver/main/setup/preserved_loader_schemas.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/preserved_loader_schemas.rs @@ -1,10 +1,11 @@ //! Raise when the selected target's schema is `preserve` and a loader or Python node has no schema. use crate::compile_attachment::main::resolved_target::resolved_target; +use crate::compile_driver::_helpers::setup::node_results::require_node_results_schema; use crate::compile_driver::_helpers::setup::targets::planner_error; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use sqlbuild_analysis::assembly::target_checks::constants::{ - MANAGED_WRITE_SCHEMA_CODE, PRESERVE_TARGET_VALUE, PRESERVED_NODE_RESULTS_CODE, + MANAGED_WRITE_SCHEMA_CODE, PRESERVE_TARGET_VALUE, }; use sqlbuild_analysis::assembly::target_checks::errors::TargetError; use sqlbuild_analysis::assembly::target_checks::main::target_error_message::target_error_message; @@ -13,16 +14,7 @@ use sqlbuild_python_base::boundary::main::optional_text::optional_text; use sqlbuild_python_base::records::main::fields::field::field as native_field; use std::collections::HashSet; -/// The discovered inputs' Python node functions, each kind of which keeps node results. -const PYTHON_NODE_FIELDS: [&str; 4] = [ - "loader_functions", - "task_functions", - "asset_functions", - "check_functions", -]; - -/// Raise when the selected target's schema is `preserve` and one of `loaders` names no schema, or -/// the project has Python nodes and the target names no loader schema to keep their results in. +/// Raise when a `preserve` target leaves one of `loaders`, or the Python node results, without a schema. pub fn preserved_loader_schemas( discovered: &Bound<'_, PyAny>, selected_target: &Bound<'_, PyAny>, @@ -56,32 +48,5 @@ pub fn preserved_loader_schemas( )), )); } - let loader_schema: Option = - optional_text(&native_field(&target_config, "loader_schema")?)? - .map(|schema| python_strip(&schema).to_owned()) - .filter(|schema| !schema.is_empty() && schema != PRESERVE_TARGET_VALUE); - if loader_schema.is_some() || !has_python_nodes(discovered)? { - return Ok(()); - } - let target: String = target_name.extract()?; - Err(planner_error( - py, - target_error_message(&TargetError::PreservedNodeResults(target.clone())), - PRESERVED_NODE_RESULTS_CODE, - Some(format!( - "to keep Python node results in a schema, set this in sqlbuild_project.toml:\n \ - [targets.{target}]\n \ - loader_schema = \"my_schema\"" - )), - )) -} - -/// Whether the project declares any loader, task, asset or check written in Python. -fn has_python_nodes(discovered: &Bound<'_, PyAny>) -> PyResult { - for field in PYTHON_NODE_FIELDS { - if native_field(discovered, field)?.len()? > 0 { - return Ok(true); - } - } - Ok(false) + require_node_results_schema(discovered, (&target_name, &target_config)) } diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/setup/project_model_free_locations.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/project_model_free_locations.rs deleted file mode 100644 index 58d8ed401c..0000000000 --- a/crates/sqlbuild-python-compile/src/compile_driver/main/setup/project_model_free_locations.rs +++ /dev/null @@ -1,22 +0,0 @@ -//! The compiled project's database and schema for Python nodes outside any model. - -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; -use sqlbuild_python_base::records::main::fields::field::field as native_field; - -use crate::compile_driver::main::setup::model_free_locations::model_free_locations; - -/// The compiled project's `model_free_locations`, falling back to the adapter's defaults. -pub fn project_model_free_locations<'py>( - project: &Bound<'py, PyAny>, - loader_schema: Option<&str>, - adapter: &Bound<'py, PyAny>, -) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { - model_free_locations( - ( - native_field(project, "effective_target_database")?, - native_field(project, "effective_target_schema")?, - ), - loader_schema, - |method| adapter.call_method0(method), - ) -} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/models.rs b/crates/sqlbuild-python-compile/src/compile_driver/models.rs index c400be8c3b..f2e42b435b 100644 --- a/crates/sqlbuild-python-compile/src/compile_driver/models.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/models.rs @@ -1,5 +1,6 @@ //! Public records of compile driver. +use crate::compile_driver::_helpers::setup::model_free_locations::model_free_locations; use pyo3::IntoPyObjectExt; use pyo3::prelude::Bound; use pyo3::prelude::Py; @@ -83,6 +84,22 @@ impl TargetSettingsValues { self.loader_schema.as_deref() } + /// Python nodes' `(database, schema)` under these settings, as `model_free_locations` picks. + pub fn node_locations<'py>( + &self, + adapter: &Bound<'py, PyAny>, + ) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { + let py: Python<'py> = adapter.py(); + model_free_locations( + ( + self.database.as_deref().into_bound_py_any(py)?, + self.schema.as_deref().into_bound_py_any(py)?, + ), + self.loader_schema(), + adapter, + ) + } + /// The `TargetSettings` record: a `preserve` location reads `None` with its flag set. pub fn record<'py>(&self, py: Python<'py>) -> PyResult> { let located = |value: &Option| -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs index ff98f45069..f4caf1a4a6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs @@ -43,7 +43,6 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_compile::compile_driver::main::setup::build_project_relation_targets::build_project_relation_targets; -use sqlbuild_python_compile::compile_driver::main::setup::project_model_free_locations::project_model_free_locations; use sqlbuild_python_compile::compile_driver::main::setup::project_target_settings::project_target_settings; use sqlbuild_python_compile::compile_driver::models::TargetSettingsValues; use sqlbuild_python_front::cost_context::models::CostContext; @@ -340,8 +339,7 @@ pub(crate) fn execute_build_plan<'py>( &project, (&connection_config, &adapter), )?; - let (default_database, default_schema) = - project_model_free_locations(&project, target_config.loader_schema(), &adapter)?; + let (default_database, default_schema) = target_config.node_locations(&adapter)?; let runtime: Bound<'py, PyAny> = call_with( &host_entry(py, ExecutorHost::BuildRuntimeParams)?, &[ @@ -641,8 +639,7 @@ fn check_runtime<'py>( &project, (&context.get("connection_config")?, &adapter), )?; - let (default_database, default_schema) = - project_model_free_locations(&project, target_config.loader_schema(), &adapter)?; + let (default_database, default_schema) = target_config.node_locations(&adapter)?; call_with( &host_entry(py, ExecutorHost::PythonNodeRuntime)?, &[ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs index 591968e64b..f5bdab0275 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs @@ -28,7 +28,6 @@ use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_key use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_compile::compile_driver::main::setup::build_project_relation_targets::build_project_relation_targets; use sqlbuild_python_compile::compile_driver::main::setup::build_python_relation_targets::build_python_relation_targets; -use sqlbuild_python_compile::compile_driver::main::setup::project_model_free_locations::project_model_free_locations; use sqlbuild_python_compile::compile_driver::main::setup::project_target_settings::project_target_settings; use sqlbuild_python_compile::compile_driver::models::TargetSettingsValues; use sqlbuild_python_project::graph::main::python_nodes::build_python_sql_run_lifecycle::build_python_sql_run_lifecycle; @@ -439,7 +438,7 @@ pub(crate) fn prepare_direct_python_lifecycle<'py>( connection_config: &connection_config, project: &project, reload: context.request_flag("reload_sources")?, - defaults: project_model_free_locations(&project, target_config.loader_schema(), &adapter)?, + defaults: target_config.node_locations(&adapter)?, target_config, relations: (relation_targets, project_relations), inputs, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs index f9c9b8ce21..74337e8a27 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs @@ -25,7 +25,6 @@ use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_compile::compile_driver::main::setup::build_project_relation_targets::build_project_relation_targets; use sqlbuild_python_compile::compile_driver::main::setup::build_python_relation_targets::build_python_relation_targets; use sqlbuild_python_compile::compile_driver::main::setup::build_static_pipeline_result::build_static_pipeline_result; -use sqlbuild_python_compile::compile_driver::main::setup::project_model_free_locations::project_model_free_locations; use sqlbuild_python_compile::compile_driver::main::setup::project_target_settings::project_target_settings; use sqlbuild_python_compile::compile_driver::models::TargetSettingsValues; use sqlbuild_python_front::discovery::main::sources::consumer_uses::consumer_uses; @@ -347,7 +346,7 @@ fn prepare<'py>( lifecycle_plan, relation_targets, relation_refs, - defaults: project_model_free_locations(&project, target_config.loader_schema(), &adapter)?, + defaults: target_config.node_locations(&adapter)?, target_config, project_relations: object( py, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/check_core.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/check_core.rs index bd31197b5e..e2b2068d11 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/check/check_core.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/check_core.rs @@ -10,8 +10,8 @@ use pyo3::prelude::{ use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyFrozenSetMethods, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_python_base::boundary::main::keywords::keywords; +use sqlbuild_python_base::warehouse_access::main::adapter_default_location::adapter_default_location; use sqlbuild_python_compile::compile_driver::main::setup::build_python_relation_targets::build_python_relation_targets; -use sqlbuild_python_compile::compile_driver::main::setup::project_model_free_locations::project_model_free_locations; use sqlbuild_python_project::graph::main::python_nodes::resolve_python_check_excludes::resolve_python_check_excludes; use sqlbuild_python_project::planner_phases::main::object::object; @@ -447,14 +447,13 @@ fn read_side_references<'py>( fn node_runtime<'py>(inputs: &ReadSideInputs<'py>) -> PyResult> { let py: Python<'py> = inputs.adapter.py(); let project: Bound<'py, PyAny> = native_field(&inputs.pipeline_result, "project")?; - let (default_database, default_schema) = project_model_free_locations( - &project, - inputs - .target_config - .as_ref() - .and_then(TargetSettingsValues::loader_schema), - &inputs.adapter, - )?; + let (default_database, default_schema) = match &inputs.target_config { + Some(settings) => settings.node_locations(&inputs.adapter)?, + None => ( + adapter_default_location(&inputs.adapter, "default_database")?, + adapter_default_location(&inputs.adapter, "default_schema")?, + ), + }; call_with( &host_entry(py, ExecutorHost::PythonNodeRuntime)?, &[ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs index 2f8ae9a18f..8bfde4354e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs @@ -35,7 +35,6 @@ use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; -use sqlbuild_python_compile::compile_driver::main::setup::model_free_locations::model_free_locations; use sqlbuild_python_compile::compile_driver::main::setup::selected_target_settings::selected_target_settings; use sqlbuild_python_compile::compile_driver::main::stages::discover_project_inputs::discover_project_inputs; use sqlbuild_python_compile::compile_driver::models::TargetSettingsValues; @@ -68,7 +67,6 @@ struct Invocation<'py> { selected: Vec>, references: Vec>, relation_sources: Vec>, - defaults: LoaderDefaults<'py>, style: CliStyle, stream: Bound<'py, PyAny>, } @@ -183,7 +181,6 @@ fn invocation<'py>(py: Python<'py>, request: &Bound<'py, PyAny>) -> PyResult( let py: Python<'py> = request.py(); let (target_name, effective_vars, run_id) = &preparation.runtime; let overrides: &Bound<'py, PyAny> = &preparation.cursor_overrides; - let (default_database, default_schema) = model_free_locations( - ( - invocation.defaults.database.clone(), - invocation.defaults.schema.clone(), - ), - preparation.target_config.loader_schema(), - |method| preparation.adapter.call_method0(method), - )?; + let (default_database, default_schema) = preparation + .target_config + .node_locations(&preparation.adapter)?; call_with( &host_entry(py, ExecutorHost::LoadRuntimeParams)?, &[ diff --git a/fensu.toml b/fensu.toml index bb7e60942e..46b500d54c 100644 --- a/fensu.toml +++ b/fensu.toml @@ -2693,13 +2693,13 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs=16", - "crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs=11", + "crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs=20", + "crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs=19", "crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/build_command/policies.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs=19", - "crates/sqlbuild-python/src/bindings/_helpers/check/check_core.rs=8", + "crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs=18", + "crates/sqlbuild-python/src/bindings/_helpers/check/check_core.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/check/check_results.rs=13", "crates/sqlbuild-python/src/bindings/_helpers/cli/argv.rs=18", "crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs=17", diff --git a/src/sqlbuild/executor/build/_helpers/source_node.py b/src/sqlbuild/executor/build/_helpers/source_node.py index 475fa9ec99..9cc8971d4c 100644 --- a/src/sqlbuild/executor/build/_helpers/source_node.py +++ b/src/sqlbuild/executor/build/_helpers/source_node.py @@ -62,6 +62,7 @@ def execute_build_source_node( connection=connection, database=runtime.default_database, schema=runtime.default_schema, + target_config=runtime.target_config, ) result: LoadExecutionResult = execute_source_load( source_entry=source_entry, diff --git a/src/sqlbuild/executor/load/_helpers/dag_runtime.py b/src/sqlbuild/executor/load/_helpers/dag_runtime.py index a26bdfddcc..87e0509bbe 100644 --- a/src/sqlbuild/executor/load/_helpers/dag_runtime.py +++ b/src/sqlbuild/executor/load/_helpers/dag_runtime.py @@ -191,6 +191,7 @@ def _execute_ready_dag_source( connection=connection, database=runtime.default_database, schema=runtime.default_schema, + target_config=runtime.target_config, ) return execute_source_load( source_entry=source, diff --git a/src/sqlbuild/executor/node_results/classes/direct_store.py b/src/sqlbuild/executor/node_results/classes/direct_store.py index 3e42718578..54f5d0723f 100644 --- a/src/sqlbuild/executor/node_results/classes/direct_store.py +++ b/src/sqlbuild/executor/node_results/classes/direct_store.py @@ -27,11 +27,11 @@ def __init__( connection: Any, database: str | None, schema: str | None, - previous_location: tuple[str | None, str | None] | None = None, + previous_locations: tuple[tuple[str | None, str | None], ...] = (), ) -> None: self.adapter: BaseAdapter = adapter self.connection: Any = connection - self.previous_location: tuple[str | None, str | None] | None = previous_location + self.previous_locations: tuple[tuple[str | None, str | None], ...] = previous_locations super().__init__(database=database, schema=schema) def write(self, record: NodeResultRecord) -> None: @@ -57,19 +57,22 @@ def write(self, record: NodeResultRecord) -> None: self._cache_record(record=record) def _read_persisted(self, *, query: NodeResultQuery) -> tuple[NodeResultEnvelope, ...]: + """Read this store's location, then each earlier location until one has the node.""" + results: tuple[NodeResultEnvelope, ...] = self._read_at( database=self.database, schema=self.schema, query=query ) - if results or self.previous_location is None: - return results - # Results written before this release's location move: read them for one release only, - # and remove this fallback with the control-store migrations. - database, schema = self.previous_location - return self._read_at( - database=database, - schema=schema, - query=replace(query, target_database=database, target_schema=schema), - ) + location: tuple[str | None, str | None] + for location in self.previous_locations: + if results: + break + database, schema = location + results = self._read_at( + database=database, + schema=schema, + query=replace(query, target_database=database, target_schema=schema), + ) + return results def _read_at( self, *, database: str | None, schema: str | None, query: NodeResultQuery diff --git a/src/sqlbuild/executor/node_results/constants.py b/src/sqlbuild/executor/node_results/constants.py index 3b2d792021..a4b072a926 100644 --- a/src/sqlbuild/executor/node_results/constants.py +++ b/src/sqlbuild/executor/node_results/constants.py @@ -43,3 +43,4 @@ ("__sqlbuild_asset__", PythonNodeKind.ASSET.value), ("__sqlbuild_loader__", PythonNodeKind.LOADER.value), ) +PRESERVED_SCHEMA_NAME: str = "preserve" diff --git a/src/sqlbuild/executor/node_results/main/_direct_store.py b/src/sqlbuild/executor/node_results/main/_direct_store.py index 94a71d239c..5b989c81d9 100644 --- a/src/sqlbuild/executor/node_results/main/_direct_store.py +++ b/src/sqlbuild/executor/node_results/main/_direct_store.py @@ -4,8 +4,10 @@ from typing import Any +from sqlbuild._native import TargetSettings from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.executor.node_results.classes.direct_store import DirectNodeResultStore +from sqlbuild.executor.node_results.constants import PRESERVED_SCHEMA_NAME def build_direct_node_result_store( @@ -14,14 +16,21 @@ def build_direct_node_result_store( connection: Any, database: str | None, schema: str | None, + target_config: TargetSettings | None = None, ) -> DirectNodeResultStore: - """Build a direct-mode node result store that also reads the adapter's default location.""" + """Build a direct-mode node result store that, for one release, also reads older locations.""" - previous: tuple[str | None, str | None] = (adapter.default_database(), adapter.default_schema()) + earlier: list[tuple[str | None, str | None]] = [ + (adapter.default_database(), adapter.default_schema()) + ] + if target_config is not None and target_config.schema_preserved: + earlier.append((database, PRESERVED_SCHEMA_NAME)) return DirectNodeResultStore( adapter=adapter, connection=connection, database=database, schema=schema, - previous_location=previous if previous != (database, schema) else None, + previous_locations=tuple( + location for location in earlier if location != (database, schema) + ), ) diff --git a/src/sqlbuild/executor/python_nodes/_helpers/execution.py b/src/sqlbuild/executor/python_nodes/_helpers/execution.py index 194ea2ca61..74426fec85 100644 --- a/src/sqlbuild/executor/python_nodes/_helpers/execution.py +++ b/src/sqlbuild/executor/python_nodes/_helpers/execution.py @@ -82,6 +82,7 @@ def execute_ready_python_node( connection=runtime.connection, database=runtime.default_database, schema=runtime.default_schema, + target_config=runtime.target_config, ) if runtime.persist_node_results else None diff --git a/src/sqlbuild/executor/python_nodes/_helpers/ingress_execution.py b/src/sqlbuild/executor/python_nodes/_helpers/ingress_execution.py index ab671bae2f..34f58948e6 100644 --- a/src/sqlbuild/executor/python_nodes/_helpers/ingress_execution.py +++ b/src/sqlbuild/executor/python_nodes/_helpers/ingress_execution.py @@ -99,6 +99,7 @@ def execute_ingress_python_loader_nodes( connection=runtime.connection, database=runtime.default_database, schema=runtime.default_schema, + target_config=runtime.target_config, ) if runtime.persist_node_results else None diff --git a/src/sqlbuild/executor/python_nodes/_helpers/python_checks.py b/src/sqlbuild/executor/python_nodes/_helpers/python_checks.py index 508a4b4d03..70715affd9 100644 --- a/src/sqlbuild/executor/python_nodes/_helpers/python_checks.py +++ b/src/sqlbuild/executor/python_nodes/_helpers/python_checks.py @@ -91,6 +91,7 @@ def execute_python_check_nodes( connection=connection, database=default_database, schema=default_schema, + target_config=runtime.target_config, ) if runtime.persist_node_results else None diff --git a/src/sqlbuild/executor/python_nodes/classes/read_side_python_execution_tracker.py b/src/sqlbuild/executor/python_nodes/classes/read_side_python_execution_tracker.py index b85237fa24..b063f8a60c 100644 --- a/src/sqlbuild/executor/python_nodes/classes/read_side_python_execution_tracker.py +++ b/src/sqlbuild/executor/python_nodes/classes/read_side_python_execution_tracker.py @@ -55,6 +55,7 @@ def __init__( connection=runtime.connection, database=runtime.default_database, schema=runtime.default_schema, + target_config=runtime.target_config, ) if runtime.persist_node_results else None diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py index 165db667db..8d7d1b52cb 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py @@ -1684,3 +1684,15 @@ class PreservedTargetRefusalE2ETestCase: description: str python_files: dict[str, str] expected_error: str + + +@dataclass(frozen=True) +class AdapterDefaultResultsFallbackE2ETestCase: + """Results left in the adapter default by an earlier build, and the loader rebuilt after it.""" + + description: str + target_schema: str + rebuilt_loader: str + kept_loader: str + reading_task: str + expected_payload_keys: tuple[str, str] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py index 62caae7019..c6a0ec51e3 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py @@ -1489,3 +1489,46 @@ def changed_build_steps(warehouse: ChangedWarehouse) -> tuple[ChangedBuildStep, ChangedBuildStep("after the plain build", (), {}, (CHANGED_ANY,)), ChangedBuildStep("full refresh", (), {}, (CHANGED_ANY, "--full-refresh")), ) + + +def node_result_source_yml(location: str) -> str: + """A managed `raw_orders` source at `location`, for node result location tests.""" + + return ( + "sources:\n - name: raw_orders\n description: Raw orders.\n" + f" managed: true\n{location} write_strategy: table\n" + " columns:\n - name: order_id\n type: INTEGER\n" + ) + + +def node_result_project(*, tmp_path: Path, target_lines: str, files: dict[str, str]) -> Path: + """A DuckDB orders project whose `dev` target has `target_lines`, plus `files`.""" + + return prepare_inline_project( + tmp_path=tmp_path, + project_name="orders", + repo_files={ + "sqlbuild_project.toml": ( + 'name = "orders"\nadapter = "duckdb"\ndefault_target = "dev"\n\n' + '[connection]\ndatabase = "dev.duckdb"\n\n' + f'[defaults]\ndatabase = "dev"\nschema = "mart"\n\n[targets.dev]\n{target_lines}' + ), + "models/orders.sql": ( + "MODEL (description 'Orders.', materialized table);\n\n" + 'SELECT order_id FROM __source("raw_orders")\n' + ), + } + | files, + ) + + +def node_results_in(*, db_path: Path, schema: str, node_type: str) -> list[tuple[str, str, str]]: + """`(node_name, run_id, payload_json_b64)` of one node type's results in `schema`.""" + + return query_duckdb( + db_path=db_path, + sql=( + f"SELECT node_name, run_id, payload_json_b64 FROM {schema}._sqlbuild_node_results " + f"WHERE node_type = '{node_type}' ORDER BY node_name" + ), + ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_node_result_location.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_node_result_location.py index fc68b2141d..b1a18240ff 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_node_result_location.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_node_result_location.py @@ -10,15 +10,22 @@ import pytest from tests.e2e.src.sqlbuild.cli.commands.main.build._test_types import ( + AdapterDefaultResultsFallbackE2ETestCase, PreservedTargetNodeResultsE2ETestCase, PreservedTargetRefusalE2ETestCase, ) +from tests.e2e.src.sqlbuild.cli.commands.main.build.helpers import ( + node_result_project, + node_result_source_yml, + node_results_in, +) from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( prepare_inline_project, query_duckdb, run_sqb, ) +_EDGE_CASES: Path = Path("benchmarks/duckshop/edge_cases/loaders") _LOADER: str = ( "from sqlbuild.loaders import loader\n\n\n@loader\n" "def raw_orders(ctx: object) -> list[dict[str, object]]:\n" @@ -31,33 +38,22 @@ ' """Announce the loaded orders."""\n' " return ctx.result(payload={'announced': True})\n" ) - - -def _source_yml(location: str) -> str: - return ( - "sources:\n - name: raw_orders\n description: Raw orders.\n" - f" managed: true\n{location} write_strategy: table\n" - " columns:\n - name: order_id\n type: INTEGER\n" - ) - - -def _project(*, tmp_path: Path, target_lines: str, files: dict[str, str]) -> Path: - return prepare_inline_project( - tmp_path=tmp_path, - project_name="orders", - repo_files={ - "sqlbuild_project.toml": ( - 'name = "orders"\nadapter = "duckdb"\ndefault_target = "dev"\n\n' - '[connection]\ndatabase = "dev.duckdb"\n\n' - f'[defaults]\ndatabase = "dev"\nschema = "mart"\n\n[targets.dev]\n{target_lines}' - ), - "models/orders.sql": ( - "MODEL (description 'Orders.', materialized table);\n\n" - 'SELECT order_id FROM __source("raw_orders")\n' - ), - } - | files, - ) +_CHECK: str = ( + "from python.tasks.announce import announce_orders\nfrom sqlbuild.checks import check\n\n\n" + "@check(depends_on=announce_orders)\n" + "def check_orders_announced(ctx):\n" + ' """The task announced the orders."""\n' + " announced = ctx.result_of(node_function=announce_orders).payload['announced']\n" + " return ctx.pass_(message='announced') if announced else ctx.fail(message='silent')\n" +) +_COPYING_LOADER: str = ( + "from python.loaders.orders import raw_orders\nfrom sqlbuild.loaders import loader\n\n\n" + "@loader\n" + "def raw_order_copies(ctx) -> list[dict[str, object]]:\n" + ' """One order when the orders loader has a result to read, none otherwise."""\n' + " found = ctx.result_of(node_function=raw_orders, default=None) is not None\n" + " return [{'order_id': 1}] if found else []\n" +) @pytest.mark.parametrize( @@ -83,18 +79,28 @@ def _project(*, tmp_path: Path, target_lines: str, files: dict[str, str]) -> Pat def test_given_preserved_target_when_building_twice_then_node_results_stay_in_one_real_location( test_case: PreservedTargetNodeResultsE2ETestCase, tmp_path: Path ) -> None: - project_dir: Path = _project( + project_dir: Path = node_result_project( tmp_path=tmp_path, target_lines=test_case.target_lines, files={ - "sources/raw.yml": _source_yml(test_case.source_location), + "sources/raw.yml": node_result_source_yml(test_case.source_location), + "sources/copies.yml": node_result_source_yml(test_case.source_location).replace( + "raw_orders", "raw_order_copies" + ), "python/loaders/orders.py": _LOADER, + "python/loaders/copies.py": _COPYING_LOADER, "python/tasks/announce.py": _TASK, + "python/checks/orders.py": _CHECK, }, ) results: list[subprocess.CompletedProcess[str]] = [ run_sqb(command=("--no-color", "build"), project_dir=project_dir) for _ in range(2) + ] + [ + run_sqb( + command=("--no-color", "load", "--select", "raw_order_copies"), + project_dir=project_dir, + ) ] for result in results: @@ -125,9 +131,12 @@ def test_given_preserved_target_when_building_twice_then_node_results_stay_in_on "GROUP BY ALL ORDER BY ALL" ), ) == [ + ("check", "check_orders_announced", test_case.expected_runs_per_node), + ("loader", "raw_order_copies", test_case.expected_runs_per_node), ("loader", "raw_orders", test_case.expected_runs_per_node), ("task", "announce_orders", test_case.expected_runs_per_node), ] + assert query_duckdb(db_path=db_path, sql="SELECT order_id FROM raw.raw_order_copies") == [(1,)] @pytest.mark.parametrize( @@ -168,21 +177,22 @@ def test_given_preserved_schema_without_loader_schema_when_compiling_then_python assert test_case.expected_error in result.stderr -_EDGE_CASES: Path = Path("benchmarks/duckshop/edge_cases/loaders") - - -def _results(*, db_path: Path, schema: str, node_type: str) -> list[tuple[str, str, str]]: - return query_duckdb( - db_path=db_path, - sql=( - f"SELECT node_name, run_id, payload_json_b64 FROM {schema}._sqlbuild_node_results " - f"WHERE node_type = '{node_type}' ORDER BY node_name" +@pytest.mark.parametrize( + "test_case", + [ + AdapterDefaultResultsFallbackE2ETestCase( + description="a task reads a rebuilt loader and one whose result is only in main", + target_schema="mart", + rebuilt_loader="raw_order_events", + kept_loader="raw_customer_events", + reading_task="announce_events", + expected_payload_keys=("orders", "customers"), ), - ) - - + ], + ids=lambda case: case.description, +) def test_given_results_in_the_adapter_default_when_one_loader_rebuilds_then_both_are_still_read( - tmp_path: Path, + test_case: AdapterDefaultResultsFallbackE2ETestCase, tmp_path: Path ) -> None: project_dir: Path = prepare_inline_project( tmp_path=tmp_path, @@ -202,28 +212,45 @@ def test_given_results_in_the_adapter_default_when_one_loader_rebuilds_then_both ) assert first.returncode == 0, first.stdout + first.stderr old_runs: dict[str, str] = { - name: run for name, run, _ in _results(db_path=db_path, schema="main", node_type="loader") + name: run + for name, run, _ in node_results_in(db_path=db_path, schema="main", node_type="loader") } toml: Path = project_dir / "sqlbuild_project.toml" - toml.write_text(toml.read_text(encoding="utf-8") + 'schema = "mart"\n', encoding="utf-8") + toml.write_text( + toml.read_text(encoding="utf-8") + f'schema = "{test_case.target_schema}"\n', + encoding="utf-8", + ) second: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build", "--select", "raw_order_events", "announce_events"), + command=( + "--no-color", + "build", + "--select", + test_case.rebuilt_loader, + test_case.reading_task, + ), project_dir=project_dir, ) assert second.returncode == 0, second.stdout + second.stderr new_runs: dict[str, str] = { - name: run for name, run, _ in _results(db_path=db_path, schema="mart", node_type="loader") + name: run + for name, run, _ in node_results_in( + db_path=db_path, schema=test_case.target_schema, node_type="loader" + ) } - assert set(new_runs) == {"raw_order_events"} - assert new_runs["raw_order_events"] != old_runs["raw_order_events"] - [(_, task_run, payload)] = _results(db_path=db_path, schema="mart", node_type="task") - assert task_run == new_runs["raw_order_events"] + assert set(new_runs) == {test_case.rebuilt_loader} + assert new_runs[test_case.rebuilt_loader] != old_runs[test_case.rebuilt_loader] + [(_, task_run, payload)] = node_results_in( + db_path=db_path, schema=test_case.target_schema, node_type="task" + ) + assert task_run == new_runs[test_case.rebuilt_loader] + rebuilt_key, kept_key = test_case.expected_payload_keys assert json.loads(base64.b64decode(payload)) == { - "orders": new_runs["raw_order_events"], - "customers": old_runs["raw_customer_events"], + rebuilt_key: new_runs[test_case.rebuilt_loader], + kept_key: old_runs[test_case.kept_loader], } assert { - name: run for name, run, _ in _results(db_path=db_path, schema="main", node_type="loader") + name: run + for name, run, _ in node_results_in(db_path=db_path, schema="main", node_type="loader") } == old_runs diff --git a/tests/integration/src/sqlbuild/executor/node_results/main/_test_types.py b/tests/integration/src/sqlbuild/executor/node_results/main/_test_types.py index 77ba572b7e..39d34e3c46 100644 --- a/tests/integration/src/sqlbuild/executor/node_results/main/_test_types.py +++ b/tests/integration/src/sqlbuild/executor/node_results/main/_test_types.py @@ -6,3 +6,14 @@ class NodeResultReadWriteIntegrationTestCase: description: str expected_payload: dict[str, object] expected_metadata: dict[str, object] + + +@dataclass(frozen=True) +class EarlierNodeResultLocationIntegrationTestCase: + """A result left in an earlier location, results in the store's own, and which one is read.""" + + description: str + earlier_schema: str + schema_preserved: bool + new_run_ids: tuple[str, ...] + expected_run_id: str | None diff --git a/tests/integration/src/sqlbuild/executor/node_results/main/helpers.py b/tests/integration/src/sqlbuild/executor/node_results/main/helpers.py new file mode 100644 index 0000000000..19ddf21d7f --- /dev/null +++ b/tests/integration/src/sqlbuild/executor/node_results/main/helpers.py @@ -0,0 +1,21 @@ +"""Shared builders for node result integration tests.""" + +from __future__ import annotations + +from sqlbuild.executor.node_results.models import NodeResultRecord + + +def task_result_record(*, run_id: str, schema: str) -> NodeResultRecord: + """A successful `produce_result` task result recorded in `schema`.""" + + return NodeResultRecord( + node_type="task", + node_name="produce_result", + target_database=None, + target_schema=schema, + target_name=None, + run_id=run_id, + status="success", + payload={"run": run_id}, + metadata={}, + ) diff --git a/tests/integration/src/sqlbuild/executor/node_results/main/test_read_write.py b/tests/integration/src/sqlbuild/executor/node_results/main/test_read_write.py index eb6d7c7a94..d433c4082d 100644 --- a/tests/integration/src/sqlbuild/executor/node_results/main/test_read_write.py +++ b/tests/integration/src/sqlbuild/executor/node_results/main/test_read_write.py @@ -7,7 +7,10 @@ import pytest +from sqlbuild._native import TargetSettings from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.executor.node_results.classes.direct_store import DirectNodeResultStore +from sqlbuild.executor.node_results.main._direct_store import build_direct_node_result_store from sqlbuild.executor.node_results.main._read import read_node_results from sqlbuild.executor.node_results.main._write import write_node_result_record from sqlbuild.executor.node_results.models import ( @@ -16,8 +19,10 @@ NodeResultRecord, ) from tests.integration.src.sqlbuild.executor.node_results.main._test_types import ( + EarlierNodeResultLocationIntegrationTestCase, NodeResultReadWriteIntegrationTestCase, ) +from tests.integration.src.sqlbuild.executor.node_results.main.helpers import task_result_record @pytest.mark.parametrize( @@ -83,3 +88,72 @@ def test_given_node_result_record_when_writing_and_reading_then_round_trips_enve assert len(results) == 1 assert results[0].payload == test_case.expected_payload assert results[0].metadata == test_case.expected_metadata + + +@pytest.mark.parametrize( + "test_case", + [ + EarlierNodeResultLocationIntegrationTestCase( + description="a result only in the adapter default schema is read", + earlier_schema="main", + schema_preserved=False, + new_run_ids=(), + expected_run_id="run_old", + ), + EarlierNodeResultLocationIntegrationTestCase( + description="a preserved target reads a result left in a schema named preserve", + earlier_schema="preserve", + schema_preserved=True, + new_run_ids=(), + expected_run_id="run_old", + ), + EarlierNodeResultLocationIntegrationTestCase( + description="a target that is not preserved ignores a schema named preserve", + earlier_schema="preserve", + schema_preserved=False, + new_run_ids=(), + expected_run_id=None, + ), + EarlierNodeResultLocationIntegrationTestCase( + description="a result in the store's own schema wins over an earlier one", + earlier_schema="main", + schema_preserved=False, + new_run_ids=("run_new",), + expected_run_id="run_new", + ), + ], + ids=lambda case: case.description, +) +def test_given_result_in_earlier_location_when_reading_then_store_falls_back_only_when_empty( + test_case: EarlierNodeResultLocationIntegrationTestCase, + tmp_path: Path, +) -> None: + adapter: DuckDbAdapter = DuckDbAdapter() + connection: Any = adapter.connect({"database": str(tmp_path / "node_results.duckdb")}) + try: + earlier: DirectNodeResultStore = DirectNodeResultStore( + adapter=adapter, connection=connection, database=None, schema=test_case.earlier_schema + ) + earlier.write(task_result_record(run_id="run_old", schema=test_case.earlier_schema)) + store: DirectNodeResultStore = build_direct_node_result_store( + adapter=adapter, + connection=connection, + database=None, + schema="raw", + target_config=TargetSettings(schema_preserved=test_case.schema_preserved), + ) + run_id: str + for run_id in test_case.new_run_ids: + store.write(task_result_record(run_id=run_id, schema="raw")) + reader: DirectNodeResultStore = build_direct_node_result_store( + adapter=adapter, + connection=connection, + database=None, + schema="raw", + target_config=TargetSettings(schema_preserved=test_case.schema_preserved), + ) + found: object = reader.result_of(node_type="task", node_name="produce_result", default=None) + finally: + adapter.close(connection) + + assert getattr(found, "run_id", None) == test_case.expected_run_id diff --git a/tests/unit/src/sqlbuild/executor/build/_helpers/_test_types.py b/tests/unit/src/sqlbuild/executor/build/_helpers/_test_types.py index 22f4b0e984..51d726594c 100644 --- a/tests/unit/src/sqlbuild/executor/build/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/executor/build/_helpers/_test_types.py @@ -10,6 +10,7 @@ ) from sqlbuild.adapter.contract.models import RelationInfo from sqlbuild.adapter.contract.types import RetentionChangePhase +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.executor.build.types import BuildStatus from sqlbuild.executor.scheduling.types import ExecutionStatus from sqlbuild.runtime.contracts.types import ExecutionResourceKind @@ -126,7 +127,7 @@ class BuildSourceNodeResultLocationTestCase: """Where a scheduled loader's result goes for an adapter default and a runtime location.""" description: str - adapter_default_schema: str | None + adapter_class: type[DuckDbAdapter] runtime_schema: str expected_result_schemas: tuple[str, ...] diff --git a/tests/unit/src/sqlbuild/executor/build/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/build/_helpers/helpers.py index 2e2800504b..59942559cf 100644 --- a/tests/unit/src/sqlbuild/executor/build/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/executor/build/_helpers/helpers.py @@ -24,6 +24,7 @@ SourceWriteStrategy, ) from sqlbuild.adapter.contract.classes.retention_adapter import RetentionAdapterMixin +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.auditing.types import ( AuditAttachmentKind, AuditOutcome, @@ -213,3 +214,10 @@ def build_model_key_edges( for name, children in edges.items(): key_edges[model_key(name)] = tuple(model_key(child) for child in children) return key_edges + + +class NoDefaultSchemaDuckDbAdapter(DuckDbAdapter): + """A DuckDB adapter that, like Snowflake, has no default schema.""" + + def default_schema(self) -> str | None: + return None diff --git a/tests/unit/src/sqlbuild/executor/build/_helpers/test_source_node.py b/tests/unit/src/sqlbuild/executor/build/_helpers/test_source_node.py index cfea281a4c..c86dc86caf 100644 --- a/tests/unit/src/sqlbuild/executor/build/_helpers/test_source_node.py +++ b/tests/unit/src/sqlbuild/executor/build/_helpers/test_source_node.py @@ -18,6 +18,7 @@ BuildSourceNodeResultLocationTestCase, ) from tests.unit.src.sqlbuild.executor.build._helpers.helpers import ( + NoDefaultSchemaDuckDbAdapter, build_discovered_source_loader, build_source_load_plan_output, ) @@ -89,25 +90,18 @@ def test_given_build_source_node_when_executing_then_runs_loader_runtime( assert start_events == [test_case.expected_start_event] -class _NoDefaultSchemaAdapter(DuckDbAdapter): - """A DuckDB adapter that, like Snowflake, has no default schema.""" - - def default_schema(self) -> str | None: - return None - - @pytest.mark.parametrize( "test_case", [ BuildSourceNodeResultLocationTestCase( description="an adapter without a default schema still records the result", - adapter_default_schema=None, + adapter_class=NoDefaultSchemaDuckDbAdapter, runtime_schema="analytics", expected_result_schemas=("analytics",), ), BuildSourceNodeResultLocationTestCase( description="the runtime location wins over the adapter default schema", - adapter_default_schema="main", + adapter_class=DuckDbAdapter, runtime_schema="analytics", expected_result_schemas=("analytics",), ), @@ -118,9 +112,7 @@ def test_given_runtime_location_when_executing_source_node_then_result_is_writte test_case: BuildSourceNodeResultLocationTestCase, tmp_path: Path, ) -> None: - adapter: DuckDbAdapter = ( - DuckDbAdapter() if test_case.adapter_default_schema else _NoDefaultSchemaAdapter() - ) + adapter: DuckDbAdapter = test_case.adapter_class() connection: object = adapter.connect({"database": str(tmp_path / "source_node.duckdb")}) try: result: LoadExecutionResult = execute_build_source_node( From 2636a15938a8197fac2a84d22df4381f67209bf5 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 17 Nov 2026 13:31:06 +0000 Subject: [PATCH 3/6] fix: resolve sqb load's result location with CLI vars and share loader result writes sqb load now takes its node-result location from the CLI-var-aware loader defaults, as build does, so a schema set with --vars is read from the same place both commands write. Build and ingress loaders write their results through one load operation. --- Makefile | 1 + .../src/compile_driver/models.rs | 22 +++++++--- .../src/bindings/_helpers/check/check_core.rs | 13 +++--- .../bindings/_helpers/load/load_command.rs | 5 ++- .../executor/build/_helpers/source_node.py | 36 +--------------- .../executor/load/main/_persist_result.py | 42 +++++++++++++++++++ .../_helpers/ingress_execution.py | 38 ++--------------- .../cli/commands/main/build/_test_types.py | 3 +- .../main/build/test_node_result_location.py | 15 +++++-- .../executor/build/_helpers/_test_types.py | 3 +- .../executor/build/_helpers/helpers.py | 8 ---- .../build/_helpers/test_source_node.py | 9 ++-- .../executor/python_nodes/_helpers/helpers.py | 1 + 13 files changed, 95 insertions(+), 101 deletions(-) create mode 100644 src/sqlbuild/executor/load/main/_persist_result.py diff --git a/Makefile b/Makefile index b3dad765e5..f3e8c2144c 100644 --- a/Makefile +++ b/Makefile @@ -151,6 +151,7 @@ E2E_DUCKDB_BUILD_CORE_PATHS := \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_defer_migration_origin.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_sql_hook_reads.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_identity.py \ + tests/e2e/src/sqlbuild/cli/commands/main/build/test_node_result_location.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_sql_hook_identity.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_query_change_tracking.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_query_fingerprint_changes.py \ diff --git a/crates/sqlbuild-python-compile/src/compile_driver/models.rs b/crates/sqlbuild-python-compile/src/compile_driver/models.rs index f2e42b435b..5838cd685d 100644 --- a/crates/sqlbuild-python-compile/src/compile_driver/models.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/models.rs @@ -68,8 +68,8 @@ impl InputsRequest { } } -/// The active target's locations as SQLBuild resolves them; `preserve` stays as written until `record`. -#[derive(Debug, Clone)] +/// The active target's locations as SQLBuild resolves them (default: unset); `preserve` stays as written until `record`. +#[derive(Debug, Clone, Default)] pub struct TargetSettingsValues { pub(crate) name: Option, pub(crate) database: Option, @@ -90,11 +90,23 @@ impl TargetSettingsValues { adapter: &Bound<'py, PyAny>, ) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { let py: Python<'py> = adapter.py(); - model_free_locations( + self.node_locations_from( ( - self.database.as_deref().into_bound_py_any(py)?, - self.schema.as_deref().into_bound_py_any(py)?, + &self.database.as_deref().into_bound_py_any(py)?, + &self.schema.as_deref().into_bound_py_any(py)?, ), + adapter, + ) + } + + /// `node_locations` from a caller-resolved `(database, schema)`, such as `sqb load`'s, which applies CLI vars. + pub fn node_locations_from<'py>( + &self, + (database, schema): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + adapter: &Bound<'py, PyAny>, + ) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { + model_free_locations( + (database.clone(), schema.clone()), self.loader_schema(), adapter, ) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/check_core.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/check_core.rs index e2b2068d11..04442d4156 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/check/check_core.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/check_core.rs @@ -10,7 +10,6 @@ use pyo3::prelude::{ use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyFrozenSetMethods, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_python_base::boundary::main::keywords::keywords; -use sqlbuild_python_base::warehouse_access::main::adapter_default_location::adapter_default_location; use sqlbuild_python_compile::compile_driver::main::setup::build_python_relation_targets::build_python_relation_targets; use sqlbuild_python_project::graph::main::python_nodes::resolve_python_check_excludes::resolve_python_check_excludes; use sqlbuild_python_project::planner_phases::main::object::object; @@ -447,13 +446,11 @@ fn read_side_references<'py>( fn node_runtime<'py>(inputs: &ReadSideInputs<'py>) -> PyResult> { let py: Python<'py> = inputs.adapter.py(); let project: Bound<'py, PyAny> = native_field(&inputs.pipeline_result, "project")?; - let (default_database, default_schema) = match &inputs.target_config { - Some(settings) => settings.node_locations(&inputs.adapter)?, - None => ( - adapter_default_location(&inputs.adapter, "default_database")?, - adapter_default_location(&inputs.adapter, "default_schema")?, - ), - }; + let (default_database, default_schema) = inputs + .target_config + .clone() + .unwrap_or_default() + .node_locations(&inputs.adapter)?; call_with( &host_entry(py, ExecutorHost::PythonNodeRuntime)?, &[ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs index 8bfde4354e..5449d2ddce 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs @@ -67,6 +67,7 @@ struct Invocation<'py> { selected: Vec>, references: Vec>, relation_sources: Vec>, + defaults: LoaderDefaults<'py>, style: CliStyle, stream: Bound<'py, PyAny>, } @@ -181,6 +182,7 @@ fn invocation<'py>(py: Python<'py>, request: &Bound<'py, PyAny>) -> PyResult( let py: Python<'py> = request.py(); let (target_name, effective_vars, run_id) = &preparation.runtime; let overrides: &Bound<'py, PyAny> = &preparation.cursor_overrides; + let defaults = (&invocation.defaults.database, &invocation.defaults.schema); let (default_database, default_schema) = preparation .target_config - .node_locations(&preparation.adapter)?; + .node_locations_from(defaults, &preparation.adapter)?; call_with( &host_entry(py, ExecutorHost::LoadRuntimeParams)?, &[ diff --git a/src/sqlbuild/executor/build/_helpers/source_node.py b/src/sqlbuild/executor/build/_helpers/source_node.py index 9cc8971d4c..46ce2b0ae4 100644 --- a/src/sqlbuild/executor/build/_helpers/source_node.py +++ b/src/sqlbuild/executor/build/_helpers/source_node.py @@ -19,10 +19,10 @@ from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.executor.build.models import BuildCallbacks, BuildRuntimeParams from sqlbuild.executor.load.main._execute import execute_source_load +from sqlbuild.executor.load.main._persist_result import persist_loader_result from sqlbuild.executor.load.models import LoadExecutionResult, LoadRuntimeParams from sqlbuild.executor.node_results.classes.direct_store import DirectNodeResultStore from sqlbuild.executor.node_results.main._direct_store import build_direct_node_result_store -from sqlbuild.executor.node_results.models import NodeResultRecord from sqlbuild.runtime.contracts.types import ExecutionResourceKind @@ -101,42 +101,10 @@ def execute_build_source_node( resource_name=loader_name, phase="finalize", ): - _persist_loader_result( + persist_loader_result( loader_name=loader_name, result=timed_result, run_id=runtime.run_id, result_store=result_store, ) return timed_result - - -def _persist_loader_result( - *, - loader_name: str, - result: LoadExecutionResult, - run_id: str, - result_store: DirectNodeResultStore, -) -> None: - if result_store.connection is None: - return - result_store.write( - NodeResultRecord( - node_type="loader", - node_name=loader_name, - target_database=result_store.database, - target_schema=result_store.schema, - target_name=None, - run_id=run_id, - status=result.status.value, - payload=result.result_payload, - metadata={ - "source_name": result.source_name, - "loader_name": result.loader_name, - "rows_loaded": result.rows_loaded, - "target": result.target, - **result.result_metadata, - }, - error_message=result.error_message or result.skip_reason, - materialized=result.result_materialized, - ) - ) diff --git a/src/sqlbuild/executor/load/main/_persist_result.py b/src/sqlbuild/executor/load/main/_persist_result.py new file mode 100644 index 0000000000..9e12c08a0d --- /dev/null +++ b/src/sqlbuild/executor/load/main/_persist_result.py @@ -0,0 +1,42 @@ +"""Public operation for recording a loader's result in the node-result store.""" + +from __future__ import annotations + +from sqlbuild.compiler.python_nodes.types import PythonNodeKind +from sqlbuild.executor.load.models import LoadExecutionResult +from sqlbuild.executor.node_results.classes.direct_store import DirectNodeResultStore +from sqlbuild.executor.node_results.models import NodeResultRecord + + +def persist_loader_result( + *, + result_store: DirectNodeResultStore | None, + loader_name: str, + result: LoadExecutionResult, + run_id: str, +) -> None: + """Write one loader result to the store's location; do nothing without a live store.""" + + if result_store is None or result_store.connection is None: + return + result_store.write( + NodeResultRecord( + node_type=PythonNodeKind.LOADER.value, + node_name=loader_name, + target_database=result_store.database, + target_schema=result_store.schema, + target_name=None, + run_id=run_id, + status=result.status.value, + payload=result.result_payload, + metadata={ + "source_name": result.source_name, + "loader_name": result.loader_name, + "rows_loaded": result.rows_loaded, + "target": result.target, + **result.result_metadata, + }, + error_message=result.error_message or result.skip_reason, + materialized=result.result_materialized, + ) + ) diff --git a/src/sqlbuild/executor/python_nodes/_helpers/ingress_execution.py b/src/sqlbuild/executor/python_nodes/_helpers/ingress_execution.py index 34f58948e6..383e915f9e 100644 --- a/src/sqlbuild/executor/python_nodes/_helpers/ingress_execution.py +++ b/src/sqlbuild/executor/python_nodes/_helpers/ingress_execution.py @@ -22,11 +22,11 @@ from sqlbuild.compiler.python_nodes.types import PythonNodeKind, PythonNodeStatus, SkipMode from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.load.main._execute import execute_source_load +from sqlbuild.executor.load.main._persist_result import persist_loader_result from sqlbuild.executor.load.main._resource_kind import load_resource_kind from sqlbuild.executor.load.main._skipped_result import skipped_load_result from sqlbuild.executor.load.models import LoadExecutionResult, LoadRuntimeParams from sqlbuild.executor.node_results.main._direct_store import build_direct_node_result_store -from sqlbuild.executor.node_results.models import NodeResultRecord from sqlbuild.executor.python_nodes._helpers.fingerprinting import ( try_write_python_node_identity_fingerprint, ) @@ -306,7 +306,7 @@ def _execute_ingress_loader( source_ref_entries=source_map, ) results.record_load_result(name=node.name, result=result) - _persist_loader_result( + persist_loader_result( result_store=runtime.result_store, loader_name=loader.name, result=result, @@ -339,38 +339,6 @@ def _execute_ingress_loader( return _load_result_to_lifecycle_result(node_name=node.name, result=result) -def _persist_loader_result( - *, - result_store: Any | None, - loader_name: str, - result: LoadExecutionResult, - run_id: str, -) -> None: - if result_store is None: - return - result_store.write( - NodeResultRecord( - node_type=PythonNodeKind.LOADER.value, - node_name=loader_name, - target_database=result_store.database, - target_schema=result_store.schema, - target_name=None, - run_id=run_id, - status=result.status.value, - payload=result.result_payload, - metadata={ - "source_name": result.source_name, - "loader_name": result.loader_name, - "rows_loaded": result.rows_loaded, - "target": result.target, - **result.result_metadata, - }, - error_message=result.error_message or result.skip_reason, - materialized=result.result_materialized, - ) - ) - - def _record_scheduler_skips( *, scheduler_result: LifecycleSchedulerResult, @@ -406,7 +374,7 @@ def _record_scheduler_skips( resource_name=result.name, phase="finalize", ): - _persist_loader_result( + persist_loader_result( result_store=result_store, loader_name=result.name, result=skipped_result, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py index 8d7d1b52cb..7c6b8f8fc6 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py @@ -1668,13 +1668,14 @@ class CtxTargetSchemaE2ETestCase: @dataclass(frozen=True) class PreservedTargetNodeResultsE2ETestCase: - """A target that keeps its database or schema, and where two builds' node results land.""" + """A target's location settings, and where two builds' and a load's node results land.""" description: str target_lines: str source_location: str expected_results_schema: str expected_runs_per_node: int + cli_args: tuple[str, ...] = () @dataclass(frozen=True) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_node_result_location.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_node_result_location.py index b1a18240ff..02de124e27 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_node_result_location.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_node_result_location.py @@ -73,10 +73,18 @@ expected_results_schema="analytics", expected_runs_per_node=2, ), + PreservedTargetNodeResultsE2ETestCase( + description="a schema set by --vars holds the node results build and load share", + target_lines='schema = "${results_schema}"\n\n[vars]\nresults_schema = "mart"\n', + source_location=" schema: raw\n", + expected_results_schema="ops", + expected_runs_per_node=2, + cli_args=("--vars", json.dumps({"results_schema": "ops"})), + ), ], ids=lambda case: case.description, ) -def test_given_preserved_target_when_building_twice_then_node_results_stay_in_one_real_location( +def test_given_target_location_when_building_and_loading_then_node_results_stay_in_one_place( test_case: PreservedTargetNodeResultsE2ETestCase, tmp_path: Path ) -> None: project_dir: Path = node_result_project( @@ -95,10 +103,11 @@ def test_given_preserved_target_when_building_twice_then_node_results_stay_in_on ) results: list[subprocess.CompletedProcess[str]] = [ - run_sqb(command=("--no-color", "build"), project_dir=project_dir) for _ in range(2) + run_sqb(command=("--no-color", "build", *test_case.cli_args), project_dir=project_dir) + for _ in range(2) ] + [ run_sqb( - command=("--no-color", "load", "--select", "raw_order_copies"), + command=("--no-color", "load", "--select", "raw_order_copies", *test_case.cli_args), project_dir=project_dir, ) ] diff --git a/tests/unit/src/sqlbuild/executor/build/_helpers/_test_types.py b/tests/unit/src/sqlbuild/executor/build/_helpers/_test_types.py index 51d726594c..22f4b0e984 100644 --- a/tests/unit/src/sqlbuild/executor/build/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/executor/build/_helpers/_test_types.py @@ -10,7 +10,6 @@ ) from sqlbuild.adapter.contract.models import RelationInfo from sqlbuild.adapter.contract.types import RetentionChangePhase -from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.executor.build.types import BuildStatus from sqlbuild.executor.scheduling.types import ExecutionStatus from sqlbuild.runtime.contracts.types import ExecutionResourceKind @@ -127,7 +126,7 @@ class BuildSourceNodeResultLocationTestCase: """Where a scheduled loader's result goes for an adapter default and a runtime location.""" description: str - adapter_class: type[DuckDbAdapter] + adapter_default_schema: str | None runtime_schema: str expected_result_schemas: tuple[str, ...] diff --git a/tests/unit/src/sqlbuild/executor/build/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/build/_helpers/helpers.py index 59942559cf..2e2800504b 100644 --- a/tests/unit/src/sqlbuild/executor/build/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/executor/build/_helpers/helpers.py @@ -24,7 +24,6 @@ SourceWriteStrategy, ) from sqlbuild.adapter.contract.classes.retention_adapter import RetentionAdapterMixin -from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.auditing.types import ( AuditAttachmentKind, AuditOutcome, @@ -214,10 +213,3 @@ def build_model_key_edges( for name, children in edges.items(): key_edges[model_key(name)] = tuple(model_key(child) for child in children) return key_edges - - -class NoDefaultSchemaDuckDbAdapter(DuckDbAdapter): - """A DuckDB adapter that, like Snowflake, has no default schema.""" - - def default_schema(self) -> str | None: - return None diff --git a/tests/unit/src/sqlbuild/executor/build/_helpers/test_source_node.py b/tests/unit/src/sqlbuild/executor/build/_helpers/test_source_node.py index c86dc86caf..38ce1f18cb 100644 --- a/tests/unit/src/sqlbuild/executor/build/_helpers/test_source_node.py +++ b/tests/unit/src/sqlbuild/executor/build/_helpers/test_source_node.py @@ -18,7 +18,6 @@ BuildSourceNodeResultLocationTestCase, ) from tests.unit.src.sqlbuild.executor.build._helpers.helpers import ( - NoDefaultSchemaDuckDbAdapter, build_discovered_source_loader, build_source_load_plan_output, ) @@ -95,13 +94,13 @@ def test_given_build_source_node_when_executing_then_runs_loader_runtime( [ BuildSourceNodeResultLocationTestCase( description="an adapter without a default schema still records the result", - adapter_class=NoDefaultSchemaDuckDbAdapter, + adapter_default_schema=None, runtime_schema="analytics", expected_result_schemas=("analytics",), ), BuildSourceNodeResultLocationTestCase( description="the runtime location wins over the adapter default schema", - adapter_class=DuckDbAdapter, + adapter_default_schema="main", runtime_schema="analytics", expected_result_schemas=("analytics",), ), @@ -111,8 +110,10 @@ def test_given_build_source_node_when_executing_then_runs_loader_runtime( def test_given_runtime_location_when_executing_source_node_then_result_is_written_there( test_case: BuildSourceNodeResultLocationTestCase, tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, ) -> None: - adapter: DuckDbAdapter = test_case.adapter_class() + adapter: DuckDbAdapter = DuckDbAdapter() + monkeypatch.setattr(adapter, "default_schema", lambda: test_case.adapter_default_schema) connection: object = adapter.connect({"database": str(tmp_path / "source_node.duckdb")}) try: result: LoadExecutionResult = execute_build_source_node( diff --git a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/helpers.py index d775f4e8ce..6a7767909d 100644 --- a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/helpers.py @@ -149,6 +149,7 @@ class PythonNodeContextTestResultStore: def __init__(self, results: dict[tuple[str, str], tuple[NodeResultEnvelope, ...]]) -> None: self._results: dict[tuple[str, str], tuple[NodeResultEnvelope, ...]] = results self.written_records: list[NodeResultRecord] = [] + self.connection: object = object() self.database: str | None = None self.schema: str | None = "default_schema" From 10f42c65c7c0e19c79dc0b275d14988a2909ea51 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 17 Nov 2026 14:16:09 +0000 Subject: [PATCH 4/6] test: add the preserved-schema refusal to the compiler failure corpus --- .../_helpers/analysis_corpus/failure_cases.py | 18 ++++++++++++++++++ ...ed-schema-without-node-results-schema.json | 19 +++++++++++++++++++ 2 files changed, 37 insertions(+) create mode 100644 tests/goldens/compiler/failure/target-preserved-schema-without-node-results-schema.json diff --git a/scripts/compiler_differential/_helpers/analysis_corpus/failure_cases.py b/scripts/compiler_differential/_helpers/analysis_corpus/failure_cases.py index 50d5556bc2..e42b0a6ff3 100644 --- a/scripts/compiler_differential/_helpers/analysis_corpus/failure_cases.py +++ b/scripts/compiler_differential/_helpers/analysis_corpus/failure_cases.py @@ -457,6 +457,24 @@ def _target_cases() -> tuple[FailureCase, ...]: + 'database = "ANALYTICS"\n\n[targets.dev]\ndatabase = "ANALYTICS"\n' }, ), + failure_case( + name="target-preserved-schema-without-node-results-schema", + expected_code="S105", + expected_message="Target 'dev' keeps each model's own schema", + files={ + FAILURE_CONFIG_PATH: FAILURE_BASE_CONFIG.replace( + 'adapter = "duckdb"\n', 'adapter = "duckdb"\ndefault_target = "dev"\n' + ) + + '\n[defaults]\nschema = "mart"\n\n[targets.dev]\nschema = "preserve"\n', + "python/tasks/announce.py": ( + "from sqlbuild.tasks import task\n\n\n" + "@task\n" + "def announce_orders(ctx):\n" + ' """Announce the loaded orders."""\n' + " return ctx.result(payload={'announced': True})\n" + ), + }, + ), ) diff --git a/tests/goldens/compiler/failure/target-preserved-schema-without-node-results-schema.json b/tests/goldens/compiler/failure/target-preserved-schema-without-node-results-schema.json new file mode 100644 index 0000000000..3b47051dde --- /dev/null +++ b/tests/goldens/compiler/failure/target-preserved-schema-without-node-results-schema.json @@ -0,0 +1,19 @@ +{ + "commands": [ + { + "label": "compile", + "exit_code": 1, + "diagnostics": [ + { + "phase": "compile", + "severity": "error", + "code": "S105", + "message": "Target 'dev' keeps each model's own schema, so it has no schema for the results of loaders, tasks, assets and checks. Set targets.dev.loader_schema; Python node results are kept there.", + "help": "to keep Python node results in a schema, set this in sqlbuild_project.toml:\n [targets.dev]\n loader_schema = \"my_schema\"" + } + ] + } + ], + "compiled": {}, + "manifest": null +} From 88b22daa7c981fda4fcce06463e17cd7491ad70a Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 17 Nov 2026 15:42:01 +0000 Subject: [PATCH 5/6] test: give the preserved-target load cases a loader schema A preserve target with a loader now needs loader_schema (S105), so the managed-source load cases set one. The source still lands in its own schema. --- .../cli/commands/main/compile/test_missing_target_database.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_missing_target_database.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_missing_target_database.py index 3ee1d8dc14..25598b9427 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_missing_target_database.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_missing_target_database.py @@ -21,6 +21,7 @@ 'name = "orders"\nadapter = "duckdb"\ndefault_target = "dev"\n\n' '[connection]\ndatabase = "dev.duckdb"\n\n' '[defaults]\nschema = "analytics"\n\n[targets.dev]\nschema = "preserve"\n' + 'loader_schema = "landing"\n' ) From 152cdc9102364bef4695be7693dfcffafe11595f Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 17 Nov 2026 15:43:21 +0000 Subject: [PATCH 6/6] fix!: require a loader schema for preserve targets with Python nodes BREAKING CHANGE: a preserve target with Python loaders, tasks, assets or checks now needs loader_schema, which is where their results are kept. Without it, compile fails with S105, which says to set targets..loader_schema.