diff --git a/Makefile b/Makefile index b3dad765e5..f3e8c2144c 100644 --- a/Makefile +++ b/Makefile @@ -151,6 +151,7 @@ E2E_DUCKDB_BUILD_CORE_PATHS := \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_defer_migration_origin.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_sql_hook_reads.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_identity.py \ + tests/e2e/src/sqlbuild/cli/commands/main/build/test_node_result_location.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_sql_hook_identity.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_query_change_tracking.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_query_fingerprint_changes.py \ diff --git a/benchmarks/duckshop/edge_cases/loaders/loader_results.py b/benchmarks/duckshop/edge_cases/loaders/loader_results.py new file mode 100644 index 0000000000..4c000683c4 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/loaders/loader_results.py @@ -0,0 +1,34 @@ +"""Two loaders and a task that reads both of their results. + +A build that selects only one loader still reads the other loader's latest result, which may come +from an earlier build. On Snowflake the adapter has no default schema, so the results must go to +the target's schema for the task to find them. +""" + +from sqlbuild.executor.load.models import LoaderContext +from sqlbuild.loaders import loader +from sqlbuild.refs import source +from sqlbuild.tasks import task + + +@loader +def raw_order_events(ctx: LoaderContext) -> list[dict[str, object]]: + """Order events.""" + return [{"event_id": 1}] + + +@loader +def raw_customer_events(ctx: LoaderContext) -> list[dict[str, object]]: + """Customer events.""" + return [{"event_id": 1}] + + +@task(depends_on=[source("raw_order_events"), source("raw_customer_events")]) +def announce_events(ctx): + """Record the run each loader result came from.""" + return ctx.result( + payload={ + "orders": ctx.result_of(node_function=raw_order_events).run_id, + "customers": ctx.result_of(node_function=raw_customer_events).run_id, + } + ) diff --git a/benchmarks/duckshop/edge_cases/loaders/loader_results.yml b/benchmarks/duckshop/edge_cases/loaders/loader_results.yml new file mode 100644 index 0000000000..1caf61e111 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/loaders/loader_results.yml @@ -0,0 +1,9 @@ +sources: + - name: raw_order_events + description: Order events, read by announce_events. + managed: true + write_strategy: table + - name: raw_customer_events + description: Customer events, read by announce_events. + managed: true + write_strategy: table diff --git a/crates/sqlbuild-analysis/src/assembly/target_checks/constants.rs b/crates/sqlbuild-analysis/src/assembly/target_checks/constants.rs index 947493d5bb..01df1688b4 100644 --- a/crates/sqlbuild-analysis/src/assembly/target_checks/constants.rs +++ b/crates/sqlbuild-analysis/src/assembly/target_checks/constants.rs @@ -10,5 +10,7 @@ pub const LOADER_COLLISION_CODE: &str = "S102"; pub const MANAGED_WRITE_SCHEMA_CODE: &str = "S103"; /// A named target chooses no schema strategy. pub const NAMED_TARGET_SCHEMA_CODE: &str = "S104"; +/// A target that keeps each model's schema has no schema for Python node results. +pub const PRESERVED_NODE_RESULTS_CODE: &str = "S105"; pub(crate) const DATABASE_PART: &str = "database"; pub(crate) const SCHEMA_PART: &str = "schema"; diff --git a/crates/sqlbuild-analysis/src/assembly/target_checks/errors.rs b/crates/sqlbuild-analysis/src/assembly/target_checks/errors.rs index 9c1dcf2dc0..d65fe743e9 100644 --- a/crates/sqlbuild-analysis/src/assembly/target_checks/errors.rs +++ b/crates/sqlbuild-analysis/src/assembly/target_checks/errors.rs @@ -15,4 +15,6 @@ pub enum TargetError { NamedTargetSchema(String), /// An intermediate loader, by name, whose schema resolves to the target's `preserve`. PreservedLoaderSchema(String), + /// A target, by name, whose schema is `preserve` and that names no loader schema. + PreservedNodeResults(String), } diff --git a/crates/sqlbuild-analysis/src/assembly/target_checks/main/target_error_message.rs b/crates/sqlbuild-analysis/src/assembly/target_checks/main/target_error_message.rs index ba4b45c4a6..7cfb39622b 100644 --- a/crates/sqlbuild-analysis/src/assembly/target_checks/main/target_error_message.rs +++ b/crates/sqlbuild-analysis/src/assembly/target_checks/main/target_error_message.rs @@ -48,5 +48,10 @@ pub fn target_error_message(error: &TargetError) -> String { '{PRESERVE_TARGET_VALUE}'. The target's schema '{PRESERVE_TARGET_VALUE}' keeps each \ model's own schema, and a loader has none, so its destination must name a schema." ), + TargetError::PreservedNodeResults(target) => format!( + "Target '{target}' keeps each model's own schema, so it has no schema for the \ + results of loaders, tasks, assets and checks. Set targets.{target}.loader_schema; \ + Python node results are kept there." + ), } } diff --git a/crates/sqlbuild-python-base/src/warehouse_access/main/adapter_default_location.rs b/crates/sqlbuild-python-base/src/warehouse_access/main/adapter_default_location.rs new file mode 100644 index 0000000000..69a56b834e --- /dev/null +++ b/crates/sqlbuild-python-base/src/warehouse_access/main/adapter_default_location.rs @@ -0,0 +1,11 @@ +//! The adapter's own default database or schema. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; + +/// The adapter's `default_database` or `default_schema`, named by `method`. +pub fn adapter_default_location<'py>( + adapter: &Bound<'py, PyAny>, + method: &str, +) -> PyResult> { + adapter.call_method0(method) +} diff --git a/crates/sqlbuild-python-base/src/warehouse_access/main/mod.rs b/crates/sqlbuild-python-base/src/warehouse_access/main/mod.rs index b0a2d9b65e..e9d3d3e481 100644 --- a/crates/sqlbuild-python-base/src/warehouse_access/main/mod.rs +++ b/crates/sqlbuild-python-base/src/warehouse_access/main/mod.rs @@ -1,3 +1,4 @@ +pub mod adapter_default_location; pub mod ensure_event_id_column; pub mod is_warehouse_access_refused; pub mod read_catalog; diff --git a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/mod.rs b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/mod.rs index 34b8da1b5a..4f83d12b05 100644 --- a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/mod.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/mod.rs @@ -1,4 +1,6 @@ pub(crate) mod inputs; +pub(crate) mod model_free_locations; +pub(crate) mod node_results; pub(crate) mod pipeline; pub(crate) mod pipeline_entries; pub(crate) mod project; diff --git a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/model_free_locations.rs b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/model_free_locations.rs new file mode 100644 index 0000000000..e001269c72 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/model_free_locations.rs @@ -0,0 +1,35 @@ +//! The target database and schema that Python nodes use outside any model. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::PyString; +use sqlbuild_analysis::assembly::target_checks::constants::PRESERVE_TARGET_VALUE; +use sqlbuild_model_config::templates::main::holds_context_placeholder::holds_context_placeholder; +use sqlbuild_python_base::warehouse_access::main::adapter_default_location::adapter_default_location; + +/// Python nodes' `(database, schema)`: a `preserve` schema reads `loader_schema`; unusable values the adapter's. +pub(crate) fn model_free_locations<'py>( + (database, schema): (Bound<'py, PyAny>, Bound<'py, PyAny>), + loader_schema: Option<&str>, + adapter: &Bound<'py, PyAny>, +) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { + let kept_schema: Option<&str> = + loader_schema.filter(|name| !name.is_empty() && *name != PRESERVE_TARGET_VALUE); + let location = |value: Bound<'py, PyAny>, method: &str, preserved: Option<&str>| match ( + value.extract::>()?.as_deref(), + preserved, + ) { + (Some(PRESERVE_TARGET_VALUE), Some(name)) => Ok(PyString::new(value.py(), name).into_any()), + (Some(text), _) + if !text.is_empty() + && text != PRESERVE_TARGET_VALUE + && !holds_context_placeholder(text) => + { + Ok(value) + } + _ => adapter_default_location(adapter, method), + }; + Ok(( + location(database, "default_database", None)?, + location(schema, "default_schema", kept_schema)?, + )) +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/node_results.rs b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/node_results.rs new file mode 100644 index 0000000000..d302abe94b --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/node_results.rs @@ -0,0 +1,53 @@ +//! The schema a `preserve` target keeps Python node results in. + +use crate::compile_driver::_helpers::setup::targets::planner_error; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_analysis::assembly::target_checks::constants::{ + PRESERVE_TARGET_VALUE, PRESERVED_NODE_RESULTS_CODE, +}; +use sqlbuild_analysis::assembly::target_checks::errors::TargetError; +use sqlbuild_analysis::assembly::target_checks::main::target_error_message::target_error_message; +use sqlbuild_core::text::main::python_strip::python_strip; +use sqlbuild_python_base::boundary::main::optional_text::optional_text; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The discovered inputs' Python node functions, each kind of which keeps node results. +const PYTHON_NODE_FIELDS: [&str; 4] = [ + "loader_functions", + "task_functions", + "asset_functions", + "check_functions", +]; + +/// Raise when a `preserve` target has Python nodes but no loader schema to keep their results in. +pub(crate) fn require_node_results_schema( + discovered: &Bound<'_, PyAny>, + (target_name, target_config): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), +) -> PyResult<()> { + let loader_schema: Option = + optional_text(&native_field(target_config, "loader_schema")?)? + .map(|schema| python_strip(&schema).to_owned()) + .filter(|schema| !schema.is_empty() && schema != PRESERVE_TARGET_VALUE); + if loader_schema.is_some() || python_node_count(discovered)? == 0 { + return Ok(()); + } + let target: String = target_name.extract()?; + Err(planner_error( + discovered.py(), + target_error_message(&TargetError::PreservedNodeResults(target.clone())), + PRESERVED_NODE_RESULTS_CODE, + Some(format!( + "to keep Python node results in a schema, set this in sqlbuild_project.toml:\n \ + [targets.{target}]\n \ + loader_schema = \"my_schema\"" + )), + )) +} + +/// How many loaders, tasks, assets and checks the project declares in Python. +fn python_node_count(discovered: &Bound<'_, PyAny>) -> PyResult { + PYTHON_NODE_FIELDS + .iter() + .map(|field| native_field(discovered, field)?.len()) + .sum() +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/setup/mod.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/mod.rs index 2e7fb32797..7217f6eb02 100644 --- a/crates/sqlbuild-python-compile/src/compile_driver/main/setup/mod.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/mod.rs @@ -11,7 +11,6 @@ pub mod compile_project_phase; pub mod compiled_phase; pub mod expanded_loader_namespace; pub mod managed_write_schema; -pub mod model_free_locations; pub mod plan_has_executable_work; pub mod preserved_loader_schemas; pub mod progress; diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/setup/model_free_locations.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/model_free_locations.rs deleted file mode 100644 index 38e17b630f..0000000000 --- a/crates/sqlbuild-python-compile/src/compile_driver/main/setup/model_free_locations.rs +++ /dev/null @@ -1,23 +0,0 @@ -//! The target database and schema that Python nodes use outside any model. - -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; -use sqlbuild_model_config::templates::main::holds_context_placeholder::holds_context_placeholder; -use sqlbuild_python_base::records::main::fields::field::field as native_field; - -/// The compiled project's target database and schema, each from `default` with the adapter method's name when it is unset, empty, or resolves only inside a model, such as a `${CTX:...}` schema. -pub fn model_free_locations<'py>( - project: &Bound<'py, PyAny>, - default: impl Fn(&str) -> PyResult>, -) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { - let location = |field: &str, fallback: &str| { - let value = native_field(project, field)?; - match value.extract::>()? { - Some(text) if !text.is_empty() && !holds_context_placeholder(&text) => Ok(value), - _ => default(fallback), - } - }; - Ok(( - location("effective_target_database", "default_database")?, - location("effective_target_schema", "default_schema")?, - )) -} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/setup/preserved_loader_schemas.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/preserved_loader_schemas.rs index 54162399ed..9a0c3d649b 100644 --- a/crates/sqlbuild-python-compile/src/compile_driver/main/setup/preserved_loader_schemas.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/preserved_loader_schemas.rs @@ -1,6 +1,7 @@ -//! Raise when the selected target's schema is `preserve` and one of `loaders` names no schema. +//! Raise when the selected target's schema is `preserve` and a loader or Python node has no schema. use crate::compile_attachment::main::resolved_target::resolved_target; +use crate::compile_driver::_helpers::setup::node_results::require_node_results_schema; use crate::compile_driver::_helpers::setup::targets::planner_error; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use sqlbuild_analysis::assembly::target_checks::constants::{ @@ -13,7 +14,7 @@ use sqlbuild_python_base::boundary::main::optional_text::optional_text; use sqlbuild_python_base::records::main::fields::field::field as native_field; use std::collections::HashSet; -/// Raise when the selected target's schema is `preserve` and one of `loaders` names no schema. +/// Raise when a `preserve` target leaves one of `loaders`, or the Python node results, without a schema. pub fn preserved_loader_schemas( discovered: &Bound<'_, PyAny>, selected_target: &Bound<'_, PyAny>, @@ -47,5 +48,5 @@ pub fn preserved_loader_schemas( )), )); } - Ok(()) + require_node_results_schema(discovered, (&target_name, &target_config)) } diff --git a/crates/sqlbuild-python-compile/src/compile_driver/models.rs b/crates/sqlbuild-python-compile/src/compile_driver/models.rs index aece8e4552..5838cd685d 100644 --- a/crates/sqlbuild-python-compile/src/compile_driver/models.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/models.rs @@ -1,5 +1,6 @@ //! Public records of compile driver. +use crate::compile_driver::_helpers::setup::model_free_locations::model_free_locations; use pyo3::IntoPyObjectExt; use pyo3::prelude::Bound; use pyo3::prelude::Py; @@ -67,8 +68,8 @@ impl InputsRequest { } } -/// The active target's locations as SQLBuild resolves them; `preserve` stays as written until `record`. -#[derive(Debug, Clone)] +/// The active target's locations as SQLBuild resolves them (default: unset); `preserve` stays as written until `record`. +#[derive(Debug, Clone, Default)] pub struct TargetSettingsValues { pub(crate) name: Option, pub(crate) database: Option, @@ -77,6 +78,40 @@ pub struct TargetSettingsValues { } impl TargetSettingsValues { + /// The resolved schema managed loader data goes to. + #[must_use] + pub fn loader_schema(&self) -> Option<&str> { + self.loader_schema.as_deref() + } + + /// Python nodes' `(database, schema)` under these settings, as `model_free_locations` picks. + pub fn node_locations<'py>( + &self, + adapter: &Bound<'py, PyAny>, + ) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { + let py: Python<'py> = adapter.py(); + self.node_locations_from( + ( + &self.database.as_deref().into_bound_py_any(py)?, + &self.schema.as_deref().into_bound_py_any(py)?, + ), + adapter, + ) + } + + /// `node_locations` from a caller-resolved `(database, schema)`, such as `sqb load`'s, which applies CLI vars. + pub fn node_locations_from<'py>( + &self, + (database, schema): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + adapter: &Bound<'py, PyAny>, + ) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { + model_free_locations( + (database.clone(), schema.clone()), + self.loader_schema(), + adapter, + ) + } + /// The `TargetSettings` record: a `preserve` location reads `None` with its flag set. pub fn record<'py>(&self, py: Python<'py>) -> PyResult> { let located = |value: &Option| -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs index c4bebd360d..f4caf1a4a6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs @@ -44,6 +44,7 @@ use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_compile::compile_driver::main::setup::build_project_relation_targets::build_project_relation_targets; use sqlbuild_python_compile::compile_driver::main::setup::project_target_settings::project_target_settings; +use sqlbuild_python_compile::compile_driver::models::TargetSettingsValues; use sqlbuild_python_front::cost_context::models::CostContext; use sqlbuild_python_front::cursor::main::overrides::cursor_integer::cursor_integer; use sqlbuild_python_front::cursor::main::overrides::cursor_timestamp::cursor_timestamp; @@ -333,6 +334,12 @@ pub(crate) fn execute_build_plan<'py>( let lifecycle: &DirectPythonLifecycle = preparation.lifecycle.get(); let [start_ts, end_ts, start_int, end_int] = &preparation.cursors; let request = |name: &str| native_field(&context.request, name); + let target_config: TargetSettingsValues = project_target_settings( + &context.get("discovered_inputs")?, + &project, + (&connection_config, &adapter), + )?; + let (default_database, default_schema) = target_config.node_locations(&adapter)?; let runtime: Bound<'py, PyAny> = call_with( &host_entry(py, ExecutorHost::BuildRuntimeParams)?, &[ @@ -360,15 +367,9 @@ pub(crate) fn execute_build_plan<'py>( preparation.concurrency.into_pyobject(py)?.into_any(), ), ("loader_is_reload", request("reload_sources")?), - ( - "target_config", - project_target_settings( - &context.get("discovered_inputs")?, - &project, - (&connection_config, &adapter), - )? - .record(py)?, - ), + ("target_config", target_config.record(py)?), + ("default_database", default_database), + ("default_schema", default_schema), ("start_cursor_ts", start_ts.clone()), ("end_cursor_ts", end_ts.clone()), ("start_cursor_int", start_int.clone()), @@ -633,6 +634,12 @@ fn check_runtime<'py>( .iter() .map(|check| native_field(check, "name")) .collect::>()?; + let target_config: TargetSettingsValues = project_target_settings( + &context.get("discovered_inputs")?, + &project, + (&context.get("connection_config")?, &adapter), + )?; + let (default_database, default_schema) = target_config.node_locations(&adapter)?; call_with( &host_entry(py, ExecutorHost::PythonNodeRuntime)?, &[ @@ -646,20 +653,9 @@ fn check_runtime<'py>( "is_reload", native_field(&context.request, "reload_sources")?, ), - ( - "target_config", - project_target_settings( - &context.get("discovered_inputs")?, - &project, - (&context.get("connection_config")?, &adapter), - )? - .record(py)?, - ), - ( - "default_database", - adapter.call_method0("default_database")?, - ), - ("default_schema", adapter.call_method0("default_schema")?), + ("target_config", target_config.record(py)?), + ("default_database", default_database), + ("default_schema", default_schema), ( "relation_targets", build_check_relation_targets(&adapter, pipeline, graph, frozenset(py, &names)?)?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs index c0dbe28177..f5bdab0275 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs @@ -28,7 +28,6 @@ use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_key use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_compile::compile_driver::main::setup::build_project_relation_targets::build_project_relation_targets; use sqlbuild_python_compile::compile_driver::main::setup::build_python_relation_targets::build_python_relation_targets; -use sqlbuild_python_compile::compile_driver::main::setup::model_free_locations::model_free_locations; use sqlbuild_python_compile::compile_driver::main::setup::project_target_settings::project_target_settings; use sqlbuild_python_compile::compile_driver::models::TargetSettingsValues; use sqlbuild_python_project::graph::main::python_nodes::build_python_sql_run_lifecycle::build_python_sql_run_lifecycle; @@ -432,17 +431,15 @@ pub(crate) fn prepare_direct_python_lifecycle<'py>( )?), ) .and_then(|value| pyo3::IntoPyObjectExt::into_bound_py_any(value, py))?; + let target_config: TargetSettingsValues = + project_target_settings(&discovered, &project, (&connection_config, &adapter))?; let parts: RuntimeParts<'_, 'py> = RuntimeParts { adapter: &adapter, connection_config: &connection_config, project: &project, reload: context.request_flag("reload_sources")?, - defaults: model_free_locations(&project, |method| adapter.call_method0(method))?, - target_config: project_target_settings( - &discovered, - &project, - (&connection_config, &adapter), - )?, + defaults: target_config.node_locations(&adapter)?, + target_config, relations: (relation_targets, project_relations), inputs, }; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs index 7b544a29f9..74337e8a27 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs @@ -25,7 +25,6 @@ use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_compile::compile_driver::main::setup::build_project_relation_targets::build_project_relation_targets; use sqlbuild_python_compile::compile_driver::main::setup::build_python_relation_targets::build_python_relation_targets; use sqlbuild_python_compile::compile_driver::main::setup::build_static_pipeline_result::build_static_pipeline_result; -use sqlbuild_python_compile::compile_driver::main::setup::model_free_locations::model_free_locations; use sqlbuild_python_compile::compile_driver::main::setup::project_target_settings::project_target_settings; use sqlbuild_python_compile::compile_driver::models::TargetSettingsValues; use sqlbuild_python_front::discovery::main::sources::consumer_uses::consumer_uses; @@ -343,12 +342,12 @@ fn prepare<'py>( (&native_field(invocation, "connection_config")?, &adapter), )?; Ok(Prepared { - target_config, check_functions: PyTuple::new(py, check_functions)?.into_any(), lifecycle_plan, relation_targets, relation_refs, - defaults: model_free_locations(&project, |method| adapter.call_method0(method))?, + defaults: target_config.node_locations(&adapter)?, + target_config, project_relations: object( py, build_project_relation_targets( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/check_core.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/check_core.rs index 98b2b61bb4..04442d4156 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/check/check_core.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/check_core.rs @@ -446,6 +446,11 @@ fn read_side_references<'py>( fn node_runtime<'py>(inputs: &ReadSideInputs<'py>) -> PyResult> { let py: Python<'py> = inputs.adapter.py(); let project: Bound<'py, PyAny> = native_field(&inputs.pipeline_result, "project")?; + let (default_database, default_schema) = inputs + .target_config + .clone() + .unwrap_or_default() + .node_locations(&inputs.adapter)?; call_with( &host_entry(py, ExecutorHost::PythonNodeRuntime)?, &[ @@ -463,14 +468,8 @@ fn node_runtime<'py>(inputs: &ReadSideInputs<'py>) -> PyResult None => py.None().into_bound(py), }, ), - ( - "default_database", - inputs.adapter.call_method0("default_database")?, - ), - ( - "default_schema", - inputs.adapter.call_method0("default_schema")?, - ), + ("default_database", default_database), + ("default_schema", default_schema), ("relation_targets", inputs.relation_targets.clone()), ("project_relations", inputs.project_relations.clone()), ("providers", inputs.providers.clone()), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs index 4ea2a1ffd1..5449d2ddce 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs @@ -67,6 +67,7 @@ struct Invocation<'py> { selected: Vec>, references: Vec>, relation_sources: Vec>, + defaults: LoaderDefaults<'py>, style: CliStyle, stream: Bound<'py, PyAny>, } @@ -181,6 +182,7 @@ fn invocation<'py>(py: Python<'py>, request: &Bound<'py, PyAny>) -> PyResult( let py: Python<'py> = request.py(); let (target_name, effective_vars, run_id) = &preparation.runtime; let overrides: &Bound<'py, PyAny> = &preparation.cursor_overrides; + let defaults = (&invocation.defaults.database, &invocation.defaults.schema); + let (default_database, default_schema) = preparation + .target_config + .node_locations_from(defaults, &preparation.adapter)?; call_with( &host_entry(py, ExecutorHost::LoadRuntimeParams)?, &[ @@ -408,6 +414,8 @@ fn runtime_params<'py>( ("vars", effective_vars.clone()), ("is_reload", native_field(request, "reload")?), ("target_config", preparation.target_config.record(py)?), + ("default_database", default_database), + ("default_schema", default_schema), ( "start_cursor_ts", cursor_timestamp(&native_field(overrides, "start_ts")?)?, diff --git a/fensu.toml b/fensu.toml index 0719fd7fa5..3b0780ec42 100644 --- a/fensu.toml +++ b/fensu.toml @@ -2650,13 +2650,13 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs=16", - "crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs=11", + "crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs=20", + "crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs=19", "crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/build_command/policies.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs=19", - "crates/sqlbuild-python/src/bindings/_helpers/check/check_core.rs=8", + "crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs=18", + "crates/sqlbuild-python/src/bindings/_helpers/check/check_core.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/check/check_results.rs=13", "crates/sqlbuild-python/src/bindings/_helpers/cli/argv.rs=18", "crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs=17", diff --git a/scripts/compiler_differential/_helpers/analysis_corpus/failure_cases.py b/scripts/compiler_differential/_helpers/analysis_corpus/failure_cases.py index 50d5556bc2..e42b0a6ff3 100644 --- a/scripts/compiler_differential/_helpers/analysis_corpus/failure_cases.py +++ b/scripts/compiler_differential/_helpers/analysis_corpus/failure_cases.py @@ -457,6 +457,24 @@ def _target_cases() -> tuple[FailureCase, ...]: + 'database = "ANALYTICS"\n\n[targets.dev]\ndatabase = "ANALYTICS"\n' }, ), + failure_case( + name="target-preserved-schema-without-node-results-schema", + expected_code="S105", + expected_message="Target 'dev' keeps each model's own schema", + files={ + FAILURE_CONFIG_PATH: FAILURE_BASE_CONFIG.replace( + 'adapter = "duckdb"\n', 'adapter = "duckdb"\ndefault_target = "dev"\n' + ) + + '\n[defaults]\nschema = "mart"\n\n[targets.dev]\nschema = "preserve"\n', + "python/tasks/announce.py": ( + "from sqlbuild.tasks import task\n\n\n" + "@task\n" + "def announce_orders(ctx):\n" + ' """Announce the loaded orders."""\n' + " return ctx.result(payload={'announced': True})\n" + ), + }, + ), ) diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md index 15338d5372..98b07f8260 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md @@ -177,7 +177,7 @@ A target is a named build context - the database and schema you build into, plus | Field | Description | |-------|-------------| | `schema` | Schema for all models in this target; required for named targets | -| `loader_schema` | Default write schema for managed source loaders; falls back to `schema` | +| `loader_schema` | Default write schema for managed source loaders; falls back to `schema`. Required when `schema` is `preserve` and the project has loaders, tasks, assets or checks, because their [results](python-nodes/overview.md#result-persistence) go there | | `database` | Database for all models in this target | | `connection` | Name from `[connections.]` used for endpoint, authentication, and compute; optional when exactly one named connection is defined | | `vars` | Target-specific project variables | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md index bd1768654e..bbac4abad1 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md @@ -172,7 +172,19 @@ Within one run, downstream nodes run only if at least one upstream succeeded. If ## Result persistence -Node results (payload, metadata, status, errors) are persisted after each execution. Results are stored in `_sqlbuild_node_results` in the warehouse alongside your data. Results persist across runs, so they are available for observability, debugging, and downstream consumption. +Node results (payload, metadata, status, errors) are persisted after each execution in a +`_sqlbuild_node_results` table. Results persist across runs, so they are available for +observability, debugging, and downstream consumption through `ctx.result_of`. + +Results of loaders, tasks, assets and checks all go to one schema, chosen by the target's `schema`: + +| Target `schema` | Results table schema | +|-----------------|----------------------| +| A schema name, such as `analytics` | That schema, in the target's database | +| `preserve` | The target's `loader_schema`; commands that compile the project stop with `S105` until it is set | +| Unset, or resolved per model, such as `${CTX:...}` | The adapter's default schema, such as `main` on DuckDB or `public` on PostgreSQL | + +A target `database` of `preserve` keeps results in the connection's database. ## Selection diff --git a/src/sqlbuild/executor/build/_helpers/source_node.py b/src/sqlbuild/executor/build/_helpers/source_node.py index 146df17c03..46ce2b0ae4 100644 --- a/src/sqlbuild/executor/build/_helpers/source_node.py +++ b/src/sqlbuild/executor/build/_helpers/source_node.py @@ -19,10 +19,10 @@ from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.executor.build.models import BuildCallbacks, BuildRuntimeParams from sqlbuild.executor.load.main._execute import execute_source_load +from sqlbuild.executor.load.main._persist_result import persist_loader_result from sqlbuild.executor.load.models import LoadExecutionResult, LoadRuntimeParams from sqlbuild.executor.node_results.classes.direct_store import DirectNodeResultStore from sqlbuild.executor.node_results.main._direct_store import build_direct_node_result_store -from sqlbuild.executor.node_results.models import NodeResultRecord from sqlbuild.runtime.contracts.types import ExecutionResourceKind @@ -60,8 +60,9 @@ def execute_build_source_node( result_store: DirectNodeResultStore = build_direct_node_result_store( adapter=adapter, connection=connection, - database=adapter.default_database(), - schema=adapter.default_schema(), + database=runtime.default_database, + schema=runtime.default_schema, + target_config=runtime.target_config, ) result: LoadExecutionResult = execute_source_load( source_entry=source_entry, @@ -73,6 +74,8 @@ def execute_build_source_node( run_id=runtime.run_id, target=runtime.target, target_config=runtime.target_config, + default_database=runtime.default_database, + default_schema=runtime.default_schema, vars=runtime.effective_vars or {}, is_reload=runtime.loader_is_reload, runtime_dir=runtime.runtime_dir, @@ -98,52 +101,10 @@ def execute_build_source_node( resource_name=loader_name, phase="finalize", ): - _persist_loader_result( - adapter=adapter, - connection=connection, + persist_loader_result( loader_name=loader_name, result=timed_result, run_id=runtime.run_id, result_store=result_store, ) return timed_result - - -def _persist_loader_result( - *, - adapter: BaseAdapter, - connection: Any, - loader_name: str, - result: LoadExecutionResult, - run_id: str, - result_store: DirectNodeResultStore | None = None, -) -> None: - if connection is None and (result_store is None or result_store.connection is None): - return - resolved_result_store: DirectNodeResultStore = result_store or build_direct_node_result_store( - adapter=adapter, - connection=connection, - database=adapter.default_database(), - schema=adapter.default_schema(), - ) - resolved_result_store.write( - NodeResultRecord( - node_type="loader", - node_name=loader_name, - target_database=resolved_result_store.database, - target_schema=resolved_result_store.schema, - target_name=None, - run_id=run_id, - status=result.status.value, - payload=result.result_payload, - metadata={ - "source_name": result.source_name, - "loader_name": result.loader_name, - "rows_loaded": result.rows_loaded, - "target": result.target, - **result.result_metadata, - }, - error_message=result.error_message or result.skip_reason, - materialized=result.result_materialized, - ) - ) diff --git a/src/sqlbuild/executor/build/models.py b/src/sqlbuild/executor/build/models.py index b7d4ecc4f3..8afc1983ef 100644 --- a/src/sqlbuild/executor/build/models.py +++ b/src/sqlbuild/executor/build/models.py @@ -80,6 +80,8 @@ class BuildRuntimeParams: end_cursor_int: int | None = None target: str = "" target_config: TargetSettings | None = None + default_database: str | None = None + default_schema: str | None = None effective_vars: dict[str, object] | None = None use_color: bool = False providers: ProviderContainer | None = None diff --git a/src/sqlbuild/executor/load/_helpers/dag_runtime.py b/src/sqlbuild/executor/load/_helpers/dag_runtime.py index 0695bf42ec..87e0509bbe 100644 --- a/src/sqlbuild/executor/load/_helpers/dag_runtime.py +++ b/src/sqlbuild/executor/load/_helpers/dag_runtime.py @@ -189,8 +189,9 @@ def _execute_ready_dag_source( resolved_result_store = build_direct_node_result_store( adapter=adapter, connection=connection, - database=adapter.default_database(), - schema=adapter.default_schema(), + database=runtime.default_database, + schema=runtime.default_schema, + target_config=runtime.target_config, ) return execute_source_load( source_entry=source, diff --git a/src/sqlbuild/executor/load/main/_persist_result.py b/src/sqlbuild/executor/load/main/_persist_result.py new file mode 100644 index 0000000000..9e12c08a0d --- /dev/null +++ b/src/sqlbuild/executor/load/main/_persist_result.py @@ -0,0 +1,42 @@ +"""Public operation for recording a loader's result in the node-result store.""" + +from __future__ import annotations + +from sqlbuild.compiler.python_nodes.types import PythonNodeKind +from sqlbuild.executor.load.models import LoadExecutionResult +from sqlbuild.executor.node_results.classes.direct_store import DirectNodeResultStore +from sqlbuild.executor.node_results.models import NodeResultRecord + + +def persist_loader_result( + *, + result_store: DirectNodeResultStore | None, + loader_name: str, + result: LoadExecutionResult, + run_id: str, +) -> None: + """Write one loader result to the store's location; do nothing without a live store.""" + + if result_store is None or result_store.connection is None: + return + result_store.write( + NodeResultRecord( + node_type=PythonNodeKind.LOADER.value, + node_name=loader_name, + target_database=result_store.database, + target_schema=result_store.schema, + target_name=None, + run_id=run_id, + status=result.status.value, + payload=result.result_payload, + metadata={ + "source_name": result.source_name, + "loader_name": result.loader_name, + "rows_loaded": result.rows_loaded, + "target": result.target, + **result.result_metadata, + }, + error_message=result.error_message or result.skip_reason, + materialized=result.result_materialized, + ) + ) diff --git a/src/sqlbuild/executor/load/models.py b/src/sqlbuild/executor/load/models.py index 1632edc59b..7e236f68d5 100644 --- a/src/sqlbuild/executor/load/models.py +++ b/src/sqlbuild/executor/load/models.py @@ -283,6 +283,8 @@ class LoadRuntimeParams: vars: dict[str, object] is_reload: bool target_config: TargetSettings | None = None + default_database: str | None = None + default_schema: str | None = None runtime_dir: Path = Path("target") start_cursor_ts: datetime | None = None end_cursor_ts: datetime | None = None diff --git a/src/sqlbuild/executor/node_results/classes/direct_store.py b/src/sqlbuild/executor/node_results/classes/direct_store.py index 020a015a93..54f5d0723f 100644 --- a/src/sqlbuild/executor/node_results/classes/direct_store.py +++ b/src/sqlbuild/executor/node_results/classes/direct_store.py @@ -2,6 +2,7 @@ from __future__ import annotations +from dataclasses import replace from typing import Any from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter @@ -26,9 +27,11 @@ def __init__( connection: Any, database: str | None, schema: str | None, + previous_locations: tuple[tuple[str | None, str | None], ...] = (), ) -> None: self.adapter: BaseAdapter = adapter self.connection: Any = connection + self.previous_locations: tuple[tuple[str | None, str | None], ...] = previous_locations super().__init__(database=database, schema=schema) def write(self, record: NodeResultRecord) -> None: @@ -54,14 +57,34 @@ def write(self, record: NodeResultRecord) -> None: self._cache_record(record=record) def _read_persisted(self, *, query: NodeResultQuery) -> tuple[NodeResultEnvelope, ...]: - if self.schema is None: + """Read this store's location, then each earlier location until one has the node.""" + + results: tuple[NodeResultEnvelope, ...] = self._read_at( + database=self.database, schema=self.schema, query=query + ) + location: tuple[str | None, str | None] + for location in self.previous_locations: + if results: + break + database, schema = location + results = self._read_at( + database=database, + schema=schema, + query=replace(query, target_database=database, target_schema=schema), + ) + return results + + def _read_at( + self, *, database: str | None, schema: str | None, query: NodeResultQuery + ) -> tuple[NodeResultEnvelope, ...]: + if schema is None: return () return read_node_results( connection=self.connection, execute=self.adapter.execute, relation_exists=self.adapter.relation_exists, - database=self.database, - schema=self.schema, + database=database, + schema=schema, query=query, render_qualified_name=self.adapter.render_qualified_name, ) diff --git a/src/sqlbuild/executor/node_results/constants.py b/src/sqlbuild/executor/node_results/constants.py index 3b2d792021..a4b072a926 100644 --- a/src/sqlbuild/executor/node_results/constants.py +++ b/src/sqlbuild/executor/node_results/constants.py @@ -43,3 +43,4 @@ ("__sqlbuild_asset__", PythonNodeKind.ASSET.value), ("__sqlbuild_loader__", PythonNodeKind.LOADER.value), ) +PRESERVED_SCHEMA_NAME: str = "preserve" diff --git a/src/sqlbuild/executor/node_results/main/_direct_store.py b/src/sqlbuild/executor/node_results/main/_direct_store.py index 9c1b9edb8b..5b989c81d9 100644 --- a/src/sqlbuild/executor/node_results/main/_direct_store.py +++ b/src/sqlbuild/executor/node_results/main/_direct_store.py @@ -4,8 +4,10 @@ from typing import Any +from sqlbuild._native import TargetSettings from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.executor.node_results.classes.direct_store import DirectNodeResultStore +from sqlbuild.executor.node_results.constants import PRESERVED_SCHEMA_NAME def build_direct_node_result_store( @@ -14,12 +16,21 @@ def build_direct_node_result_store( connection: Any, database: str | None, schema: str | None, + target_config: TargetSettings | None = None, ) -> DirectNodeResultStore: - """Build a direct-mode runtime node result store.""" + """Build a direct-mode node result store that, for one release, also reads older locations.""" + earlier: list[tuple[str | None, str | None]] = [ + (adapter.default_database(), adapter.default_schema()) + ] + if target_config is not None and target_config.schema_preserved: + earlier.append((database, PRESERVED_SCHEMA_NAME)) return DirectNodeResultStore( adapter=adapter, connection=connection, database=database, schema=schema, + previous_locations=tuple( + location for location in earlier if location != (database, schema) + ), ) diff --git a/src/sqlbuild/executor/python_nodes/_helpers/execution.py b/src/sqlbuild/executor/python_nodes/_helpers/execution.py index 194ea2ca61..74426fec85 100644 --- a/src/sqlbuild/executor/python_nodes/_helpers/execution.py +++ b/src/sqlbuild/executor/python_nodes/_helpers/execution.py @@ -82,6 +82,7 @@ def execute_ready_python_node( connection=runtime.connection, database=runtime.default_database, schema=runtime.default_schema, + target_config=runtime.target_config, ) if runtime.persist_node_results else None diff --git a/src/sqlbuild/executor/python_nodes/_helpers/ingress_execution.py b/src/sqlbuild/executor/python_nodes/_helpers/ingress_execution.py index d364544c48..383e915f9e 100644 --- a/src/sqlbuild/executor/python_nodes/_helpers/ingress_execution.py +++ b/src/sqlbuild/executor/python_nodes/_helpers/ingress_execution.py @@ -22,11 +22,11 @@ from sqlbuild.compiler.python_nodes.types import PythonNodeKind, PythonNodeStatus, SkipMode from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.load.main._execute import execute_source_load +from sqlbuild.executor.load.main._persist_result import persist_loader_result from sqlbuild.executor.load.main._resource_kind import load_resource_kind from sqlbuild.executor.load.main._skipped_result import skipped_load_result from sqlbuild.executor.load.models import LoadExecutionResult, LoadRuntimeParams from sqlbuild.executor.node_results.main._direct_store import build_direct_node_result_store -from sqlbuild.executor.node_results.models import NodeResultRecord from sqlbuild.executor.python_nodes._helpers.fingerprinting import ( try_write_python_node_identity_fingerprint, ) @@ -99,6 +99,7 @@ def execute_ingress_python_loader_nodes( connection=runtime.connection, database=runtime.default_database, schema=runtime.default_schema, + target_config=runtime.target_config, ) if runtime.persist_node_results else None @@ -284,6 +285,8 @@ def _execute_ingress_loader( run_id=runtime.run_id, target=runtime.target, target_config=runtime.target_config, + default_database=runtime.default_database, + default_schema=runtime.default_schema, vars=runtime.vars, is_reload=runtime.is_reload, start_cursor_ts=runtime.start_cursor_ts, @@ -303,7 +306,7 @@ def _execute_ingress_loader( source_ref_entries=source_map, ) results.record_load_result(name=node.name, result=result) - _persist_loader_result( + persist_loader_result( result_store=runtime.result_store, loader_name=loader.name, result=result, @@ -320,8 +323,8 @@ def _execute_ingress_loader( adapter=runtime.adapter, connection=runtime.connection, run_id=runtime.run_id, - database=runtime.adapter.default_database(), - schema=runtime.adapter.default_schema(), + database=runtime.default_database, + schema=runtime.default_schema, target_name=source_entry.name, ) if attempt is not None and result.status == ExecutionStatus.FAILED: @@ -336,38 +339,6 @@ def _execute_ingress_loader( return _load_result_to_lifecycle_result(node_name=node.name, result=result) -def _persist_loader_result( - *, - result_store: Any | None, - loader_name: str, - result: LoadExecutionResult, - run_id: str, -) -> None: - if result_store is None: - return - result_store.write( - NodeResultRecord( - node_type=PythonNodeKind.LOADER.value, - node_name=loader_name, - target_database=result_store.database, - target_schema=result_store.schema, - target_name=None, - run_id=run_id, - status=result.status.value, - payload=result.result_payload, - metadata={ - "source_name": result.source_name, - "loader_name": result.loader_name, - "rows_loaded": result.rows_loaded, - "target": result.target, - **result.result_metadata, - }, - error_message=result.error_message or result.skip_reason, - materialized=result.result_materialized, - ) - ) - - def _record_scheduler_skips( *, scheduler_result: LifecycleSchedulerResult, @@ -403,7 +374,7 @@ def _record_scheduler_skips( resource_name=result.name, phase="finalize", ): - _persist_loader_result( + persist_loader_result( result_store=result_store, loader_name=result.name, result=skipped_result, diff --git a/src/sqlbuild/executor/python_nodes/_helpers/python_checks.py b/src/sqlbuild/executor/python_nodes/_helpers/python_checks.py index 508a4b4d03..70715affd9 100644 --- a/src/sqlbuild/executor/python_nodes/_helpers/python_checks.py +++ b/src/sqlbuild/executor/python_nodes/_helpers/python_checks.py @@ -91,6 +91,7 @@ def execute_python_check_nodes( connection=connection, database=default_database, schema=default_schema, + target_config=runtime.target_config, ) if runtime.persist_node_results else None diff --git a/src/sqlbuild/executor/python_nodes/classes/read_side_python_execution_tracker.py b/src/sqlbuild/executor/python_nodes/classes/read_side_python_execution_tracker.py index b85237fa24..b063f8a60c 100644 --- a/src/sqlbuild/executor/python_nodes/classes/read_side_python_execution_tracker.py +++ b/src/sqlbuild/executor/python_nodes/classes/read_side_python_execution_tracker.py @@ -55,6 +55,7 @@ def __init__( connection=runtime.connection, database=runtime.default_database, schema=runtime.default_schema, + target_config=runtime.target_config, ) if runtime.persist_node_results else None diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py index a444f30373..7c6b8f8fc6 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py @@ -1664,3 +1664,36 @@ class CtxTargetSchemaE2ETestCase: target_schema: str expected_model_schema: str expected_loader_schema: str | None + + +@dataclass(frozen=True) +class PreservedTargetNodeResultsE2ETestCase: + """A target's location settings, and where two builds' and a load's node results land.""" + + description: str + target_lines: str + source_location: str + expected_results_schema: str + expected_runs_per_node: int + cli_args: tuple[str, ...] = () + + +@dataclass(frozen=True) +class PreservedTargetRefusalE2ETestCase: + """A preserved target without a loader schema and the compile error it must print.""" + + description: str + python_files: dict[str, str] + expected_error: str + + +@dataclass(frozen=True) +class AdapterDefaultResultsFallbackE2ETestCase: + """Results left in the adapter default by an earlier build, and the loader rebuilt after it.""" + + description: str + target_schema: str + rebuilt_loader: str + kept_loader: str + reading_task: str + expected_payload_keys: tuple[str, str] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py index 62caae7019..c6a0ec51e3 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py @@ -1489,3 +1489,46 @@ def changed_build_steps(warehouse: ChangedWarehouse) -> tuple[ChangedBuildStep, ChangedBuildStep("after the plain build", (), {}, (CHANGED_ANY,)), ChangedBuildStep("full refresh", (), {}, (CHANGED_ANY, "--full-refresh")), ) + + +def node_result_source_yml(location: str) -> str: + """A managed `raw_orders` source at `location`, for node result location tests.""" + + return ( + "sources:\n - name: raw_orders\n description: Raw orders.\n" + f" managed: true\n{location} write_strategy: table\n" + " columns:\n - name: order_id\n type: INTEGER\n" + ) + + +def node_result_project(*, tmp_path: Path, target_lines: str, files: dict[str, str]) -> Path: + """A DuckDB orders project whose `dev` target has `target_lines`, plus `files`.""" + + return prepare_inline_project( + tmp_path=tmp_path, + project_name="orders", + repo_files={ + "sqlbuild_project.toml": ( + 'name = "orders"\nadapter = "duckdb"\ndefault_target = "dev"\n\n' + '[connection]\ndatabase = "dev.duckdb"\n\n' + f'[defaults]\ndatabase = "dev"\nschema = "mart"\n\n[targets.dev]\n{target_lines}' + ), + "models/orders.sql": ( + "MODEL (description 'Orders.', materialized table);\n\n" + 'SELECT order_id FROM __source("raw_orders")\n' + ), + } + | files, + ) + + +def node_results_in(*, db_path: Path, schema: str, node_type: str) -> list[tuple[str, str, str]]: + """`(node_name, run_id, payload_json_b64)` of one node type's results in `schema`.""" + + return query_duckdb( + db_path=db_path, + sql=( + f"SELECT node_name, run_id, payload_json_b64 FROM {schema}._sqlbuild_node_results " + f"WHERE node_type = '{node_type}' ORDER BY node_name" + ), + ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_node_result_location.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_node_result_location.py new file mode 100644 index 0000000000..02de124e27 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_node_result_location.py @@ -0,0 +1,265 @@ +"""CLI e2e coverage: where Python node results go, so later builds read them back.""" + +from __future__ import annotations + +import base64 +import json +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.build._test_types import ( + AdapterDefaultResultsFallbackE2ETestCase, + PreservedTargetNodeResultsE2ETestCase, + PreservedTargetRefusalE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.build.helpers import ( + node_result_project, + node_result_source_yml, + node_results_in, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( + prepare_inline_project, + query_duckdb, + run_sqb, +) + +_EDGE_CASES: Path = Path("benchmarks/duckshop/edge_cases/loaders") +_LOADER: str = ( + "from sqlbuild.loaders import loader\n\n\n@loader\n" + "def raw_orders(ctx: object) -> list[dict[str, object]]:\n" + ' """Raw orders."""\n _ = ctx\n return [{"order_id": 1}]\n' +) +_TASK: str = ( + "from sqlbuild.refs import source\nfrom sqlbuild.tasks import task\n\n\n" + '@task(depends_on=source("raw_orders"))\n' + "def announce_orders(ctx):\n" + ' """Announce the loaded orders."""\n' + " return ctx.result(payload={'announced': True})\n" +) +_CHECK: str = ( + "from python.tasks.announce import announce_orders\nfrom sqlbuild.checks import check\n\n\n" + "@check(depends_on=announce_orders)\n" + "def check_orders_announced(ctx):\n" + ' """The task announced the orders."""\n' + " announced = ctx.result_of(node_function=announce_orders).payload['announced']\n" + " return ctx.pass_(message='announced') if announced else ctx.fail(message='silent')\n" +) +_COPYING_LOADER: str = ( + "from python.loaders.orders import raw_orders\nfrom sqlbuild.loaders import loader\n\n\n" + "@loader\n" + "def raw_order_copies(ctx) -> list[dict[str, object]]:\n" + ' """One order when the orders loader has a result to read, none otherwise."""\n' + " found = ctx.result_of(node_function=raw_orders, default=None) is not None\n" + " return [{'order_id': 1}] if found else []\n" +) + + +@pytest.mark.parametrize( + "test_case", + [ + PreservedTargetNodeResultsE2ETestCase( + description="a preserved schema keeps node results in the loader schema", + target_lines='schema = "preserve"\nloader_schema = "raw"\n', + source_location=" schema: raw\n", + expected_results_schema="raw", + expected_runs_per_node=2, + ), + PreservedTargetNodeResultsE2ETestCase( + description="a preserved database keeps node results in the connection database", + target_lines='database = "preserve"\nschema = "analytics"\nloader_schema = "raw"\n', + source_location=" database: dev\n schema: raw\n", + expected_results_schema="analytics", + expected_runs_per_node=2, + ), + PreservedTargetNodeResultsE2ETestCase( + description="a schema set by --vars holds the node results build and load share", + target_lines='schema = "${results_schema}"\n\n[vars]\nresults_schema = "mart"\n', + source_location=" schema: raw\n", + expected_results_schema="ops", + expected_runs_per_node=2, + cli_args=("--vars", json.dumps({"results_schema": "ops"})), + ), + ], + ids=lambda case: case.description, +) +def test_given_target_location_when_building_and_loading_then_node_results_stay_in_one_place( + test_case: PreservedTargetNodeResultsE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = node_result_project( + tmp_path=tmp_path, + target_lines=test_case.target_lines, + files={ + "sources/raw.yml": node_result_source_yml(test_case.source_location), + "sources/copies.yml": node_result_source_yml(test_case.source_location).replace( + "raw_orders", "raw_order_copies" + ), + "python/loaders/orders.py": _LOADER, + "python/loaders/copies.py": _COPYING_LOADER, + "python/tasks/announce.py": _TASK, + "python/checks/orders.py": _CHECK, + }, + ) + + results: list[subprocess.CompletedProcess[str]] = [ + run_sqb(command=("--no-color", "build", *test_case.cli_args), project_dir=project_dir) + for _ in range(2) + ] + [ + run_sqb( + command=("--no-color", "load", "--select", "raw_order_copies", *test_case.cli_args), + project_dir=project_dir, + ) + ] + + for result in results: + assert result.returncode == 0, result.stdout + result.stderr + db_path: Path = project_dir / "dev.duckdb" + assert ( + query_duckdb( + db_path=db_path, + sql=( + "SELECT catalog_name, schema_name FROM information_schema.schemata " + "WHERE lower(schema_name) = 'preserve' OR lower(catalog_name) = 'preserve'" + ), + ) + == [] + ) + assert query_duckdb( + db_path=db_path, + sql=( + "SELECT table_schema FROM information_schema.tables " + "WHERE table_name = '_sqlbuild_node_results'" + ), + ) == [(test_case.expected_results_schema,)] + assert query_duckdb( + db_path=db_path, + sql=( + "SELECT node_type, node_name, count(DISTINCT run_id) " + f"FROM {test_case.expected_results_schema}._sqlbuild_node_results " + "GROUP BY ALL ORDER BY ALL" + ), + ) == [ + ("check", "check_orders_announced", test_case.expected_runs_per_node), + ("loader", "raw_order_copies", test_case.expected_runs_per_node), + ("loader", "raw_orders", test_case.expected_runs_per_node), + ("task", "announce_orders", test_case.expected_runs_per_node), + ] + assert query_duckdb(db_path=db_path, sql="SELECT order_id FROM raw.raw_order_copies") == [(1,)] + + +@pytest.mark.parametrize( + "test_case", + [ + PreservedTargetRefusalE2ETestCase( + description="a task under a preserved schema needs a loader schema", + python_files={ + "python/tasks/announce.py": _TASK.replace('depends_on=source("raw_orders")', "") + }, + expected_error="error[S105]: Target 'dev' keeps each model's own schema", + ), + ], + ids=lambda case: case.description, +) +def test_given_preserved_schema_without_loader_schema_when_compiling_then_python_nodes_are_refused( + test_case: PreservedTargetRefusalE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="orders", + repo_files={ + "sqlbuild_project.toml": ( + 'name = "orders"\nadapter = "duckdb"\ndefault_target = "dev"\n\n' + '[connection]\ndatabase = "dev.duckdb"\n\n' + '[defaults]\nschema = "mart"\n\n[targets.dev]\nschema = "preserve"\n' + ), + "models/orders.sql": "MODEL (description 'Orders.');\n\nSELECT 1 AS order_id\n", + } + | test_case.python_files, + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "compile"), project_dir=project_dir + ) + + assert result.returncode == 1, result.stdout + result.stderr + assert test_case.expected_error in result.stderr + + +@pytest.mark.parametrize( + "test_case", + [ + AdapterDefaultResultsFallbackE2ETestCase( + description="a task reads a rebuilt loader and one whose result is only in main", + target_schema="mart", + rebuilt_loader="raw_order_events", + kept_loader="raw_customer_events", + reading_task="announce_events", + expected_payload_keys=("orders", "customers"), + ), + ], + ids=lambda case: case.description, +) +def test_given_results_in_the_adapter_default_when_one_loader_rebuilds_then_both_are_still_read( + test_case: AdapterDefaultResultsFallbackE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="orders", + repo_files={ + "sqlbuild_project.toml": ( + 'name = "orders"\nadapter = "duckdb"\ndefault_target = "dev"\n\n' + '[connection]\ndatabase = "dev.duckdb"\n\n[targets.dev]\n' + ), + "sources/events.yml": (_EDGE_CASES / "loader_results.yml").read_text(), + "python/loaders/events.py": (_EDGE_CASES / "loader_results.py").read_text(), + }, + ) + db_path: Path = project_dir / "dev.duckdb" + first: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + assert first.returncode == 0, first.stdout + first.stderr + old_runs: dict[str, str] = { + name: run + for name, run, _ in node_results_in(db_path=db_path, schema="main", node_type="loader") + } + toml: Path = project_dir / "sqlbuild_project.toml" + toml.write_text( + toml.read_text(encoding="utf-8") + f'schema = "{test_case.target_schema}"\n', + encoding="utf-8", + ) + + second: subprocess.CompletedProcess[str] = run_sqb( + command=( + "--no-color", + "build", + "--select", + test_case.rebuilt_loader, + test_case.reading_task, + ), + project_dir=project_dir, + ) + + assert second.returncode == 0, second.stdout + second.stderr + new_runs: dict[str, str] = { + name: run + for name, run, _ in node_results_in( + db_path=db_path, schema=test_case.target_schema, node_type="loader" + ) + } + assert set(new_runs) == {test_case.rebuilt_loader} + assert new_runs[test_case.rebuilt_loader] != old_runs[test_case.rebuilt_loader] + [(_, task_run, payload)] = node_results_in( + db_path=db_path, schema=test_case.target_schema, node_type="task" + ) + assert task_run == new_runs[test_case.rebuilt_loader] + rebuilt_key, kept_key = test_case.expected_payload_keys + assert json.loads(base64.b64decode(payload)) == { + rebuilt_key: new_runs[test_case.rebuilt_loader], + kept_key: old_runs[test_case.kept_loader], + } + assert { + name: run + for name, run, _ in node_results_in(db_path=db_path, schema="main", node_type="loader") + } == old_runs diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_missing_target_database.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_missing_target_database.py index 3ee1d8dc14..25598b9427 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_missing_target_database.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_missing_target_database.py @@ -21,6 +21,7 @@ 'name = "orders"\nadapter = "duckdb"\ndefault_target = "dev"\n\n' '[connection]\ndatabase = "dev.duckdb"\n\n' '[defaults]\nschema = "analytics"\n\n[targets.dev]\nschema = "preserve"\n' + 'loader_schema = "landing"\n' ) diff --git a/tests/goldens/compiler/failure/target-preserved-schema-without-node-results-schema.json b/tests/goldens/compiler/failure/target-preserved-schema-without-node-results-schema.json new file mode 100644 index 0000000000..3b47051dde --- /dev/null +++ b/tests/goldens/compiler/failure/target-preserved-schema-without-node-results-schema.json @@ -0,0 +1,19 @@ +{ + "commands": [ + { + "label": "compile", + "exit_code": 1, + "diagnostics": [ + { + "phase": "compile", + "severity": "error", + "code": "S105", + "message": "Target 'dev' keeps each model's own schema, so it has no schema for the results of loaders, tasks, assets and checks. Set targets.dev.loader_schema; Python node results are kept there.", + "help": "to keep Python node results in a schema, set this in sqlbuild_project.toml:\n [targets.dev]\n loader_schema = \"my_schema\"" + } + ] + } + ], + "compiled": {}, + "manifest": null +} diff --git a/tests/integration/src/sqlbuild/executor/node_results/main/_test_types.py b/tests/integration/src/sqlbuild/executor/node_results/main/_test_types.py index 77ba572b7e..39d34e3c46 100644 --- a/tests/integration/src/sqlbuild/executor/node_results/main/_test_types.py +++ b/tests/integration/src/sqlbuild/executor/node_results/main/_test_types.py @@ -6,3 +6,14 @@ class NodeResultReadWriteIntegrationTestCase: description: str expected_payload: dict[str, object] expected_metadata: dict[str, object] + + +@dataclass(frozen=True) +class EarlierNodeResultLocationIntegrationTestCase: + """A result left in an earlier location, results in the store's own, and which one is read.""" + + description: str + earlier_schema: str + schema_preserved: bool + new_run_ids: tuple[str, ...] + expected_run_id: str | None diff --git a/tests/integration/src/sqlbuild/executor/node_results/main/helpers.py b/tests/integration/src/sqlbuild/executor/node_results/main/helpers.py new file mode 100644 index 0000000000..19ddf21d7f --- /dev/null +++ b/tests/integration/src/sqlbuild/executor/node_results/main/helpers.py @@ -0,0 +1,21 @@ +"""Shared builders for node result integration tests.""" + +from __future__ import annotations + +from sqlbuild.executor.node_results.models import NodeResultRecord + + +def task_result_record(*, run_id: str, schema: str) -> NodeResultRecord: + """A successful `produce_result` task result recorded in `schema`.""" + + return NodeResultRecord( + node_type="task", + node_name="produce_result", + target_database=None, + target_schema=schema, + target_name=None, + run_id=run_id, + status="success", + payload={"run": run_id}, + metadata={}, + ) diff --git a/tests/integration/src/sqlbuild/executor/node_results/main/test_read_write.py b/tests/integration/src/sqlbuild/executor/node_results/main/test_read_write.py index eb6d7c7a94..d433c4082d 100644 --- a/tests/integration/src/sqlbuild/executor/node_results/main/test_read_write.py +++ b/tests/integration/src/sqlbuild/executor/node_results/main/test_read_write.py @@ -7,7 +7,10 @@ import pytest +from sqlbuild._native import TargetSettings from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.executor.node_results.classes.direct_store import DirectNodeResultStore +from sqlbuild.executor.node_results.main._direct_store import build_direct_node_result_store from sqlbuild.executor.node_results.main._read import read_node_results from sqlbuild.executor.node_results.main._write import write_node_result_record from sqlbuild.executor.node_results.models import ( @@ -16,8 +19,10 @@ NodeResultRecord, ) from tests.integration.src.sqlbuild.executor.node_results.main._test_types import ( + EarlierNodeResultLocationIntegrationTestCase, NodeResultReadWriteIntegrationTestCase, ) +from tests.integration.src.sqlbuild.executor.node_results.main.helpers import task_result_record @pytest.mark.parametrize( @@ -83,3 +88,72 @@ def test_given_node_result_record_when_writing_and_reading_then_round_trips_enve assert len(results) == 1 assert results[0].payload == test_case.expected_payload assert results[0].metadata == test_case.expected_metadata + + +@pytest.mark.parametrize( + "test_case", + [ + EarlierNodeResultLocationIntegrationTestCase( + description="a result only in the adapter default schema is read", + earlier_schema="main", + schema_preserved=False, + new_run_ids=(), + expected_run_id="run_old", + ), + EarlierNodeResultLocationIntegrationTestCase( + description="a preserved target reads a result left in a schema named preserve", + earlier_schema="preserve", + schema_preserved=True, + new_run_ids=(), + expected_run_id="run_old", + ), + EarlierNodeResultLocationIntegrationTestCase( + description="a target that is not preserved ignores a schema named preserve", + earlier_schema="preserve", + schema_preserved=False, + new_run_ids=(), + expected_run_id=None, + ), + EarlierNodeResultLocationIntegrationTestCase( + description="a result in the store's own schema wins over an earlier one", + earlier_schema="main", + schema_preserved=False, + new_run_ids=("run_new",), + expected_run_id="run_new", + ), + ], + ids=lambda case: case.description, +) +def test_given_result_in_earlier_location_when_reading_then_store_falls_back_only_when_empty( + test_case: EarlierNodeResultLocationIntegrationTestCase, + tmp_path: Path, +) -> None: + adapter: DuckDbAdapter = DuckDbAdapter() + connection: Any = adapter.connect({"database": str(tmp_path / "node_results.duckdb")}) + try: + earlier: DirectNodeResultStore = DirectNodeResultStore( + adapter=adapter, connection=connection, database=None, schema=test_case.earlier_schema + ) + earlier.write(task_result_record(run_id="run_old", schema=test_case.earlier_schema)) + store: DirectNodeResultStore = build_direct_node_result_store( + adapter=adapter, + connection=connection, + database=None, + schema="raw", + target_config=TargetSettings(schema_preserved=test_case.schema_preserved), + ) + run_id: str + for run_id in test_case.new_run_ids: + store.write(task_result_record(run_id=run_id, schema="raw")) + reader: DirectNodeResultStore = build_direct_node_result_store( + adapter=adapter, + connection=connection, + database=None, + schema="raw", + target_config=TargetSettings(schema_preserved=test_case.schema_preserved), + ) + found: object = reader.result_of(node_type="task", node_name="produce_result", default=None) + finally: + adapter.close(connection) + + assert getattr(found, "run_id", None) == test_case.expected_run_id diff --git a/tests/unit/src/sqlbuild/executor/build/_helpers/_test_types.py b/tests/unit/src/sqlbuild/executor/build/_helpers/_test_types.py index b9fa3b4e93..1cbf267c7a 100644 --- a/tests/unit/src/sqlbuild/executor/build/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/executor/build/_helpers/_test_types.py @@ -121,6 +121,16 @@ class BuildSourceNodeExecutionTestCase: expected_rows: tuple[tuple[object, ...], ...] +@dataclass(frozen=True) +class BuildSourceNodeResultLocationTestCase: + """Where a scheduled loader's result goes for an adapter default and a runtime location.""" + + description: str + adapter_default_schema: str | None + runtime_schema: str + expected_result_schemas: tuple[str, ...] + + @dataclass(frozen=True) class MaterializationRecreatesRelationTestCase: description: str diff --git a/tests/unit/src/sqlbuild/executor/build/_helpers/test_source_node.py b/tests/unit/src/sqlbuild/executor/build/_helpers/test_source_node.py index 1d1d63143e..38ce1f18cb 100644 --- a/tests/unit/src/sqlbuild/executor/build/_helpers/test_source_node.py +++ b/tests/unit/src/sqlbuild/executor/build/_helpers/test_source_node.py @@ -15,6 +15,7 @@ from sqlbuild.runtime.contracts.types import ExecutionResourceKind from tests.unit.src.sqlbuild.executor.build._helpers._test_types import ( BuildSourceNodeExecutionTestCase, + BuildSourceNodeResultLocationTestCase, ) from tests.unit.src.sqlbuild.executor.build._helpers.helpers import ( build_discovered_source_loader, @@ -86,3 +87,68 @@ def test_given_build_source_node_when_executing_then_runs_loader_runtime( assert rows == test_case.expected_rows assert progress_events == [test_case.expected_progress_event] assert start_events == [test_case.expected_start_event] + + +@pytest.mark.parametrize( + "test_case", + [ + BuildSourceNodeResultLocationTestCase( + description="an adapter without a default schema still records the result", + adapter_default_schema=None, + runtime_schema="analytics", + expected_result_schemas=("analytics",), + ), + BuildSourceNodeResultLocationTestCase( + description="the runtime location wins over the adapter default schema", + adapter_default_schema="main", + runtime_schema="analytics", + expected_result_schemas=("analytics",), + ), + ], + ids=lambda case: case.description, +) +def test_given_runtime_location_when_executing_source_node_then_result_is_written_there( + test_case: BuildSourceNodeResultLocationTestCase, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + adapter: DuckDbAdapter = DuckDbAdapter() + monkeypatch.setattr(adapter, "default_schema", lambda: test_case.adapter_default_schema) + connection: object = adapter.connect({"database": str(tmp_path / "source_node.duckdb")}) + try: + result: LoadExecutionResult = execute_build_source_node( + key=CompiledObjectKey(CompiledResourceType.SOURCE, "raw_orders"), + plan=build_source_load_plan_output( + source_name="raw_orders", loader_name="raw_orders_loader" + ), + loader_functions_by_name={ + "raw_orders_loader": build_discovered_source_loader(loader_name="raw_orders_loader") + }, + loader_ref_entries={}, + adapter=adapter, + connection_config={"database": str(tmp_path / "source_node.duckdb")}, + connection=connection, + runtime=BuildRuntimeParams( + run_id="run-1", + target="dev", + default_schema=test_case.runtime_schema, + ), + callbacks=BuildCallbacks(), + ) + result_schemas: tuple[str, ...] = tuple( + row[0] + for row in connection.execute( + "SELECT table_schema FROM information_schema.tables " + "WHERE table_name = '_sqlbuild_node_results' ORDER BY ALL" + ).fetchall() + ) + recorded: list[tuple[object, ...]] = connection.execute( + f"SELECT node_type, target_schema, run_id " + f"FROM {test_case.runtime_schema}._sqlbuild_node_results" + ).fetchall() + finally: + adapter.close(connection) + + assert result.status == ExecutionStatus.SUCCESS + assert result_schemas == test_case.expected_result_schemas + assert recorded == [("loader", test_case.runtime_schema, "run-1")] diff --git a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/helpers.py index d775f4e8ce..6a7767909d 100644 --- a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/helpers.py @@ -149,6 +149,7 @@ class PythonNodeContextTestResultStore: def __init__(self, results: dict[tuple[str, str], tuple[NodeResultEnvelope, ...]]) -> None: self._results: dict[tuple[str, str], tuple[NodeResultEnvelope, ...]] = results self.written_records: list[NodeResultRecord] = [] + self.connection: object = object() self.database: str | None = None self.schema: str | None = "default_schema" diff --git a/website/src/content/docs/docs/concepts/project-configuration.mdx b/website/src/content/docs/docs/concepts/project-configuration.mdx index b43a5fbcc9..6c556efbf0 100644 --- a/website/src/content/docs/docs/concepts/project-configuration.mdx +++ b/website/src/content/docs/docs/concepts/project-configuration.mdx @@ -152,7 +152,7 @@ A target is a named build context - the database and schema you build into, plus | Field | Description | |-------|-------------| | `schema` | Schema for all models in this target; required for named targets | -| `loader_schema` | Default write schema for managed source loaders; falls back to `schema` | +| `loader_schema` | Default write schema for managed source loaders; falls back to `schema`. Required when `schema` is `preserve` and the project has loaders, tasks, assets or checks, because their [results](/docs/concepts/python-nodes/overview#result-persistence) go there | | `database` | Database for all models in this target | | `connection` | Name from `[connections.]` used for endpoint, authentication, and compute; optional when exactly one named connection is defined | | `vars` | Target-specific project variables | diff --git a/website/src/content/docs/docs/concepts/python-nodes/overview.mdx b/website/src/content/docs/docs/concepts/python-nodes/overview.mdx index 87c93836f3..5954224c00 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/overview.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/overview.mdx @@ -156,7 +156,19 @@ Within one run, downstream nodes run only if at least one upstream succeeded. If ## Result persistence -Node results (payload, metadata, status, errors) are persisted after each execution. Results are stored in `_sqlbuild_node_results` in the warehouse alongside your data. Results persist across runs, so they are available for observability, debugging, and downstream consumption. +Node results (payload, metadata, status, errors) are persisted after each execution in a +`_sqlbuild_node_results` table. Results persist across runs, so they are available for +observability, debugging, and downstream consumption through `ctx.result_of`. + +Results of loaders, tasks, assets and checks all go to one schema, chosen by the target's `schema`: + +| Target `schema` | Results table schema | +|-----------------|----------------------| +| A schema name, such as `analytics` | That schema, in the target's database | +| `preserve` | The target's `loader_schema`; commands that compile the project stop with `S105` until it is set | +| Unset, or resolved per model, such as `${CTX:...}` | The adapter's default schema, such as `main` on DuckDB or `public` on PostgreSQL | + +A target `database` of `preserve` keeps results in the connection's database. ## Selection