From 6b180cdfc04c06963e9ea3dc0694cad57a1d53f4 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 16 Nov 2026 23:16:06 +0000 Subject: [PATCH 1/6] fix: keep Python node results out of per-model target schemas A target schema such as s_${CTX:model.name} resolves only inside a model. sqb build passed the raw template to Python tasks and loaders as their default location, so persisting a node result ran CREATE SCHEMA on the template and the build failed. Build and check now fall back to the adapter's default schema and database when the target location holds a context placeholder. --- .../src/templates/_helpers/deferred.rs | 2 +- .../main/holds_context_placeholder.rs | 13 ++++++++ .../src/templates/main/mod.rs | 1 + .../_helpers/build_command/lifecycle.rs | 7 ++--- .../bindings/_helpers/check/check_command.rs | 31 +++++-------------- .../_helpers/compile_driver/targets.rs | 12 +++++++ .../cli/commands/main/build/test_build.py | 24 ++++++++++++-- 7 files changed, 59 insertions(+), 31 deletions(-) create mode 100644 crates/sqlbuild-model-config/src/templates/main/holds_context_placeholder.rs diff --git a/crates/sqlbuild-model-config/src/templates/_helpers/deferred.rs b/crates/sqlbuild-model-config/src/templates/_helpers/deferred.rs index 94b4fa3e42..d0b9543eb0 100644 --- a/crates/sqlbuild-model-config/src/templates/_helpers/deferred.rs +++ b/crates/sqlbuild-model-config/src/templates/_helpers/deferred.rs @@ -1,7 +1,7 @@ //! The `${CTX:key}` and `${name}` placeholders of a deferred target's schema or database. /// The deferred-target context marker inside `${...}`. -const CONTEXT_PREFIX: &str = "CTX:"; +pub(crate) const CONTEXT_PREFIX: &str = "CTX:"; /// The opening of a `${...}` placeholder. pub(crate) const PLACEHOLDER_OPEN: &str = "${"; const PLACEHOLDER_CLOSE: char = '}'; diff --git a/crates/sqlbuild-model-config/src/templates/main/holds_context_placeholder.rs b/crates/sqlbuild-model-config/src/templates/main/holds_context_placeholder.rs new file mode 100644 index 0000000000..8a9e7bc07a --- /dev/null +++ b/crates/sqlbuild-model-config/src/templates/main/holds_context_placeholder.rs @@ -0,0 +1,13 @@ +//! Whether a target's schema or database resolves only inside a model. + +use crate::templates::_helpers::deferred::{CONTEXT_PREFIX, PLACEHOLDER_OPEN, placeholders}; + +/// Whether `template` holds a `${CTX:key}` placeholder, so it names a location only inside a model. +pub fn holds_context_placeholder(template: &str) -> bool { + !placeholders( + template, + &format!("{PLACEHOLDER_OPEN}{CONTEXT_PREFIX}"), + None, + ) + .is_empty() +} diff --git a/crates/sqlbuild-model-config/src/templates/main/mod.rs b/crates/sqlbuild-model-config/src/templates/main/mod.rs index 90ea55da0e..1ec97c9001 100644 --- a/crates/sqlbuild-model-config/src/templates/main/mod.rs +++ b/crates/sqlbuild-model-config/src/templates/main/mod.rs @@ -1,4 +1,5 @@ pub mod deferred_target_segments; pub mod expand_template_string; pub mod expand_template_text; +pub mod holds_context_placeholder; pub mod template_error_message; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs index 7e7b13b14b..f7c4301d26 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs @@ -15,7 +15,7 @@ use sqlbuild_host::entries::models::ExecutorHost; use crate::bindings::_helpers::compile_driver::relation_targets::build_project_relation_targets; use crate::bindings::_helpers::compile_driver::relation_targets::build_python_relation_targets; use crate::bindings::_helpers::compile_driver::targets::{ - TargetSettingsValues, project_target_settings, + TargetSettingsValues, model_free_location, project_target_settings, }; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; @@ -23,7 +23,6 @@ use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::build_command::main::boolean::boolean; use sqlbuild_python_base::build_command::main::emit::emit; use sqlbuild_python_base::build_command::main::frozenset::frozenset; -use sqlbuild_python_base::build_command::main::or_none::or_none; use sqlbuild_python_base::build_command::main::text::text; use sqlbuild_python_base::build_command::main::tuple::tuple; use sqlbuild_python_base::native_host::main::host_entry::host_entry; @@ -439,10 +438,10 @@ pub(crate) fn prepare_direct_python_lifecycle<'py>( project: &project, reload: context.request_flag("reload_sources")?, defaults: ( - or_none(native_field(&project, "effective_target_database")?, || { + model_free_location(native_field(&project, "effective_target_database")?, || { adapter.call_method0("default_database") })?, - or_none(native_field(&project, "effective_target_schema")?, || { + model_free_location(native_field(&project, "effective_target_schema")?, || { adapter.call_method0("default_schema") })?, ), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs index 8f084c260f..f14c4236d3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs @@ -8,7 +8,7 @@ use crate::bindings::_helpers::compile_driver::relation_targets::build_project_r use crate::bindings::_helpers::compile_driver::relation_targets::build_python_relation_targets; use crate::bindings::_helpers::compile_driver::static_context::compile_static_command_context; use crate::bindings::_helpers::compile_driver::targets::{ - TargetSettingsValues, project_target_settings, + TargetSettingsValues, model_free_location, project_target_settings, }; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; use crate::bindings::_helpers::result_output::documents::write_execution_json_output; @@ -280,19 +280,6 @@ fn compile_pipeline<'py>( ) } -fn target_default<'py>( - project: &Bound<'py, PyAny>, - adapter: &Bound<'py, PyAny>, - (field, fallback): (&str, &str), -) -> PyResult> { - let value: Bound<'py, PyAny> = native_field(project, field)?; - if value.is_none() { - adapter.call_method0(fallback) - } else { - Ok(value) - } -} - /// The discovered Python graph, after refusing unit-test selectors, and the selected checks. fn python_selection<'py>( request: &Bound<'py, PyAny>, @@ -370,16 +357,12 @@ fn prepare<'py>( relation_targets, relation_refs, defaults: ( - target_default( - &project, - &adapter, - ("effective_target_database", "default_database"), - )?, - target_default( - &project, - &adapter, - ("effective_target_schema", "default_schema"), - )?, + model_free_location(native_field(&project, "effective_target_database")?, || { + adapter.call_method0("default_database") + })?, + model_free_location(native_field(&project, "effective_target_schema")?, || { + adapter.call_method0("default_schema") + })?, ), project_relations: object( py, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs index f797a8508e..aa3fa20119 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs @@ -21,6 +21,7 @@ use sqlbuild_analysis::assembly::target_checks::main::target_error_message::targ use sqlbuild_analysis::assembly::target_checks::models::{LoaderCollision, MissingParts}; use sqlbuild_core::text::main::python_strip::python_strip; +use sqlbuild_model_config::templates::main::holds_context_placeholder::holds_context_placeholder; use sqlbuild_model_config::templates::models::TemplateOptions; use sqlbuild_python_base::boundary::main::keywords::keywords; @@ -502,6 +503,17 @@ pub(crate) fn target_settings<'py>( }) } +/// A target `location` that Python nodes can use outside any model: `default` when it is unset, empty, or resolves only inside a model, such as a `${CTX:...}` schema. +pub(crate) fn model_free_location<'py>( + location: Bound<'py, PyAny>, + default: impl FnOnce() -> PyResult>, +) -> PyResult> { + match location.extract::>()? { + Some(text) if !text.is_empty() && !holds_context_placeholder(&text) => Ok(location), + _ => default(), + } +} + /// The selected target's settings, with its database and schema resolved as `sqb debug` resolves them. pub(crate) fn selected_target_settings<'py>( (discovered, selected_target): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_build.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_build.py index e9178ec58a..47335bcc61 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_build.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_build.py @@ -1991,6 +1991,15 @@ def test_given_ctx_templated_target_schema_when_running_then_python_sees_no_load " record(ctx, 'loader')\n" " return [{'order_id': 1}]\n" ), + "python/tasks/configured.py": ( + "from python.recorder import record\n" + "from sqlbuild.tasks import task\n\n" + "@task\n" + "def configured_task(ctx):\n" + ' """Test task configured_task."""\n' + " record(ctx, 'task')\n" + " return ctx.result()\n" + ), "sources/raw.yml": ( "sources:\n" " - name: raw_orders\n description: Test source raw_orders.\n" @@ -2006,7 +2015,7 @@ def test_given_ctx_templated_target_schema_when_running_then_python_sees_no_load results: list[subprocess.CompletedProcess[str]] = [ run_sqb(command=("--no-color", *command), project_dir=project_dir) - for command in (("build", "--select", "orders"), ("load",), ("check",)) + for command in (("build",), ("load",), ("check",)) ] for result in results: @@ -2021,8 +2030,19 @@ def test_given_ctx_templated_target_schema_when_running_then_python_sees_no_load for line in (project_dir / "target_config.jsonl").read_text(encoding="utf-8").splitlines() ] assert [(line["node"], line["name"], line["loader_schema"]) for line in recorded] == [ - ("loader", "dev", test_case.expected_loader_schema) + ("loader", "dev", test_case.expected_loader_schema), + ("task", "dev", test_case.expected_loader_schema), + ("loader", "dev", test_case.expected_loader_schema), ], recorded + # Python node results land in the adapter's default schema; no schema keeps the template. + assert table_exists(db_path=db_path, table_name="_sqlbuild_node_results", schema="main") + schemas: list[str] = [ + str(row[0]) + for row in query_duckdb( + db_path=db_path, sql="SELECT schema_name FROM information_schema.schemata" + ) + ] + assert not [schema for schema in schemas if "${" in schema], schemas @pytest.mark.parametrize( From fb815c530de5eeb35103f8c254a8f8592ba83816 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 16 Nov 2026 23:30:25 +0000 Subject: [PATCH 2/6] feat: set up exporter, sink and sensor providers with their target Event exporters and command output sinks bind their providers when the command starts, and that session is shared with build, check and load. It was set up without the target, so a provider used by a sink got setup(None) even inside sqb build. The scope now resolves the selected target first and passes its settings; when the project can't resolve them, ctx stays None and the command still runs. A custom sensor's providers get the settings of the target the sensor evaluates for. Calendars keep ctx None and the providers page says so. --- .../src/event_exporting/models.rs | 12 ++++-- .../bindings/_helpers/cli/observability.rs | 20 ++++++++- .../_helpers/compile_driver/targets.rs | 42 +++++++++++++++++++ .../native_host/orchestration_sensors.rs | 33 +++++++++++---- .../docs/concepts/python-nodes/providers.md | 15 ++++++- src/sqlbuild/_native.pyi | 1 + .../cli/commands/main/build/_test_types.py | 1 + .../cli/commands/main/build/test_build.py | 24 +++++++++++ .../publication/test_custom_sensor.py | 8 +++- .../docs/concepts/python-nodes/providers.mdx | 15 ++++++- 10 files changed, 153 insertions(+), 18 deletions(-) diff --git a/crates/sqlbuild-python-base/src/event_exporting/models.rs b/crates/sqlbuild-python-base/src/event_exporting/models.rs index 6815c2e197..eff0abdada 100644 --- a/crates/sqlbuild-python-base/src/event_exporting/models.rs +++ b/crates/sqlbuild-python-base/src/event_exporting/models.rs @@ -380,7 +380,7 @@ impl EventExporterCommandScope { pub(crate) fn bind_extensions<'py>( &self, py: Python<'py>, - project_dir: Option>, + (project_dir, setup_context): (Option>, Bound<'py, PyAny>), (providers, event_exporters, command_output_sinks): ( Bound<'py, PyAny>, Bound<'py, PyAny>, @@ -413,6 +413,7 @@ impl EventExporterCommandScope { &host_entry(py, UserCodeHost::BuildProviderSession)?, &[ ("discovered_providers", providers), + ("setup_context", setup_context), ("allow_shared", PyBool::new(py, false).to_owned().into_any()), ], )?; @@ -508,7 +509,7 @@ impl EventExporterCommandScope { ) -> PyResult<()> { self.bind_extensions( py, - None, + (None, py.None().into_bound(py)), ( native_field(discovered_inputs, "providers")?, native_field(discovered_inputs, "event_exporters")?, @@ -530,7 +531,7 @@ impl EventExporterCommandScope { options, ( &["providers", "event_exporters"], - &["project_dir", "command_output_sinks"], + &["project_dir", "command_output_sinks", "setup_context"], ), )?; let project_dir: Bound<'_, PyAny> = options.value("project_dir")?; @@ -542,7 +543,10 @@ impl EventExporterCommandScope { }; self.bind_extensions( py, - (!project_dir.is_none()).then_some(project_dir), + ( + (!project_dir.is_none()).then_some(project_dir), + options.value("setup_context")?, + ), ( options.value("providers")?, options.value("event_exporters")?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs index ec77c89452..28ff5127f0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs @@ -15,6 +15,7 @@ use sqlbuild_cli::dispatch::models::{ExportSummary, ExporterLoss}; use sqlbuild_core::event_export::constants::DEFAULT_EVENT_EXPORT_SHUTDOWN_TIMEOUT_SECONDS; use sqlbuild_host::entries::models::{ExecutorHost, UserCodeHost}; +use crate::bindings::_helpers::compile_driver::targets::project_setup_context; use crate::bindings::_helpers::event_exporting::dispatcher::EventExporterDispatcher; use crate::bindings::_helpers::recall::console::write_stderr; use crate::bindings::_helpers::result_output::terminal_index::TerminalEventIndex; @@ -407,13 +408,30 @@ impl Observed { ) .map(pyo3::Bound::into_any)?; self.exporter_scope = Some(scope.clone().unbind()); + let providers: Bound<'_, PyAny> = native_field(&extensions, "providers")?; + let setup_context: Bound<'_, PyAny> = if providers.is_truthy()? { + project_setup_context( + project_dir, + ( + &args + .getattr("target") + .unwrap_or_else(|_| py.None().into_bound(py)), + &args + .getattr("vars") + .unwrap_or_else(|_| py.None().into_bound(py)), + ), + )? + } else { + py.None().into_bound(py) + }; call_with( &native_field(&scope, "configure_extensions")?, &[ ("project_dir", project_dir.clone()), - ("providers", native_field(&extensions, "providers")?), + ("providers", providers), ("event_exporters", event_exporters), ("command_output_sinks", sinks), + ("setup_context", setup_context), ], )?; self.unsubscribe_exporters = diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs index aa3fa20119..6dafed1852 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs @@ -43,7 +43,11 @@ const EXPLICIT_TARGET_ADAPTERS: [&str; 3] = ["snowflake", "bigquery", "databrick const LOADER_NAMESPACE_LABEL: &str = "managed loader target namespace"; const DEFAULTS_HELP_PURPOSE: &str = "to set project-wide defaults"; const NAMED_TARGET_HELP_PURPOSE: &str = "to build this target into that schema"; +use crate::bindings::_helpers::command_runtime::connection::project_connection_config; use crate::bindings::_helpers::compile_attachment::target::resolved_target; +use crate::bindings::_helpers::query_command::project::{ + ProjectAdapter, ProjectInputs, project_adapter, +}; use pyo3::IntoPyObjectExt; use crate::bindings::_helpers::compile_attachment::target::effective_target_namespace; @@ -503,6 +507,44 @@ pub(crate) fn target_settings<'py>( }) } +/// The `ctx` a provider's `setup` receives outside a command's own session, such as in an event exporter or a sensor: the settings of `target` (the default target when `None`) for the project at `project_dir`, or `None` when the project cannot resolve them, such as without connection settings. +pub(crate) fn project_setup_context<'py>( + project_dir: &Bound<'py, PyAny>, + (target, cli_vars): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult> { + let py: Python<'py> = project_dir.py(); + let resolved = || -> PyResult> { + let ProjectAdapter { + project_dir, + discovered, + adapter, + } = project_adapter(py, Some(project_dir.clone()), ProjectInputs::Configuration)?; + let connection: Bound<'py, PyAny> = project_connection_config( + &discovered, + (&project_dir, target), + (cli_vars, false), + false, + )?; + let (_, effective_vars, _): (Bound<'py, PyAny>, Bound<'py, PyAny>, Bound<'py, PyAny>) = + build_effective_runtime_config( + &discovered, + Some(target.clone()).filter(|value| !value.is_none()), + Some(cli_vars.clone()).filter(|value| !value.is_none()), + )? + .extract()?; + selected_target_settings( + (&discovered, target), + &adapter.getattr("adapter_name")?.extract::()?, + (&connection, &effective_vars, &adapter), + )? + .setup_context(py) + }; + match resolved() { + Err(error) if error.is_instance_of::(py) => Ok(py.None().into_bound(py)), + context => context, + } +} + /// A target `location` that Python nodes can use outside any model: `default` when it is unset, empty, or resolves only inside a model, such as a `${CTX:...}` schema. pub(crate) fn model_free_location<'py>( location: Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_sensors.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_sensors.rs index f05239f1ee..715241bb8a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_sensors.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_sensors.rs @@ -11,6 +11,7 @@ use sqlbuild_orchestration::sensors::models::{ CustomSensorOutcome, SensorCondition, SensorEvaluationRequest, }; +use crate::bindings::_helpers::compile_driver::targets::project_setup_context; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::within::within; @@ -78,24 +79,40 @@ pub(crate) fn evaluate_sensor( SecretPolicy::Defer.name(), )?; let by_name = provider_by_name(&providers)?; + let mut usages: Vec> = Vec::new(); + for usage in provider_usages(&function, &by_name)?.try_iter()? { + let usage = usage?; + if usage.getattr("parameter_name")?.extract::()? != SENSOR_CONTEXT_PARAMETER { + usages.push(usage); + } + } + let setup_context = if usages.is_empty() { + py.None().into_bound(py) + } else { + project_setup_context( + &imported(py, "pathlib", "Path")?.call1((project_dir,))?, + ( + &evaluation.target.as_str().into_pyobject(py)?.into_any(), + &py.None().into_bound(py), + ), + )? + }; let session = call_with( &entry(py, UserCodeHost::BuildProviderSession)?, &[ ("discovered_providers", providers), + ("setup_context", setup_context), ("allow_shared", PyBool::new(py, false).to_owned().into_any()), ], )?; within(&session, |_| { let arguments = PyDict::new(py); arguments.set_item("ctx", &context)?; - for usage in provider_usages(&function, &by_name)?.try_iter()? { - let usage = usage?; - if usage.getattr("parameter_name")?.extract::()? != SENSOR_CONTEXT_PARAMETER { - arguments.set_item( - usage.getattr("parameter_name")?, - session.call_method1("get", (usage.getattr("provider_name")?,))?, - )?; - } + for usage in &usages { + arguments.set_item( + usage.getattr("parameter_name")?, + session.call_method1("get", (usage.getattr("provider_name")?,))?, + )?; } let result = function.call((), Some(&arguments))?; if result.is_instance(&authoring.getattr("SkipReason")?)? { diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/providers.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/providers.md index c9217cb9a4..24863f2d3c 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/providers.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/providers.md @@ -173,7 +173,7 @@ Both `setup` and `teardown` are optional. A provider with only field declaration ### The setup context -In `sqb build`, `sqb check` and `sqb load`, `setup(ctx)` receives the active target's resolved settings as `ctx.target_config`, the same values Python nodes see (see [Target settings](overview.md#target-settings)): +`setup(ctx)` receives the active target's resolved settings as `ctx.target_config`, the same values Python nodes see (see [Target settings](overview.md#target-settings)): ```python class PartnerStage(Provider): @@ -184,9 +184,20 @@ class PartnerStage(Provider): self.stage = f"{target.database}.{target.loader_schema}.partner_stage" ``` +Which target `ctx.target_config` describes depends on what uses the provider: + +| Used by | `ctx` | +| --- | --- | +| Python nodes in `sqb build`, `sqb check` and `sqb load` | The command's target | +| Event exporters and command output sinks | The command's target: `--target`, or the project's default target | +| Orchestration sensors | The target the sensor evaluates for | +| Orchestration calendars | `None` | + Details: -- A provider first used by an event exporter or command output sink, or by an orchestration sensor or calendar, is set up with `ctx` set to `None`. Check for `None` when such a provider also reads the target. +- An exporter or sink provider gets `ctx` set to `None` when the project can't resolve the target's settings, for example when the target names a connection that isn't defined. +- Commands that don't need the target, such as `sqb format`, still run in that case. +- Check for `None` in a provider that calendars, exporters or sinks use. ## Configuration from environment variables diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 11632aced0..1af14e21b3 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -2878,6 +2878,7 @@ class EventExporterCommandScope: providers: tuple[DiscoveredProvider, ...], event_exporters: tuple[DiscoveredEventExporter, ...], command_output_sinks: tuple[DiscoveredCommandOutputSink, ...] | None = None, + setup_context: ProviderSetupContext | None = None, ) -> None: ... def cached_extensions( self, *, project_dir: Path diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py index b20aad918f..3bb3b7ca58 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py @@ -1642,6 +1642,7 @@ class PythonTargetConfigBuildE2ETestCase: project_target_lines: str local_target_lines: str expected_settings: dict[str, object] + lifecycle_sink: bool = False @dataclass(frozen=True) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_build.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_build.py index 47335bcc61..e61eaf31ff 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_build.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_build.py @@ -1835,6 +1835,20 @@ def test_given_loader_status_result_when_building_then_persists_loader_status_ro "schema_preserved": False, }, ), + PythonTargetConfigBuildE2ETestCase( + description="a lifecycle sink shares the provider and its target", + project_target_lines='schema = "dev"\nloader_schema = "raw_dev"\n', + local_target_lines='schema = "dev_local"\nloader_schema = "raw_local"\n', + expected_settings={ + "name": "dev", + "database": None, + "schema": "dev_local", + "loader_schema": "raw_local", + "database_preserved": False, + "schema_preserved": False, + }, + lifecycle_sink=True, + ), PythonTargetConfigBuildE2ETestCase( description="loader_schema falls back to the target schema", project_target_lines='schema = "dev"\n', @@ -1921,6 +1935,16 @@ def test_given_local_target_override_when_running_python_then_every_context_sees (project_dir / "sqlbuild_local.toml").write_text( "[targets.dev]\n" + test_case.local_target_lines, encoding="utf-8" ) + if test_case.lifecycle_sink: + (project_dir / "sinks").mkdir() + (project_dir / "sinks" / "events.py").write_text( + "from providers.target_probe import TargetProbe\n" + "from sqlbuild.sinks import LifecycleEvent, lifecycle_event_sink\n\n" + "@lifecycle_event_sink\n" + "def probe_events(event: LifecycleEvent, target_probe: TargetProbe) -> None:\n" + " del event, target_probe\n", + encoding="utf-8", + ) results: list[subprocess.CompletedProcess[str]] = [ run_sqb(command=("--no-color", *command), project_dir=project_dir) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_custom_sensor.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_custom_sensor.py index 6b543dd1c8..702547713b 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_custom_sensor.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_custom_sensor.py @@ -54,7 +54,9 @@ " def record(self, value):\n" " with Path(os.environ['SQLBUILD_SENSOR_CALLS']).open('a') as out:\n" " out.write(json.dumps(value | {'pid': os.getpid()}) + '\\n')\n" - " def setup(self, ctx):\n self.record({'phase': 'setup'})\n" + " def setup(self, ctx):\n" + " target = None if ctx is None else ctx.target_config\n" + " self.record({'phase': 'setup', 'target_config': target and [target.name, target.schema]})\n" " def teardown(self):\n self.record({'phase': 'teardown'})\n" " def poll(self, ctx):\n" " mode = json.loads(Path(os.environ['SQLBUILD_FULFILLMENT_DATA']).read_text())['mode']\n" @@ -180,6 +182,10 @@ def test_given_a_custom_polling_sensor_when_a_callback_fails_or_times_out_then_a assert skipped["outcome"]["cursor"]["custom"] == "orders/page-1" evaluated: list[dict[str, Any]] = sensor_calls(calls, {"phase": "evaluate"}) assert {record["target"] for record in evaluated} == {"development"} + # The provider is set up with the settings of the target the sensor evaluates for. + assert [ + record["target_config"] for record in sensor_calls(calls, {"phase": "setup"}) + ][: len(evaluated)] == [["development", "main"]] * len(evaluated) assert len({record["pid"] for record in evaluated}) == len(evaluated) assert { record["cursor"] diff --git a/website/src/content/docs/docs/concepts/python-nodes/providers.mdx b/website/src/content/docs/docs/concepts/python-nodes/providers.mdx index 3f6481302b..05cc2c8c3e 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/providers.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/providers.mdx @@ -155,7 +155,7 @@ Both `setup` and `teardown` are optional. A provider with only field declaration ### The setup context -In `sqb build`, `sqb check` and `sqb load`, `setup(ctx)` receives the active target's resolved settings as `ctx.target_config`, the same values Python nodes see (see [Target settings](/docs/concepts/python-nodes/overview#target-settings)): +`setup(ctx)` receives the active target's resolved settings as `ctx.target_config`, the same values Python nodes see (see [Target settings](/docs/concepts/python-nodes/overview#target-settings)): ```python class PartnerStage(Provider): @@ -166,9 +166,20 @@ class PartnerStage(Provider): self.stage = f"{target.database}.{target.loader_schema}.partner_stage" ``` +Which target `ctx.target_config` describes depends on what uses the provider: + +| Used by | `ctx` | +| --- | --- | +| Python nodes in `sqb build`, `sqb check` and `sqb load` | The command's target | +| Event exporters and command output sinks | The command's target: `--target`, or the project's default target | +| Orchestration sensors | The target the sensor evaluates for | +| Orchestration calendars | `None` | + Details: -- A provider first used by an event exporter or command output sink, or by an orchestration sensor or calendar, is set up with `ctx` set to `None`. Check for `None` when such a provider also reads the target. +- An exporter or sink provider gets `ctx` set to `None` when the project can't resolve the target's settings, for example when the target names a connection that isn't defined. +- Commands that don't need the target, such as `sqb format`, still run in that case. +- Check for `None` in a provider that calendars, exporters or sinks use. ## Configuration from environment variables From 4aa4bc0ba29141fc4652471ac29302f0c4aa50c0 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 16 Nov 2026 23:52:15 +0000 Subject: [PATCH 3/6] fix: fail build, check and load when the exporter target can't be resolved Exporter and sink providers fell back to setup(None) whenever the selected target couldn't be resolved, so a build with an undefined connection set them up without a target before failing. build, check and load now fail at startup instead, before any provider is set up. Other commands, such as sqb format, still run and pass None. A sensor evaluation fails rather than set its providers up without its target. Target resolution at startup also runs with stdout sent to stderr in machine-output modes, like extension discovery next to it. --- .../bindings/_helpers/cli/observability.rs | 35 ++++++--- .../_helpers/command_runtime/setup_context.rs | 7 +- .../native_host/orchestration_sensors.rs | 1 + .../docs/concepts/python-nodes/providers.md | 5 +- .../cli/commands/main/build/_test_types.py | 10 +++ .../cli/commands/main/build/test_build.py | 76 +++++++++++++++++++ .../docs/concepts/python-nodes/providers.mdx | 5 +- 7 files changed, 122 insertions(+), 17 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs index 023e8b710d..88219e48c5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs @@ -39,6 +39,8 @@ const DEBUG_COMMAND: &str = "debug"; const LINEAGE_COMMAND: &str = "lineage"; /// Compile's phase operations stay out of project lifecycle sinks; its invocation events do not. const COMPILE_COMMAND: &str = "compile"; +/// Commands whose own provider sessions get the target: their exporter and sink providers fail startup, rather than get `ctx` set to `None`, when the target can't be resolved. +const TARGET_COMMANDS: [&str; 3] = ["build", "check", "load"]; const COMPILE_WITHHELD_KINDS: &[&str] = &["operation"]; const INVOCATION_COMPLETED: &str = "invocation_completed"; const INVOCATION_FAILED: &str = "invocation_failed"; @@ -410,17 +412,28 @@ impl Observed { self.exporter_scope = Some(scope.clone().unbind()); let providers: Bound<'_, PyAny> = native_field(&extensions, "providers")?; let setup_context: Bound<'_, PyAny> = if providers.is_truthy()? { - project_setup_context( - project_dir, - ( - &args - .getattr("target") - .unwrap_or_else(|_| py.None().into_bound(py)), - &args - .getattr("vars") - .unwrap_or_else(|_| py.None().into_bound(py)), - ), - )? + let command: Option = native_field(args, "command")?.extract()?; + let resolve = || { + project_setup_context( + project_dir, + ( + &args + .getattr("target") + .unwrap_or_else(|_| py.None().into_bound(py)), + &args + .getattr("vars") + .unwrap_or_else(|_| py.None().into_bound(py)), + ), + command + .is_some_and(|command| TARGET_COMMANDS.contains(&command.as_str())), + ) + }; + // Resolving the target imports project adapters, which must not write to JSON stdout. + if machine_output_mode(args)? { + within(&redirect_stdout_to_stderr(py)?, |_| resolve())? + } else { + resolve()? + } } else { py.None().into_bound(py) }; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/setup_context.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/setup_context.rs index 69f4f0328f..0ff95fd171 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/setup_context.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/setup_context.rs @@ -10,10 +10,11 @@ use crate::bindings::_helpers::query_command::project::{ ProjectAdapter, ProjectInputs, project_adapter, }; -/// The settings of `target` (the default target when `None`) for the project at `project_dir`, as a provider setup context, such as for an event exporter or a sensor, or `None` when the project can't resolve them. +/// The settings of `target` (the default target when `None`) for the project at `project_dir`, as a provider setup context, such as for an event exporter or a sensor. When the project can't resolve them, the error propagates if the target is `required`, and the context is `None` otherwise. pub(crate) fn project_setup_context<'py>( project_dir: &Bound<'py, PyAny>, (target, cli_vars): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + required: bool, ) -> PyResult> { let py: Python<'py> = project_dir.py(); let resolved = || -> PyResult> { @@ -43,7 +44,9 @@ pub(crate) fn project_setup_context<'py>( .setup_context(py) }; match resolved() { - Err(error) if error.is_instance_of::(py) => Ok(py.None().into_bound(py)), + Err(error) if !required && error.is_instance_of::(py) => { + Ok(py.None().into_bound(py)) + } context => context, } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_sensors.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_sensors.rs index 677eaf92f9..63d695c952 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_sensors.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_sensors.rs @@ -95,6 +95,7 @@ pub(crate) fn evaluate_sensor( &evaluation.target.as_str().into_pyobject(py)?.into_any(), &py.None().into_bound(py), ), + true, )? }; let session = call_with( diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/providers.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/providers.md index 24863f2d3c..d93cd2dd59 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/providers.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/providers.md @@ -195,8 +195,9 @@ Which target `ctx.target_config` describes depends on what uses the provider: Details: -- An exporter or sink provider gets `ctx` set to `None` when the project can't resolve the target's settings, for example when the target names a connection that isn't defined. -- Commands that don't need the target, such as `sqb format`, still run in that case. +- When the project can't resolve the target's settings, for example because the target names a connection that isn't defined, `sqb build`, `sqb check` and `sqb load` fail before any exporter or sink provider is set up. +- Other commands still run in that case, and their exporter and sink providers get `ctx` set to `None`. `sqb format` is one example. +- A sensor evaluation whose providers can't get its target's settings fails, and the sensor's cursor doesn't advance. - Check for `None` in a provider that calendars, exporters or sinks use. ## Configuration from environment variables diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py index 3bb3b7ca58..cb3ed2e9ec 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py @@ -1645,6 +1645,16 @@ class PythonTargetConfigBuildE2ETestCase: lifecycle_sink: bool = False +@dataclass(frozen=True) +class UnresolvedTargetSinkE2ETestCase: + """Test case for a sink provider under a target the project can't resolve.""" + + description: str + command: tuple[str, ...] + expected_success: bool + expected_setups: list[object] + + @dataclass(frozen=True) class CtxTargetSchemaE2ETestCase: """Test case for commands under a target schema that resolves only inside a model.""" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_build.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_build.py index e61eaf31ff..6f94a6a2cf 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_build.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_build.py @@ -24,6 +24,7 @@ PythonTargetConfigBuildE2ETestCase, PythonTargetIsolationBuildE2ETestCase, SelectionAwareStalenessBuildE2ETestCase, + UnresolvedTargetSinkE2ETestCase, ) from tests.e2e.src.sqlbuild.cli.commands.main.build.helpers import ( assert_defer_clone_build_case, @@ -1975,6 +1976,81 @@ def test_given_local_target_override_when_running_python_then_every_context_sees assert all(line == test_case.expected_settings for line in recorded), recorded +@pytest.mark.parametrize( + "test_case", + [ + UnresolvedTargetSinkE2ETestCase( + description="build fails before any sink provider is set up", + command=("build",), + expected_success=False, + expected_setups=[], + ), + UnresolvedTargetSinkE2ETestCase( + description="format needs no target and sets the provider up with None", + command=("format", "--check"), + expected_success=True, + expected_setups=[None], + ), + ], + ids=lambda case: case.description, +) +def test_given_unresolvable_target_when_running_with_a_sink_then_only_target_free_commands_run( + test_case: UnresolvedTargetSinkE2ETestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="unresolved_target_project", + repo_files={ + "sqlbuild_project.toml": ( + 'name = "unresolved_target_project"\n' + 'adapter = "duckdb"\n' + 'default_target = "dev"\n\n' + "[targets.dev]\n" + 'connection = "warehouse"\n' + 'schema = "dev"\n' + ), + "models/orders.sql": ( + "MODEL (description 'Test model orders.', materialized table);\n\n" + "SELECT 1 AS order_id\n" + ), + "providers/target_probe.py": ( + "import json\n" + "from pathlib import Path\n" + "from sqlbuild.providers import Provider\n\n" + "class TargetProbe(Provider):\n" + ' """Test provider TargetProbe."""\n\n' + " def setup(self, ctx):\n" + " line = None if ctx is None else ctx.target_config.name\n" + " with Path(__file__).parents[1].joinpath('setups.jsonl').open('a') as out:\n" + " out.write(json.dumps(line) + '\\n')\n" + ), + "sinks/events.py": ( + "from providers.target_probe import TargetProbe\n" + "from sqlbuild.sinks import LifecycleEvent, lifecycle_event_sink\n\n" + "@lifecycle_event_sink\n" + "def probe_events(event: LifecycleEvent, target_probe: TargetProbe) -> None:\n" + " del event, target_probe\n" + ), + }, + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", *test_case.command), project_dir=project_dir + ) + + assert (result.returncode == 0) == test_case.expected_success, result.stdout + result.stderr + if not test_case.expected_success: + assert "Unknown connection 'warehouse'" in result.stdout + result.stderr + setups_path: Path = project_dir / "setups.jsonl" + setups: list[object] = ( + [json.loads(line) for line in setups_path.read_text(encoding="utf-8").splitlines()] + if setups_path.exists() + else [] + ) + assert setups == test_case.expected_setups + + @pytest.mark.parametrize( "test_case", [ diff --git a/website/src/content/docs/docs/concepts/python-nodes/providers.mdx b/website/src/content/docs/docs/concepts/python-nodes/providers.mdx index 05cc2c8c3e..368e622808 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/providers.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/providers.mdx @@ -177,8 +177,9 @@ Which target `ctx.target_config` describes depends on what uses the provider: Details: -- An exporter or sink provider gets `ctx` set to `None` when the project can't resolve the target's settings, for example when the target names a connection that isn't defined. -- Commands that don't need the target, such as `sqb format`, still run in that case. +- When the project can't resolve the target's settings, for example because the target names a connection that isn't defined, `sqb build`, `sqb check` and `sqb load` fail before any exporter or sink provider is set up. +- Other commands still run in that case, and their exporter and sink providers get `ctx` set to `None`. `sqb format` is one example. +- A sensor evaluation whose providers can't get its target's settings fails, and the sensor's cursor doesn't advance. - Check for `None` in a provider that calendars, exporters or sinks use. ## Configuration from environment variables From 79ba8044dc941fa3cd11c53181560b7ae2aa40cf Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 17 Nov 2026 00:23:54 +0000 Subject: [PATCH 4/6] refactor: resolve exporter and sensor setup contexts in one helper Exporter, sink and sensor providers now get their setup context from one native host helper. It returns None when the project has no providers, fails build, check and load on an unresolvable target, and ignores the model name that mv, rename and scope take as TARGET. --- .../src/compile_driver/main/setup/mod.rs | 2 +- .../main/setup/model_free_location.rs | 15 ---- .../main/setup/model_free_locations.rs | 23 ++++++ .../_helpers/build_command/lifecycle.rs | 11 +-- .../bindings/_helpers/check/check_command.rs | 11 +-- .../bindings/_helpers/cli/observability.rs | 31 +------- .../bindings/_helpers/command_runtime/mod.rs | 1 - .../_helpers/command_runtime/setup_context.rs | 52 ------------- .../src/bindings/_helpers/native_host/mod.rs | 1 + .../native_host/orchestration_sensors.rs | 47 +++++------- .../_helpers/native_host/setup_context.rs | 73 +++++++++++++++++++ fensu.toml | 5 +- .../docs/concepts/python-nodes/providers.md | 2 +- .../cli/commands/main/build/_test_types.py | 5 +- .../cli/commands/main/build/test_build.py | 62 +++++++--------- .../docs/concepts/python-nodes/providers.mdx | 2 +- 16 files changed, 156 insertions(+), 187 deletions(-) delete mode 100644 crates/sqlbuild-python-compile/src/compile_driver/main/setup/model_free_location.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/main/setup/model_free_locations.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/command_runtime/setup_context.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/native_host/setup_context.rs diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/setup/mod.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/mod.rs index cf0424c240..2e7fb32797 100644 --- a/crates/sqlbuild-python-compile/src/compile_driver/main/setup/mod.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/mod.rs @@ -11,7 +11,7 @@ pub mod compile_project_phase; pub mod compiled_phase; pub mod expanded_loader_namespace; pub mod managed_write_schema; -pub mod model_free_location; +pub mod model_free_locations; pub mod plan_has_executable_work; pub mod preserved_loader_schemas; pub mod progress; diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/setup/model_free_location.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/model_free_location.rs deleted file mode 100644 index 1b9dd80177..0000000000 --- a/crates/sqlbuild-python-compile/src/compile_driver/main/setup/model_free_location.rs +++ /dev/null @@ -1,15 +0,0 @@ -//! A target location that Python nodes can use outside any model. - -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; -use sqlbuild_model_config::templates::main::holds_context_placeholder::holds_context_placeholder; - -/// `location`, or `default` when it is unset, empty, or resolves only inside a model, such as a `${CTX:...}` schema. -pub fn model_free_location<'py>( - location: Bound<'py, PyAny>, - default: impl FnOnce() -> PyResult>, -) -> PyResult> { - match location.extract::>()? { - Some(text) if !text.is_empty() && !holds_context_placeholder(&text) => Ok(location), - _ => default(), - } -} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/setup/model_free_locations.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/model_free_locations.rs new file mode 100644 index 0000000000..38e17b630f --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/setup/model_free_locations.rs @@ -0,0 +1,23 @@ +//! The target database and schema that Python nodes use outside any model. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_model_config::templates::main::holds_context_placeholder::holds_context_placeholder; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The compiled project's target database and schema, each from `default` with the adapter method's name when it is unset, empty, or resolves only inside a model, such as a `${CTX:...}` schema. +pub fn model_free_locations<'py>( + project: &Bound<'py, PyAny>, + default: impl Fn(&str) -> PyResult>, +) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { + let location = |field: &str, fallback: &str| { + let value = native_field(project, field)?; + match value.extract::>()? { + Some(text) if !text.is_empty() && !holds_context_placeholder(&text) => Ok(value), + _ => default(fallback), + } + }; + Ok(( + location("effective_target_database", "default_database")?, + location("effective_target_schema", "default_schema")?, + )) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs index 2d8a769536..c0dbe28177 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs @@ -28,7 +28,7 @@ use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_key use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_compile::compile_driver::main::setup::build_project_relation_targets::build_project_relation_targets; use sqlbuild_python_compile::compile_driver::main::setup::build_python_relation_targets::build_python_relation_targets; -use sqlbuild_python_compile::compile_driver::main::setup::model_free_location::model_free_location; +use sqlbuild_python_compile::compile_driver::main::setup::model_free_locations::model_free_locations; use sqlbuild_python_compile::compile_driver::main::setup::project_target_settings::project_target_settings; use sqlbuild_python_compile::compile_driver::models::TargetSettingsValues; use sqlbuild_python_project::graph::main::python_nodes::build_python_sql_run_lifecycle::build_python_sql_run_lifecycle; @@ -437,14 +437,7 @@ pub(crate) fn prepare_direct_python_lifecycle<'py>( connection_config: &connection_config, project: &project, reload: context.request_flag("reload_sources")?, - defaults: ( - model_free_location(native_field(&project, "effective_target_database")?, || { - adapter.call_method0("default_database") - })?, - model_free_location(native_field(&project, "effective_target_schema")?, || { - adapter.call_method0("default_schema") - })?, - ), + defaults: model_free_locations(&project, |method| adapter.call_method0(method))?, target_config: project_target_settings( &discovered, &project, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs index 9606b65442..5d0df371b9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs @@ -26,7 +26,7 @@ use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_compile::compile_driver::main::setup::build_project_relation_targets::build_project_relation_targets; use sqlbuild_python_compile::compile_driver::main::setup::build_python_relation_targets::build_python_relation_targets; use sqlbuild_python_compile::compile_driver::main::setup::build_static_pipeline_result::build_static_pipeline_result; -use sqlbuild_python_compile::compile_driver::main::setup::model_free_location::model_free_location; +use sqlbuild_python_compile::compile_driver::main::setup::model_free_locations::model_free_locations; use sqlbuild_python_compile::compile_driver::main::setup::project_target_settings::project_target_settings; use sqlbuild_python_compile::compile_driver::models::TargetSettingsValues; use sqlbuild_python_front::discovery::main::sources::consumer_uses::consumer_uses; @@ -356,14 +356,7 @@ fn prepare<'py>( lifecycle_plan, relation_targets, relation_refs, - defaults: ( - model_free_location(native_field(&project, "effective_target_database")?, || { - adapter.call_method0("default_database") - })?, - model_free_location(native_field(&project, "effective_target_schema")?, || { - adapter.call_method0("default_schema") - })?, - ), + defaults: model_free_locations(&project, |method| adapter.call_method0(method))?, project_relations: object( py, build_project_relation_targets( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs index 88219e48c5..7cdd01251c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs @@ -15,8 +15,8 @@ use sqlbuild_cli::dispatch::models::{ExportSummary, ExporterLoss}; use sqlbuild_core::event_export::constants::DEFAULT_EVENT_EXPORT_SHUTDOWN_TIMEOUT_SECONDS; use sqlbuild_host::entries::models::{ExecutorHost, UserCodeHost}; -use crate::bindings::_helpers::command_runtime::setup_context::project_setup_context; use crate::bindings::_helpers::event_exporting::dispatcher::EventExporterDispatcher; +use crate::bindings::_helpers::native_host::setup_context::invocation_setup_context; use crate::bindings::_helpers::recall::console::write_stderr; use crate::bindings::_helpers::result_output::terminal_index::TerminalEventIndex; use crate::bindings::_helpers::result_output::terminal_index::terminal_event_index_scope; @@ -39,8 +39,6 @@ const DEBUG_COMMAND: &str = "debug"; const LINEAGE_COMMAND: &str = "lineage"; /// Compile's phase operations stay out of project lifecycle sinks; its invocation events do not. const COMPILE_COMMAND: &str = "compile"; -/// Commands whose own provider sessions get the target: their exporter and sink providers fail startup, rather than get `ctx` set to `None`, when the target can't be resolved. -const TARGET_COMMANDS: [&str; 3] = ["build", "check", "load"]; const COMPILE_WITHHELD_KINDS: &[&str] = &["operation"]; const INVOCATION_COMPLETED: &str = "invocation_completed"; const INVOCATION_FAILED: &str = "invocation_failed"; @@ -411,32 +409,7 @@ impl Observed { .map(pyo3::Bound::into_any)?; self.exporter_scope = Some(scope.clone().unbind()); let providers: Bound<'_, PyAny> = native_field(&extensions, "providers")?; - let setup_context: Bound<'_, PyAny> = if providers.is_truthy()? { - let command: Option = native_field(args, "command")?.extract()?; - let resolve = || { - project_setup_context( - project_dir, - ( - &args - .getattr("target") - .unwrap_or_else(|_| py.None().into_bound(py)), - &args - .getattr("vars") - .unwrap_or_else(|_| py.None().into_bound(py)), - ), - command - .is_some_and(|command| TARGET_COMMANDS.contains(&command.as_str())), - ) - }; - // Resolving the target imports project adapters, which must not write to JSON stdout. - if machine_output_mode(args)? { - within(&redirect_stdout_to_stderr(py)?, |_| resolve())? - } else { - resolve()? - } - } else { - py.None().into_bound(py) - }; + let setup_context = invocation_setup_context(args, (project_dir, &providers))?; call_with( &native_field(&scope, "configure_extensions")?, &[ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/mod.rs index 34d9be14a1..4d947aefd7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/mod.rs @@ -3,6 +3,5 @@ pub(crate) mod invocation; pub(crate) mod preview_target; pub(crate) mod registration; pub(crate) mod runtime_errors; -pub(crate) mod setup_context; pub(crate) mod warehouse_scope; pub(crate) mod warehouses; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/setup_context.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/setup_context.rs deleted file mode 100644 index 0ff95fd171..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/setup_context.rs +++ /dev/null @@ -1,52 +0,0 @@ -//! The `ctx` a provider's `setup` receives outside a command's own session. - -use pyo3::exceptions::PyException; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use sqlbuild_python_compile::compile_driver::main::setup::selected_target_settings::selected_target_settings; -use sqlbuild_python_front::compile_attachment::main::build_effective_runtime_config::build_effective_runtime_config; - -use crate::bindings::_helpers::command_runtime::connection::project_connection_config; -use crate::bindings::_helpers::query_command::project::{ - ProjectAdapter, ProjectInputs, project_adapter, -}; - -/// The settings of `target` (the default target when `None`) for the project at `project_dir`, as a provider setup context, such as for an event exporter or a sensor. When the project can't resolve them, the error propagates if the target is `required`, and the context is `None` otherwise. -pub(crate) fn project_setup_context<'py>( - project_dir: &Bound<'py, PyAny>, - (target, cli_vars): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), - required: bool, -) -> PyResult> { - let py: Python<'py> = project_dir.py(); - let resolved = || -> PyResult> { - let ProjectAdapter { - project_dir, - discovered, - adapter, - } = project_adapter(py, Some(project_dir.clone()), ProjectInputs::Configuration)?; - let connection: Bound<'py, PyAny> = project_connection_config( - &discovered, - (&project_dir, target), - (cli_vars, false), - false, - )?; - let (_, effective_vars, _): (Bound<'py, PyAny>, Bound<'py, PyAny>, Bound<'py, PyAny>) = - build_effective_runtime_config( - &discovered, - Some(target.clone()).filter(|value| !value.is_none()), - Some(cli_vars.clone()).filter(|value| !value.is_none()), - )? - .extract()?; - selected_target_settings( - (&discovered, target), - &adapter.getattr("adapter_name")?.extract::()?, - (&connection, &effective_vars, &adapter), - )? - .setup_context(py) - }; - match resolved() { - Err(error) if !required && error.is_instance_of::(py) => { - Ok(py.None().into_bound(py)) - } - context => context, - } -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs index bf73124167..d1d5094f01 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs @@ -3,5 +3,6 @@ pub(crate) mod orchestration_sensors; pub(crate) mod registration; pub(crate) mod resource_costs_state; +pub(crate) mod setup_context; pub(crate) mod statement_report; pub(crate) mod warehouse_metadata; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_sensors.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_sensors.rs index 63d695c952..18ade8b1b8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_sensors.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_sensors.rs @@ -5,13 +5,13 @@ use std::path::Path; use pyo3::IntoPyObject; use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::{PyBool, PyDict, PyDictMethods}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyString}; use sqlbuild_host::entries::models::UserCodeHost; use sqlbuild_orchestration::sensors::models::{ CustomSensorOutcome, SensorCondition, SensorEvaluationRequest, }; -use crate::bindings::_helpers::command_runtime::setup_context::project_setup_context; +use crate::bindings::_helpers::native_host::setup_context::project_setup_context; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::within::within; @@ -74,30 +74,14 @@ pub(crate) fn evaluate_sensor( .import("sqlbuild.orchestration")? .getattr("SensorContext")? .call((), Some(&fields))?; - let providers = discover_provider_classes( - imported(py, "pathlib", "Path")?.call1((project_dir,))?, - SecretPolicy::Defer.name(), - )?; + let project_path = imported(py, "pathlib", "Path")?.call1((project_dir,))?; + let providers = discover_provider_classes(project_path.clone(), SecretPolicy::Defer.name())?; let by_name = provider_by_name(&providers)?; - let mut usages: Vec> = Vec::new(); - for usage in provider_usages(&function, &by_name)?.try_iter()? { - let usage = usage?; - if usage.getattr("parameter_name")?.extract::()? != SENSOR_CONTEXT_PARAMETER { - usages.push(usage); - } - } - let setup_context = if usages.is_empty() { - py.None().into_bound(py) - } else { - project_setup_context( - &imported(py, "pathlib", "Path")?.call1((project_dir,))?, - ( - &evaluation.target.as_str().into_pyobject(py)?.into_any(), - &py.None().into_bound(py), - ), - true, - )? - }; + let target = ( + &PyString::new(py, &evaluation.target).into_any(), + &py.None().into_bound(py), + ); + let setup_context = project_setup_context((&project_path, &providers), target, true)?; let session = call_with( &entry(py, UserCodeHost::BuildProviderSession)?, &[ @@ -109,11 +93,14 @@ pub(crate) fn evaluate_sensor( within(&session, |_| { let arguments = PyDict::new(py); arguments.set_item("ctx", &context)?; - for usage in &usages { - arguments.set_item( - usage.getattr("parameter_name")?, - session.call_method1("get", (usage.getattr("provider_name")?,))?, - )?; + for usage in provider_usages(&function, &by_name)?.try_iter()? { + let usage = usage?; + if usage.getattr("parameter_name")?.extract::()? != SENSOR_CONTEXT_PARAMETER { + arguments.set_item( + usage.getattr("parameter_name")?, + session.call_method1("get", (usage.getattr("provider_name")?,))?, + )?; + } } let result = function.call((), Some(&arguments))?; if result.is_instance(&authoring.getattr("SkipReason")?)? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/setup_context.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/setup_context.rs new file mode 100644 index 0000000000..7a01178c87 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/setup_context.rs @@ -0,0 +1,73 @@ +//! The `ctx` a provider's `setup` receives outside a command's own session. + +use pyo3::exceptions::PyException; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::boundary::main::within::within; +use sqlbuild_python_base::native_host::main::redirect_stdout_to_stderr::redirect_stdout_to_stderr; +use sqlbuild_python_compile::compile_driver::main::setup::selected_target_settings::selected_target_settings; +use sqlbuild_python_front::compile_attachment::main::build_effective_runtime_config::build_effective_runtime_config; + +use crate::bindings::_helpers::command_runtime::connection::project_connection_config; +use crate::bindings::_helpers::query_command::project::{ProjectInputs, project_adapter}; + +/// Commands whose own provider sessions get the target, so their exporter and sink providers fail startup rather than get `None`. +const TARGET_COMMANDS: [&str; 3] = ["build", "check", "load"]; +/// Commands whose `TARGET` names a model or scope rather than a warehouse target. +const MODEL_TARGET_COMMANDS: [&str; 3] = ["mv", "rename", "scope"]; + +/// The setup context of the CLI invocation `args`'s exporter and sink providers: the settings of its `--target`, or of the default target. +pub(crate) fn invocation_setup_context<'py>( + args: &Bound<'py, PyAny>, + (project_dir, providers): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult> { + let command: Option = args.getattr("command")?.extract()?; + let command = command.as_deref().unwrap_or_default(); + let target = if MODEL_TARGET_COMMANDS.contains(&command) { + args.py().None().into_bound(args.py()) + } else { + args.getattr("target")? + }; + let target_settings = (&target, &args.getattr("vars")?); + project_setup_context( + (project_dir, providers), + target_settings, + TARGET_COMMANDS.contains(&command), + ) +} + +/// The settings of `target` (the default target when `None`) for the project at `project_dir`, as a setup context for its discovered `providers`, such as for an event exporter or a sensor; `None` when it has no providers. When the project can't resolve them, the error propagates if the target is `required`, and the context is `None` otherwise. Project code imported on the way writes to stderr, so machine-readable stdout stays valid. +pub(crate) fn project_setup_context<'py>( + (project_dir, providers): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (target, cli_vars): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + required: bool, +) -> PyResult> { + let py = project_dir.py(); + if !providers.is_truthy()? { + return Ok(py.None().into_bound(py)); + } + let resolved = within(&redirect_stdout_to_stderr(py)?, |_| { + let project = project_adapter(py, Some(project_dir.clone()), ProjectInputs::Configuration)?; + let connection = project_connection_config( + &project.discovered, + (&project.project_dir, target), + (cli_vars, false), + false, + )?; + let vars = build_effective_runtime_config( + &project.discovered, + Some(target.clone()).filter(|value| !value.is_none()), + Some(cli_vars.clone()).filter(|value| !value.is_none()), + )? + .get_item(1)?; + let adapter_name: String = project.adapter.getattr("adapter_name")?.extract()?; + let settings = (&connection, &vars, &project.adapter); + selected_target_settings((&project.discovered, target), &adapter_name, settings)? + .setup_context(py) + }); + match resolved { + Err(error) if !required && error.is_instance_of::(py) => { + Ok(py.None().into_bound(py)) + } + context => context, + } +} diff --git a/fensu.toml b/fensu.toml index 2fe267411b..3ce398a99b 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1670,7 +1670,7 @@ select = ["FPRS", "XSB072", "XSB073", "XSB075", "XSB076", "XSB077", "XSB078", "X [[targets.rust.threshold_overrides]] paths = ["crates/sqlbuild-python/Cargo.toml"] -thresholds = { max_crate_code_lines = 66831 } +thresholds = { max_crate_code_lines = 66875 } reason = "The Python bindings crate is above the code-line limit; this caps it at its current size until it is split into crates by area. Lower the cap as code moves out." [[targets.rust.threshold_overrides]] @@ -2627,7 +2627,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs=16", "crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs=11", "crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs=21", + "crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs=20", "crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/build_command/policies.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/build_progress/audit_progress.rs=1", @@ -3119,6 +3119,7 @@ module_layers = [ "model_config=registry", "native_host=front", "native_host/orchestration_sensors=commands", + "native_host/setup_context=commands", "native_host/resource_costs_state=front", "native_host/warehouse_metadata=commands", "observations=plan", diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/providers.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/providers.md index d93cd2dd59..b1421053e7 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/providers.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/providers.md @@ -197,7 +197,7 @@ Details: - When the project can't resolve the target's settings, for example because the target names a connection that isn't defined, `sqb build`, `sqb check` and `sqb load` fail before any exporter or sink provider is set up. - Other commands still run in that case, and their exporter and sink providers get `ctx` set to `None`. `sqb format` is one example. -- A sensor evaluation whose providers can't get its target's settings fails, and the sensor's cursor doesn't advance. +- In a project that defines providers, a sensor evaluation whose target's settings can't be resolved fails, and the sensor's cursor doesn't advance. - Check for `None` in a provider that calendars, exporters or sinks use. ## Configuration from environment variables diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py index cb3ed2e9ec..a444f30373 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py @@ -1642,7 +1642,7 @@ class PythonTargetConfigBuildE2ETestCase: project_target_lines: str local_target_lines: str expected_settings: dict[str, object] - lifecycle_sink: bool = False + extra_files: tuple[tuple[str, str], ...] = () @dataclass(frozen=True) @@ -1651,7 +1651,8 @@ class UnresolvedTargetSinkE2ETestCase: description: str command: tuple[str, ...] - expected_success: bool + expected_returncode: int + expected_output: str expected_setups: list[object] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_build.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_build.py index 6f94a6a2cf..acab6ff2d2 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_build.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_build.py @@ -47,6 +47,13 @@ " with Path(__file__).parents[1].joinpath('target_config.jsonl').open('a') as out:\n" " out.write(json.dumps(line) + '\\n')\n" ) +_TARGET_PROBE_SINK: str = ( + "from providers.target_probe import TargetProbe\n" + "from sqlbuild.sinks import LifecycleEvent, lifecycle_event_sink\n\n" + "@lifecycle_event_sink\n" + "def probe_events(event: LifecycleEvent, target_probe: TargetProbe) -> None:\n" + " del event, target_probe\n" +) @pytest.mark.parametrize( @@ -1848,7 +1855,7 @@ def test_given_loader_status_result_when_building_then_persists_loader_status_ro "database_preserved": False, "schema_preserved": False, }, - lifecycle_sink=True, + extra_files=(("sinks/events.py", _TARGET_PROBE_SINK),), ), PythonTargetConfigBuildE2ETestCase( description="loader_schema falls back to the target schema", @@ -1931,21 +1938,12 @@ def test_given_local_target_override_when_running_python_then_every_context_sees " - name: order_id\n" " type: INTEGER\n" ), + **dict(test_case.extra_files), }, ) (project_dir / "sqlbuild_local.toml").write_text( "[targets.dev]\n" + test_case.local_target_lines, encoding="utf-8" ) - if test_case.lifecycle_sink: - (project_dir / "sinks").mkdir() - (project_dir / "sinks" / "events.py").write_text( - "from providers.target_probe import TargetProbe\n" - "from sqlbuild.sinks import LifecycleEvent, lifecycle_event_sink\n\n" - "@lifecycle_event_sink\n" - "def probe_events(event: LifecycleEvent, target_probe: TargetProbe) -> None:\n" - " del event, target_probe\n", - encoding="utf-8", - ) results: list[subprocess.CompletedProcess[str]] = [ run_sqb(command=("--no-color", *command), project_dir=project_dir) @@ -1982,13 +1980,15 @@ def test_given_local_target_override_when_running_python_then_every_context_sees UnresolvedTargetSinkE2ETestCase( description="build fails before any sink provider is set up", command=("build",), - expected_success=False, + expected_returncode=1, + expected_output="Unknown connection 'warehouse' selected by target", expected_setups=[], ), UnresolvedTargetSinkE2ETestCase( description="format needs no target and sets the provider up with None", command=("format", "--check"), - expected_success=True, + expected_returncode=0, + expected_output="Completed.", expected_setups=[None], ), ], @@ -2025,13 +2025,8 @@ def test_given_unresolvable_target_when_running_with_a_sink_then_only_target_fre " with Path(__file__).parents[1].joinpath('setups.jsonl').open('a') as out:\n" " out.write(json.dumps(line) + '\\n')\n" ), - "sinks/events.py": ( - "from providers.target_probe import TargetProbe\n" - "from sqlbuild.sinks import LifecycleEvent, lifecycle_event_sink\n\n" - "@lifecycle_event_sink\n" - "def probe_events(event: LifecycleEvent, target_probe: TargetProbe) -> None:\n" - " del event, target_probe\n" - ), + "sinks/events.py": _TARGET_PROBE_SINK, + "setups.jsonl": "", }, ) @@ -2039,15 +2034,12 @@ def test_given_unresolvable_target_when_running_with_a_sink_then_only_target_fre command=("--no-color", *test_case.command), project_dir=project_dir ) - assert (result.returncode == 0) == test_case.expected_success, result.stdout + result.stderr - if not test_case.expected_success: - assert "Unknown connection 'warehouse'" in result.stdout + result.stderr - setups_path: Path = project_dir / "setups.jsonl" - setups: list[object] = ( - [json.loads(line) for line in setups_path.read_text(encoding="utf-8").splitlines()] - if setups_path.exists() - else [] - ) + assert result.returncode == test_case.expected_returncode, result.stdout + result.stderr + assert test_case.expected_output in result.stdout + result.stderr + setups: list[object] = [ + json.loads(line) + for line in (project_dir / "setups.jsonl").read_text(encoding="utf-8").splitlines() + ] assert setups == test_case.expected_setups @@ -2136,13 +2128,13 @@ def test_given_ctx_templated_target_schema_when_running_then_python_sees_no_load ], recorded # Python node results land in the adapter's default schema; no schema keeps the template. assert table_exists(db_path=db_path, table_name="_sqlbuild_node_results", schema="main") - schemas: list[str] = [ - str(row[0]) - for row in query_duckdb( - db_path=db_path, sql="SELECT schema_name FROM information_schema.schemata" + assert ( + query_duckdb( + db_path=db_path, + sql="SELECT schema_name FROM information_schema.schemata WHERE schema_name LIKE '%${%'", ) - ] - assert not [schema for schema in schemas if "${" in schema], schemas + == [] + ) @pytest.mark.parametrize( diff --git a/website/src/content/docs/docs/concepts/python-nodes/providers.mdx b/website/src/content/docs/docs/concepts/python-nodes/providers.mdx index 368e622808..f9fc12b6b2 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/providers.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/providers.mdx @@ -179,7 +179,7 @@ Details: - When the project can't resolve the target's settings, for example because the target names a connection that isn't defined, `sqb build`, `sqb check` and `sqb load` fail before any exporter or sink provider is set up. - Other commands still run in that case, and their exporter and sink providers get `ctx` set to `None`. `sqb format` is one example. -- A sensor evaluation whose providers can't get its target's settings fails, and the sensor's cursor doesn't advance. +- In a project that defines providers, a sensor evaluation whose target's settings can't be resolved fails, and the sensor's cursor doesn't advance. - Check for `None` in a provider that calendars, exporters or sinks use. ## Configuration from environment variables From 4fec8373524a8ef5b58f1a1630b5a9c7a856e2fd Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 17 Nov 2026 00:41:01 +0000 Subject: [PATCH 5/6] fix: fail every target command when exporter providers can't get the target Compile, dag, plan, test, seed, query, freshness, audit and debug need the target too, and already fail on an unresolvable one. They now fail before setting up exporter and sink providers, like build, check and load, so only target-free commands give those providers None. --- .../_helpers/native_host/setup_context.rs | 34 ++++++++++--------- fensu.toml | 2 +- .../docs/concepts/python-nodes/providers.md | 4 +-- .../cli/commands/main/build/test_build.py | 7 ++++ .../docs/concepts/python-nodes/providers.mdx | 4 +-- 5 files changed, 30 insertions(+), 21 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/setup_context.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/setup_context.rs index 7a01178c87..747ca1ac92 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/setup_context.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/setup_context.rs @@ -10,10 +10,21 @@ use sqlbuild_python_front::compile_attachment::main::build_effective_runtime_con use crate::bindings::_helpers::command_runtime::connection::project_connection_config; use crate::bindings::_helpers::query_command::project::{ProjectInputs, project_adapter}; -/// Commands whose own provider sessions get the target, so their exporter and sink providers fail startup rather than get `None`. -const TARGET_COMMANDS: [&str; 3] = ["build", "check", "load"]; -/// Commands whose `TARGET` names a model or scope rather than a warehouse target. -const MODEL_TARGET_COMMANDS: [&str; 3] = ["mv", "rename", "scope"]; +/// Commands that use the target, so their exporter and sink providers fail startup on an unresolvable target rather than get `None`. +const TARGET_COMMANDS: [&str; 12] = [ + "audit", + "build", + "check", + "compile", + "dag", + "debug", + "freshness", + "load", + "plan", + "query", + "seed", + "test", +]; /// The setup context of the CLI invocation `args`'s exporter and sink providers: the settings of its `--target`, or of the default target. pub(crate) fn invocation_setup_context<'py>( @@ -21,18 +32,9 @@ pub(crate) fn invocation_setup_context<'py>( (project_dir, providers): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), ) -> PyResult> { let command: Option = args.getattr("command")?.extract()?; - let command = command.as_deref().unwrap_or_default(); - let target = if MODEL_TARGET_COMMANDS.contains(&command) { - args.py().None().into_bound(args.py()) - } else { - args.getattr("target")? - }; - let target_settings = (&target, &args.getattr("vars")?); - project_setup_context( - (project_dir, providers), - target_settings, - TARGET_COMMANDS.contains(&command), - ) + let required = TARGET_COMMANDS.contains(&command.as_deref().unwrap_or_default()); + let target_settings = (&args.getattr("target")?, &args.getattr("vars")?); + project_setup_context((project_dir, providers), target_settings, required) } /// The settings of `target` (the default target when `None`) for the project at `project_dir`, as a setup context for its discovered `providers`, such as for an event exporter or a sensor; `None` when it has no providers. When the project can't resolve them, the error propagates if the target is `required`, and the context is `None` otherwise. Project code imported on the way writes to stderr, so machine-readable stdout stays valid. diff --git a/fensu.toml b/fensu.toml index 3ce398a99b..c9f1a2266f 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1670,7 +1670,7 @@ select = ["FPRS", "XSB072", "XSB073", "XSB075", "XSB076", "XSB077", "XSB078", "X [[targets.rust.threshold_overrides]] paths = ["crates/sqlbuild-python/Cargo.toml"] -thresholds = { max_crate_code_lines = 66875 } +thresholds = { max_crate_code_lines = 66878 } reason = "The Python bindings crate is above the code-line limit; this caps it at its current size until it is split into crates by area. Lower the cap as code moves out." [[targets.rust.threshold_overrides]] diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/providers.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/providers.md index b1421053e7..23bb7bb11b 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/providers.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/providers.md @@ -195,8 +195,8 @@ Which target `ctx.target_config` describes depends on what uses the provider: Details: -- When the project can't resolve the target's settings, for example because the target names a connection that isn't defined, `sqb build`, `sqb check` and `sqb load` fail before any exporter or sink provider is set up. -- Other commands still run in that case, and their exporter and sink providers get `ctx` set to `None`. `sqb format` is one example. +- When the project can't resolve the target's settings, for example because the target names a connection that isn't defined, every command that uses the target, such as `sqb build`, `sqb compile` or `sqb query`, fails before any exporter or sink provider is set up. +- Commands that don't use the target, such as `sqb format`, still run, and their exporter and sink providers get `ctx` set to `None`. - In a project that defines providers, a sensor evaluation whose target's settings can't be resolved fails, and the sensor's cursor doesn't advance. - Check for `None` in a provider that calendars, exporters or sinks use. diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_build.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_build.py index acab6ff2d2..3ccb67b8a2 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_build.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_build.py @@ -1984,6 +1984,13 @@ def test_given_local_target_override_when_running_python_then_every_context_sees expected_output="Unknown connection 'warehouse' selected by target", expected_setups=[], ), + UnresolvedTargetSinkE2ETestCase( + description="compile also needs the target and fails before provider setup", + command=("compile",), + expected_returncode=1, + expected_output="Unknown connection 'warehouse' selected by target", + expected_setups=[], + ), UnresolvedTargetSinkE2ETestCase( description="format needs no target and sets the provider up with None", command=("format", "--check"), diff --git a/website/src/content/docs/docs/concepts/python-nodes/providers.mdx b/website/src/content/docs/docs/concepts/python-nodes/providers.mdx index f9fc12b6b2..05e8e8eab6 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/providers.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/providers.mdx @@ -177,8 +177,8 @@ Which target `ctx.target_config` describes depends on what uses the provider: Details: -- When the project can't resolve the target's settings, for example because the target names a connection that isn't defined, `sqb build`, `sqb check` and `sqb load` fail before any exporter or sink provider is set up. -- Other commands still run in that case, and their exporter and sink providers get `ctx` set to `None`. `sqb format` is one example. +- When the project can't resolve the target's settings, for example because the target names a connection that isn't defined, every command that uses the target, such as `sqb build`, `sqb compile` or `sqb query`, fails before any exporter or sink provider is set up. +- Commands that don't use the target, such as `sqb format`, still run, and their exporter and sink providers get `ctx` set to `None`. - In a project that defines providers, a sensor evaluation whose target's settings can't be resolved fails, and the sensor's cursor doesn't advance. - Check for `None` in a provider that calendars, exporters or sinks use. From fbd2135a1c55f02dbadc07d35ba35be4aa279f70 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 17 Nov 2026 00:46:18 +0000 Subject: [PATCH 6/6] style: format the custom sensor test --- .../compiled_project/publication/test_custom_sensor.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_custom_sensor.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_custom_sensor.py index 702547713b..4ada34b1eb 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_custom_sensor.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_custom_sensor.py @@ -183,9 +183,9 @@ def test_given_a_custom_polling_sensor_when_a_callback_fails_or_times_out_then_a evaluated: list[dict[str, Any]] = sensor_calls(calls, {"phase": "evaluate"}) assert {record["target"] for record in evaluated} == {"development"} # The provider is set up with the settings of the target the sensor evaluates for. - assert [ - record["target_config"] for record in sensor_calls(calls, {"phase": "setup"}) - ][: len(evaluated)] == [["development", "main"]] * len(evaluated) + assert [record["target_config"] for record in sensor_calls(calls, {"phase": "setup"})][ + : len(evaluated) + ] == [["development", "main"]] * len(evaluated) assert len({record["pid"] for record in evaluated}) == len(evaluated) assert { record["cursor"]