Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
//! The `${CTX:key}` and `${name}` placeholders of a deferred target's schema or database.

/// The deferred-target context marker inside `${...}`.
const CONTEXT_PREFIX: &str = "CTX:";
pub(crate) const CONTEXT_PREFIX: &str = "CTX:";
/// The opening of a `${...}` placeholder.
pub(crate) const PLACEHOLDER_OPEN: &str = "${";
const PLACEHOLDER_CLOSE: char = '}';
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
//! Whether a target's schema or database resolves only inside a model.

use crate::templates::_helpers::deferred::{CONTEXT_PREFIX, PLACEHOLDER_OPEN, placeholders};

/// Whether `template` holds a `${CTX:key}` placeholder, so it names a location only inside a model.
pub fn holds_context_placeholder(template: &str) -> bool {
!placeholders(
template,
&format!("{PLACEHOLDER_OPEN}{CONTEXT_PREFIX}"),
None,
)
.is_empty()
}
1 change: 1 addition & 0 deletions crates/sqlbuild-model-config/src/templates/main/mod.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
pub mod deferred_target_segments;
pub mod expand_template_string;
pub mod expand_template_text;
pub mod holds_context_placeholder;
pub mod template_error_message;
12 changes: 8 additions & 4 deletions crates/sqlbuild-python-base/src/event_exporting/models.rs
Original file line number Diff line number Diff line change
Expand Up @@ -380,7 +380,7 @@ impl EventExporterCommandScope {
pub(crate) fn bind_extensions<'py>(
&self,
py: Python<'py>,
project_dir: Option<Bound<'py, PyAny>>,
(project_dir, setup_context): (Option<Bound<'py, PyAny>>, Bound<'py, PyAny>),
(providers, event_exporters, command_output_sinks): (
Bound<'py, PyAny>,
Bound<'py, PyAny>,
Expand Down Expand Up @@ -413,6 +413,7 @@ impl EventExporterCommandScope {
&host_entry(py, UserCodeHost::BuildProviderSession)?,
&[
("discovered_providers", providers),
("setup_context", setup_context),
("allow_shared", PyBool::new(py, false).to_owned().into_any()),
],
)?;
Expand Down Expand Up @@ -508,7 +509,7 @@ impl EventExporterCommandScope {
) -> PyResult<()> {
self.bind_extensions(
py,
None,
(None, py.None().into_bound(py)),
(
native_field(discovered_inputs, "providers")?,
native_field(discovered_inputs, "event_exporters")?,
Expand All @@ -530,7 +531,7 @@ impl EventExporterCommandScope {
options,
(
&["providers", "event_exporters"],
&["project_dir", "command_output_sinks"],
&["project_dir", "command_output_sinks", "setup_context"],
),
)?;
let project_dir: Bound<'_, PyAny> = options.value("project_dir")?;
Expand All @@ -542,7 +543,10 @@ impl EventExporterCommandScope {
};
self.bind_extensions(
py,
(!project_dir.is_none()).then_some(project_dir),
(
(!project_dir.is_none()).then_some(project_dir),
options.value("setup_context")?,
),
(
options.value("providers")?,
options.value("event_exporters")?,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ pub mod compile_project_phase;
pub mod compiled_phase;
pub mod expanded_loader_namespace;
pub mod managed_write_schema;
pub mod model_free_locations;
pub mod plan_has_executable_work;
pub mod preserved_loader_schemas;
pub mod progress;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
//! The target database and schema that Python nodes use outside any model.

use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult};
use sqlbuild_model_config::templates::main::holds_context_placeholder::holds_context_placeholder;
use sqlbuild_python_base::records::main::fields::field::field as native_field;

/// The compiled project's target database and schema, each from `default` with the adapter method's name when it is unset, empty, or resolves only inside a model, such as a `${CTX:...}` schema.
pub fn model_free_locations<'py>(
project: &Bound<'py, PyAny>,
default: impl Fn(&str) -> PyResult<Bound<'py, PyAny>>,
) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> {
let location = |field: &str, fallback: &str| {
let value = native_field(project, field)?;
match value.extract::<Option<String>>()? {
Some(text) if !text.is_empty() && !holds_context_placeholder(&text) => Ok(value),
_ => default(fallback),
}
};
Ok((
location("effective_target_database", "default_database")?,
location("effective_target_schema", "default_schema")?,
))
}
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ use sqlbuild_python_base::boundary::main::keywords::keywords;
use sqlbuild_python_base::build_command::main::boolean::boolean;
use sqlbuild_python_base::build_command::main::emit::emit;
use sqlbuild_python_base::build_command::main::frozenset::frozenset;
use sqlbuild_python_base::build_command::main::or_none::or_none;
use sqlbuild_python_base::build_command::main::text::text;
use sqlbuild_python_base::build_command::main::tuple::tuple;
use sqlbuild_python_base::native_host::main::host_entry::host_entry;
Expand All @@ -29,6 +28,7 @@ use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_key
use sqlbuild_python_base::records::types::RecordId;
use sqlbuild_python_compile::compile_driver::main::setup::build_project_relation_targets::build_project_relation_targets;
use sqlbuild_python_compile::compile_driver::main::setup::build_python_relation_targets::build_python_relation_targets;
use sqlbuild_python_compile::compile_driver::main::setup::model_free_locations::model_free_locations;
use sqlbuild_python_compile::compile_driver::main::setup::project_target_settings::project_target_settings;
use sqlbuild_python_compile::compile_driver::models::TargetSettingsValues;
use sqlbuild_python_project::graph::main::python_nodes::build_python_sql_run_lifecycle::build_python_sql_run_lifecycle;
Expand Down Expand Up @@ -437,14 +437,7 @@ pub(crate) fn prepare_direct_python_lifecycle<'py>(
connection_config: &connection_config,
project: &project,
reload: context.request_flag("reload_sources")?,
defaults: (
or_none(native_field(&project, "effective_target_database")?, || {
adapter.call_method0("default_database")
})?,
or_none(native_field(&project, "effective_target_schema")?, || {
adapter.call_method0("default_schema")
})?,
),
defaults: model_free_locations(&project, |method| adapter.call_method0(method))?,
target_config: project_target_settings(
&discovered,
&project,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ use sqlbuild_python_base::records::main::construct::record_class::record_class;
use sqlbuild_python_compile::compile_driver::main::setup::build_project_relation_targets::build_project_relation_targets;
use sqlbuild_python_compile::compile_driver::main::setup::build_python_relation_targets::build_python_relation_targets;
use sqlbuild_python_compile::compile_driver::main::setup::build_static_pipeline_result::build_static_pipeline_result;
use sqlbuild_python_compile::compile_driver::main::setup::model_free_locations::model_free_locations;
use sqlbuild_python_compile::compile_driver::main::setup::project_target_settings::project_target_settings;
use sqlbuild_python_compile::compile_driver::models::TargetSettingsValues;
use sqlbuild_python_front::discovery::main::sources::consumer_uses::consumer_uses;
Expand Down Expand Up @@ -271,19 +272,6 @@ fn compile_pipeline<'py>(
)
}

fn target_default<'py>(
project: &Bound<'py, PyAny>,
adapter: &Bound<'py, PyAny>,
(field, fallback): (&str, &str),
) -> PyResult<Bound<'py, PyAny>> {
let value: Bound<'py, PyAny> = native_field(project, field)?;
if value.is_none() {
adapter.call_method0(fallback)
} else {
Ok(value)
}
}

/// The discovered Python graph, after refusing unit-test selectors, and the selected checks.
fn python_selection<'py>(
request: &Bound<'py, PyAny>,
Expand Down Expand Up @@ -360,18 +348,7 @@ fn prepare<'py>(
lifecycle_plan,
relation_targets,
relation_refs,
defaults: (
target_default(
&project,
&adapter,
("effective_target_database", "default_database"),
)?,
target_default(
&project,
&adapter,
("effective_target_schema", "default_schema"),
)?,
),
defaults: model_free_locations(&project, |method| adapter.call_method0(method))?,
project_relations: object(
py,
build_project_relation_targets(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ use sqlbuild_core::event_export::constants::DEFAULT_EVENT_EXPORT_SHUTDOWN_TIMEOU
use sqlbuild_host::entries::models::{ExecutorHost, UserCodeHost};

use crate::bindings::_helpers::event_exporting::dispatcher::EventExporterDispatcher;
use crate::bindings::_helpers::native_host::setup_context::invocation_setup_context;
use crate::bindings::_helpers::recall::console::write_stderr;
use crate::bindings::_helpers::result_output::terminal_index::TerminalEventIndex;
use crate::bindings::_helpers::result_output::terminal_index::terminal_event_index_scope;
Expand Down Expand Up @@ -407,13 +408,16 @@ impl Observed {
)
.map(pyo3::Bound::into_any)?;
self.exporter_scope = Some(scope.clone().unbind());
let providers: Bound<'_, PyAny> = native_field(&extensions, "providers")?;
let setup_context = invocation_setup_context(args, (project_dir, &providers))?;
call_with(
&native_field(&scope, "configure_extensions")?,
&[
("project_dir", project_dir.clone()),
("providers", native_field(&extensions, "providers")?),
("providers", providers),
("event_exporters", event_exporters),
("command_output_sinks", sinks),
("setup_context", setup_context),
],
)?;
self.unsubscribe_exporters =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,5 +3,6 @@
pub(crate) mod orchestration_sensors;
pub(crate) mod registration;
pub(crate) mod resource_costs_state;
pub(crate) mod setup_context;
pub(crate) mod statement_report;
pub(crate) mod warehouse_metadata;
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,13 @@ use std::path::Path;
use pyo3::IntoPyObject;
use pyo3::exceptions::PyValueError;
use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python};
use pyo3::types::{PyBool, PyDict, PyDictMethods};
use pyo3::types::{PyBool, PyDict, PyDictMethods, PyString};
use sqlbuild_host::entries::models::UserCodeHost;
use sqlbuild_orchestration::sensors::models::{
CustomSensorOutcome, SensorCondition, SensorEvaluationRequest,
};

use crate::bindings::_helpers::native_host::setup_context::project_setup_context;
use sqlbuild_python_base::boundary::main::call_with::call_with;
use sqlbuild_python_base::boundary::main::imported::imported;
use sqlbuild_python_base::boundary::main::within::within;
Expand Down Expand Up @@ -73,15 +74,19 @@ pub(crate) fn evaluate_sensor(
.import("sqlbuild.orchestration")?
.getattr("SensorContext")?
.call((), Some(&fields))?;
let providers = discover_provider_classes(
imported(py, "pathlib", "Path")?.call1((project_dir,))?,
SecretPolicy::Defer.name(),
)?;
let project_path = imported(py, "pathlib", "Path")?.call1((project_dir,))?;
let providers = discover_provider_classes(project_path.clone(), SecretPolicy::Defer.name())?;
let by_name = provider_by_name(&providers)?;
let target = (
&PyString::new(py, &evaluation.target).into_any(),
&py.None().into_bound(py),
);
let setup_context = project_setup_context((&project_path, &providers), target, true)?;
let session = call_with(
&entry(py, UserCodeHost::BuildProviderSession)?,
&[
("discovered_providers", providers),
("setup_context", setup_context),
("allow_shared", PyBool::new(py, false).to_owned().into_any()),
],
)?;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
//! The `ctx` a provider's `setup` receives outside a command's own session.

use pyo3::exceptions::PyException;
use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult};
use sqlbuild_python_base::boundary::main::within::within;
use sqlbuild_python_base::native_host::main::redirect_stdout_to_stderr::redirect_stdout_to_stderr;
use sqlbuild_python_compile::compile_driver::main::setup::selected_target_settings::selected_target_settings;
use sqlbuild_python_front::compile_attachment::main::build_effective_runtime_config::build_effective_runtime_config;

use crate::bindings::_helpers::command_runtime::connection::project_connection_config;
use crate::bindings::_helpers::query_command::project::{ProjectInputs, project_adapter};

/// Commands that use the target, so their exporter and sink providers fail startup on an unresolvable target rather than get `None`.
const TARGET_COMMANDS: [&str; 12] = [
"audit",
"build",
"check",
"compile",
"dag",
"debug",
"freshness",
"load",
"plan",
"query",
"seed",
"test",
];

/// The setup context of the CLI invocation `args`'s exporter and sink providers: the settings of its `--target`, or of the default target.
pub(crate) fn invocation_setup_context<'py>(
args: &Bound<'py, PyAny>,
(project_dir, providers): (&Bound<'py, PyAny>, &Bound<'py, PyAny>),
) -> PyResult<Bound<'py, PyAny>> {
let command: Option<String> = args.getattr("command")?.extract()?;
let required = TARGET_COMMANDS.contains(&command.as_deref().unwrap_or_default());
let target_settings = (&args.getattr("target")?, &args.getattr("vars")?);
project_setup_context((project_dir, providers), target_settings, required)
}

/// The settings of `target` (the default target when `None`) for the project at `project_dir`, as a setup context for its discovered `providers`, such as for an event exporter or a sensor; `None` when it has no providers. When the project can't resolve them, the error propagates if the target is `required`, and the context is `None` otherwise. Project code imported on the way writes to stderr, so machine-readable stdout stays valid.
pub(crate) fn project_setup_context<'py>(
(project_dir, providers): (&Bound<'py, PyAny>, &Bound<'py, PyAny>),
(target, cli_vars): (&Bound<'py, PyAny>, &Bound<'py, PyAny>),
required: bool,
) -> PyResult<Bound<'py, PyAny>> {
let py = project_dir.py();
if !providers.is_truthy()? {
return Ok(py.None().into_bound(py));
}
let resolved = within(&redirect_stdout_to_stderr(py)?, |_| {
let project = project_adapter(py, Some(project_dir.clone()), ProjectInputs::Configuration)?;
let connection = project_connection_config(
&project.discovered,
(&project.project_dir, target),
(cli_vars, false),
false,
)?;
let vars = build_effective_runtime_config(
&project.discovered,
Some(target.clone()).filter(|value| !value.is_none()),
Some(cli_vars.clone()).filter(|value| !value.is_none()),
)?
.get_item(1)?;
let adapter_name: String = project.adapter.getattr("adapter_name")?.extract()?;
let settings = (&connection, &vars, &project.adapter);
selected_target_settings((&project.discovered, target), &adapter_name, settings)?
.setup_context(py)
});
match resolved {
Err(error) if !required && error.is_instance_of::<PyException>(py) => {
Ok(py.None().into_bound(py))
}
context => context,
}
}
5 changes: 3 additions & 2 deletions fensu.toml
Original file line number Diff line number Diff line change
Expand Up @@ -1670,7 +1670,7 @@ select = ["FPRS", "XSB072", "XSB073", "XSB075", "XSB076", "XSB077", "XSB078", "X

[[targets.rust.threshold_overrides]]
paths = ["crates/sqlbuild-python/Cargo.toml"]
thresholds = { max_crate_code_lines = 66709 }
thresholds = { max_crate_code_lines = 66756 }
reason = "The Python bindings crate is above the code-line limit; this caps it at its current size until it is split into crates by area. Lower the cap as code moves out."

[[targets.rust.threshold_overrides]]
Expand Down Expand Up @@ -2625,7 +2625,7 @@ pending_budgets = [
"crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs=16",
"crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs=11",
"crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs=5",
"crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs=21",
"crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs=20",
"crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs=8",
"crates/sqlbuild-python/src/bindings/_helpers/build_command/policies.rs=4",
"crates/sqlbuild-python/src/bindings/_helpers/build_progress/audit_progress.rs=1",
Expand Down Expand Up @@ -3116,6 +3116,7 @@ module_layers = [
"model_config=registry",
"native_host=front",
"native_host/orchestration_sensors=commands",
"native_host/setup_context=commands",
"native_host/resource_costs_state=front",
"native_host/warehouse_metadata=commands",
"observations=plan",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -173,7 +173,7 @@ Both `setup` and `teardown` are optional. A provider with only field declaration

### The setup context

In `sqb build`, `sqb check` and `sqb load`, `setup(ctx)` receives the active target's resolved settings as `ctx.target_config`, the same values Python nodes see (see [Target settings](overview.md#target-settings)):
`setup(ctx)` receives the active target's resolved settings as `ctx.target_config`, the same values Python nodes see (see [Target settings](overview.md#target-settings)):

```python
class PartnerStage(Provider):
Expand All @@ -184,9 +184,21 @@ class PartnerStage(Provider):
self.stage = f"{target.database}.{target.loader_schema}.partner_stage"
```

Which target `ctx.target_config` describes depends on what uses the provider:

| Used by | `ctx` |
| --- | --- |
| Python nodes in `sqb build`, `sqb check` and `sqb load` | The command's target |
| Event exporters and command output sinks | The command's target: `--target`, or the project's default target |
| Orchestration sensors | The target the sensor evaluates for |
| Orchestration calendars | `None` |

Details:

- A provider first used by an event exporter or command output sink, or by an orchestration sensor or calendar, is set up with `ctx` set to `None`. Check for `None` when such a provider also reads the target.
- When the project can't resolve the target's settings, for example because the target names a connection that isn't defined, every command that uses the target, such as `sqb build`, `sqb compile` or `sqb query`, fails before any exporter or sink provider is set up.
- Commands that don't use the target, such as `sqb format`, still run, and their exporter and sink providers get `ctx` set to `None`.
- In a project that defines providers, a sensor evaluation whose target's settings can't be resolved fails, and the sensor's cursor doesn't advance.
- Check for `None` in a provider that calendars, exporters or sinks use.

## Configuration from environment variables

Expand Down
Loading
Loading