Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,7 @@ E2E_DUCKDB_BUILD_CORE_PATHS := \
tests/e2e/src/sqlbuild/cli/commands/main/build/test_defer_migration_origin.py \
tests/e2e/src/sqlbuild/cli/commands/main/build/test_sql_hook_reads.py \
tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_identity.py \
tests/e2e/src/sqlbuild/cli/commands/main/build/test_node_result_location.py \
tests/e2e/src/sqlbuild/cli/commands/main/build/test_sql_hook_identity.py \
tests/e2e/src/sqlbuild/cli/commands/main/build/test_query_change_tracking.py \
tests/e2e/src/sqlbuild/cli/commands/main/build/test_query_fingerprint_changes.py \
Expand Down
34 changes: 34 additions & 0 deletions benchmarks/duckshop/edge_cases/loaders/loader_results.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
"""Two loaders and a task that reads both of their results.
A build that selects only one loader still reads the other loader's latest result, which may come
from an earlier build. On Snowflake the adapter has no default schema, so the results must go to
the target's schema for the task to find them.
"""

from sqlbuild.executor.load.models import LoaderContext
from sqlbuild.loaders import loader
from sqlbuild.refs import source
from sqlbuild.tasks import task


@loader
def raw_order_events(ctx: LoaderContext) -> list[dict[str, object]]:
"""Order events."""
return [{"event_id": 1}]


@loader
def raw_customer_events(ctx: LoaderContext) -> list[dict[str, object]]:
"""Customer events."""
return [{"event_id": 1}]


@task(depends_on=[source("raw_order_events"), source("raw_customer_events")])
def announce_events(ctx):
"""Record the run each loader result came from."""
return ctx.result(
payload={
"orders": ctx.result_of(node_function=raw_order_events).run_id,
"customers": ctx.result_of(node_function=raw_customer_events).run_id,
}
)
9 changes: 9 additions & 0 deletions benchmarks/duckshop/edge_cases/loaders/loader_results.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
sources:
- name: raw_order_events
description: Order events, read by announce_events.
managed: true
write_strategy: table
- name: raw_customer_events
description: Customer events, read by announce_events.
managed: true
write_strategy: table
Original file line number Diff line number Diff line change
Expand Up @@ -10,5 +10,7 @@ pub const LOADER_COLLISION_CODE: &str = "S102";
pub const MANAGED_WRITE_SCHEMA_CODE: &str = "S103";
/// A named target chooses no schema strategy.
pub const NAMED_TARGET_SCHEMA_CODE: &str = "S104";
/// A target that keeps each model's schema has no schema for Python node results.
pub const PRESERVED_NODE_RESULTS_CODE: &str = "S105";
pub(crate) const DATABASE_PART: &str = "database";
pub(crate) const SCHEMA_PART: &str = "schema";
2 changes: 2 additions & 0 deletions crates/sqlbuild-analysis/src/assembly/target_checks/errors.rs
Original file line number Diff line number Diff line change
Expand Up @@ -15,4 +15,6 @@ pub enum TargetError {
NamedTargetSchema(String),
/// An intermediate loader, by name, whose schema resolves to the target's `preserve`.
PreservedLoaderSchema(String),
/// A target, by name, whose schema is `preserve` and that names no loader schema.
PreservedNodeResults(String),
}
Original file line number Diff line number Diff line change
Expand Up @@ -48,5 +48,10 @@ pub fn target_error_message(error: &TargetError) -> String {
'{PRESERVE_TARGET_VALUE}'. The target's schema '{PRESERVE_TARGET_VALUE}' keeps each \
model's own schema, and a loader has none, so its destination must name a schema."
),
TargetError::PreservedNodeResults(target) => format!(
"Target '{target}' keeps each model's own schema, so it has no schema for the \
results of loaders, tasks, assets and checks. Set targets.{target}.loader_schema; \
Python node results are kept there."
),
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
//! The adapter's own default database or schema.

use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult};

/// The adapter's `default_database` or `default_schema`, named by `method`.
pub fn adapter_default_location<'py>(
adapter: &Bound<'py, PyAny>,
method: &str,
) -> PyResult<Bound<'py, PyAny>> {
adapter.call_method0(method)
}
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
pub mod adapter_default_location;
pub mod ensure_event_id_column;
pub mod is_warehouse_access_refused;
pub mod read_catalog;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
pub(crate) mod inputs;
pub(crate) mod model_free_locations;
pub(crate) mod node_results;
pub(crate) mod pipeline;
pub(crate) mod pipeline_entries;
pub(crate) mod project;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
//! The target database and schema that Python nodes use outside any model.
use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult};
use pyo3::types::PyString;
use sqlbuild_analysis::assembly::target_checks::constants::PRESERVE_TARGET_VALUE;
use sqlbuild_model_config::templates::main::holds_context_placeholder::holds_context_placeholder;
use sqlbuild_python_base::warehouse_access::main::adapter_default_location::adapter_default_location;

/// Python nodes' `(database, schema)`: a `preserve` schema reads `loader_schema`; unusable values the adapter's.
pub(crate) fn model_free_locations<'py>(
(database, schema): (Bound<'py, PyAny>, Bound<'py, PyAny>),
loader_schema: Option<&str>,
adapter: &Bound<'py, PyAny>,
) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> {
let kept_schema: Option<&str> =
loader_schema.filter(|name| !name.is_empty() && *name != PRESERVE_TARGET_VALUE);
let location = |value: Bound<'py, PyAny>, method: &str, preserved: Option<&str>| match (
value.extract::<Option<String>>()?.as_deref(),
preserved,
) {
(Some(PRESERVE_TARGET_VALUE), Some(name)) => Ok(PyString::new(value.py(), name).into_any()),
(Some(text), _)
if !text.is_empty()
&& text != PRESERVE_TARGET_VALUE
&& !holds_context_placeholder(text) =>
{
Ok(value)
}
_ => adapter_default_location(adapter, method),
};
Ok((
location(database, "default_database", None)?,
location(schema, "default_schema", kept_schema)?,
))
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
//! The schema a `preserve` target keeps Python node results in.
use crate::compile_driver::_helpers::setup::targets::planner_error;
use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult};
use sqlbuild_analysis::assembly::target_checks::constants::{
PRESERVE_TARGET_VALUE, PRESERVED_NODE_RESULTS_CODE,
};
use sqlbuild_analysis::assembly::target_checks::errors::TargetError;
use sqlbuild_analysis::assembly::target_checks::main::target_error_message::target_error_message;
use sqlbuild_core::text::main::python_strip::python_strip;
use sqlbuild_python_base::boundary::main::optional_text::optional_text;
use sqlbuild_python_base::records::main::fields::field::field as native_field;

/// The discovered inputs' Python node functions, each kind of which keeps node results.
const PYTHON_NODE_FIELDS: [&str; 4] = [
"loader_functions",
"task_functions",
"asset_functions",
"check_functions",
];

/// Raise when a `preserve` target has Python nodes but no loader schema to keep their results in.
pub(crate) fn require_node_results_schema(
discovered: &Bound<'_, PyAny>,
(target_name, target_config): (&Bound<'_, PyAny>, &Bound<'_, PyAny>),
) -> PyResult<()> {
let loader_schema: Option<String> =
optional_text(&native_field(target_config, "loader_schema")?)?
.map(|schema| python_strip(&schema).to_owned())
.filter(|schema| !schema.is_empty() && schema != PRESERVE_TARGET_VALUE);
if loader_schema.is_some() || python_node_count(discovered)? == 0 {
return Ok(());
}
let target: String = target_name.extract()?;
Err(planner_error(
discovered.py(),
target_error_message(&TargetError::PreservedNodeResults(target.clone())),
PRESERVED_NODE_RESULTS_CODE,
Some(format!(
"to keep Python node results in a schema, set this in sqlbuild_project.toml:\n \
[targets.{target}]\n \
loader_schema = \"my_schema\""
)),
))
}

/// How many loaders, tasks, assets and checks the project declares in Python.
fn python_node_count(discovered: &Bound<'_, PyAny>) -> PyResult<usize> {
PYTHON_NODE_FIELDS
.iter()
.map(|field| native_field(discovered, field)?.len())
.sum()
}
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ pub mod compile_project_phase;
pub mod compiled_phase;
pub mod expanded_loader_namespace;
pub mod managed_write_schema;
pub mod model_free_locations;
pub mod plan_has_executable_work;
pub mod preserved_loader_schemas;
pub mod progress;
Expand Down

This file was deleted.

Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
//! Raise when the selected target's schema is `preserve` and one of `loaders` names no schema.
//! Raise when the selected target's schema is `preserve` and a loader or Python node has no schema.

use crate::compile_attachment::main::resolved_target::resolved_target;
use crate::compile_driver::_helpers::setup::node_results::require_node_results_schema;
use crate::compile_driver::_helpers::setup::targets::planner_error;
use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python};
use sqlbuild_analysis::assembly::target_checks::constants::{
Expand All @@ -13,7 +14,7 @@ use sqlbuild_python_base::boundary::main::optional_text::optional_text;
use sqlbuild_python_base::records::main::fields::field::field as native_field;
use std::collections::HashSet;

/// Raise when the selected target's schema is `preserve` and one of `loaders` names no schema.
/// Raise when a `preserve` target leaves one of `loaders`, or the Python node results, without a schema.
pub fn preserved_loader_schemas(
discovered: &Bound<'_, PyAny>,
selected_target: &Bound<'_, PyAny>,
Expand Down Expand Up @@ -47,5 +48,5 @@ pub fn preserved_loader_schemas(
)),
));
}
Ok(())
require_node_results_schema(discovered, (&target_name, &target_config))
}
39 changes: 37 additions & 2 deletions crates/sqlbuild-python-compile/src/compile_driver/models.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
//! Public records of compile driver.

use crate::compile_driver::_helpers::setup::model_free_locations::model_free_locations;
use pyo3::IntoPyObjectExt;
use pyo3::prelude::Bound;
use pyo3::prelude::Py;
Expand Down Expand Up @@ -67,8 +68,8 @@ impl InputsRequest {
}
}

/// The active target's locations as SQLBuild resolves them; `preserve` stays as written until `record`.
#[derive(Debug, Clone)]
/// The active target's locations as SQLBuild resolves them (default: unset); `preserve` stays as written until `record`.
#[derive(Debug, Clone, Default)]
pub struct TargetSettingsValues {
pub(crate) name: Option<String>,
pub(crate) database: Option<String>,
Expand All @@ -77,6 +78,40 @@ pub struct TargetSettingsValues {
}

impl TargetSettingsValues {
/// The resolved schema managed loader data goes to.
#[must_use]
pub fn loader_schema(&self) -> Option<&str> {
self.loader_schema.as_deref()
}

/// Python nodes' `(database, schema)` under these settings, as `model_free_locations` picks.
pub fn node_locations<'py>(
&self,
adapter: &Bound<'py, PyAny>,
) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> {
let py: Python<'py> = adapter.py();
self.node_locations_from(
(
&self.database.as_deref().into_bound_py_any(py)?,
&self.schema.as_deref().into_bound_py_any(py)?,
),
adapter,
)
}

/// `node_locations` from a caller-resolved `(database, schema)`, such as `sqb load`'s, which applies CLI vars.
pub fn node_locations_from<'py>(
&self,
(database, schema): (&Bound<'py, PyAny>, &Bound<'py, PyAny>),
adapter: &Bound<'py, PyAny>,
) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> {
model_free_locations(
(database.clone(), schema.clone()),
self.loader_schema(),
adapter,
)
}

/// The `TargetSettings` record: a `preserve` location reads `None` with its flag set.
pub fn record<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyAny>> {
let located = |value: &Option<String>| -> PyResult<Bound<'py, PyAny>> {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get;
use sqlbuild_python_base::records::types::RecordId;
use sqlbuild_python_compile::compile_driver::main::setup::build_project_relation_targets::build_project_relation_targets;
use sqlbuild_python_compile::compile_driver::main::setup::project_target_settings::project_target_settings;
use sqlbuild_python_compile::compile_driver::models::TargetSettingsValues;
use sqlbuild_python_front::cost_context::models::CostContext;
use sqlbuild_python_front::cursor::main::overrides::cursor_integer::cursor_integer;
use sqlbuild_python_front::cursor::main::overrides::cursor_timestamp::cursor_timestamp;
Expand Down Expand Up @@ -333,6 +334,12 @@ pub(crate) fn execute_build_plan<'py>(
let lifecycle: &DirectPythonLifecycle = preparation.lifecycle.get();
let [start_ts, end_ts, start_int, end_int] = &preparation.cursors;
let request = |name: &str| native_field(&context.request, name);
let target_config: TargetSettingsValues = project_target_settings(
&context.get("discovered_inputs")?,
&project,
(&connection_config, &adapter),
)?;
let (default_database, default_schema) = target_config.node_locations(&adapter)?;
let runtime: Bound<'py, PyAny> = call_with(
&host_entry(py, ExecutorHost::BuildRuntimeParams)?,
&[
Expand Down Expand Up @@ -360,15 +367,9 @@ pub(crate) fn execute_build_plan<'py>(
preparation.concurrency.into_pyobject(py)?.into_any(),
),
("loader_is_reload", request("reload_sources")?),
(
"target_config",
project_target_settings(
&context.get("discovered_inputs")?,
&project,
(&connection_config, &adapter),
)?
.record(py)?,
),
("target_config", target_config.record(py)?),
("default_database", default_database),
("default_schema", default_schema),
("start_cursor_ts", start_ts.clone()),
("end_cursor_ts", end_ts.clone()),
("start_cursor_int", start_int.clone()),
Expand Down Expand Up @@ -633,6 +634,12 @@ fn check_runtime<'py>(
.iter()
.map(|check| native_field(check, "name"))
.collect::<PyResult<_>>()?;
let target_config: TargetSettingsValues = project_target_settings(
&context.get("discovered_inputs")?,
&project,
(&context.get("connection_config")?, &adapter),
)?;
let (default_database, default_schema) = target_config.node_locations(&adapter)?;
call_with(
&host_entry(py, ExecutorHost::PythonNodeRuntime)?,
&[
Expand All @@ -646,20 +653,9 @@ fn check_runtime<'py>(
"is_reload",
native_field(&context.request, "reload_sources")?,
),
(
"target_config",
project_target_settings(
&context.get("discovered_inputs")?,
&project,
(&context.get("connection_config")?, &adapter),
)?
.record(py)?,
),
(
"default_database",
adapter.call_method0("default_database")?,
),
("default_schema", adapter.call_method0("default_schema")?),
("target_config", target_config.record(py)?),
("default_database", default_database),
("default_schema", default_schema),
(
"relation_targets",
build_check_relation_targets(&adapter, pipeline, graph, frozenset(py, &names)?)?,
Expand Down
Loading
Loading