Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -6,14 +6,20 @@ use sqlbuild_analysis::assembly::target_checks::constants::PRESERVE_TARGET_VALUE
use sqlbuild_model_config::templates::main::holds_context_placeholder::holds_context_placeholder;
use sqlbuild_python_base::warehouse_access::main::adapter_default_location::adapter_default_location;

/// The resolved `loader_schema` a `preserve` schema keeps Python node results in, unless it is unusable.
pub(crate) fn kept_loader_schema(loader_schema: Option<&str>) -> Option<&str> {
loader_schema.filter(|name| {
!name.is_empty() && *name != PRESERVE_TARGET_VALUE && !holds_context_placeholder(name)
})
}

/// Python nodes' `(database, schema)`: a `preserve` schema reads `loader_schema`; unusable values the adapter's.
pub(crate) fn model_free_locations<'py>(
(database, schema): (Bound<'py, PyAny>, Bound<'py, PyAny>),
loader_schema: Option<&str>,
adapter: &Bound<'py, PyAny>,
) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> {
let kept_schema: Option<&str> =
loader_schema.filter(|name| !name.is_empty() && *name != PRESERVE_TARGET_VALUE);
let kept_schema: Option<&str> = kept_loader_schema(loader_schema);
let location = |value: Bound<'py, PyAny>, method: &str, preserved: Option<&str>| match (
value.extract::<Option<String>>()?.as_deref(),
preserved,
Expand Down
Original file line number Diff line number Diff line change
@@ -1,10 +1,12 @@
//! The schema a `preserve` target keeps Python node results in.

use crate::compile_driver::_helpers::setup::model_free_locations::kept_loader_schema;
use crate::compile_driver::_helpers::setup::targets::planner_error;
use crate::compile_driver::main::setup::expanded_loader_namespace::expanded_loader_namespace;
use pyo3::exceptions::PyException;
use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult};
use sqlbuild_analysis::assembly::target_checks::constants::{
PRESERVE_TARGET_VALUE, PRESERVED_NODE_RESULTS_CODE,
};
use pyo3::types::{PyDict, PyDictMethods};
use sqlbuild_analysis::assembly::target_checks::constants::PRESERVED_NODE_RESULTS_CODE;
use sqlbuild_analysis::assembly::target_checks::errors::TargetError;
use sqlbuild_analysis::assembly::target_checks::main::target_error_message::target_error_message;
use sqlbuild_core::text::main::python_strip::python_strip;
Expand All @@ -19,16 +21,15 @@ const PYTHON_NODE_FIELDS: [&str; 4] = [
"check_functions",
];

/// Raise when a `preserve` target has Python nodes but no loader schema to keep their results in.
/// Raise when a `preserve` target has Python nodes but no usable loader schema, after `effective_vars`, to keep their results in.
pub(crate) fn require_node_results_schema(
discovered: &Bound<'_, PyAny>,
(target_name, target_config): (&Bound<'_, PyAny>, &Bound<'_, PyAny>),
effective_vars: &Bound<'_, PyAny>,
) -> PyResult<()> {
let loader_schema: Option<String> =
optional_text(&native_field(target_config, "loader_schema")?)?
.map(|schema| python_strip(&schema).to_owned())
.filter(|schema| !schema.is_empty() && schema != PRESERVE_TARGET_VALUE);
if loader_schema.is_some() || python_node_count(discovered)? == 0 {
let loader_schema: Option<String> = resolved_loader_schema(target_config, effective_vars)?;
if kept_loader_schema(loader_schema.as_deref()).is_some() || python_node_count(discovered)? == 0
{
return Ok(());
}
let target: String = target_name.extract()?;
Expand All @@ -44,8 +45,31 @@ pub(crate) fn require_node_results_schema(
))
}

/// The target's `loader_schema` as runtime resolves it: vars expanded, `None` when it cannot resolve outside a model.
fn resolved_loader_schema(
target_config: &Bound<'_, PyAny>,
effective_vars: &Bound<'_, PyAny>,
) -> PyResult<Option<String>> {
let Some(schema) = optional_text(&native_field(target_config, "loader_schema")?)? else {
return Ok(None);
};
let py = target_config.py();
let namespace: Bound<'_, PyDict> = PyDict::new(py);
namespace.set_item("schema", python_strip(&schema))?;
match expanded_loader_namespace(namespace.as_any(), effective_vars) {
Ok(expanded) => {
let schema: Bound<'_, PyAny> = expanded.get_item("schema")?;
Ok((!schema.is_none())
.then(|| schema.str()?.extract())
.transpose()?)
}
Err(error) if error.is_instance_of::<PyException>(py) => Ok(None),
Err(error) => Err(error),
}
}

/// How many loaders, tasks, assets and checks the project declares in Python.
fn python_node_count(discovered: &Bound<'_, PyAny>) -> PyResult<usize> {
pub(crate) fn python_node_count(discovered: &Bound<'_, PyAny>) -> PyResult<usize> {
PYTHON_NODE_FIELDS
.iter()
.map(|field| native_field(discovered, field)?.len())
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -205,7 +205,10 @@ fn targeted_project<'py>(
}
preserved_loader_schemas(
discovered,
request.selected_target.bind(py),
(
request.selected_target.bind(py),
&native_field(project, "effective_vars")?,
),
&intermediate_loaders,
)?;
managed_write_schemas(adapter, project)?;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ pub mod preserved_loader_schemas;
pub mod progress;
pub mod project_graph;
pub mod project_target_settings;
pub mod python_node_count;
pub mod python_read_source_names_for;
pub mod register_inputs;
pub mod register_pipeline;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,10 @@ use sqlbuild_python_base::boundary::main::optional_text::optional_text;
use sqlbuild_python_base::records::main::fields::field::field as native_field;
use std::collections::HashSet;

/// Raise when a `preserve` target leaves one of `loaders`, or the Python node results, without a schema.
/// Raise when a `preserve` target leaves `loaders` or Python node results without a schema, after `effective_vars`.
pub fn preserved_loader_schemas(
discovered: &Bound<'_, PyAny>,
selected_target: &Bound<'_, PyAny>,
(selected_target, effective_vars): (&Bound<'_, PyAny>, &Bound<'_, PyAny>),
loaders: &HashSet<String>,
) -> PyResult<()> {
let py: Python<'_> = discovered.py();
Expand Down Expand Up @@ -48,5 +48,5 @@ pub fn preserved_loader_schemas(
)),
));
}
require_node_results_schema(discovered, (&target_name, &target_config))
require_node_results_schema(discovered, (&target_name, &target_config), effective_vars)
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
//! How many Python nodes a project declares.

use crate::compile_driver::_helpers::setup::node_results;
use pyo3::prelude::{Bound, PyAny, PyResult};

/// How many loaders, tasks, assets and checks the project declares in Python.
pub fn python_node_count(discovered: &Bound<'_, PyAny>) -> PyResult<usize> {
node_results::python_node_count(discovered)
}
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,12 @@ pub fn build_execution_plan<'py>(
"deferral",
"policies",
],
&["on_progress", "project_config", "local_config"],
&[
"on_progress",
"project_config",
"local_config",
"node_results_location",
],
),
)?;
let project = args.value("project")?;
Expand All @@ -54,6 +59,7 @@ pub fn build_execution_plan<'py>(
local_config: args.value("local_config")?.unbind(),
on_progress: args.value("on_progress")?.unbind(),
invocation_time: utc_now(py)?.unbind(),
node_results_location: args.value("node_results_location")?.extract()?,
},
)?
.into_any();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ pub(crate) fn read_planner_runtime<'py>(
"local_config" => (&this.local_config).into_bound_py_any(py),
"on_progress" => (&this.on_progress).into_bound_py_any(py),
"invocation_time" => (&this.invocation_time).into_bound_py_any(py),
"node_results_location" => (&this.node_results_location).into_bound_py_any(py),
_ => return None,
})
}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
//! The planner runtime and warehouse state: built from keyword arguments and read by planner phases.

use crate::planner_warehouse::models::MigrationFingerprintCache;
use pyo3::prelude::{Bound, Py, PyAny, PyResult, Python};
use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python};
use pyo3::pymethods;
use pyo3::types::{PyDict, PyTuple};

Expand All @@ -25,6 +25,7 @@ impl PlannerRuntime {
"local_config",
"on_progress",
"invocation_time",
"node_results_location",
],
),
)?;
Expand All @@ -41,6 +42,7 @@ impl PlannerRuntime {
local_config: args.value("local_config")?.unbind(),
on_progress: args.value("on_progress")?.unbind(),
invocation_time: invocation_time.unbind(),
node_results_location: args.value("node_results_location")?.extract()?,
})
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ use crate::planner_migrations::models::{ColumnMigrationInputs, MigrationPhase};
use crate::planner_phases::_helpers::inputs::contexts::class_field;
use crate::planner_phases::_helpers::inputs::runtime::runtime_field;
use crate::planner_phases::main::rebuilds::effectively_full_refreshed_model_names::effectively_full_refreshed_model_names;
use crate::planner_phases::models::PlannerWarehouseState;
use crate::planner_phases::models::{PlannerRuntime, PlannerWarehouseState};
use crate::planner_warehouse::main::inspection::warehouse_snapshot::warehouse_snapshot;
use crate::planner_warehouse::models::MigrationFingerprintCache;
use crate::planner_warehouse::models::SnapshotRequest;
Expand Down Expand Up @@ -104,6 +104,11 @@ pub(crate) fn gather_planner_warehouse_state<'py>(
on_progress: runtime_field(runtime, |runtime| &runtime.on_progress)?,
deferred_locations: native_field(deferral, "deferred_locations")?,
cursor_scope,
node_results_location: runtime
.cast::<PlannerRuntime>()?
.get()
.node_results_location
.clone(),
})?;
let phase = MigrationPhase {
runtime: runtime.clone(),
Expand Down
2 changes: 2 additions & 0 deletions crates/sqlbuild-python-planner/src/planner_phases/models.rs
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,8 @@ pub struct PlannerRuntime {
pub local_config: Py<PyAny>,
pub on_progress: Py<PyAny>,
pub invocation_time: Py<PyAny>,
/// Python nodes' `(database, schema)`, where their results and fingerprints live; `None` when unknown.
pub node_results_location: Option<(Option<String>, Option<String>)>,
}

/// Warehouse snapshot, inspection relations and migration plans gathered once per plan.
Expand Down
Loading
Loading